Line data Source code
1 : // This file (periodic_cmd_stats.cc) was created by Ron Rechenmacher <ron@fnal.gov> on
2 : // Jan 5, 2017. "TERMS AND CONDITIONS" governing this file are in the README
3 : // or COPYING file. If you do not have such a file, one can be obtained by
4 : // contacting Ron or Fermi Lab in Batavia IL, 60510, phone: 630-840-3000.
5 : // $RCSfile: periodic_cmd_stats.cc,v $
6 : static const char *rev = "$Revision: 1.19 $$Date: 2018/06/28 21:14:28 $";
7 :
8 : // make periodic_cmd_stats
9 : // OR make periodic_cmd_stats CXX="g++ -std=c++0x -DDO_TRACE"
10 :
11 : #define USAGE \
12 : "\
13 : usage: %s --cmd=<cmd>\n\
14 : examples: %s --cmd='sleep 25' # this will take about a minute\n\
15 : %s --cmd='taskset -c 4 dd if=/dev/zero of=/dev/sdf bs=50M count=500 oflag=direct' --disk=sdf\n\
16 : %s --cmd='dd if=/dev/zero of=t.dat count=500' --stat='md*MB/s?/proc/diskstats?/md[0-3]/?$10?(1.0/2048)?yes'\n\
17 : gnuplot -e png=0 `/bin/ls -t periodic_*_stats.out|head -1`\n\
18 : For each cmd, record CPU info. Additionally, record total system CPU\n\
19 : (can be >100 w/ multicore), file system Cached and Dirty %%, plus any\n\
20 : other stats specified by --stats options\n\
21 : options:\n\
22 : --cmd= can have multiple\n\
23 : --Cmd= run cmd, but dont graph CPU (can have multiple)\n\
24 : --pid= graph comma or space sep. list of pids (getting cmd from /proc/)\n\
25 : --disk= automatically build --stat for disk(s)\n\
26 : --stat= desc?file?linespec?fieldspec?multiplier?rate\n\
27 : builtin = CPUnode,Cached,Dirty\n\
28 : cmd-builtin = CPUcmdN, CPU+cmdN\n\
29 : --out-dir= output dir (for 2+ output files; stats and cmd+)\n\
30 : \n\
31 : --period= (float) default=%s\n\
32 : --pre= the number of periods wait before exec of --cmd\n\
33 : --post= the number of periods to loop after cmd exits\n\
34 : --graph= add to graph any possibly included non-graphed metrics\n\
35 : \n\
36 : --no-defaults for no default stats (ie. cpu-parent,cpu-children)\n\
37 : --init= default dropcache if root. NOT implemented yet\n\
38 : --duration=\n\
39 : --comment=\n\
40 : --yrange=400:1400\n\
41 : --y2max=\n\
42 : --y2incr=\n\
43 : --pause=0\n\
44 : --sys-iowait,-w include the system iowait on the graph\n\
45 : --cmd-iowait include cmd iowait on the graph\n\
46 : --fault,-f\n\
47 : ", \
48 : basename(argv[0]), basename(argv[0]), basename(argv[0]), basename(argv[0]), opt_period
49 :
50 : enum
51 : {
52 : s_desc,
53 : s_file,
54 : s_linespec,
55 : s_fieldspec,
56 : s_multiplier,
57 : s_rate
58 : };
59 :
60 : #include <fcntl.h> // O_WRONLY|O_CREAT, open
61 : #include <getopt.h> // getopt_long, {no,required,optional}_argument, extern char *optarg; extern int opt{ind,err,opt}
62 : #include <sys/time.h> /* gettimeofday, timeval */
63 : #include <sys/utsname.h> // uname
64 : #include <sys/wait.h> // wait
65 : #include <unistd.h> // getpid, sysconf
66 : #include <csignal> /* sigaction, siginfo_t, sigset_t */
67 : #include <cstdio> // printf
68 : #include <fstream> // std::ifstream
69 : #include <sstream> // std::stringstream
70 : #include <string>
71 : #include <thread>
72 : #include <vector>
73 :
74 : #ifdef DO_TRACE
75 : #define TRACE_NAME "periodic_cmd_stats"
76 : #include "trace.h"
77 : #else
78 : #include <cstdarg> /* va_list */
79 : #include <cstring> /* memcpy */
80 : #include <string>
81 0 : static void trace_ap(const char *msg, va_list ap)
82 : {
83 : char m_[1024];
84 0 : unsigned len = strlen(msg);
85 0 : len = len < (sizeof(m_) - 2) ? len : (sizeof(m_) - 2);
86 0 : memcpy(m_, msg, len + 1);
87 0 : if (m_[len - 1] != '\n')
88 : {
89 0 : memcpy(&(m_[len]), "\n", 2);
90 : }
91 0 : vprintf(m_, ap);
92 0 : va_end(ap);
93 0 : }
94 : static void trace_p(const char *msg, ...) __attribute__((format(printf, 1, 2)));
95 0 : static void trace_p(const char *msg, ...)
96 : {
97 : va_list ap;
98 0 : va_start(ap, msg); // NOLINT
99 0 : trace_ap(msg, ap); // NOLINT
100 0 : va_end(ap); // NOLINT
101 0 : }
102 0 : static void trace_p(const std::string msg, ...)
103 : {
104 : va_list ap;
105 0 : va_start(ap, msg);
106 0 : trace_ap(msg.c_str(), ap);
107 0 : va_end(ap); // NOLINT
108 0 : }
109 : enum
110 : {
111 : TLVL_ERROR,
112 : TLVL_WARNING,
113 : TLVL_INFO,
114 : TLVL_DEBUG
115 : };
116 : #define TRACE(lvl, ...) \
117 : do \
118 : if (lvl <= TLVL_WARNING) trace_p(__VA_ARGS__); \
119 : while (0)
120 : #define TRACE_CNTL(xyzz, ...)
121 : #endif
122 :
123 : /* GLOBALS */
124 : int opt_v = 1;
125 : char *opt_init = nullptr;
126 : std::vector<std::string> opt_cmd;
127 : std::vector<std::string> opt_Cmd;
128 : std::string opt_pid;
129 : std::string opt_disk;
130 : std::string opt_stats;
131 : std::string opt_outdir;
132 : std::string opt_graph("CPUnode,Cached,Dirty,Free"); // CPU+ will always be graphed
133 : const char *opt_period = "5.0";
134 : std::string opt_comment;
135 : int opt_pre = 6; // number of periods to sleepB4exec
136 : int opt_post = 6;
137 : int opt_ymin = 0;
138 : int opt_ymax = 2000;
139 : int opt_yincr = 200;
140 : int opt_y2max = 200;
141 : int opt_y2incr = 20;
142 : int opt_sys_iowait = 0;
143 : int opt_cmd_iowait = 0;
144 : int opt_fault = 0;
145 :
146 : std::vector<pid_t> g_pid_vec;
147 :
148 0 : void charreplace(char *instr, char oldc, char newc)
149 : {
150 0 : while (*instr != 0)
151 : {
152 0 : if (*instr == oldc)
153 : {
154 0 : *instr = newc;
155 : }
156 0 : ++instr;
157 : }
158 0 : }
159 :
160 0 : void parse_args(int argc, char *argv[])
161 : {
162 : char *cp;
163 : // parse opt, optargs, and args
164 : while (true)
165 : {
166 : int opt;
167 : static struct option long_options[] = {
168 : // name has_arg *flag val
169 : {"help", no_argument, nullptr, 'h'},
170 : {"init", required_argument, nullptr, 'i'},
171 : {"cmd", required_argument, nullptr, 'c'},
172 : {"Cmd", required_argument, nullptr, 'C'},
173 : {"disk", required_argument, nullptr, 'd'},
174 : {"stat", required_argument, nullptr, 's'},
175 : {"out-dir", required_argument, nullptr, 'o'},
176 : {"period", required_argument, nullptr, 'p'},
177 : {"sys-iowait", no_argument, nullptr, 'w'},
178 : {"fault", no_argument, nullptr, 'f'},
179 : {"pid", required_argument, nullptr, 'P'},
180 : {"ymax", required_argument, nullptr, 1},
181 : {"yincr", required_argument, nullptr, 2},
182 : {"y2max", required_argument, nullptr, 3},
183 : {"y2incr", required_argument, nullptr, 4},
184 : {"pre", required_argument, nullptr, 5},
185 : {"post", required_argument, nullptr, 6},
186 : {"graph", required_argument, nullptr, 7},
187 : {"yrange", required_argument, nullptr, 8},
188 : {"comment", required_argument, nullptr, 9},
189 : {"cmd-iowait", no_argument, nullptr, 10},
190 : {nullptr, 0, nullptr, 0}};
191 0 : opt = getopt_long(argc, argv, "?hvqVi:c:C:d:s:o:p:P:wf",
192 : long_options, nullptr);
193 0 : if (opt == -1)
194 : {
195 0 : break;
196 : }
197 0 : switch (opt)
198 : {
199 0 : case '?':
200 : case 'h':
201 0 : printf(USAGE);
202 0 : exit(0);
203 : break;
204 0 : case 'V':
205 0 : printf("%s\n", rev);
206 0 : exit(0);
207 : break;
208 0 : case 'v':
209 0 : ++opt_v;
210 0 : break;
211 0 : case 'q':
212 0 : --opt_v;
213 0 : break;
214 0 : case 'i':
215 0 : opt_init = optarg;
216 0 : break;
217 0 : case 'c':
218 0 : opt_cmd.emplace_back(optarg);
219 0 : break;
220 0 : case 'C':
221 0 : opt_Cmd.emplace_back(optarg);
222 0 : break;
223 0 : case 'd':
224 0 : if (!opt_disk.empty())
225 : {
226 0 : opt_disk = opt_disk + "," + optarg;
227 : }
228 : else
229 : {
230 0 : opt_disk = optarg;
231 : }
232 0 : break;
233 0 : case 's':
234 0 : if (!opt_stats.empty())
235 : {
236 0 : opt_stats += std::string(",") + optarg;
237 : }
238 : else
239 : {
240 0 : opt_stats = optarg;
241 : }
242 0 : break;
243 0 : case 'o':
244 0 : opt_outdir = std::string(optarg) + "/";
245 0 : break;
246 0 : case 'p':
247 0 : opt_period = optarg;
248 0 : break;
249 0 : case 'w':
250 0 : opt_sys_iowait = 1;
251 0 : break;
252 0 : case 'f':
253 0 : opt_fault = 1;
254 0 : break;
255 0 : case 'P':
256 0 : charreplace(optarg, ' ', ',');
257 0 : if (!opt_pid.empty() != 0u)
258 : {
259 0 : opt_pid = opt_pid + "," + optarg;
260 : }
261 : else
262 : {
263 0 : opt_pid = optarg;
264 : }
265 0 : break;
266 0 : case 1:
267 0 : opt_ymax = strtoul(optarg, nullptr, 0);
268 0 : break;
269 0 : case 2:
270 0 : opt_yincr = strtoul(optarg, nullptr, 0);
271 0 : break;
272 0 : case 3:
273 0 : opt_y2max = strtoul(optarg, nullptr, 0);
274 0 : break;
275 0 : case 4:
276 0 : opt_y2incr = strtoul(optarg, nullptr, 0);
277 0 : break;
278 0 : case 5:
279 0 : opt_pre = strtoul(optarg, nullptr, 0);
280 0 : break;
281 0 : case 6:
282 0 : opt_post = strtoul(optarg, nullptr, 0);
283 0 : break;
284 0 : case 7:
285 0 : opt_graph += std::string(",") + optarg;
286 0 : break;
287 0 : case 8:
288 0 : opt_ymin = strtoul(optarg, nullptr, 0);
289 0 : cp = strstr(optarg, ":") + 1;
290 0 : opt_ymax = strtoul(cp, nullptr, 0);
291 0 : if ((cp = strstr(cp, ":")) != nullptr)
292 : {
293 0 : ++cp;
294 0 : opt_yincr = strtoul(strstr(cp, ":") + 1, nullptr, 0);
295 : }
296 : else
297 : {
298 0 : opt_yincr = (opt_ymax - opt_ymin) / 5;
299 : }
300 0 : break;
301 0 : case 9:
302 0 : opt_comment = optarg;
303 0 : break;
304 0 : case 10:
305 0 : opt_cmd_iowait = 1;
306 0 : break;
307 0 : default:
308 0 : printf("?? getopt returned character code 0%o ??\n", opt);
309 0 : exit(1);
310 : }
311 0 : }
312 0 : } /* parse_args */
313 :
314 0 : void perror_exit(const char *msg, ...)
315 : {
316 : char buf[1024];
317 : va_list ap;
318 0 : va_start(ap, msg);
319 0 : vsnprintf(buf, sizeof(buf), msg, ap);
320 0 : va_end(ap);
321 0 : TRACE(TLVL_ERROR, "%s", buf);
322 0 : perror(buf);
323 0 : exit(1);
324 : }
325 :
326 : // void atfork_trace(void) { TRACE( 3, "process %d forking", getpid() ); }
327 : /* iofd is in/out
328 : if iofd[x]==-1 then create a pipe for that index, x, and return the appropriate pipe fd in iofd[x]
329 : else if iofd[x]!=x, dup2(iofd[x],x)
330 : else inherit
331 : Could add ==-2, then close???
332 : */
333 0 : pid_t fork_execv(int close_start, int close_cnt, int sleepB4exec_us, int iofd[3], const char *cmd, char *const argv[], char *const env[])
334 : {
335 : int pipes[3][2];
336 : int lcl_iofd[3];
337 0 : for (auto ii = 0; ii < 3; ++ii)
338 : {
339 0 : lcl_iofd[ii] = iofd[ii];
340 0 : if (iofd[ii] == -1)
341 : {
342 0 : pipe(pipes[ii]); /* pipes[ii][0] refers to the read end */
343 0 : iofd[ii] = ii == 0 ? pipes[ii][1] : pipes[ii][0];
344 : }
345 : }
346 0 : pid_t pid = fork();
347 0 : if (pid < 0)
348 : {
349 0 : perror_exit("fork");
350 : }
351 0 : else if (pid == 0)
352 : { /* child */
353 0 : if (lcl_iofd[0] == -1)
354 : { // deal with child stdin
355 0 : close(pipes[0][1]); // child closes write end of pipe which will be it's stdin
356 0 : int fd = dup2(pipes[0][0], 0); // NOLINT
357 : TRACE(TLVL_DEBUG + 0, "fork_execv dupped(%d) onto %d (should be 0)", pipes[0][0], fd);
358 0 : close(pipes[0][0]);
359 : }
360 0 : if (sleepB4exec_us != 0)
361 : {
362 : // Do sleep before dealing with stdout/err incase we want TRACE to go to console
363 : // int sts=pthread_atfork( atfork_trace, NULL, NULL );
364 0 : usleep(sleepB4exec_us);
365 0 : TRACE(TLVL_WARNING, "fork_execv sleep complete. sleepB4exec_us=%d sts=%d", sleepB4exec_us, 0 /*sts*/);
366 : }
367 0 : for (auto ii = 1; ii < 3; ++ii)
368 : { // deal with child stdout/err
369 0 : if (lcl_iofd[ii] == -1)
370 : {
371 0 : close(pipes[ii][0]);
372 0 : int fd = dup2(pipes[ii][1], ii); // NOLINT
373 : TRACE(TLVL_DEBUG + 0, "fork_execv dupped(%d) onto %d (should be %d)", pipes[ii][1], fd, ii);
374 0 : close(pipes[ii][1]);
375 : }
376 0 : else if (lcl_iofd[ii] != ii)
377 : {
378 0 : int fd = dup2(lcl_iofd[ii], ii); // NOLINT
379 : TRACE(TLVL_DEBUG + 0, "fork_execv dupped(%d) onto %d (should be %d)", pipes[ii][1], fd, ii);
380 : }
381 : }
382 0 : for (auto ii = close_start; ii < (close_start + close_cnt); ++ii)
383 : {
384 0 : close(ii);
385 : }
386 0 : if (env != nullptr)
387 : {
388 0 : execve(cmd, argv, env);
389 : }
390 : else
391 : {
392 0 : execv(cmd, argv);
393 : }
394 0 : exit(1);
395 : }
396 : else
397 : { // parent
398 0 : for (auto ii = 0; ii < 3; ++ii)
399 : {
400 0 : if (lcl_iofd[ii] == -1)
401 : {
402 0 : close(ii == 0 ? pipes[ii][0] : pipes[ii][1]);
403 : }
404 : }
405 : }
406 :
407 : TRACE(TLVL_DEBUG + 0, "fork_execv pid=%d", pid);
408 0 : return pid;
409 : } // fork_execv
410 :
411 0 : uint64_t swapPtr(void *X)
412 : {
413 0 : auto x = (uint64_t)X;
414 0 : x = (x & 0x00000000ffffffff) << 32 | (x & 0xffffffff00000000) >> 32;
415 0 : x = (x & 0x0000ffff0000ffff) << 16 | (x & 0xfff0000fffff0000) >> 16;
416 0 : x = (x & 0x00ff00ff00ff00ff) << 8 | (x & 0xff00ff00ff00ff00) >> 8;
417 0 : return x;
418 : }
419 :
420 : /*
421 : * Input to AWK can either be a file spec or a string.
422 : * If input is string, the fork_execv call is told to create pipe for input.
423 : *
424 : * The run time duration of the AWK prooces can be determined via TRACE:
425 : /home/ron/src
426 : mu2edaq01 :^) tshow|egrep 'AWK b4 |AWK after read' |tdelta -d 1 -post /b4/ -stats | tail
427 : 1013 1489724640538688 2047 1116418481 13521 0 6 3 . AWK b4 fork_execv input=(nil)
428 : 1018 1489724640536624 1969 1111669678 13521 0 6 3 . AWK b4 fork_execv input=(nil)
429 : 1023 1489724640534717 1866 1107283893 13521 0 6 3 . AWK b4 fork_execv input=(nil)
430 : 1032 1489724640531756 2289 1100474359 13521 0 13 3 . AWK b4 fork_execv input=(nil)
431 : cpu="0"
432 : min 1821
433 : max 49210
434 : tot 293610
435 : ave 2645.1351
436 : cnt 111
437 : --2017-03-17_08:13:23--
438 : */
439 : static int g_devnullfd = -1;
440 :
441 : // Run the awk script specified in awk_cmd on the file
442 0 : std::string AWK(std::string const &awk_cmd, const char *file, const char *input)
443 : {
444 : char readbuf[1024];
445 0 : ssize_t bytes = 0, tot_bytes = 0;
446 0 : char *const argv_[4] = {(char *)"/bin/gawk",
447 0 : (char *)awk_cmd.c_str(),
448 : (char *)file,
449 0 : nullptr};
450 : pid_t pid;
451 : ;
452 0 : int infd = 0;
453 0 : if (g_devnullfd == -1)
454 : {
455 0 : g_devnullfd = open("/dev/null", O_WRONLY);
456 : }
457 0 : if (input != nullptr)
458 : {
459 0 : infd = -1;
460 : }
461 : // int iofd[3]={infd,-1,g_devnullfd};
462 0 : int iofd[3] = {infd, -1, 2}; // make stdin=infd, create pipr for stdout, inherit stderr
463 : TRACE(TLVL_DEBUG + 0, "AWK b4 fork_execv input=%p", (void *)input);
464 : char *env[1];
465 0 : env[0] = nullptr; // mainly do not want big LD_LIBRARY_PATH
466 0 : pid = fork_execv(0, 0 /*closeCnt*/, 0, iofd, "/bin/gawk", argv_, env); // NOLINT
467 0 : if (input != nullptr /*||iofd[0]!=0*/)
468 : {
469 0 : int xx = strlen(input);
470 0 : int sts = write(iofd[0], input, xx);
471 0 : if (sts != xx)
472 : {
473 0 : perror("write AWK stdin");
474 : }
475 0 : close(iofd[0]);
476 0 : while ((bytes = read(iofd[1], &readbuf[tot_bytes], sizeof(readbuf) - tot_bytes)) != 0)
477 : {
478 : TRACE(TLVL_DEBUG + 0, "AWK while bytes=read > 0 bytes=%zd readbuf=0x%016lx errno=%d", bytes, swapPtr(&readbuf[tot_bytes]), errno);
479 0 : if (bytes == -1)
480 : {
481 0 : if (errno == EINTR)
482 : {
483 0 : continue;
484 : }
485 0 : break;
486 : }
487 0 : tot_bytes += bytes;
488 : }
489 : TRACE(TLVL_DEBUG + 0, "AWK after read tot=" + std::to_string((long long unsigned)tot_bytes) + " bytes=" + std::to_string((long long unsigned)bytes) + " input=" + std::string(input));
490 : }
491 : else
492 : {
493 0 : while ((bytes = read(iofd[1], &readbuf[tot_bytes], sizeof(readbuf) - tot_bytes)) > 0)
494 : {
495 0 : tot_bytes += bytes;
496 : }
497 : TRACE(TLVL_DEBUG + 0, "AWK after read tot=%zd bytes=%zd [0]=0x%x input=%p", tot_bytes, bytes, readbuf[0], (void *)input);
498 : }
499 0 : readbuf[tot_bytes >= 0 ? tot_bytes : 0] = '\0';
500 0 : close(iofd[1]);
501 : TRACE(TLVL_DEBUG + 0, "AWK after close child stdout. child pid=%d", pid);
502 : #if 0
503 : int status;
504 : pid_t done_pid = waitpid(pid,&status,0);
505 : TRACE( 3, "AWK after wait pid=%d done_pid=%d status=%d(0x%x)"
506 : , pid, done_pid, status, status );
507 : #endif
508 0 : return std::string(readbuf);
509 : } // AWK
510 :
511 : // separate string and _add_to_ vector
512 0 : void string_addto_vector(std::string &instr, std::vector<std::string> &outvec, char delim)
513 : {
514 0 : std::stringstream ss(instr);
515 0 : while (ss.good())
516 : {
517 0 : std::string substr;
518 0 : std::getline(ss, substr, delim);
519 0 : outvec.push_back(substr);
520 0 : }
521 0 : }
522 :
523 0 : uint64_t gettimeofday_us() // struct timespec *ts )
524 : {
525 : struct timeval tv;
526 0 : gettimeofday(&tv, nullptr);
527 : // if (ts) {
528 : // ts->tv_sec = tv.tv_sec;
529 : // ts->tv_nsec = tv.tv_usec * 1000;
530 : // }
531 0 : return (uint64_t)tv.tv_sec * 1000000 + tv.tv_usec;
532 : } /* gettimeofday_us */
533 :
534 : #define DATA_START " DATA START"
535 : #define GNUPLOT_PREFIX (const char *) \
536 : "\
537 : #!/usr/bin/env gnuplot\n\
538 : # ./$0\n\
539 : # OR\n\
540 : # gnuplot -e 'ymin=400;ymax=1400' ./$0\n\
541 : # OR try\n\
542 : # gnuplot -e 'duration_s=35;set multiplot' ./gnuplot.gnuplot ./gnuplot.1.gnuplot -e 'set nomultiplot;pause -1'\n\
543 : if(!exists('ARG0')) ARG0='' # for version 4, use: gnuplot -e ARG0=hello\n\
544 : print 'ARG0=',ARG0 # ARG0.. automatically define in gnuplot version 5+\n\
545 : if(!exists('ymin')) ymin=%d\n\
546 : if(!exists('ymax')) ymax=%d\n\
547 : if(!exists('yincr')) yincr=%d\n\
548 : if(!exists('y2max')) y2max=%d\n\
549 : if(!exists('y2incr')) y2incr=%d\n\
550 : if(!exists('png')) png=1\n\
551 : if(!exists('duration_s')) duration_s=0\n\
552 : if(!exists('width')) width=512\n\
553 : if(!exists('height')) height=384\n\
554 : thisPid=system('echo `ps -p$$ -oppid=`')\n\
555 : thisFile=system('ls -l /proc/'.thisPid.\"/fd | grep -v pipe: | tail -1 | sed -e 's/.*-> //'\")\n\
556 : \n\
557 : set title \"Disk Write Rate and %%CPU vs. time\\n%s %s %s%s\" # cmd and/or comment at end\n\
558 : set xdata time\n\
559 : tfmt='%%Y-%%m-%%dT%%H:%%M:%%S' # try to use consistent format\n\
560 : set timefmt '%%Y-%%m-%%dT%%H:%%M:%%S'\n\
561 : set xlabel 'time'\n\
562 : set grid xtics back\n\
563 : xstart=system(\"awk '/^....-..-..T/{print$1;exit}' \".thisFile)\n\
564 : xend=system(\"awk 'END{print$1}' \".thisFile)\n\
565 : print 'xstart='.xstart.' xend='.xend.' duration=',strptime(tfmt,xend)-strptime(tfmt,xstart)\n\
566 : if(duration_s>0) end_t=strptime(tfmt,xstart)+duration_s; else end_t=strptime(tfmt,xend)\n\
567 : set xrange [xstart:end_t]\n\
568 : \n\
569 : set ylabel '%s'\n\
570 : set ytics nomirror\n\
571 : if(ymax==0) set yrange [ymin:*];\\\n\
572 : else set yrange [ymin:ymax];set ytics yincr\n\
573 : set grid ytics back\n\
574 : \n\
575 : set y2label '%%CPU, %%MemTotal'\n\
576 : set y2tics autofreq\n\
577 : if(y2max==0) set y2range [0:*];\\\n\
578 : else set y2range [0:y2max];set y2tics y2incr\n\
579 : set pointsize .6\n\
580 : \n\
581 : if(png==1) set terminal png size width,height;\\\n\
582 : pngfile=system( 'echo `basename '.thisFile.' .out`.png' );\\\n\
583 : set output pngfile;\\\n\
584 : else set terminal x11 size width,height\n\
585 : \n\
586 : plot \"< awk '/^#" DATA_START "/,/NEVER HAPPENS/' \".thisFile "
587 :
588 0 : void sigchld_sigaction(int signo, siginfo_t *info, void *context __attribute__((__unused__)))
589 : {
590 : /* see man sigaction for description of siginfo_t */
591 0 : for (int pid : g_pid_vec)
592 : {
593 0 : if (pid == info->si_pid)
594 : {
595 : TRACE(TLVL_INFO, "sigchld_sigaction signo=%d status=%d(0x%x) code=%d(0x%x) sending_pid=%d", signo, info->si_status, info->si_status, info->si_code, info->si_code, info->si_pid);
596 0 : return;
597 : }
598 : }
599 : TRACE(TLVL_DEBUG + 0, "sigchld_sigaction signo=%d status=%d(0x%x) code=%d(0x%x) sending_pid=%d", signo, info->si_status, info->si_status, info->si_code, info->si_code, info->si_pid);
600 : }
601 :
602 0 : void read_proc_file(const char *file, char *buffer, int buffer_size)
603 : {
604 : TRACE(TLVL_DEBUG + 1, "read_proc_file b4 open proc file" + std::string(file));
605 0 : int fd = open(file, O_RDONLY);
606 0 : int offset = 0, sts = 0;
607 : while (true)
608 : {
609 0 : sts = read(fd, &buffer[offset], buffer_size - offset);
610 0 : if (sts <= 0)
611 : {
612 0 : sts = 0;
613 0 : break;
614 : }
615 0 : offset += sts;
616 : }
617 0 : buffer[sts + offset] = '\0';
618 0 : close(fd);
619 : TRACE(TLVL_DEBUG + 1, "read_proc_file after close " + std::string(file) + " read=%d offset=%d", sts, offset);
620 0 : }
621 :
622 0 : pid_t check_pid_vec()
623 : {
624 0 : for (size_t ii = 0; ii < g_pid_vec.size();)
625 : {
626 0 : pid_t pid = g_pid_vec[ii];
627 : int status;
628 0 : pid_t pp = waitpid(pid, &status, WNOHANG);
629 : TRACE(TLVL_DEBUG + 0, "check_pid_vec %d=waitpid(pid=%d) errno=%d", pp, pid, errno);
630 0 : if (pp > 0)
631 : {
632 0 : g_pid_vec.erase(g_pid_vec.begin() + ii);
633 : }
634 0 : else if (pp == -1)
635 : {
636 0 : if (errno == ECHILD && kill(pid, 0) == 0)
637 : {
638 : // there is a process, but not my child process
639 0 : ++ii;
640 : }
641 : else
642 : {
643 : // some other error
644 0 : g_pid_vec.erase(g_pid_vec.begin() + ii);
645 : }
646 : }
647 : else
648 : {
649 0 : ++ii;
650 : }
651 : }
652 0 : if (g_pid_vec.empty())
653 : {
654 0 : return -1;
655 : }
656 : {
657 0 : return 0;
658 : }
659 : }
660 :
661 0 : void cleanup()
662 : {
663 0 : TRACE(TLVL_WARNING, "atexit cleanup g_pid_vec.size()=%zd\n", g_pid_vec.size());
664 0 : for (int &pid : g_pid_vec)
665 : {
666 0 : kill(pid, SIGHUP);
667 : }
668 0 : }
669 : #if (defined(__cplusplus) && (__cplusplus >= 201103L)) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L))
670 : #pragma GCC diagnostic push
671 : #pragma GCC diagnostic ignored "-Wunused-parameter" /* b/c of TRACE_XTRA_UNUSED */
672 : #endif
673 0 : void sigint_sigaction(int signo, siginfo_t *info, void *context)
674 : {
675 0 : cleanup();
676 0 : exit(1);
677 : }
678 : #if (defined(__cplusplus) && (__cplusplus >= 201103L)) || (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L))
679 : #pragma GCC diagnostic pop
680 : #endif
681 :
682 0 : int main(int argc, char *argv[])
683 : {
684 : struct timeval tv;
685 0 : int post_periods_completed = 0;
686 0 : parse_args(argc, argv);
687 0 : if ((argc - optind) != 0 || (opt_cmd.empty() && opt_pid.empty()))
688 : { //(argc-optind) is the number of non-opt args supplied.
689 : int ii;
690 0 : printf("unexpected argument(s) %d!=0\n", argc - optind);
691 0 : for (ii = 0; (optind + ii) < argc; ++ii)
692 : {
693 0 : printf("arg%d=%s\n", ii + 1, argv[optind + ii]);
694 : }
695 0 : printf(USAGE);
696 0 : exit(0);
697 : }
698 :
699 0 : std::vector<std::string> graphs;
700 0 : string_addto_vector(opt_graph, graphs, ',');
701 :
702 0 : char motherboard[1024] = {0};
703 0 : if (getuid() == 0)
704 : {
705 0 : FILE *fp = popen("dmidecode | grep -m2 'Product Name:' | tail -1", "r");
706 0 : fread(motherboard, 1, sizeof(motherboard), fp);
707 0 : pclose(fp);
708 : }
709 0 : TRACE(TLVL_WARNING, "main - motherboard=" + std::string(motherboard));
710 :
711 : /* Note, when doing "waitpid" the wait would sometimes take a "long"
712 : time (10's to 100's milliseconds; rcu???) If signal is generated
713 : (i.e SA_NOCLDWAIT w/ sigchld_sigaction (not SIG_IGN)), it would
714 : sometimes effect the read or write calls for the following AWK forks.
715 : So, use SIG_IGN+SA_NOCLDWAIT.
716 : */
717 : struct sigaction sigaction_s;
718 : #ifndef DO_SIGCHLD
719 : #define DO_SIGCHLD 1
720 : #endif
721 : #if DO_SIGCHLD
722 0 : sigaction_s.sa_sigaction = sigchld_sigaction;
723 0 : sigaction_s.sa_flags = SA_SIGINFO | SA_NOCLDWAIT;
724 : #else
725 : sigaction_s.sa_handler = SIG_IGN;
726 : sigaction_s.sa_flags = SA_NOCLDWAIT;
727 : #endif
728 0 : sigemptyset(&sigaction_s.sa_mask);
729 0 : sigaction(SIGCHLD, &sigaction_s, nullptr);
730 :
731 0 : sigaction_s.sa_sigaction = sigint_sigaction;
732 0 : sigaction_s.sa_flags = SA_SIGINFO;
733 0 : sigaction(SIGINT, &sigaction_s, nullptr);
734 :
735 : // may return 0 when not able to detect
736 : // long long unsigned concurentThreadsSupported = std::thread::hardware_concurrency();
737 0 : long long unsigned concurentThreadsSupported = sysconf(_SC_NPROCESSORS_ONLN);
738 : // TRACE_CNTL( "reset" ); TRACE_CNTL( "modeM", 1L );
739 0 : TRACE(TLVL_ERROR, "main concurentThreadsSupported=%u opt_stats=" + opt_stats, concurentThreadsSupported);
740 :
741 : char run_time[80];
742 0 : gettimeofday(&tv, nullptr);
743 0 : strftime(run_time, sizeof(run_time), "%FT%H%M%S", localtime(&tv.tv_sec));
744 0 : TRACE(TLVL_ERROR, "main run_time=" + std::string(run_time));
745 :
746 : // get hostname
747 : struct utsname ubuf;
748 0 : uname(&ubuf);
749 : char *dot;
750 0 : if ((dot = strchr(ubuf.nodename, '.')) != nullptr)
751 : {
752 0 : *dot = '\0';
753 : }
754 0 : std::string hostname(ubuf.nodename);
755 0 : TRACE(TLVL_WARNING, "release=" + std::string(ubuf.release) + " version=" + std::string(ubuf.version));
756 :
757 : // get system mem (KB)
758 0 : std::string memKB = AWK("NR==1{print$2;exit}", "/proc/meminfo", nullptr);
759 0 : memKB = memKB.substr(0, memKB.size() - 1); // remove trailing newline
760 :
761 0 : std::string dat_file_out(opt_outdir + "periodic_" + run_time + "_" + hostname + "_stats.out");
762 :
763 0 : double period = atof(opt_period);
764 :
765 0 : atexit(cleanup);
766 : pid_t pp;
767 0 : std::vector<std::string> pidfile;
768 :
769 0 : std::vector<std::string> stats;
770 :
771 : // For each cmd: create out file, fork process (with delay param),
772 : // add to stats vec to get CPU info, add to graphs vec to plot cmd CPU
773 0 : for (size_t ii = 0; ii < opt_cmd.size(); ++ii)
774 : {
775 : char cmd_file_out[1024];
776 0 : snprintf(cmd_file_out, sizeof(cmd_file_out), "%speriodic_%s_%s_cmd%zd.out", opt_outdir.c_str(), run_time, hostname.c_str(), ii);
777 0 : int fd = open(cmd_file_out, O_WRONLY | O_CREAT, 0666);
778 0 : TRACE(TLVL_ERROR, "main fd=%d opt_cmd=" + opt_cmd[ii] + " cmd_file_out=" + std::string(cmd_file_out), fd);
779 0 : int iofd[3] = {0, fd, fd}; // redirect stdout/err to the cmd-out-file
780 0 : char *const argv_[4] = {(char *)"/bin/sh",
781 : (char *)"-c",
782 0 : (char *)opt_cmd[ii].c_str(),
783 0 : nullptr};
784 0 : g_pid_vec.push_back(fork_execv(0, 0, (int)(period * opt_pre * 1e6), iofd, "/bin/sh", argv_, nullptr));
785 0 : close(fd); // the output file has been given to the subprocess
786 0 : std::string pidstr = std::to_string((long long int)g_pid_vec[ii]);
787 0 : pidfile.push_back("/proc/" + pidstr + "/stat");
788 : // pidfile.push_back( "/proc/"+pidstr+"/task/"+pidstr+"/stat" );
789 : char desc[128], ss[1024];
790 : // field 14-17: Documentation/filesystems/proc.txt Table 1-4: utime stime cutime cstime
791 0 : snprintf(ss, sizeof(ss), "CPUcmd%zd?%s?NR==1?$14+$15?1?yes", ii, pidfile[ii].c_str());
792 0 : stats.emplace_back(ss);
793 :
794 0 : snprintf(desc, sizeof(desc), "CPU+cmd%zd", ii);
795 0 : graphs.emplace_back(desc); // cmd0 is in the GNUPLOT_PREFIX
796 0 : snprintf(ss, sizeof(ss), "%s?%s?NR==1?$14+$15+16+$17?1?yes", desc, pidfile[ii].c_str());
797 0 : stats.emplace_back(ss);
798 :
799 0 : snprintf(desc, sizeof(desc), "WaitBlkIOcmd%zd", ii);
800 0 : if (opt_cmd_iowait != 0)
801 : {
802 0 : graphs.emplace_back(desc);
803 : }
804 0 : snprintf(ss, sizeof(ss), "%s?%s?NR==1?$42?1?yes", desc, pidfile[ii].c_str());
805 0 : stats.emplace_back(ss);
806 :
807 0 : snprintf(desc, sizeof(desc), "Faultcmd%zd", ii);
808 0 : if (opt_fault != 0)
809 : {
810 0 : graphs.emplace_back(desc);
811 : }
812 0 : snprintf(ss, sizeof(ss), "%s?%s?NR==1?$10+$11+$12+$13?4096.0/1048576?yes", desc, pidfile[ii].c_str());
813 0 : stats.emplace_back(ss);
814 0 : }
815 0 : for (size_t ii = 0; ii < opt_Cmd.size(); ++ii)
816 : {
817 : char cmd_file_out[1024];
818 0 : snprintf(cmd_file_out, sizeof(cmd_file_out), "%speriodic_%s_%s_cmd%zd.out", opt_outdir.c_str(), run_time, hostname.c_str(), ii + opt_cmd.size());
819 0 : int fd = open(cmd_file_out, O_WRONLY | O_CREAT, 0666);
820 0 : TRACE(TLVL_ERROR, "main fd=%d opt_Cmd=" + opt_Cmd[ii] + " cmd_file_out=" + std::string(cmd_file_out), fd);
821 0 : int iofd[3] = {0, fd, fd}; // redirect stdout/err to the cmd-out-file
822 0 : char *const argv_[4] = {(char *)"/bin/sh",
823 : (char *)"-c",
824 0 : (char *)opt_Cmd[ii].c_str(),
825 0 : nullptr};
826 0 : g_pid_vec.push_back(fork_execv(0, 0, (int)(period * opt_pre * 1e6), iofd, "/bin/sh", argv_, nullptr));
827 0 : close(fd); // the output file has been given to the subprocess
828 0 : std::string pidstr = std::to_string((long long int)g_pid_vec[ii]);
829 0 : pidfile.push_back("/proc/" + pidstr + "/stat");
830 : // pidfile.push_back( "/proc/"+pidstr+"/task/"+pidstr+"/stat" );
831 : char desc[128], ss[1024];
832 0 : snprintf(desc, sizeof(desc), "CPU+cmd%zd", ii + opt_cmd.size());
833 0 : snprintf(ss, sizeof(ss), "CPUcmd%zd?%s?NR==1?$14+$15?1?yes", ii + opt_cmd.size(), pidfile[ii].c_str());
834 0 : stats.emplace_back(ss);
835 0 : snprintf(ss, sizeof(ss), "CPU+cmd%zd?%s?NR==1?$14+$15+16+$17?1?yes", ii + opt_cmd.size(), pidfile[ii].c_str());
836 0 : stats.emplace_back(ss);
837 : // JUST DONT ADD THESE TO graphs
838 0 : }
839 0 : std::vector<std::string> pids;
840 0 : if (!opt_pid.empty() != 0u)
841 : {
842 0 : string_addto_vector(opt_pid, pids, ',');
843 : }
844 0 : for (size_t ii = 0; ii < pids.size(); ++ii)
845 : {
846 0 : g_pid_vec.push_back(std::stoi(pids[ii]));
847 0 : TRACE(TLVL_WARNING, "pid=%s g_pid_vec.size()=%ld", pids[ii].c_str(), g_pid_vec.size());
848 0 : pidfile.push_back("/proc/" + pids[ii] + "/stat");
849 : char desc[128], ss[1024];
850 : // field 14-17: Documentation/filesystems/proc.txt Table 1-4: utime stime cutime cstime
851 0 : snprintf(ss, sizeof(ss), "CPUpid%zd?%s?NR==1?$14+$15?1?yes", ii, pidfile[ii].c_str());
852 0 : stats.emplace_back(ss);
853 :
854 0 : std::ifstream t("/proc/" + pids[ii] + "/comm");
855 : std::string comm((std::istreambuf_iterator<char>(t)),
856 0 : std::istreambuf_iterator<char>());
857 0 : comm = comm.substr(0, comm.size() - 1); // strip nl
858 :
859 0 : snprintf(desc, sizeof(desc), "CPU+pid%zd_%s", ii, comm.c_str());
860 0 : graphs.emplace_back(desc); // cmd0 is in the GNUPLOT_PREFIX
861 0 : snprintf(ss, sizeof(ss), "%s?%s?NR==1?$14+$15+16+$17?1?yes", desc, pidfile[ii].c_str());
862 0 : stats.emplace_back(ss);
863 :
864 0 : snprintf(desc, sizeof(desc), "WaitBlkIOpid%zd", ii);
865 0 : if (opt_cmd_iowait != 0)
866 : {
867 0 : graphs.emplace_back(desc);
868 : }
869 0 : snprintf(ss, sizeof(ss), "%s?%s?NR==1?$42?1?yes", desc, pidfile[ii].c_str());
870 0 : stats.emplace_back(ss);
871 :
872 0 : snprintf(desc, sizeof(desc), "Faultpid%zd", ii);
873 0 : if (opt_fault != 0)
874 : {
875 0 : graphs.emplace_back(desc);
876 : }
877 0 : snprintf(ss, sizeof(ss), "%s?%s?NR==1?$10+$11+$12+$13?4096.0/1048576?yes", desc, pidfile[ii].c_str());
878 0 : stats.emplace_back(ss);
879 0 : }
880 :
881 0 : stats.emplace_back("CPUnode");
882 0 : stats.emplace_back("IOWait");
883 0 : if (opt_sys_iowait != 0) { graphs.emplace_back("IOWait"); }
884 0 : stats.emplace_back("Cached");
885 0 : stats.emplace_back("Dirty");
886 0 : stats.emplace_back("Free");
887 :
888 0 : if (!opt_disk.empty() != 0u)
889 : {
890 0 : std::vector<std::string> tmp;
891 0 : string_addto_vector(opt_disk, tmp, ',');
892 0 : for (auto &dk : tmp)
893 : {
894 : // /proc/diskstat has 11 field after an initial 3 (14 total) for each device
895 : // The 7th field after the device name (the 10th field total) is # of sectors written.
896 : // Sectors appear to be 512 bytes. So, deviding by 2048 converts to MBs.
897 0 : std::string statstr = dk + "_wrMB/s?/proc/diskstats?/" + dk + "/?$10?(1.0/2048)?yes";
898 0 : stats.push_back(statstr);
899 0 : std::vector<std::string> stat_spec;
900 0 : string_addto_vector(statstr, stat_spec, '?');
901 0 : graphs.push_back(stat_spec[s_desc]);
902 :
903 0 : statstr = dk + "_rdMB/s?/proc/diskstats?/" + dk + "/?$6?(1.0/2048)?yes";
904 0 : stats.push_back(statstr);
905 0 : stat_spec.clear();
906 0 : string_addto_vector(statstr, stat_spec, '?');
907 : // graphs.push_back( stat_spec[s_desc] ); // don't add read by default -- can be added with --graph
908 0 : }
909 0 : }
910 :
911 0 : if (!opt_stats.empty() != 0u)
912 : {
913 0 : std::vector<std::string> tmp_stats;
914 0 : string_addto_vector(opt_stats, tmp_stats, ',');
915 0 : for (auto &tmp_stat : tmp_stats)
916 : {
917 0 : stats.push_back(tmp_stat);
918 0 : std::vector<std::string> stat_spec;
919 0 : string_addto_vector(tmp_stat, stat_spec, '?');
920 0 : graphs.push_back(stat_spec[s_desc]);
921 0 : }
922 0 : }
923 :
924 0 : std::vector<long> pre_vals;
925 0 : std::vector<double> multipliers;
926 0 : std::vector<std::vector<std::string>> spec2(stats.size());
927 0 : std::vector<std::string> awkCmd;
928 :
929 0 : std::string header_str("#" DATA_START "\n#_______time_______");
930 :
931 0 : int outfd = open(dat_file_out.c_str(), O_WRONLY | O_CREAT, 0777);
932 : // FILE *outfp=stdout;
933 0 : FILE *outfp = fdopen(outfd, "w");
934 :
935 0 : std::string cmd_comment;
936 0 : if (!opt_cmd.empty() != 0u)
937 : {
938 0 : cmd_comment += "\\ncmd: " + opt_cmd[0];
939 : }
940 0 : if (!opt_comment.empty() != 0u)
941 : {
942 0 : cmd_comment += "\\ncomment: " + opt_comment;
943 : }
944 0 : fprintf(outfp, GNUPLOT_PREFIX, opt_ymin, opt_ymax, opt_yincr, opt_y2max, opt_y2incr, run_time, hostname.c_str(), ubuf.release, cmd_comment.c_str(), "disk write MB/s");
945 :
946 0 : uint64_t t_start = gettimeofday_us();
947 :
948 : // build header string and get initial values for "rate" stats
949 0 : bool first_graph_spec_added = false;
950 0 : for (size_t ii = 0; ii < stats.size(); ++ii)
951 : {
952 0 : std::vector<std::string> stat_spec;
953 0 : string_addto_vector(stats[ii], stat_spec, '?');
954 0 : if (stat_spec[s_desc] == "CPUnode" && stat_spec.size() == 1)
955 : {
956 : // Ref. Documentation/filesystems/proc.txt: user+nice+system (skip idle) +iowait+irq+softirq+steal (skip guest)
957 0 : stats[ii] += "?/proc/stat?/^cpu[^0-9]/?$2+$3+$4+$6+$7+$8+$9?1.0/" + std::to_string(concurentThreadsSupported) + "?yes";
958 : }
959 0 : else if (stat_spec[s_desc] == "IOWait" && stat_spec.size() == 1)
960 : {
961 0 : stats[ii] += "?/proc/stat?/^cpu[^0-9]/?$6?1.0/" + std::to_string(concurentThreadsSupported) + "?yes";
962 : }
963 0 : else if (stat_spec[s_desc] == "Cached" && stat_spec.size() == 1)
964 : {
965 0 : stats[ii] += "?/proc/meminfo?/^(Cached|Buffers):/?$2?1?no";
966 : }
967 0 : else if (stat_spec[s_desc] == "Dirty" && stat_spec.size() == 1)
968 : {
969 0 : stats[ii] += "?/proc/meminfo?/^Dirty:/?$2?1?no";
970 : }
971 0 : else if (stat_spec[s_desc] == "Free" && stat_spec.size() == 1)
972 : {
973 0 : stats[ii] += "?/proc/meminfo?/^MemFree:/?$2?1?no";
974 : }
975 :
976 0 : header_str += " " + stat_spec[s_desc];
977 :
978 0 : string_addto_vector(stats[ii], spec2[ii], '?');
979 : char awk_cmd[1024];
980 0 : snprintf(awk_cmd, sizeof(awk_cmd), "%s{vv+=%s}END{print vv}"
981 : // snprintf( awk_cmd, sizeof(awk_cmd), "%s{vv+=%s;print \"vv now\",vv > \"/dev/stderr\";}END{print vv}"
982 : ,
983 0 : spec2[ii][s_linespec].c_str(), spec2[ii][s_fieldspec].c_str());
984 0 : awkCmd.emplace_back(awk_cmd);
985 :
986 0 : std::string stat = AWK(awkCmd.back(), spec2[ii][s_file].c_str(), nullptr);
987 :
988 0 : pre_vals.push_back(atol(stat.c_str()));
989 0 : multipliers.push_back(atof(AWK("BEGIN{print " + spec2[ii][s_multiplier] + "}", "/dev/null", nullptr).c_str()));
990 : // fprintf( stderr, " l=%s", spec2[ii][s_linespec].c_str() );
991 0 : for (const auto &graph : graphs)
992 : {
993 0 : if (graph == stat_spec[s_desc])
994 : {
995 0 : if (first_graph_spec_added)
996 : {
997 0 : fprintf(outfp, ",\\\n '' ");
998 : }
999 0 : if (strncmp(stat_spec[s_desc].c_str(), "CPU", 3) == 0)
1000 : {
1001 0 : fprintf(outfp, "using 1:%zd title '%s' w linespoints axes x1y2", ii + 2, stat_spec[s_desc].c_str());
1002 : }
1003 0 : else if (stat_spec[s_desc] == "Cached" || stat_spec[s_desc] == "Dirty" || stat_spec[s_desc] == "Free")
1004 : {
1005 0 : fprintf(outfp, "using 1:($%zd/%s*100) title '%s%%' w linespoints axes x1y2", ii + 2, memKB.c_str(), stat_spec[s_desc].c_str());
1006 : }
1007 0 : else if (stat_spec[s_desc].substr(0, 6) == "CPUcmd" || stat_spec[s_desc].substr(0, 6) == "CPU+cm")
1008 : {
1009 0 : fprintf(outfp, "using 1:%zd title '%s' w linespoints axes x1y2", ii + 2, stat_spec[s_desc].c_str());
1010 : }
1011 0 : else if (stat_spec[s_desc].substr(0, 12) == "WaitBlkIOcmd")
1012 : {
1013 0 : fprintf(outfp, "using 1:%zd title '%s' w linespoints axes x1y2", ii + 2, stat_spec[s_desc].c_str());
1014 : }
1015 : else
1016 : {
1017 0 : fprintf(outfp, "using 1:%zd title '%s' w linespoints axes x1y1", ii + 2, stat_spec[s_desc].c_str());
1018 : }
1019 0 : first_graph_spec_added = true;
1020 : }
1021 : }
1022 0 : }
1023 0 : header_str += " #\n";
1024 :
1025 0 : fprintf(outfp,
1026 : "\nif(png==0) pause -1 'Press Enter/Return or ^C to finish'\n\
1027 : exit\n");
1028 :
1029 : // print the cmds
1030 0 : fprintf(outfp, "cmds:\n");
1031 0 : for (const auto &ii : opt_cmd)
1032 : {
1033 0 : std::string ss = ii + "\n";
1034 0 : fprintf(outfp, "%s", ss.c_str());
1035 0 : }
1036 :
1037 : // print the specs
1038 0 : fprintf(outfp, "stats:\n");
1039 0 : for (const auto &stat : stats)
1040 : {
1041 0 : std::string ss = stat + "\n";
1042 0 : fprintf(outfp, "%s", ss.c_str());
1043 0 : }
1044 :
1045 : // now print header
1046 0 : fprintf(outfp, "%s", header_str.c_str());
1047 0 : fflush(outfp);
1048 :
1049 0 : std::string tmpdbg("main lp=%d done stat%zd=%ld rate=%f ");
1050 : // char tmpdbgbuf[128];
1051 : char proc_stats[8192];
1052 : char *awk_in;
1053 : int lp;
1054 :
1055 : // - - - - - - - - - - - - - - - - - - - - - - - -
1056 : // wait a period and then start collecting the stats
1057 0 : eintr1:
1058 0 : int64_t t_sleep = (t_start + (uint64_t)(period * 1e6)) - gettimeofday_us();
1059 0 : if (t_sleep > 0)
1060 : {
1061 0 : int sts = usleep(t_sleep); // NOLINT
1062 : TRACE(TLVL_DEBUG + 0, "main usleep sts=%d errno=%d", sts, errno);
1063 0 : if (errno == EINTR)
1064 : {
1065 0 : goto eintr1;
1066 : }
1067 : }
1068 :
1069 : #define MAX_LP 600
1070 0 : for (lp = 2; lp < MAX_LP; ++lp)
1071 : {
1072 : char str[80];
1073 0 : gettimeofday(&tv, nullptr);
1074 0 : strftime(str, sizeof(str), "%FT%T", localtime(&tv.tv_sec));
1075 : // fprintf(outfp, "%s.%ld", str, tv.tv_usec/100000 );
1076 0 : fprintf(outfp, "%s", str);
1077 0 : std::string prv_file;
1078 0 : for (size_t ii = 0; ii < stats.size(); ++ii)
1079 : {
1080 : TRACE(TLVL_DEBUG + 0, "main lp=%d start stat%zd", lp, ii);
1081 : char const *awk_file;
1082 0 : if (ii < (2 * opt_cmd.size()))
1083 : { // For each cmd, the
1084 : // /proc/<pid>/stat file
1085 : // will be referenced twice.
1086 0 : if ((ii & 1) == 0)
1087 : {
1088 0 : read_proc_file(pidfile[ii / 2].c_str(), proc_stats, sizeof(proc_stats));
1089 : }
1090 0 : awk_in = proc_stats;
1091 0 : awk_file = nullptr;
1092 : }
1093 0 : else if (spec2[ii][s_file] != prv_file)
1094 : {
1095 0 : prv_file = spec2[ii][s_file];
1096 0 : read_proc_file(spec2[ii][s_file].c_str(), proc_stats, sizeof(proc_stats));
1097 0 : awk_in = proc_stats;
1098 0 : awk_file = nullptr;
1099 : }
1100 :
1101 0 : std::string stat_str = AWK(awkCmd[ii], awk_file, awk_in); // NOLINT
1102 :
1103 0 : long stat = atol(stat_str.c_str());
1104 :
1105 0 : if (spec2[ii][s_rate] == "yes")
1106 : {
1107 : double rate;
1108 0 : if (stat_str != "\n")
1109 : {
1110 0 : rate = (stat - pre_vals[ii]) * multipliers[ii] / period;
1111 : }
1112 : else
1113 : {
1114 0 : rate = 0.0;
1115 : }
1116 : TRACE(TLVL_DEBUG + 0, tmpdbg + "stat_str[0]=0x%x stat_str.size()=%zd", lp, ii, stat, rate, stat_str[0], stat_str.size());
1117 0 : fprintf(outfp, " %.2f", rate);
1118 0 : if (rate < 0.0 && spec2[ii][s_file] == "/proc/diskstats")
1119 : {
1120 0 : TRACE(TLVL_ERROR, "main stat:" + spec2[ii][s_desc] + " rate=%f pre_val=%ld stat=%ld stat_str=\"" + stat_str + "\" awkCmd=" + awkCmd[ii] + " proc_diskstats=" + proc_stats, rate, pre_vals[ii], stat);
1121 : // TRACE_CNTL( "modeM", 0L );
1122 : }
1123 0 : pre_vals[ii] = stat;
1124 : }
1125 : else
1126 : {
1127 : TRACE(TLVL_DEBUG + 0, "main lp=%d done stat%zd=%ld", lp, ii, stat);
1128 0 : fprintf(outfp, " %.2f", stat * multipliers[ii]);
1129 : }
1130 0 : }
1131 0 : fprintf(outfp, "\n");
1132 0 : fflush(outfp);
1133 0 : eintr2:
1134 0 : int64_t t_sleep = (t_start + (uint64_t)(period * lp * 1000000)) - gettimeofday_us();
1135 0 : if (t_sleep > 0)
1136 : {
1137 0 : int sts = usleep(t_sleep); // NOLINT
1138 : TRACE(TLVL_DEBUG + 0, "main usleep sts=%d errno=%d", sts, errno);
1139 0 : if (errno == EINTR)
1140 : {
1141 0 : goto eintr2;
1142 : }
1143 : }
1144 0 : pp = check_pid_vec();
1145 : TRACE(TLVL_INFO, "main pp=%d t_sleep=%ld", pp, t_sleep);
1146 0 : if (pp == -1)
1147 : {
1148 0 : if (post_periods_completed == 0)
1149 : {
1150 0 : TRACE(TLVL_WARNING, "main processes complete - waiting %d post periods", opt_post);
1151 : }
1152 0 : if (post_periods_completed++ == opt_post)
1153 : {
1154 0 : break;
1155 : }
1156 : }
1157 0 : }
1158 0 : if (lp == MAX_LP)
1159 : {
1160 0 : fprintf(outfp, "# MAX_LP abort\n");
1161 : }
1162 :
1163 : // TRACE( 0, "main waiting for pid=%d", pid );
1164 : // wait(&status);
1165 : // TRACE( 0, "main status=%d",status );
1166 0 : TRACE(TLVL_ERROR, "main done/complete/returning");
1167 : // TRACE_CNTL( "modeM", 0L );
1168 0 : return (0);
1169 0 : } // main
|
