1 /****************************************************************************
2 * Copyright (C) 2006-2010 by Jason Ansel, Kapil Arya, and Gene Cooperman *
3 * jansel@csail.mit.edu, kapil@ccs.neu.edu, gene@ccs.neu.edu *
4 * *
5 * This file is part of the dmtcp/src module of DMTCP (DMTCP:dmtcp/src). *
6 * *
7 * DMTCP:dmtcp/src is free software: you can redistribute it and/or *
8 * modify it under the terms of the GNU Lesser General Public License as *
9 * published by the Free Software Foundation, either version 3 of the *
10 * License, or (at your option) any later version. *
11 * *
12 * DMTCP:dmtcp/src is distributed in the hope that it will be useful, *
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of *
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
15 * GNU Lesser General Public License for more details. *
16 * *
17 * You should have received a copy of the GNU Lesser General Public *
18 * License along with DMTCP:dmtcp/src. If not, see *
19 * <http://www.gnu.org/licenses/>. *
20 ****************************************************************************/
21
22 #include <stdarg.h>
23 #include <stdio.h>
24 #include <stdlib.h>
25 #include <vector>
26 #include <list>
27 #include <string>
28 #include "constants.h"
29 #include "connectionmanager.h"
30 #include "uniquepid.h"
31 #include "dmtcpworker.h"
32 #include "virtualpidtable.h"
33 #include "sysvipc.h"
34 #include "syscallwrappers.h"
35 #include "syslogcheckpointer.h"
36 #include "util.h"
37 #include "../jalib/jconvert.h"
38 #include "../jalib/jassert.h"
39 #include <sys/time.h>
40 #include <sys/resource.h>
41 #include <sys/personality.h>
42
43 #define INITIAL_ARGV_MAX 32
44
45 #ifdef DEBUG
46 const static bool dbg = true;
47 #else
48 const static bool dbg = false;
49 #endif
50
51 static pid_t forkChild ( long child_host, time_t child_time )
52 {
53 while ( 1 ) {
54
55 pid_t child_pid = _real_fork();
56
57 if ( child_pid == -1 ) {
58 // fork() failed
59 return child_pid;
60 } else if ( child_pid == 0 ) {
61 /* child process */
62
63 JALIB_RESET_ON_FORK ();
64 #ifdef DEBUG
65 dmtcp::UniquePid child = dmtcp::UniquePid ( child_host, _real_getpid(), child_time );
66 //child should get new logfile
67 dmtcp::ostringstream o;
68 o << dmtcp::UniquePid::getTmpDir() << "/jassertlog." << child.toString();
69 JASSERT_INIT (o.str());
70 #endif
71
72 #ifdef PID_VIRTUALIZATION
73 if ( dmtcp::VirtualPidTable::isConflictingPid ( _real_getpid() ) ) {
74 _exit(1);
75 } else {
76 return child_pid;
77 }
78 #else
79 return child_pid;
80 #endif
81 } else {
82 /* Parent Process */
83 #ifdef PID_VIRTUALIZATION
84 if ( dmtcp::VirtualPidTable::isConflictingPid ( child_pid ) ) {
85 JTRACE( "PID Conflict, creating new child" ) (child_pid);
86 _real_waitpid ( child_pid, NULL, 0 );
87 } else {
88 return child_pid;
89 }
90 #else
91 return child_pid;
92 #endif
93 }
94 }
95 return -1;
96 }
97
98 static pid_t fork_work()
99 {
100 /* Little bit cheating here: child_time should be same for both parent and
101 * child, thus we compute it before forking the child. */
102 time_t child_time = time ( NULL );
103 long child_host = dmtcp::UniquePid::ThisProcess().hostid();
104 dmtcp::UniquePid parent = dmtcp::UniquePid::ThisProcess();
105
106 //pid_t child_pid = _real_fork();
107 pid_t child_pid = forkChild ( child_host, child_time );
108 if (child_pid < 0) {
109 return child_pid;
110 }
111
112
113 if ( child_pid == 0 ) {
114 child_pid = _real_getpid();
115
116 dmtcp::UniquePid child = dmtcp::UniquePid ( child_host, child_pid, child_time );
117
118 JTRACE ( "fork()ed [CHILD]" ) ( child ) ( parent );
119
120 //fix the mutex
121 _dmtcp_remutex_on_fork();
122
123 //update ThisProcess()
124 dmtcp::UniquePid::resetOnFork ( child );
125
126 #ifdef PID_VIRTUALIZATION
127 dmtcp::VirtualPidTable::instance().resetOnFork( );
128 #endif
129
130 dmtcp::SyslogCheckpointer::resetOnFork();
131
132 //rewrite socket table
133 // dmtcp::SocketTable::instance().onForkUpdate(parent,child);
134
135 //make new connection to coordinator
136 dmtcp::DmtcpWorker::resetOnFork();
137
138 JTRACE ( "fork() done [CHILD]" ) ( child ) ( parent );
139
140 return 0;
141 } else {
142 dmtcp::UniquePid child = dmtcp::UniquePid ( child_host, child_pid, child_time );
143
144 #ifdef PID_VIRTUALIZATION
145 dmtcp::VirtualPidTable::instance().insert ( child_pid, child );
146 #endif
147
148 JTRACE ( "fork()ed [PARENT] done" ) ( child );;
149
150 // _dmtcp_lock();
151
152 //rewrite socket table
153 // dmtcp::SocketTable::instance().onForkUpdate(parent,child);
154
155 // _dmtcp_unlock();
156
157 // JTRACE("fork() done [PARENT]")(child);
158
159 return child_pid;
160 }
161 }
162
163 static void prepareForFork()
164 {
165 dmtcp::KernelDeviceToConnection::instance().prepareForFork();
166 }
167
168 extern "C" pid_t fork()
169 {
170 /* Acquire the wrapperExeution lock to prevent checkpoint to happen while
171 * processing this system call.
172 */
173 WRAPPER_EXECUTION_DISABLE_CKPT();
174
175 prepareForFork();
176 int retVal = fork_work();
177
178 if (retVal != 0) {
179 WRAPPER_EXECUTION_ENABLE_CKPT();
180 }
181
182 return retVal;
183 }
184
185
186 extern "C" pid_t vfork()
187 {
188 JTRACE ( "vfork wrapper calling fork" );
189 // This might not preserve the full semantics of vfork.
190 // Used for checkpointing gdb.
191 return fork();
192 }
193
194 static void execLibProcessAndExit(const char *path)
195 {
196 unsetenv("LD_PRELOAD"); // /lib/ld.so won't let us preload if exec'ing lib
197 const unsigned int bufSize = 100000;
198 char *buf = (char*)JALLOC_HELPER_MALLOC(bufSize);
199 memset(buf, 0, bufSize);
200 FILE *output = popen(path, "r");
201 int numRead = fread(buf, 1, bufSize, output);
202 pclose(output); // /lib/libXXX process is now done; can checkpoint now
203 // FIXME: code currently allows wrapper to proceed without lock if
204 // it was busy because of a writer. The unlock will then fail below.
205 bool __wrapperExecutionLockAcquired = true; // needed for LOCK_UNLOCK macro
206 WRAPPER_EXECUTION_ENABLE_CKPT();
207 // We are now the new /lib/libXXX process, and it's safe for DMTCP to ckpt us.
208 printf("%s", buf); // print buf, which is what /lib/libXXX would print
209 JALLOC_HELPER_FREE(buf);
210 exit(0);
211 }
212
213 // FIXME: Unify this code with code prior to execvp in dmtcp_checkpoint.cpp
214 // Can use argument to dmtcpPrepareForExec() or getenv("DMTCP_...")
215 // from DmtcpWorker constructor, to distinguish the two cases.
216 static void dmtcpPrepareForExec(const char *path)
217 {
218 const char * libPrefix = "/lib/lib";
219 const char * lib64Prefix = "/lib64/lib";
220 if (path != NULL && dmtcp::Util::strStartsWith(path, libPrefix))
221 execLibProcessAndExit(path);
222 if (path != NULL && dmtcp::Util::strStartsWith(path, lib64Prefix))
223 execLibProcessAndExit(path);
224
225 dmtcp::string serialFile = dmtcp::UniquePid::dmtcpTableFilename();
226 jalib::JBinarySerializeWriter wr ( serialFile );
227 dmtcp::UniquePid::serialize ( wr );
228 dmtcp::KernelDeviceToConnection::instance().serialize ( wr );
229 #ifdef PID_VIRTUALIZATION
230 dmtcp::VirtualPidTable::instance().serialize ( wr );
231 #endif
232 dmtcp::SysVIPC::instance().serialize ( wr );
233
234 setenv ( ENV_VAR_SERIALFILE_INITIAL, serialFile.c_str(), 1 );
235 JTRACE ( "Preparing for Exec" ) ( path );
236
237 #ifdef __i386__
238 // This is needed in 32-bit Ubuntu 9.10, to fix bug with test/dmtcp5.c
239 // NOTE: Setting personality() is cleanest way to force legacy_va_layout,
240 // but there's currently a bug on restart in the sequence:
241 // checkpoint -> restart -> checkpoint -> restart
242 # if 0
243 { unsigned long oldPersonality = personality(0xffffffffL);
244 if ( ! (oldPersonality & ADDR_COMPAT_LAYOUT) ) {
245 // Force ADDR_COMPAT_LAYOUT for libs in high mem, to avoid vdso conflict
246 personality(oldPersonality & ADDR_COMPAT_LAYOUT);
247 JTRACE( "setting ADDR_COMPAT_LAYOUT" );
248 setenv("DMTCP_ADDR_COMPAT_LAYOUT", "temporarily is set", 1);
249 }
250 }
251 # else
252 { struct rlimit rlim;
253 getrlimit(RLIMIT_STACK, &rlim);
254 if (rlim.rlim_cur != RLIM_INFINITY) {
255 char buf[100];
256 sprintf(buf, "%lu", rlim.rlim_cur); // "%llu" for BSD/Mac OS
257 JTRACE( "setting rlim_cur for RLIMIT_STACK" ) ( rlim.rlim_cur );
258 setenv("DMTCP_RLIMIT_STACK", buf, 1);
259 // Force kernel's internal compat_va_layout to 0; Force libs to high mem.
260 rlim.rlim_cur = rlim.rlim_max;
261 // FIXME: if rlim.rlim_cur != RLIM_INFINITY, then we should warn the user.
262 setrlimit(RLIMIT_STACK, &rlim);
263 // After exec, process will restore DMTCP_RLIMIT_STACK in DmtcpWorker()
264 }
265 }
266 # endif
267 #endif
268
269 dmtcp::string preload (dmtcp::DmtcpWorker::ld_preload_c);
270 if (getenv("LD_PRELOAD")) {
271 preload = preload + ":" + getenv("LD_PRELOAD");
272 }
273 setenv("LD_PRELOAD", preload.c_str(), 1);
274 JTRACE ( "Prepared for Exec" ) ( getenv( "LD_PRELOAD" ) );
275 }
276
277 static void dmtcpProcessFailedExec(const char *path)
278 {
279 const char* str = getenv("LD_PRELOAD");
280 JASSERT(str != NULL );
281 dmtcp::string preload = getenv("LD_PRELOAD");
282 JASSERT(dmtcp::Util::strStartsWith(preload, dmtcp::DmtcpWorker::ld_preload_c));
283
284 preload.erase(0, strlen(dmtcp::DmtcpWorker::ld_preload_c) + 1);
285
286 setenv("LD_PRELOAD", preload.c_str(), 1);
287 JTRACE ( "Processed failed Exec Attempt" ) (path) ( getenv( "LD_PRELOAD" ) );
288 }
289
290 static const char* ourImportantEnvs[] =
291 {
292 "LD_PRELOAD",
293 ENV_VARS_ALL //expands to a long list
294 };
295 #define ourImportantEnvsCnt ((sizeof(ourImportantEnvs))/(sizeof(const char*)))
296
297 static bool isImportantEnv ( dmtcp::string str )
298 {
299 str = str.substr(0, str.find("="));
300
301 for ( size_t i=0; i<ourImportantEnvsCnt; ++i ) {
302 if ( str == ourImportantEnvs[i] )
303 return true;
304 }
305 return false;
306 }
307
308 static dmtcp::list<dmtcp::string>& copyUserEnv ( char *const envp[] )
309 {
310 static dmtcp::list<dmtcp::string> strStorage;
311 strStorage.clear();
312
313 JTRACE ( "Creating a copy of (non-DMTCP) user env vars..." );
314 for ( ; *envp != NULL; ++envp ) {
315 if ( isImportantEnv ( *envp ) ) {
316 if(dbg)
317 JASSERT_STDERR << " skipping: " << *envp << '\n';
318 continue;
319 }
320 strStorage.push_back ( *envp );
321 if(dbg)
322 JASSERT_STDERR << " addenv[user]:" << strStorage.back() << '\n';
323 }
324 return strStorage;
325 }
326
327 static char** patchUserEnv ( dmtcp::list<dmtcp::string> &envList )
328 {
329 static dmtcp::vector<char*> envVect;
330 envVect.clear();
331
332 dmtcp::list<dmtcp::string>::iterator i;
333 for ( i = envList.begin() ; i != envList.end(); ++i ) {
334 JASSERT ( !isImportantEnv ( *i ) );
335 JASSERT ( !dbg || &(*i)[0] == (*i).c_str());
336 envVect.push_back ( (char*)i->c_str() );
337 }
338
339 JTRACE ( "patching user envp..." ) ( getenv ( "LD_PRELOAD" ) );
340
341 //pack up our ENV into the new ENV
342 for ( size_t i=0; i<ourImportantEnvsCnt; ++i ) {
343 const char* v = getenv ( ourImportantEnvs[i] );
344 if ( v != NULL ) {
345 envList.push_back ( dmtcp::string ( ourImportantEnvs[i] ) + '=' + v );
346 envVect.push_back ( &envList.back() [0] );
347 if(dbg)
348 JASSERT_STDERR << " addenv[dmtcp]:" << envList.back() << '\n';
349 }
350 }
351
352 envVect.push_back ( NULL );
353
354 return &envVect[0];
355 }
356
357 extern "C" int execve ( const char *filename, char *const argv[], char *const envp[] )
358 {
359 JTRACE ( "execve() wrapper" ) ( filename );
360
361 /* Acquire the wrapperExeution lock to prevent checkpoint to happen while
362 * processing this system call.
363 */
364 WRAPPER_EXECUTION_DISABLE_CKPT();
365
366 dmtcp::list<dmtcp::string> origUserEnv = copyUserEnv( envp );
367
368 dmtcpPrepareForExec(filename);
369
370 int retVal = _real_execve ( filename, argv, patchUserEnv ( origUserEnv ) );
371
372 dmtcpProcessFailedExec(filename);
373
374 WRAPPER_EXECUTION_ENABLE_CKPT();
375
376 return retVal;
377 }
378
379 extern "C" int fexecve ( int fd, char *const argv[], char *const envp[] )
380 {
381 JTRACE ( "fexecve() wrapper" ) ( fd );
382 /* Acquire the wrapperExeution lock to prevent checkpoint to happen while
383 * processing this system call.
384 */
385 WRAPPER_EXECUTION_DISABLE_CKPT();
386
387 dmtcp::list<dmtcp::string> origUserEnv = copyUserEnv( envp );
388
389 // FIXME: fexecve() could have fd bound to /lib/libXXX, requiring special
390 // handling. Because arg is NULL, we won't check for it.
391 dmtcpPrepareForExec(NULL);
392
393 int retVal = _real_fexecve ( fd, argv, patchUserEnv ( origUserEnv ) );
394
395 dmtcpProcessFailedExec(argv[0]);
396
397 WRAPPER_EXECUTION_ENABLE_CKPT();
398
399 return retVal;
400 }
401
402 extern "C" int execv ( const char *path, char *const argv[] )
403 {
404 JTRACE ( "execv() wrapper" ) ( path );
405 /* Acquire the wrapperExeution lock to prevent checkpoint to happen while
406 * processing this system call.
407 */
408 WRAPPER_EXECUTION_DISABLE_CKPT();
409
410 dmtcpPrepareForExec(path);
411
412 int retVal = _real_execv ( path, argv );
413
414 dmtcpProcessFailedExec(path);
415
416 WRAPPER_EXECUTION_ENABLE_CKPT();
417
418 return retVal;
419 }
420
421 extern "C" int execvp ( const char *file, char *const argv[] )
422 {
423 JTRACE ( "execvp() wrapper" ) ( file );
424 /* Acquire the wrapperExeution lock to prevent checkpoint to happen while
425 * processing this system call.
426 */
427 WRAPPER_EXECUTION_DISABLE_CKPT();
428
429 dmtcpPrepareForExec(file);
430
431 int retVal = _real_execvp ( file, argv );
432
433 dmtcpProcessFailedExec(file);
434
435 WRAPPER_EXECUTION_ENABLE_CKPT();
436
437 return retVal;
438 }
439
440 extern "C" int execl ( const char *path, const char *arg, ... )
441 {
442 JTRACE ( "execl() wrapper" ) ( path );
443
444 size_t argv_max = INITIAL_ARGV_MAX;
445 const char *initial_argv[INITIAL_ARGV_MAX];
446 const char **argv = initial_argv;
447 va_list args;
448
449 argv[0] = arg;
450
|
Event va_init: |
Initializing va_list "args". |
| Also see events: |
[missing_va_end] |
451 va_start (args, arg);
452 unsigned int i = 0;
|
At conditional (1): "argv[i++] != NULL": Taking true branch.
|
|
At conditional (3): "argv[i++] != NULL": Taking true branch.
|
|
At conditional (5): "argv[i++] != NULL": Taking true branch.
|
|
At conditional (10): "argv[i++] != NULL": Taking true branch.
|
453 while (argv[i++] != NULL)
454 {
|
At conditional (2): "i == argv_max": Taking false branch.
|
|
At conditional (4): "i == argv_max": Taking false branch.
|
|
At conditional (6): "i == argv_max": Taking true branch.
|
|
At conditional (11): "i == argv_max": Taking true branch.
|
455 if (i == argv_max)
456 {
457 argv_max *= 2;
|
At conditional (7): "argv == initial_argv": Taking true branch.
|
|
At conditional (12): "argv == initial_argv": Taking true branch.
|
458 const char **nptr = (const char**) realloc (argv == initial_argv ? NULL : argv,
459 argv_max * sizeof (const char *));
|
At conditional (8): "nptr == NULL": Taking false branch.
|
|
At conditional (13): "nptr == NULL": Taking true branch.
|
460 if (nptr == NULL)
461 {
|
At conditional (14): "argv != initial_argv": Taking true branch.
|
462 if (argv != initial_argv)
463 free (argv);
|
Event missing_va_end: |
va_end was not called for "args". |
| Also see events: |
[va_init] |
464 return -1;
465 }
|
At conditional (9): "argv == initial_argv": Taking true branch.
|
466 if (argv == initial_argv)
467 /* We have to copy the already filled-in data ourselves. */
468 memcpy (nptr, argv, i * sizeof (const char *));
469
470 argv = nptr;
471 }
472
473 argv[i] = va_arg (args, const char *);
474 }
475 va_end (args);
476
477 int ret = execv (path, (char *const *) argv);
478 if (argv != initial_argv)
479 free (argv);
480
481 return ret;
482 }
483
484
485 extern "C" int execlp ( const char *file, const char *arg, ... )
486 {
487 JTRACE ( "execlp() wrapper" ) ( file );
488
489 size_t argv_max = INITIAL_ARGV_MAX;
490 const char *initial_argv[INITIAL_ARGV_MAX];
491 const char **argv = initial_argv;
492 va_list args;
493
494 argv[0] = arg;
495
496 va_start (args, arg);
497 unsigned int i = 0;
498 while (argv[i++] != NULL)
499 {
500 if (i == argv_max)
501 {
502 argv_max *= 2;
503 const char **nptr = (const char**) realloc (argv == initial_argv ? NULL : argv,
504 argv_max * sizeof (const char *));
505 if (nptr == NULL)
506 {
507 if (argv != initial_argv)
508 free (argv);
509 return -1;
510 }
511 if (argv == initial_argv)
512 /* We have to copy the already filled-in data ourselves. */
513 memcpy (nptr, argv, i * sizeof (const char *));
514
515 argv = nptr;
516 }
517
518 argv[i] = va_arg (args, const char *);
519 }
520 va_end (args);
521
522 int ret = execvp (file, (char *const *) argv);
523 if (argv != initial_argv)
524 free (argv);
525
526 return ret;
527 }
528
529
530 extern "C" int execle(const char *path, const char *arg, ...)
531 {
532 JTRACE ( "execle() wrapper" ) ( path );
533
534 size_t argv_max = INITIAL_ARGV_MAX;
535 const char *initial_argv[INITIAL_ARGV_MAX];
536 const char **argv = initial_argv;
537 va_list args;
538 argv[0] = arg;
539
540 va_start (args, arg);
541 unsigned int i = 0;
542 while (argv[i++] != NULL)
543 {
544 if (i == argv_max)
545 {
546 argv_max *= 2;
547 const char **nptr = (const char**) realloc (argv == initial_argv ? NULL : argv,
548 argv_max * sizeof (const char *));
549 if (nptr == NULL)
550 {
551 if (argv != initial_argv)
552 free (argv);
553 return -1;
554 }
555 if (argv == initial_argv)
556 /* We have to copy the already filled-in data ourselves. */
557 memcpy (nptr, argv, i * sizeof (const char *));
558
559 argv = nptr;
560 }
561
562 argv[i] = va_arg (args, const char *);
563 }
564
565 const char *const *envp = va_arg (args, const char *const *);
566 va_end (args);
567
568 int ret = execve (path, (char *const *) argv, (char *const *) envp);
569 if (argv != initial_argv)
570 free (argv);
571
572 return ret;
573 }
574
575 // See comment in glibcsystem.cpp for why this exists and how it works.
576 extern int do_system (const char *line);
577
578 extern "C" int system (const char *line)
579 {
580 JTRACE ( "before system(), checkpointing may not work" )
581 ( line ) ( getenv ( ENV_VAR_HIJACK_LIB ) ) ( getenv ( "LD_PRELOAD" ) );
582
583 if (line == NULL)
584 /* Check that we have a command processor available. It might
585 not be available after a chroot(), for example. */
586 return do_system ("exit 0") == 0;
587
588 int result = do_system (line);
589
590 JTRACE ( "after system()" );
591
592 return result;
593 }