Actual source code: fretrieve.c


  2: /*
  3:       Code for opening and closing files.
  4: */
  5: #include <petscsys.h>
  6: #if defined(PETSC_HAVE_PWD_H)
  7:   #include <pwd.h>
  8: #endif
  9: #include <ctype.h>
 10: #include <sys/stat.h>
 11: #if defined(PETSC_HAVE_UNISTD_H)
 12:   #include <unistd.h>
 13: #endif
 14: #if defined(PETSC_HAVE_SYS_UTSNAME_H)
 15:   #include <sys/utsname.h>
 16: #endif
 17: #include <fcntl.h>
 18: #include <time.h>
 19: #if defined(PETSC_HAVE_SYS_SYSTEMINFO_H)
 20:   #include <sys/systeminfo.h>
 21: #endif

 23: /*
 24:    Private routine to delete tmp/shared storage

 26:    This is called by MPI, not by users.

 28:    Note: this is declared extern "C" because it is passed to MPI_Comm_create_keyval()

 30: */
 31: PETSC_EXTERN PetscMPIInt MPIAPI Petsc_DelTmpShared(MPI_Comm comm, PetscMPIInt keyval, void *count_val, void *extra_state)
 32: {
 33:   PetscInfo(NULL, "Deleting tmp/shared data in an MPI_Comm %ld\n", (long)comm);
 34:   PetscFree(count_val);
 35:   return MPI_SUCCESS;
 36: }

 38: /*@C
 39:    PetscGetTmp - Gets the name of the tmp directory

 41:    Collective

 43:    Input Parameters:
 44: +  comm - MPI_Communicator that may share /tmp
 45: -  len - length of string to hold name

 47:    Output Parameter:
 48: .  dir - directory name

 50:    Options Database Keys:
 51: +    -shared_tmp  - indicates the directory is shared among the MPI ranks
 52: .    -not_shared_tmp - indicates the directory is not shared among the MPI ranks
 53: -    -tmp tmpdir - name of the directory you wish to use as /tmp

 55:    Environmental Variables:
 56: +     `PETSC_SHARED_TMP` - indicates the directory is shared among the MPI ranks
 57: .     `PETSC_NOT_SHARED_TMP` - indicates the directory is not shared among the MPI ranks
 58: -     `PETSC_TMP` - name of the directory you wish to use as /tmp

 60:    Level: developer

 62: .seealso: `PetscSharedTmp()`, `PetscSharedWorkingDirectory()`, `PetscGetWorkingDirectory()`, `PetscGetHomeDirectory()`
 63: @*/
 64: PetscErrorCode PetscGetTmp(MPI_Comm comm, char dir[], size_t len)
 65: {
 66:   PetscBool flg;

 68:   PetscOptionsGetenv(comm, "PETSC_TMP", dir, len, &flg);
 69:   if (!flg) PetscStrncpy(dir, "/tmp", len);
 70:   return 0;
 71: }

 73: /*@C
 74:    PetscSharedTmp - Determines if all processors in a communicator share a
 75:          /tmp or have different ones.

 77:    Collective

 79:    Input Parameter:
 80: .  comm - MPI_Communicator that may share /tmp

 82:    Output Parameter:
 83: .  shared - `PETSC_TRUE` or `PETSC_FALSE`

 85:    Options Database Keys:
 86: +    -shared_tmp  - indicates the directory is shared among the MPI ranks
 87: .    -not_shared_tmp - indicates the directory is not shared among the MPI ranks
 88: -    -tmp tmpdir - name of the directory you wish to use as /tmp

 90:    Environmental Variables:
 91: +     `PETSC_SHARED_TMP`  - indicates the directory is shared among the MPI ranks
 92: .     `PETSC_NOT_SHARED_TMP` - indicates the directory is not shared among the MPI ranks
 93: -     `PETSC_TMP` - name of the directory you wish to use as /tmp

 95:    Level: developer

 97:    Notes:
 98:    Stores the status as a MPI attribute so it does not have
 99:     to be redetermined each time.

101:       Assumes that all processors in a communicator either
102:        1) have a common /tmp or
103:        2) each has a separate /tmp
104:       eventually we can write a fancier one that determines which processors
105:       share a common /tmp.

107:    This will be very slow on runs with a large number of processors since
108:    it requires O(p*p) file opens.

110:    If the environmental variable PETSC_TMP is set it will use this directory
111:   as the "/tmp" directory.

113: .seealso: `PetscGetTmp()`, `PetscSharedWorkingDirectory()`, `PetscGetWorkingDirectory()`, `PetscGetHomeDirectory()`
114: @*/
115: PetscErrorCode PetscSharedTmp(MPI_Comm comm, PetscBool *shared)
116: {
117:   PetscMPIInt        size, rank, *tagvalp, sum, cnt, i;
118:   PetscBool          flg, iflg;
119:   FILE              *fd;
120:   static PetscMPIInt Petsc_Tmp_keyval = MPI_KEYVAL_INVALID;
121:   int                err;

123:   MPI_Comm_size(comm, &size);
124:   if (size == 1) {
125:     *shared = PETSC_TRUE;
126:     return 0;
127:   }

129:   PetscOptionsGetenv(comm, "PETSC_SHARED_TMP", NULL, 0, &flg);
130:   if (flg) {
131:     *shared = PETSC_TRUE;
132:     return 0;
133:   }

135:   PetscOptionsGetenv(comm, "PETSC_NOT_SHARED_TMP", NULL, 0, &flg);
136:   if (flg) {
137:     *shared = PETSC_FALSE;
138:     return 0;
139:   }

141:   if (Petsc_Tmp_keyval == MPI_KEYVAL_INVALID) MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN, Petsc_DelTmpShared, &Petsc_Tmp_keyval, NULL);

143:   MPI_Comm_get_attr(comm, Petsc_Tmp_keyval, (void **)&tagvalp, (int *)&iflg);
144:   if (!iflg) {
145:     char filename[PETSC_MAX_PATH_LEN], tmpname[PETSC_MAX_PATH_LEN];

147:     /* This communicator does not yet have a shared tmp attribute */
148:     PetscMalloc1(1, &tagvalp);
149:     MPI_Comm_set_attr(comm, Petsc_Tmp_keyval, tagvalp);

151:     PetscOptionsGetenv(comm, "PETSC_TMP", tmpname, 238, &iflg);
152:     if (!iflg) {
153:       PetscStrcpy(filename, "/tmp");
154:     } else {
155:       PetscStrcpy(filename, tmpname);
156:     }

158:     PetscStrcat(filename, "/petsctestshared");
159:     MPI_Comm_rank(comm, &rank);

161:     /* each processor creates a /tmp file and all the later ones check */
162:     /* this makes sure no subset of processors is shared */
163:     *shared = PETSC_FALSE;
164:     for (i = 0; i < size - 1; i++) {
165:       if (rank == i) {
166:         fd = fopen(filename, "w");
168:         err = fclose(fd);
170:       }
171:       MPI_Barrier(comm);
172:       if (rank >= i) {
173:         fd = fopen(filename, "r");
174:         if (fd) cnt = 1;
175:         else cnt = 0;
176:         if (fd) {
177:           err = fclose(fd);
179:         }
180:       } else cnt = 0;

182:       MPIU_Allreduce(&cnt, &sum, 1, MPI_INT, MPI_SUM, comm);
183:       if (rank == i) unlink(filename);

185:       if (sum == size) {
186:         *shared = PETSC_TRUE;
187:         break;
189:     }
190:     *tagvalp = (int)*shared;
191:     PetscInfo(NULL, "processors %s %s\n", (*shared) ? "share" : "do NOT share", (iflg ? tmpname : "/tmp"));
192:   } else *shared = (PetscBool)*tagvalp;
193:   return 0;
194: }

196: /*@C
197:   PetscSharedWorkingDirectory - Determines if all processors in a communicator share a working directory or have different ones.

199:   Collective

201:   Input Parameter:
202: . comm - MPI_Communicator that may share working directory

204:   Output Parameter:
205: . shared - `PETSC_TRUE` or `PETSC_FALSE`

207:   Options Database Keys:
208: + -shared_working_directory - indicates the directory is shared among the MPI ranks
209: - -not_shared_working_directory - indicates the directory is shared among the MPI ranks

211:   Environmental Variables:
212: + `PETSC_SHARED_WORKING_DIRECTORY` - indicates the directory is shared among the MPI ranks
213: - `PETSC_NOT_SHARED_WORKING_DIRECTORY` - indicates the directory is shared among the MPI ranks

215:   Level: developer

217:   Notes:
218:   Stores the status as a MPI attribute so it does not have to be redetermined each time.

220:   Assumes that all processors in a communicator either
221: $   1) have a common working directory or
222: $   2) each has a separate working directory
223:   eventually we can write a fancier one that determines which processors share a common working directory.

225:   This will be very slow on runs with a large number of processors since it requires O(p*p) file opens.

227: .seealso: `PetscGetTmp()`, `PetscSharedTmp()`, `PetscGetWorkingDirectory()`, `PetscGetHomeDirectory()`
228: @*/
229: PetscErrorCode PetscSharedWorkingDirectory(MPI_Comm comm, PetscBool *shared)
230: {
231:   PetscMPIInt        size, rank, *tagvalp, sum, cnt, i;
232:   PetscBool          flg, iflg;
233:   FILE              *fd;
234:   static PetscMPIInt Petsc_WD_keyval = MPI_KEYVAL_INVALID;
235:   int                err;

237:   MPI_Comm_size(comm, &size);
238:   if (size == 1) {
239:     *shared = PETSC_TRUE;
240:     return 0;
241:   }

243:   PetscOptionsGetenv(comm, "PETSC_SHARED_WORKING_DIRECTORY", NULL, 0, &flg);
244:   if (flg) {
245:     *shared = PETSC_TRUE;
246:     return 0;
247:   }

249:   PetscOptionsGetenv(comm, "PETSC_NOT_SHARED_WORKING_DIRECTORY", NULL, 0, &flg);
250:   if (flg) {
251:     *shared = PETSC_FALSE;
252:     return 0;
253:   }

255:   if (Petsc_WD_keyval == MPI_KEYVAL_INVALID) MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN, Petsc_DelTmpShared, &Petsc_WD_keyval, NULL);

257:   MPI_Comm_get_attr(comm, Petsc_WD_keyval, (void **)&tagvalp, (int *)&iflg);
258:   if (!iflg) {
259:     char filename[PETSC_MAX_PATH_LEN];

261:     /* This communicator does not yet have a shared  attribute */
262:     PetscMalloc1(1, &tagvalp);
263:     MPI_Comm_set_attr(comm, Petsc_WD_keyval, tagvalp);

265:     PetscGetWorkingDirectory(filename, 240);
266:     PetscStrcat(filename, "/petsctestshared");
267:     MPI_Comm_rank(comm, &rank);

269:     /* each processor creates a  file and all the later ones check */
270:     /* this makes sure no subset of processors is shared */
271:     *shared = PETSC_FALSE;
272:     for (i = 0; i < size - 1; i++) {
273:       if (rank == i) {
274:         fd = fopen(filename, "w");
276:         err = fclose(fd);
278:       }
279:       MPI_Barrier(comm);
280:       if (rank >= i) {
281:         fd = fopen(filename, "r");
282:         if (fd) cnt = 1;
283:         else cnt = 0;
284:         if (fd) {
285:           err = fclose(fd);
287:         }
288:       } else cnt = 0;

290:       MPIU_Allreduce(&cnt, &sum, 1, MPI_INT, MPI_SUM, comm);
291:       if (rank == i) unlink(filename);

293:       if (sum == size) {
294:         *shared = PETSC_TRUE;
295:         break;
297:     }
298:     *tagvalp = (int)*shared;
299:   } else *shared = (PetscBool)*tagvalp;
300:   PetscInfo(NULL, "processors %s working directory\n", (*shared) ? "shared" : "do NOT share");
301:   return 0;
302: }

304: /*@C
305:     PetscFileRetrieve - Obtains a file from a URL or compressed
306:         and copies into local disk space as uncompressed.

308:     Collective

310:     Input Parameters:
311: +   comm     - processors accessing the file
312: .   url      - name of file, including entire URL (with or without .gz)
313: -   llen     - length of localname

315:     Output Parameters:
316: +   localname - name of local copy of file - valid on only process zero
317: -   found - if found or retrieved the file - valid on all processes

319:     Note:
320:     if the file already exists local this function just returns without downloading it.

322:     Level: intermediate
323: @*/
324: PetscErrorCode PetscFileRetrieve(MPI_Comm comm, const char url[], char localname[], size_t llen, PetscBool *found)
325: {
326:   char        buffer[PETSC_MAX_PATH_LEN], *par, *tlocalname, name[PETSC_MAX_PATH_LEN];
327:   FILE       *fp;
328:   PetscMPIInt rank;
329:   size_t      len = 0;
330:   PetscBool   flg1, flg2, flg3, flg4, download, compressed = PETSC_FALSE;

332:   MPI_Comm_rank(comm, &rank);
333:   if (rank == 0) {
334:     *found = PETSC_FALSE;

336:     PetscStrstr(url, ".gz", &par);
337:     if (par) {
338:       PetscStrlen(par, &len);
339:       if (len == 3) compressed = PETSC_TRUE;
340:     }

342:     PetscStrncmp(url, "ftp://", 6, &flg1);
343:     PetscStrncmp(url, "http://", 7, &flg2);
344:     PetscStrncmp(url, "file://", 7, &flg3);
345:     PetscStrncmp(url, "https://", 8, &flg4);
346:     download = (PetscBool)(flg1 || flg2 || flg3 || flg4);

348:     if (!download && !compressed) {
349:       PetscStrncpy(localname, url, llen);
350:       PetscTestFile(url, 'r', found);
351:       if (*found) {
352:         PetscInfo(NULL, "Found file %s\n", url);
353:       } else {
354:         PetscInfo(NULL, "Did not find file %s\n", url);
355:       }
356:       goto done;
357:     }

359:     /* look for uncompressed file in requested directory */
360:     if (compressed) {
361:       PetscStrncpy(localname, url, llen);
362:       PetscStrstr(localname, ".gz", &par);
363:       *par = 0; /* remove .gz extension */
364:       PetscTestFile(localname, 'r', found);
365:       if (*found) goto done;
366:     }

368:     /* look for file in current directory */
369:     PetscStrrchr(url, '/', &tlocalname);
370:     PetscStrncpy(localname, tlocalname, llen);
371:     if (compressed) {
372:       PetscStrstr(localname, ".gz", &par);
373:       *par = 0; /* remove .gz extension */
374:     }
375:     PetscTestFile(localname, 'r', found);
376:     if (*found) goto done;

378:     if (download) {
379:       /* local file is not already here so use curl to get it */
380:       PetscStrncpy(localname, tlocalname, llen);
381:       PetscStrcpy(buffer, "curl --fail --silent --show-error ");
382:       PetscStrcat(buffer, url);
383:       PetscStrcat(buffer, " > ");
384:       PetscStrcat(buffer, localname);
385: #if defined(PETSC_HAVE_POPEN)
386:       PetscPOpen(PETSC_COMM_SELF, NULL, buffer, "r", &fp);
387:       PetscPClose(PETSC_COMM_SELF, fp);
388: #else
389:       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP_SYS, "Cannot run external programs on this machine");
390: #endif
391:       PetscTestFile(localname, 'r', found);
392:       if (*found) {
393:         FILE *fd;
394:         char  buf[1024], *str, *substring;

396:         /* check if the file didn't exist so it downloaded an HTML message instead */
397:         fd = fopen(localname, "r");
399:         str = fgets(buf, sizeof(buf) - 1, fd);
400:         while (str) {
401:           PetscStrstr(buf, "<!DOCTYPE html>", &substring);
403:           PetscStrstr(buf, "Not Found", &substring);
405:           str = fgets(buf, sizeof(buf) - 1, fd);
406:         }
407:         fclose(fd);
408:       }
409:     } else if (compressed) {
410:       PetscTestFile(url, 'r', found);
411:       if (!*found) goto done;
412:       PetscStrncpy(localname, url, llen);
413:     }
414:     if (compressed) {
415:       PetscStrrchr(localname, '/', &tlocalname);
416:       PetscStrncpy(name, tlocalname, PETSC_MAX_PATH_LEN);
417:       PetscStrstr(name, ".gz", &par);
418:       *par = 0; /* remove .gz extension */
419:       /* uncompress file */
420:       PetscStrcpy(buffer, "gzip -c -d ");
421:       PetscStrcat(buffer, localname);
422:       PetscStrcat(buffer, " > ");
423:       PetscStrcat(buffer, name);
424: #if defined(PETSC_HAVE_POPEN)
425:       PetscPOpen(PETSC_COMM_SELF, NULL, buffer, "r", &fp);
426:       PetscPClose(PETSC_COMM_SELF, fp);
427: #else
428:       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP_SYS, "Cannot run external programs on this machine");
429: #endif
430:       PetscStrncpy(localname, name, llen);
431:       PetscTestFile(localname, 'r', found);
432:     }
433:   }
434: done:
435:   MPI_Bcast(found, 1, MPIU_BOOL, 0, comm);
436:   MPI_Bcast(localname, llen, MPI_CHAR, 0, comm);
437:   return 0;
438: }