Actual source code: fretrieve.c
1: /*
2: Code for opening and closing files.
3: */
4: #include <petscsys.h>
5: #if defined(PETSC_HAVE_PWD_H)
6: #include <pwd.h>
7: #endif
8: #include <ctype.h>
9: #include <sys/stat.h>
10: #if defined(PETSC_HAVE_UNISTD_H)
11: #include <unistd.h>
12: #endif
13: #if defined(PETSC_HAVE_SYS_UTSNAME_H)
14: #include <sys/utsname.h>
15: #endif
16: #include <fcntl.h>
17: #include <time.h>
18: #if defined(PETSC_HAVE_SYS_SYSTEMINFO_H)
19: #include <sys/systeminfo.h>
20: #endif
21: #include <petsc/private/petscimpl.h>
23: /*
24: Private routine to delete tmp/shared storage
26: This is called by MPI, not by users, when communicator attributes are deleted
28: */
29: static PetscMPIInt MPIAPI Petsc_DelTmpShared(MPI_Comm comm, PetscMPIInt keyval, void *count_val, void *extra_state)
30: {
31: PetscFunctionBegin;
32: PetscCallReturnMPI(PetscInfo(NULL, "Deleting tmp/shared data in an MPI_Comm %ld\n", (long)comm));
33: PetscCallReturnMPI(PetscFree(count_val));
34: PetscFunctionReturn(MPI_SUCCESS);
35: }
37: // "Unknown section 'Environmental Variables'"
38: // PetscClangLinter pragma disable: -fdoc-section-header-unknown
39: /*@C
40: PetscGetTmp - Gets the name of the "tmp" directory, often this is `/tmp`
42: Collective
44: Input Parameters:
45: + comm - MPI_Communicator that may share tmp
46: - len - length of string to hold name
48: Output Parameter:
49: . dir - directory name
51: Environmental Variables:
52: + `PETSC_SHARED_TMP` - indicates the directory is known to be shared among the MPI processes
53: . `PETSC_NOT_SHARED_TMP` - indicates the directory is known to be not shared among the MPI processes
54: - `PETSC_TMP` - name of the directory you wish to use as tmp
56: Level: developer
58: .seealso: `PetscSharedTmp()`, `PetscSharedWorkingDirectory()`, `PetscGetWorkingDirectory()`, `PetscGetHomeDirectory()`
59: @*/
60: PetscErrorCode PetscGetTmp(MPI_Comm comm, char dir[], size_t len)
61: {
62: PetscBool flg;
64: PetscFunctionBegin;
65: PetscCall(PetscOptionsGetenv(comm, "PETSC_TMP", dir, len, &flg));
66: if (!flg) PetscCall(PetscStrncpy(dir, "/tmp", len));
67: PetscFunctionReturn(PETSC_SUCCESS);
68: }
70: // "Unknown section 'Environmental Variables'"
71: // PetscClangLinter pragma disable: -fdoc-section-header-unknown
72: /*@
73: PetscSharedTmp - Determines if all processors in a communicator share a
74: tmp directory or have different ones.
76: Collective
78: Input Parameter:
79: . comm - MPI_Communicator that may share tmp
81: Output Parameter:
82: . shared - `PETSC_TRUE` or `PETSC_FALSE`
84: Options Database Keys:
85: + -shared_tmp - indicates the directory is known to be shared among the MPI processes
86: . -not_shared_tmp - indicates the directory is known to be not shared among the MPI processes
87: - -tmp tmpdir - name of the directory you wish to use as tmp
89: Environmental Variables:
90: + `PETSC_SHARED_TMP` - indicates the directory is known to be shared among the MPI processes
91: . `PETSC_NOT_SHARED_TMP` - indicates the directory is known to be not shared among the MPI processes
92: - `PETSC_TMP` - name of the directory you wish to use as tmp
94: Level: developer
96: Notes:
97: Stores the status as a MPI attribute so it does not have
98: to be redetermined each time.
100: Assumes that all processors in a communicator either
101: 1) have a common tmp or
102: 2) each has a separate tmp
103: eventually we can write a fancier one that determines which processors
104: share a common tmp.
106: This will be very slow on runs with a large number of processors since
107: it requires O(p*p) file opens.
109: If the environmental variable `PETSC_TMP` is set it will use this directory
110: as the "tmp" directory.
112: .seealso: `PetscGetTmp()`, `PetscSharedWorkingDirectory()`, `PetscGetWorkingDirectory()`, `PetscGetHomeDirectory()`
113: @*/
114: PetscErrorCode PetscSharedTmp(MPI_Comm comm, PetscBool *shared)
115: {
116: PetscMPIInt size, rank, *tagvalp, sum, cnt, i;
117: PetscBool flg;
118: PetscMPIInt iflg;
119: FILE *fd;
120: int err;
122: PetscFunctionBegin;
123: PetscCallMPI(MPI_Comm_size(comm, &size));
124: if (size == 1) {
125: *shared = PETSC_TRUE;
126: PetscFunctionReturn(PETSC_SUCCESS);
127: }
129: PetscCall(PetscOptionsGetenv(comm, "PETSC_SHARED_TMP", NULL, 0, &flg));
130: if (flg) {
131: *shared = PETSC_TRUE;
132: PetscFunctionReturn(PETSC_SUCCESS);
133: }
135: PetscCall(PetscOptionsGetenv(comm, "PETSC_NOT_SHARED_TMP", NULL, 0, &flg));
136: if (flg) {
137: *shared = PETSC_FALSE;
138: PetscFunctionReturn(PETSC_SUCCESS);
139: }
141: if (Petsc_SharedTmp_keyval == MPI_KEYVAL_INVALID) PetscCallMPI(MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN, Petsc_DelTmpShared, &Petsc_SharedTmp_keyval, NULL));
143: PetscCallMPI(MPI_Comm_get_attr(comm, Petsc_SharedTmp_keyval, (void **)&tagvalp, &iflg));
144: if (!iflg) {
145: char filename[PETSC_MAX_PATH_LEN], tmpname[PETSC_MAX_PATH_LEN];
147: /* This communicator does not yet have a shared tmp attribute */
148: PetscCall(PetscMalloc1(1, &tagvalp));
149: PetscCallMPI(MPI_Comm_set_attr(comm, Petsc_SharedTmp_keyval, tagvalp));
151: PetscCall(PetscOptionsGetenv(comm, "PETSC_TMP", tmpname, 238, &flg));
152: if (!flg) {
153: PetscCall(PetscStrncpy(filename, "/tmp", sizeof(filename)));
154: } else {
155: PetscCall(PetscStrncpy(filename, tmpname, sizeof(filename)));
156: }
158: PetscCall(PetscStrlcat(filename, "/petsctestshared", sizeof(filename)));
159: PetscCallMPI(MPI_Comm_rank(comm, &rank));
161: /* each processor creates a /tmp file and all the later ones check */
162: /* this makes sure no subset of processors is shared */
163: *shared = PETSC_FALSE;
164: for (i = 0; i < size - 1; i++) {
165: if (rank == i) {
166: fd = fopen(filename, "w");
167: PetscCheck(fd, PETSC_COMM_SELF, PETSC_ERR_FILE_OPEN, "Unable to open test file %s", filename);
168: err = fclose(fd);
169: PetscCheck(!err, PETSC_COMM_SELF, PETSC_ERR_SYS, "fclose() failed on file");
170: }
171: PetscCallMPI(MPI_Barrier(comm));
172: if (rank >= i) {
173: fd = fopen(filename, "r");
174: if (fd) cnt = 1;
175: else cnt = 0;
176: if (fd) {
177: err = fclose(fd);
178: PetscCheck(!err, PETSC_COMM_SELF, PETSC_ERR_SYS, "fclose() failed on file");
179: }
180: } else cnt = 0;
182: PetscCallMPI(MPIU_Allreduce(&cnt, &sum, 1, MPI_INT, MPI_SUM, comm));
183: if (rank == i) unlink(filename);
185: if (sum == size) {
186: *shared = PETSC_TRUE;
187: break;
188: } else PetscCheck(sum == 1, PETSC_COMM_SELF, PETSC_ERR_SUP_SYS, "Subset of processes share /tmp ");
189: }
190: *tagvalp = (int)*shared;
191: PetscCall(PetscInfo(NULL, "processors %s %s\n", *shared ? "share" : "do NOT share", flg ? tmpname : "/tmp"));
192: } else *shared = (PetscBool)*tagvalp;
193: PetscFunctionReturn(PETSC_SUCCESS);
194: }
196: // "Unknown section 'Environmental Variables'"
197: // PetscClangLinter pragma disable: -fdoc-section-header-unknown
198: /*@
199: PetscSharedWorkingDirectory - Determines if all processors in a communicator share a working directory or have different ones.
201: Collective
203: Input Parameter:
204: . comm - MPI_Communicator that may share working directory
206: Output Parameter:
207: . shared - `PETSC_TRUE` or `PETSC_FALSE`
209: Options Database Keys:
210: + -shared_working_directory - indicates the directory is known to be shared among the MPI processes
211: - -not_shared_working_directory - indicates the directory is known to be not shared among the MPI processes
213: Environmental Variables:
214: + `PETSC_SHARED_WORKING_DIRECTORY` - indicates the directory is known to be shared among the MPI processes
215: - `PETSC_NOT_SHARED_WORKING_DIRECTORY` - indicates the directory is known to be not shared among the MPI processes
217: Level: developer
219: Notes:
220: Stores the status as a MPI attribute so it does not have to be redetermined each time.
222: Assumes that all processors in a communicator either
223: .vb
224: 1) have a common working directory or
225: 2) each has a separate working directory
226: .ve
227: eventually we can write a fancier one that determines which processors share a common working directory.
229: This will be very slow on runs with a large number of processors since it requires O(p*p) file opens.
231: .seealso: `PetscGetTmp()`, `PetscSharedTmp()`, `PetscGetWorkingDirectory()`, `PetscGetHomeDirectory()`
232: @*/
233: PetscErrorCode PetscSharedWorkingDirectory(MPI_Comm comm, PetscBool *shared)
234: {
235: PetscMPIInt size, rank, *tagvalp, sum, cnt, i;
236: PetscBool flg;
237: PetscMPIInt iflg;
238: FILE *fd;
239: int err;
241: PetscFunctionBegin;
242: PetscCallMPI(MPI_Comm_size(comm, &size));
243: if (size == 1) {
244: *shared = PETSC_TRUE;
245: PetscFunctionReturn(PETSC_SUCCESS);
246: }
248: PetscCall(PetscOptionsGetenv(comm, "PETSC_SHARED_WORKING_DIRECTORY", NULL, 0, &flg));
249: if (flg) {
250: *shared = PETSC_TRUE;
251: PetscFunctionReturn(PETSC_SUCCESS);
252: }
254: PetscCall(PetscOptionsGetenv(comm, "PETSC_NOT_SHARED_WORKING_DIRECTORY", NULL, 0, &flg));
255: if (flg) {
256: *shared = PETSC_FALSE;
257: PetscFunctionReturn(PETSC_SUCCESS);
258: }
260: if (Petsc_SharedWD_keyval == MPI_KEYVAL_INVALID) PetscCallMPI(MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN, Petsc_DelTmpShared, &Petsc_SharedWD_keyval, NULL));
262: PetscCallMPI(MPI_Comm_get_attr(comm, Petsc_SharedWD_keyval, (void **)&tagvalp, &iflg));
263: if (!iflg) {
264: char filename[PETSC_MAX_PATH_LEN];
266: /* This communicator does not yet have a shared attribute */
267: PetscCall(PetscMalloc1(1, &tagvalp));
268: PetscCallMPI(MPI_Comm_set_attr(comm, Petsc_SharedWD_keyval, tagvalp));
270: PetscCall(PetscGetWorkingDirectory(filename, sizeof(filename) - 16));
271: PetscCall(PetscStrlcat(filename, "/petsctestshared", sizeof(filename)));
272: PetscCallMPI(MPI_Comm_rank(comm, &rank));
274: /* each processor creates a file and all the later ones check */
275: /* this makes sure no subset of processors is shared */
276: *shared = PETSC_FALSE;
277: for (i = 0; i < size - 1; i++) {
278: if (rank == i) {
279: fd = fopen(filename, "w");
280: PetscCheck(fd, PETSC_COMM_SELF, PETSC_ERR_FILE_OPEN, "Unable to open test file %s", filename);
281: err = fclose(fd);
282: PetscCheck(!err, PETSC_COMM_SELF, PETSC_ERR_SYS, "fclose() failed on file");
283: }
284: PetscCallMPI(MPI_Barrier(comm));
285: if (rank >= i) {
286: fd = fopen(filename, "r");
287: if (fd) cnt = 1;
288: else cnt = 0;
289: if (fd) {
290: err = fclose(fd);
291: PetscCheck(!err, PETSC_COMM_SELF, PETSC_ERR_SYS, "fclose() failed on file");
292: }
293: } else cnt = 0;
295: PetscCallMPI(MPIU_Allreduce(&cnt, &sum, 1, MPI_INT, MPI_SUM, comm));
296: if (rank == i) unlink(filename);
298: if (sum == size) {
299: *shared = PETSC_TRUE;
300: break;
301: } else PetscCheck(sum == 1, PETSC_COMM_SELF, PETSC_ERR_SUP_SYS, "Subset of processes share working directory");
302: }
303: *tagvalp = (int)*shared;
304: } else *shared = (PetscBool)*tagvalp;
305: PetscCall(PetscInfo(NULL, "processors %s working directory\n", (*shared) ? "shared" : "do NOT share"));
306: PetscFunctionReturn(PETSC_SUCCESS);
307: }
309: /*@C
310: PetscFileRetrieve - Obtains a file from a URL or a compressed file
311: and copies into local disk space as uncompressed.
313: Collective
315: Input Parameters:
316: + comm - processors accessing the file
317: . url - name of file, including entire URL (with or without .gz)
318: - llen - length of `localname`
320: Output Parameters:
321: + localname - name of local copy of file - valid on only process zero
322: - found - if found or retrieved the file - valid on all processes
324: Level: developer
326: Note:
327: if the file already exists locally this function just returns without downloading it.
329: .seealso: `PetscDLLibraryRetrieve()`
330: @*/
331: PetscErrorCode PetscFileRetrieve(MPI_Comm comm, const char url[], char localname[], size_t llen, PetscBool *found)
332: {
333: char buffer[PETSC_MAX_PATH_LEN], *par = NULL, *tlocalname = NULL, name[PETSC_MAX_PATH_LEN];
334: FILE *fp;
335: PetscMPIInt rank;
336: size_t len = 0;
337: PetscBool flg1, flg2, flg3, flg4, download, compressed = PETSC_FALSE;
339: PetscFunctionBegin;
340: PetscCallMPI(MPI_Comm_rank(comm, &rank));
341: if (rank == 0) {
342: *found = PETSC_FALSE;
344: PetscCall(PetscStrstr(url, ".gz", &par));
345: if (par) {
346: PetscCall(PetscStrlen(par, &len));
347: if (len == 3) compressed = PETSC_TRUE;
348: }
350: PetscCall(PetscStrncmp(url, "ftp://", 6, &flg1));
351: PetscCall(PetscStrncmp(url, "http://", 7, &flg2));
352: PetscCall(PetscStrncmp(url, "file://", 7, &flg3));
353: PetscCall(PetscStrncmp(url, "https://", 8, &flg4));
354: download = (PetscBool)(flg1 || flg2 || flg3 || flg4);
356: if (!download && !compressed) {
357: PetscCall(PetscStrncpy(localname, url, llen));
358: PetscCall(PetscTestFile(url, 'r', found));
359: if (*found) {
360: PetscCall(PetscInfo(NULL, "Found file %s\n", url));
361: } else {
362: PetscCall(PetscInfo(NULL, "Did not find file %s\n", url));
363: }
364: goto done;
365: }
367: /* look for uncompressed file in requested directory */
368: if (compressed) {
369: PetscCall(PetscStrncpy(localname, url, llen));
370: PetscCall(PetscStrstr(localname, ".gz", &par));
371: *par = 0; /* remove .gz extension */
372: PetscCall(PetscTestFile(localname, 'r', found));
373: if (*found) goto done;
374: }
376: /* look for file in current directory */
377: PetscCall(PetscStrrchr(url, '/', &tlocalname));
378: PetscCall(PetscStrncpy(localname, tlocalname, llen));
379: if (compressed) {
380: PetscCall(PetscStrstr(localname, ".gz", &par));
381: *par = 0; /* remove .gz extension */
382: }
383: PetscCall(PetscTestFile(localname, 'r', found));
384: if (*found) goto done;
386: if (download) {
387: /* local file is not already here so use curl to get it */
388: PetscCall(PetscStrncpy(localname, tlocalname, llen));
389: PetscCall(PetscStrncpy(buffer, "curl --fail --silent --show-error ", sizeof(buffer)));
390: PetscCall(PetscStrlcat(buffer, url, sizeof(buffer)));
391: PetscCall(PetscStrlcat(buffer, " > ", sizeof(buffer)));
392: PetscCall(PetscStrlcat(buffer, localname, sizeof(buffer)));
393: #if defined(PETSC_HAVE_POPEN)
394: PetscCall(PetscPOpen(PETSC_COMM_SELF, NULL, buffer, "r", &fp));
395: PetscCall(PetscPClose(PETSC_COMM_SELF, fp));
396: #else
397: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP_SYS, "Cannot run external programs on this machine");
398: #endif
399: PetscCall(PetscTestFile(localname, 'r', found));
400: if (*found) {
401: FILE *fd;
402: char buf[1024], *str, *substring;
404: /* check if the file didn't exist so it downloaded an HTML message instead */
405: fd = fopen(localname, "r");
406: PetscCheck(fd, PETSC_COMM_SELF, PETSC_ERR_PLIB, "PetscTestFile() indicates %s exists but fopen() cannot open it", localname);
407: str = fgets(buf, sizeof(buf) - 1, fd);
408: while (str) {
409: PetscCall(PetscStrstr(buf, "<!DOCTYPE html>", &substring));
410: PetscCheck(!substring, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unable to download %s it does not appear to exist at this URL, dummy HTML file was downloaded", url);
411: PetscCall(PetscStrstr(buf, "Not Found", &substring));
412: PetscCheck(!substring, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unable to download %s it does not appear to exist at this URL, dummy HTML file was downloaded", url);
413: str = fgets(buf, sizeof(buf) - 1, fd);
414: }
415: fclose(fd);
416: }
417: } else if (compressed) {
418: PetscCall(PetscTestFile(url, 'r', found));
419: if (!*found) goto done;
420: PetscCall(PetscStrncpy(localname, url, llen));
421: }
422: if (compressed) {
423: PetscCall(PetscStrrchr(localname, '/', &tlocalname));
424: PetscCall(PetscStrncpy(name, tlocalname, PETSC_MAX_PATH_LEN));
425: PetscCall(PetscStrstr(name, ".gz", &par));
426: *par = 0; /* remove .gz extension */
427: /* uncompress file */
428: PetscCall(PetscStrncpy(buffer, "gzip -c -d ", sizeof(buffer)));
429: PetscCall(PetscStrlcat(buffer, localname, sizeof(buffer)));
430: PetscCall(PetscStrlcat(buffer, " > ", sizeof(buffer)));
431: PetscCall(PetscStrlcat(buffer, name, sizeof(buffer)));
432: #if defined(PETSC_HAVE_POPEN)
433: PetscCall(PetscPOpen(PETSC_COMM_SELF, NULL, buffer, "r", &fp));
434: PetscCall(PetscPClose(PETSC_COMM_SELF, fp));
435: #else
436: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP_SYS, "Cannot run external programs on this machine");
437: #endif
438: PetscCall(PetscStrncpy(localname, name, llen));
439: PetscCall(PetscTestFile(localname, 'r', found));
440: }
441: }
442: done:
443: PetscCallMPI(MPI_Bcast(found, 1, MPI_C_BOOL, 0, comm));
444: PetscCallMPI(MPI_Bcast(localname, (PetscMPIInt)llen, MPI_CHAR, 0, comm));
445: PetscFunctionReturn(PETSC_SUCCESS);
446: }