Actual source code: fretrieve.c
1: /*
2: Code for opening and closing files.
3: */
4: #include <petscsys.h>
5: #if defined(PETSC_HAVE_PWD_H)
6: #include <pwd.h>
7: #endif
8: #include <ctype.h>
9: #include <sys/stat.h>
10: #if defined(PETSC_HAVE_UNISTD_H)
11: #include <unistd.h>
12: #endif
13: #if defined(PETSC_HAVE_SYS_UTSNAME_H)
14: #include <sys/utsname.h>
15: #endif
16: #include <fcntl.h>
17: #include <time.h>
18: #if defined(PETSC_HAVE_SYS_SYSTEMINFO_H)
19: #include <sys/systeminfo.h>
20: #endif
21: #include <petsc/private/petscimpl.h>
23: /*
24: Private routine to delete tmp/shared storage
26: This is called by MPI, not by users, when communicator attributes are deleted
28: Note: this is declared extern "C" because it is passed to MPI_Comm_create_keyval()
30: */
31: PETSC_EXTERN PetscMPIInt MPIAPI Petsc_DelTmpShared(MPI_Comm comm, PetscMPIInt keyval, void *count_val, void *extra_state)
32: {
33: PetscFunctionBegin;
34: PetscCallReturnMPI(PetscInfo(NULL, "Deleting tmp/shared data in an MPI_Comm %ld\n", (long)comm));
35: PetscCallReturnMPI(PetscFree(count_val));
36: PetscFunctionReturn(MPI_SUCCESS);
37: }
39: // "Unknown section 'Environmental Variables'"
40: // PetscClangLinter pragma disable: -fdoc-section-header-unknown
41: /*@C
42: PetscGetTmp - Gets the name of the "tmp" directory, often this is `/tmp`
44: Collective
46: Input Parameters:
47: + comm - MPI_Communicator that may share tmp
48: - len - length of string to hold name
50: Output Parameter:
51: . dir - directory name
53: Options Database Keys:
54: + -shared_tmp - indicates the directory is known to be shared among the MPI processes
55: . -not_shared_tmp - indicates the directory is known to be not shared among the MPI processes
56: - -tmp tmpdir - name of the directory you wish to use as tmp
58: Environmental Variables:
59: + `PETSC_SHARED_TMP` - indicates the directory is known to be shared among the MPI processes
60: . `PETSC_NOT_SHARED_TMP` - indicates the directory is known to be not shared among the MPI processes
61: - `PETSC_TMP` - name of the directory you wish to use as tmp
63: Level: developer
65: .seealso: `PetscSharedTmp()`, `PetscSharedWorkingDirectory()`, `PetscGetWorkingDirectory()`, `PetscGetHomeDirectory()`
66: @*/
67: PetscErrorCode PetscGetTmp(MPI_Comm comm, char dir[], size_t len)
68: {
69: PetscBool flg;
71: PetscFunctionBegin;
72: PetscCall(PetscOptionsGetenv(comm, "PETSC_TMP", dir, len, &flg));
73: if (!flg) PetscCall(PetscStrncpy(dir, "/tmp", len));
74: PetscFunctionReturn(PETSC_SUCCESS);
75: }
77: // "Unknown section 'Environmental Variables'"
78: // PetscClangLinter pragma disable: -fdoc-section-header-unknown
79: /*@
80: PetscSharedTmp - Determines if all processors in a communicator share a
81: tmp directory or have different ones.
83: Collective
85: Input Parameter:
86: . comm - MPI_Communicator that may share tmp
88: Output Parameter:
89: . shared - `PETSC_TRUE` or `PETSC_FALSE`
91: Options Database Keys:
92: + -shared_tmp - indicates the directory is known to be shared among the MPI processes
93: . -not_shared_tmp - indicates the directory is known to be not shared among the MPI processes
94: - -tmp tmpdir - name of the directory you wish to use as tmp
96: Environmental Variables:
97: + `PETSC_SHARED_TMP` - indicates the directory is known to be shared among the MPI processes
98: . `PETSC_NOT_SHARED_TMP` - indicates the directory is known to be not shared among the MPI processes
99: - `PETSC_TMP` - name of the directory you wish to use as tmp
101: Level: developer
103: Notes:
104: Stores the status as a MPI attribute so it does not have
105: to be redetermined each time.
107: Assumes that all processors in a communicator either
108: 1) have a common tmp or
109: 2) each has a separate tmp
110: eventually we can write a fancier one that determines which processors
111: share a common tmp.
113: This will be very slow on runs with a large number of processors since
114: it requires O(p*p) file opens.
116: If the environmental variable `PETSC_TMP` is set it will use this directory
117: as the "tmp" directory.
119: .seealso: `PetscGetTmp()`, `PetscSharedWorkingDirectory()`, `PetscGetWorkingDirectory()`, `PetscGetHomeDirectory()`
120: @*/
121: PetscErrorCode PetscSharedTmp(MPI_Comm comm, PetscBool *shared)
122: {
123: PetscMPIInt size, rank, *tagvalp, sum, cnt, i;
124: PetscBool flg, iflg;
125: FILE *fd;
126: int err;
128: PetscFunctionBegin;
129: PetscCallMPI(MPI_Comm_size(comm, &size));
130: if (size == 1) {
131: *shared = PETSC_TRUE;
132: PetscFunctionReturn(PETSC_SUCCESS);
133: }
135: PetscCall(PetscOptionsGetenv(comm, "PETSC_SHARED_TMP", NULL, 0, &flg));
136: if (flg) {
137: *shared = PETSC_TRUE;
138: PetscFunctionReturn(PETSC_SUCCESS);
139: }
141: PetscCall(PetscOptionsGetenv(comm, "PETSC_NOT_SHARED_TMP", NULL, 0, &flg));
142: if (flg) {
143: *shared = PETSC_FALSE;
144: PetscFunctionReturn(PETSC_SUCCESS);
145: }
147: if (Petsc_SharedTmp_keyval == MPI_KEYVAL_INVALID) PetscCallMPI(MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN, Petsc_DelTmpShared, &Petsc_SharedTmp_keyval, NULL));
149: PetscCallMPI(MPI_Comm_get_attr(comm, Petsc_SharedTmp_keyval, (void **)&tagvalp, (int *)&iflg));
150: if (!iflg) {
151: char filename[PETSC_MAX_PATH_LEN], tmpname[PETSC_MAX_PATH_LEN];
153: /* This communicator does not yet have a shared tmp attribute */
154: PetscCall(PetscMalloc1(1, &tagvalp));
155: PetscCallMPI(MPI_Comm_set_attr(comm, Petsc_SharedTmp_keyval, tagvalp));
157: PetscCall(PetscOptionsGetenv(comm, "PETSC_TMP", tmpname, 238, &iflg));
158: if (!iflg) {
159: PetscCall(PetscStrncpy(filename, "/tmp", sizeof(filename)));
160: } else {
161: PetscCall(PetscStrncpy(filename, tmpname, sizeof(filename)));
162: }
164: PetscCall(PetscStrlcat(filename, "/petsctestshared", sizeof(filename)));
165: PetscCallMPI(MPI_Comm_rank(comm, &rank));
167: /* each processor creates a /tmp file and all the later ones check */
168: /* this makes sure no subset of processors is shared */
169: *shared = PETSC_FALSE;
170: for (i = 0; i < size - 1; i++) {
171: if (rank == i) {
172: fd = fopen(filename, "w");
173: PetscCheck(fd, PETSC_COMM_SELF, PETSC_ERR_FILE_OPEN, "Unable to open test file %s", filename);
174: err = fclose(fd);
175: PetscCheck(!err, PETSC_COMM_SELF, PETSC_ERR_SYS, "fclose() failed on file");
176: }
177: PetscCallMPI(MPI_Barrier(comm));
178: if (rank >= i) {
179: fd = fopen(filename, "r");
180: if (fd) cnt = 1;
181: else cnt = 0;
182: if (fd) {
183: err = fclose(fd);
184: PetscCheck(!err, PETSC_COMM_SELF, PETSC_ERR_SYS, "fclose() failed on file");
185: }
186: } else cnt = 0;
188: PetscCallMPI(MPIU_Allreduce(&cnt, &sum, 1, MPI_INT, MPI_SUM, comm));
189: if (rank == i) unlink(filename);
191: if (sum == size) {
192: *shared = PETSC_TRUE;
193: break;
194: } else PetscCheck(sum == 1, PETSC_COMM_SELF, PETSC_ERR_SUP_SYS, "Subset of processes share /tmp ");
195: }
196: *tagvalp = (int)*shared;
197: PetscCall(PetscInfo(NULL, "processors %s %s\n", *shared ? "share" : "do NOT share", iflg ? tmpname : "/tmp"));
198: } else *shared = (PetscBool)*tagvalp;
199: PetscFunctionReturn(PETSC_SUCCESS);
200: }
202: // "Unknown section 'Environmental Variables'"
203: // PetscClangLinter pragma disable: -fdoc-section-header-unknown
204: /*@
205: PetscSharedWorkingDirectory - Determines if all processors in a communicator share a working directory or have different ones.
207: Collective
209: Input Parameter:
210: . comm - MPI_Communicator that may share working directory
212: Output Parameter:
213: . shared - `PETSC_TRUE` or `PETSC_FALSE`
215: Options Database Keys:
216: + -shared_working_directory - indicates the directory is known to be shared among the MPI processes
217: - -not_shared_working_directory - indicates the directory is known to be not shared among the MPI processes
219: Environmental Variables:
220: + `PETSC_SHARED_WORKING_DIRECTORY` - indicates the directory is known to be shared among the MPI processes
221: - `PETSC_NOT_SHARED_WORKING_DIRECTORY` - indicates the directory is known to be not shared among the MPI processes
223: Level: developer
225: Notes:
226: Stores the status as a MPI attribute so it does not have to be redetermined each time.
228: Assumes that all processors in a communicator either
229: .vb
230: 1) have a common working directory or
231: 2) each has a separate working directory
232: .ve
233: eventually we can write a fancier one that determines which processors share a common working directory.
235: This will be very slow on runs with a large number of processors since it requires O(p*p) file opens.
237: .seealso: `PetscGetTmp()`, `PetscSharedTmp()`, `PetscGetWorkingDirectory()`, `PetscGetHomeDirectory()`
238: @*/
239: PetscErrorCode PetscSharedWorkingDirectory(MPI_Comm comm, PetscBool *shared)
240: {
241: PetscMPIInt size, rank, *tagvalp, sum, cnt, i;
242: PetscBool flg, iflg;
243: FILE *fd;
244: int err;
246: PetscFunctionBegin;
247: PetscCallMPI(MPI_Comm_size(comm, &size));
248: if (size == 1) {
249: *shared = PETSC_TRUE;
250: PetscFunctionReturn(PETSC_SUCCESS);
251: }
253: PetscCall(PetscOptionsGetenv(comm, "PETSC_SHARED_WORKING_DIRECTORY", NULL, 0, &flg));
254: if (flg) {
255: *shared = PETSC_TRUE;
256: PetscFunctionReturn(PETSC_SUCCESS);
257: }
259: PetscCall(PetscOptionsGetenv(comm, "PETSC_NOT_SHARED_WORKING_DIRECTORY", NULL, 0, &flg));
260: if (flg) {
261: *shared = PETSC_FALSE;
262: PetscFunctionReturn(PETSC_SUCCESS);
263: }
265: if (Petsc_SharedWD_keyval == MPI_KEYVAL_INVALID) PetscCallMPI(MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN, Petsc_DelTmpShared, &Petsc_SharedWD_keyval, NULL));
267: PetscCallMPI(MPI_Comm_get_attr(comm, Petsc_SharedWD_keyval, (void **)&tagvalp, (int *)&iflg));
268: if (!iflg) {
269: char filename[PETSC_MAX_PATH_LEN];
271: /* This communicator does not yet have a shared attribute */
272: PetscCall(PetscMalloc1(1, &tagvalp));
273: PetscCallMPI(MPI_Comm_set_attr(comm, Petsc_SharedWD_keyval, tagvalp));
275: PetscCall(PetscGetWorkingDirectory(filename, sizeof(filename) - 16));
276: PetscCall(PetscStrlcat(filename, "/petsctestshared", sizeof(filename)));
277: PetscCallMPI(MPI_Comm_rank(comm, &rank));
279: /* each processor creates a file and all the later ones check */
280: /* this makes sure no subset of processors is shared */
281: *shared = PETSC_FALSE;
282: for (i = 0; i < size - 1; i++) {
283: if (rank == i) {
284: fd = fopen(filename, "w");
285: PetscCheck(fd, PETSC_COMM_SELF, PETSC_ERR_FILE_OPEN, "Unable to open test file %s", filename);
286: err = fclose(fd);
287: PetscCheck(!err, PETSC_COMM_SELF, PETSC_ERR_SYS, "fclose() failed on file");
288: }
289: PetscCallMPI(MPI_Barrier(comm));
290: if (rank >= i) {
291: fd = fopen(filename, "r");
292: if (fd) cnt = 1;
293: else cnt = 0;
294: if (fd) {
295: err = fclose(fd);
296: PetscCheck(!err, PETSC_COMM_SELF, PETSC_ERR_SYS, "fclose() failed on file");
297: }
298: } else cnt = 0;
300: PetscCallMPI(MPIU_Allreduce(&cnt, &sum, 1, MPI_INT, MPI_SUM, comm));
301: if (rank == i) unlink(filename);
303: if (sum == size) {
304: *shared = PETSC_TRUE;
305: break;
306: } else PetscCheck(sum == 1, PETSC_COMM_SELF, PETSC_ERR_SUP_SYS, "Subset of processes share working directory");
307: }
308: *tagvalp = (int)*shared;
309: } else *shared = (PetscBool)*tagvalp;
310: PetscCall(PetscInfo(NULL, "processors %s working directory\n", (*shared) ? "shared" : "do NOT share"));
311: PetscFunctionReturn(PETSC_SUCCESS);
312: }
314: /*@C
315: PetscFileRetrieve - Obtains a file from a URL or a compressed file
316: and copies into local disk space as uncompressed.
318: Collective
320: Input Parameters:
321: + comm - processors accessing the file
322: . url - name of file, including entire URL (with or without .gz)
323: - llen - length of `localname`
325: Output Parameters:
326: + localname - name of local copy of file - valid on only process zero
327: - found - if found or retrieved the file - valid on all processes
329: Level: developer
331: Note:
332: if the file already exists locally this function just returns without downloading it.
334: .seealso: `PetscDLLibraryRetrieve()`
335: @*/
336: PetscErrorCode PetscFileRetrieve(MPI_Comm comm, const char url[], char localname[], size_t llen, PetscBool *found)
337: {
338: char buffer[PETSC_MAX_PATH_LEN], *par = NULL, *tlocalname = NULL, name[PETSC_MAX_PATH_LEN];
339: FILE *fp;
340: PetscMPIInt rank;
341: size_t len = 0;
342: PetscBool flg1, flg2, flg3, flg4, download, compressed = PETSC_FALSE;
344: PetscFunctionBegin;
345: PetscCallMPI(MPI_Comm_rank(comm, &rank));
346: if (rank == 0) {
347: *found = PETSC_FALSE;
349: PetscCall(PetscStrstr(url, ".gz", &par));
350: if (par) {
351: PetscCall(PetscStrlen(par, &len));
352: if (len == 3) compressed = PETSC_TRUE;
353: }
355: PetscCall(PetscStrncmp(url, "ftp://", 6, &flg1));
356: PetscCall(PetscStrncmp(url, "http://", 7, &flg2));
357: PetscCall(PetscStrncmp(url, "file://", 7, &flg3));
358: PetscCall(PetscStrncmp(url, "https://", 8, &flg4));
359: download = (PetscBool)(flg1 || flg2 || flg3 || flg4);
361: if (!download && !compressed) {
362: PetscCall(PetscStrncpy(localname, url, llen));
363: PetscCall(PetscTestFile(url, 'r', found));
364: if (*found) {
365: PetscCall(PetscInfo(NULL, "Found file %s\n", url));
366: } else {
367: PetscCall(PetscInfo(NULL, "Did not find file %s\n", url));
368: }
369: goto done;
370: }
372: /* look for uncompressed file in requested directory */
373: if (compressed) {
374: PetscCall(PetscStrncpy(localname, url, llen));
375: PetscCall(PetscStrstr(localname, ".gz", &par));
376: *par = 0; /* remove .gz extension */
377: PetscCall(PetscTestFile(localname, 'r', found));
378: if (*found) goto done;
379: }
381: /* look for file in current directory */
382: PetscCall(PetscStrrchr(url, '/', &tlocalname));
383: PetscCall(PetscStrncpy(localname, tlocalname, llen));
384: if (compressed) {
385: PetscCall(PetscStrstr(localname, ".gz", &par));
386: *par = 0; /* remove .gz extension */
387: }
388: PetscCall(PetscTestFile(localname, 'r', found));
389: if (*found) goto done;
391: if (download) {
392: /* local file is not already here so use curl to get it */
393: PetscCall(PetscStrncpy(localname, tlocalname, llen));
394: PetscCall(PetscStrncpy(buffer, "curl --fail --silent --show-error ", sizeof(buffer)));
395: PetscCall(PetscStrlcat(buffer, url, sizeof(buffer)));
396: PetscCall(PetscStrlcat(buffer, " > ", sizeof(buffer)));
397: PetscCall(PetscStrlcat(buffer, localname, sizeof(buffer)));
398: #if defined(PETSC_HAVE_POPEN)
399: PetscCall(PetscPOpen(PETSC_COMM_SELF, NULL, buffer, "r", &fp));
400: PetscCall(PetscPClose(PETSC_COMM_SELF, fp));
401: #else
402: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP_SYS, "Cannot run external programs on this machine");
403: #endif
404: PetscCall(PetscTestFile(localname, 'r', found));
405: if (*found) {
406: FILE *fd;
407: char buf[1024], *str, *substring;
409: /* check if the file didn't exist so it downloaded an HTML message instead */
410: fd = fopen(localname, "r");
411: PetscCheck(fd, PETSC_COMM_SELF, PETSC_ERR_PLIB, "PetscTestFile() indicates %s exists but fopen() cannot open it", localname);
412: str = fgets(buf, sizeof(buf) - 1, fd);
413: while (str) {
414: PetscCall(PetscStrstr(buf, "<!DOCTYPE html>", &substring));
415: PetscCheck(!substring, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unable to download %s it does not appear to exist at this URL, dummy HTML file was downloaded", url);
416: PetscCall(PetscStrstr(buf, "Not Found", &substring));
417: PetscCheck(!substring, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unable to download %s it does not appear to exist at this URL, dummy HTML file was downloaded", url);
418: str = fgets(buf, sizeof(buf) - 1, fd);
419: }
420: fclose(fd);
421: }
422: } else if (compressed) {
423: PetscCall(PetscTestFile(url, 'r', found));
424: if (!*found) goto done;
425: PetscCall(PetscStrncpy(localname, url, llen));
426: }
427: if (compressed) {
428: PetscCall(PetscStrrchr(localname, '/', &tlocalname));
429: PetscCall(PetscStrncpy(name, tlocalname, PETSC_MAX_PATH_LEN));
430: PetscCall(PetscStrstr(name, ".gz", &par));
431: *par = 0; /* remove .gz extension */
432: /* uncompress file */
433: PetscCall(PetscStrncpy(buffer, "gzip -c -d ", sizeof(buffer)));
434: PetscCall(PetscStrlcat(buffer, localname, sizeof(buffer)));
435: PetscCall(PetscStrlcat(buffer, " > ", sizeof(buffer)));
436: PetscCall(PetscStrlcat(buffer, name, sizeof(buffer)));
437: #if defined(PETSC_HAVE_POPEN)
438: PetscCall(PetscPOpen(PETSC_COMM_SELF, NULL, buffer, "r", &fp));
439: PetscCall(PetscPClose(PETSC_COMM_SELF, fp));
440: #else
441: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP_SYS, "Cannot run external programs on this machine");
442: #endif
443: PetscCall(PetscStrncpy(localname, name, llen));
444: PetscCall(PetscTestFile(localname, 'r', found));
445: }
446: }
447: done:
448: PetscCallMPI(MPI_Bcast(found, 1, MPIU_BOOL, 0, comm));
449: PetscCallMPI(MPI_Bcast(localname, (PetscMPIInt)llen, MPI_CHAR, 0, comm));
450: PetscFunctionReturn(PETSC_SUCCESS);
451: }