Actual source code: fretrieve.c
1: /*
2: Code for opening and closing files.
3: */
4: #include <petsc/private/petscimpl.h>
5: #if PetscDefined(HAVE_PWD_H)
6: #include <pwd.h>
7: #endif
8: #include <ctype.h>
9: #include <sys/stat.h>
10: #if PetscDefined(HAVE_UNISTD_H)
11: #include <unistd.h>
12: #endif
13: #if PetscDefined(HAVE_SYS_UTSNAME_H)
14: #include <sys/utsname.h>
15: #endif
16: #include <fcntl.h>
17: #include <time.h>
18: #if PetscDefined(HAVE_SYS_SYSTEMINFO_H)
19: #include <sys/systeminfo.h>
20: #endif
22: /*
23: Private routine to delete tmp/shared storage
25: This is called by MPI, not by users, when communicator attributes are deleted
27: */
28: static PetscMPIInt MPIAPI Petsc_DelTmpShared(MPI_Comm comm, PetscMPIInt keyval, void *count_val, void *extra_state)
29: {
30: PetscFunctionBegin;
31: PetscCallReturnMPI(PetscInfo(NULL, "Deleting tmp/shared data in an MPI_Comm %ld\n", (long)comm));
32: PetscCallReturnMPI(PetscFree(count_val));
33: PetscFunctionReturn(MPI_SUCCESS);
34: }
36: // "Unknown section 'Environmental Variables'"
37: // PetscClangLinter pragma disable: -fdoc-section-header-unknown
38: /*@
39: PetscGetTmp - Gets the name of the "tmp" directory, often this is `/tmp`
41: Collective
43: Input Parameters:
44: + comm - MPI_Communicator that may share tmp
45: - len - length of string to hold name
47: Output Parameter:
48: . dir - directory name
50: Environmental Variables:
51: + `PETSC_SHARED_TMP` - indicates the directory is known to be shared among the MPI processes
52: . `PETSC_NOT_SHARED_TMP` - indicates the directory is known to be not shared among the MPI processes
53: - `PETSC_TMP` - name of the directory you wish to use as tmp
55: Level: developer
57: .seealso: `PetscSharedTmp()`, `PetscSharedWorkingDirectory()`, `PetscGetWorkingDirectory()`, `PetscGetHomeDirectory()`
58: @*/
59: PetscErrorCode PetscGetTmp(MPI_Comm comm, char dir[], size_t len)
60: {
61: PetscBool flg;
63: PetscFunctionBegin;
64: PetscCall(PetscOptionsGetenv(comm, "PETSC_TMP", dir, len, &flg));
65: if (!flg) PetscCall(PetscStrncpy(dir, "/tmp", len));
66: PetscFunctionReturn(PETSC_SUCCESS);
67: }
69: // "Unknown section 'Environmental Variables'"
70: // PetscClangLinter pragma disable: -fdoc-section-header-unknown
71: /*@
72: PetscSharedTmp - Determines if all processors in a communicator share a
73: tmp directory or have different ones.
75: Collective
77: Input Parameter:
78: . comm - MPI_Communicator that may share tmp
80: Output Parameter:
81: . shared - `PETSC_TRUE` or `PETSC_FALSE`
83: Options Database Keys:
84: + -shared_tmp - indicates the directory is known to be shared among the MPI processes
85: . -not_shared_tmp - indicates the directory is known to be not shared among the MPI processes
86: - -tmp tmpdir - name of the directory you wish to use as tmp
88: Environmental Variables:
89: + `PETSC_SHARED_TMP` - indicates the directory is known to be shared among the MPI processes
90: . `PETSC_NOT_SHARED_TMP` - indicates the directory is known to be not shared among the MPI processes
91: - `PETSC_TMP` - name of the directory you wish to use as tmp
93: Level: developer
95: Notes:
96: Stores the status as a MPI attribute so it does not have
97: to be redetermined each time.
99: Assumes that all processors in a communicator either
100: 1) have a common tmp or
101: 2) each has a separate tmp
102: eventually we can write a fancier one that determines which processors
103: share a common tmp.
105: This will be very slow on runs with a large number of processors since
106: it requires O(p*p) file opens.
108: If the environmental variable `PETSC_TMP` is set it will use this directory
109: as the "tmp" directory.
111: .seealso: `PetscGetTmp()`, `PetscSharedWorkingDirectory()`, `PetscGetWorkingDirectory()`, `PetscGetHomeDirectory()`
112: @*/
113: PetscErrorCode PetscSharedTmp(MPI_Comm comm, PetscBool *shared)
114: {
115: PetscMPIInt size, rank, *tagvalp, sum, i;
116: PetscBool flg;
117: PetscMPIInt iflg;
118: FILE *fd;
119: int err;
121: PetscFunctionBegin;
122: PetscCallMPI(MPI_Comm_size(comm, &size));
123: if (size == 1) {
124: *shared = PETSC_TRUE;
125: PetscFunctionReturn(PETSC_SUCCESS);
126: }
128: PetscCall(PetscOptionsGetenv(comm, "PETSC_SHARED_TMP", NULL, 0, &flg));
129: if (flg) {
130: *shared = PETSC_TRUE;
131: PetscFunctionReturn(PETSC_SUCCESS);
132: }
134: PetscCall(PetscOptionsGetenv(comm, "PETSC_NOT_SHARED_TMP", NULL, 0, &flg));
135: if (flg) {
136: *shared = PETSC_FALSE;
137: PetscFunctionReturn(PETSC_SUCCESS);
138: }
140: if (Petsc_SharedTmp_keyval == MPI_KEYVAL_INVALID) PetscCallMPI(MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN, Petsc_DelTmpShared, &Petsc_SharedTmp_keyval, NULL));
142: PetscCallMPI(MPI_Comm_get_attr(comm, Petsc_SharedTmp_keyval, (void **)&tagvalp, &iflg));
143: if (!iflg) {
144: char filename[PETSC_MAX_PATH_LEN], tmpname[PETSC_MAX_PATH_LEN];
146: /* This communicator does not yet have a shared tmp attribute */
147: PetscCall(PetscMalloc1(1, &tagvalp));
148: PetscCallMPI(MPI_Comm_set_attr(comm, Petsc_SharedTmp_keyval, tagvalp));
150: PetscCall(PetscOptionsGetenv(comm, "PETSC_TMP", tmpname, 238, &flg));
151: if (!flg) {
152: PetscCall(PetscStrncpy(filename, "/tmp", sizeof(filename)));
153: } else {
154: PetscCall(PetscStrncpy(filename, tmpname, sizeof(filename)));
155: }
157: PetscCall(PetscStrlcat(filename, "/petsctestshared", sizeof(filename)));
158: PetscCallMPI(MPI_Comm_rank(comm, &rank));
160: /* each processor creates a /tmp file and all the later ones check */
161: /* this makes sure no subset of processors is shared */
162: *shared = PETSC_FALSE;
163: for (i = 0; i < size - 1; i++) {
164: if (rank == i) {
165: fd = fopen(filename, "w");
166: PetscCheck(fd, PETSC_COMM_SELF, PETSC_ERR_FILE_OPEN, "Unable to open test file %s", filename);
167: err = fclose(fd);
168: PetscCheck(!err, PETSC_COMM_SELF, PETSC_ERR_SYS, "fclose() failed on file");
169: }
170: PetscCallMPI(MPI_Barrier(comm));
171: if (rank >= i) {
172: fd = fopen(filename, "r");
173: if (fd) sum = 1;
174: else sum = 0;
175: if (fd) {
176: err = fclose(fd);
177: PetscCheck(!err, PETSC_COMM_SELF, PETSC_ERR_SYS, "fclose() failed on file");
178: }
179: } else sum = 0;
181: PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &sum, 1, MPI_INT, MPI_SUM, comm));
182: if (rank == i) unlink(filename);
184: if (sum == size) {
185: *shared = PETSC_TRUE;
186: break;
187: } else PetscCheck(sum == 1, PETSC_COMM_SELF, PETSC_ERR_SUP_SYS, "Subset of processes share /tmp ");
188: }
189: *tagvalp = (int)*shared;
190: PetscCall(PetscInfo(NULL, "processors %s %s\n", *shared ? "share" : "do NOT share", flg ? tmpname : "/tmp"));
191: } else *shared = (PetscBool)*tagvalp;
192: PetscFunctionReturn(PETSC_SUCCESS);
193: }
195: // "Unknown section 'Environmental Variables'"
196: // PetscClangLinter pragma disable: -fdoc-section-header-unknown
197: /*@
198: PetscSharedWorkingDirectory - Determines if all processors in a communicator share a working directory or have different ones.
200: Collective
202: Input Parameter:
203: . comm - MPI_Communicator that may share working directory
205: Output Parameter:
206: . shared - `PETSC_TRUE` or `PETSC_FALSE`
208: Options Database Keys:
209: + -shared_working_directory - indicates the directory is known to be shared among the MPI processes
210: - -not_shared_working_directory - indicates the directory is known to be not shared among the MPI processes
212: Environmental Variables:
213: + `PETSC_SHARED_WORKING_DIRECTORY` - indicates the directory is known to be shared among the MPI processes
214: - `PETSC_NOT_SHARED_WORKING_DIRECTORY` - indicates the directory is known to be not shared among the MPI processes
216: Level: developer
218: Notes:
219: Stores the status as a MPI attribute so it does not have to be redetermined each time.
221: Assumes that all processors in a communicator either
222: .vb
223: 1) have a common working directory or
224: 2) each has a separate working directory
225: .ve
226: eventually we can write a fancier one that determines which processors share a common working directory.
228: This will be very slow on runs with a large number of processors since it requires O(p*p) file opens.
230: .seealso: `PetscGetTmp()`, `PetscSharedTmp()`, `PetscGetWorkingDirectory()`, `PetscGetHomeDirectory()`
231: @*/
232: PetscErrorCode PetscSharedWorkingDirectory(MPI_Comm comm, PetscBool *shared)
233: {
234: PetscMPIInt size, rank, *tagvalp, sum, i;
235: PetscBool flg;
236: PetscMPIInt iflg;
237: FILE *fd;
238: int err;
240: PetscFunctionBegin;
241: PetscCallMPI(MPI_Comm_size(comm, &size));
242: if (size == 1) {
243: *shared = PETSC_TRUE;
244: PetscFunctionReturn(PETSC_SUCCESS);
245: }
247: PetscCall(PetscOptionsGetenv(comm, "PETSC_SHARED_WORKING_DIRECTORY", NULL, 0, &flg));
248: if (flg) {
249: *shared = PETSC_TRUE;
250: PetscFunctionReturn(PETSC_SUCCESS);
251: }
253: PetscCall(PetscOptionsGetenv(comm, "PETSC_NOT_SHARED_WORKING_DIRECTORY", NULL, 0, &flg));
254: if (flg) {
255: *shared = PETSC_FALSE;
256: PetscFunctionReturn(PETSC_SUCCESS);
257: }
259: if (Petsc_SharedWD_keyval == MPI_KEYVAL_INVALID) PetscCallMPI(MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN, Petsc_DelTmpShared, &Petsc_SharedWD_keyval, NULL));
261: PetscCallMPI(MPI_Comm_get_attr(comm, Petsc_SharedWD_keyval, (void **)&tagvalp, &iflg));
262: if (!iflg) {
263: char filename[PETSC_MAX_PATH_LEN];
265: /* This communicator does not yet have a shared attribute */
266: PetscCall(PetscMalloc1(1, &tagvalp));
267: PetscCallMPI(MPI_Comm_set_attr(comm, Petsc_SharedWD_keyval, tagvalp));
269: PetscCall(PetscGetWorkingDirectory(filename, sizeof(filename) - 16));
270: PetscCall(PetscStrlcat(filename, "/petsctestshared", sizeof(filename)));
271: PetscCallMPI(MPI_Comm_rank(comm, &rank));
273: /* each processor creates a file and all the later ones check */
274: /* this makes sure no subset of processors is shared */
275: *shared = PETSC_FALSE;
276: for (i = 0; i < size - 1; i++) {
277: if (rank == i) {
278: fd = fopen(filename, "w");
279: PetscCheck(fd, PETSC_COMM_SELF, PETSC_ERR_FILE_OPEN, "Unable to open test file %s", filename);
280: err = fclose(fd);
281: PetscCheck(!err, PETSC_COMM_SELF, PETSC_ERR_SYS, "fclose() failed on file");
282: }
283: PetscCallMPI(MPI_Barrier(comm));
284: if (rank >= i) {
285: fd = fopen(filename, "r");
286: if (fd) sum = 1;
287: else sum = 0;
288: if (fd) {
289: err = fclose(fd);
290: PetscCheck(!err, PETSC_COMM_SELF, PETSC_ERR_SYS, "fclose() failed on file");
291: }
292: } else sum = 0;
294: PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &sum, 1, MPI_INT, MPI_SUM, comm));
295: if (rank == i) unlink(filename);
297: if (sum == size) {
298: *shared = PETSC_TRUE;
299: break;
300: } else PetscCheck(sum == 1, PETSC_COMM_SELF, PETSC_ERR_SUP_SYS, "Subset of processes share working directory");
301: }
302: *tagvalp = (int)*shared;
303: } else *shared = (PetscBool)*tagvalp;
304: PetscCall(PetscInfo(NULL, "processors %s working directory\n", (*shared) ? "shared" : "do NOT share"));
305: PetscFunctionReturn(PETSC_SUCCESS);
306: }
308: /*@
309: PetscFileRetrieve - Obtains a file from a URL or a compressed file
310: and copies into local disk space as uncompressed.
312: Collective
314: Input Parameters:
315: + comm - processors accessing the file
316: . url - name of file, including entire URL (with or without .gz)
317: - llen - length of `localname`
319: Output Parameters:
320: + localname - name of local copy of file - valid on only process zero
321: - found - if found or retrieved the file - valid on all processes
323: Level: developer
325: Note:
326: if the file already exists locally this function just returns without downloading it.
328: .seealso: `PetscDLLibraryRetrieve()`
329: @*/
330: PetscErrorCode PetscFileRetrieve(MPI_Comm comm, const char url[], char localname[], size_t llen, PetscBool *found)
331: {
332: char buffer[PETSC_MAX_PATH_LEN], *par = NULL, *tlocalname = NULL, name[PETSC_MAX_PATH_LEN];
333: FILE *fp;
334: PetscMPIInt rank;
335: size_t len = 0;
336: PetscBool flg1, flg2, flg3, flg4, download, compressed = PETSC_FALSE;
338: PetscFunctionBegin;
339: PetscCallMPI(MPI_Comm_rank(comm, &rank));
340: if (rank == 0) {
341: *found = PETSC_FALSE;
343: PetscCall(PetscStrstr(url, ".gz", &par));
344: if (par) {
345: PetscCall(PetscStrlen(par, &len));
346: if (len == 3) compressed = PETSC_TRUE;
347: }
349: PetscCall(PetscStrncmp(url, "ftp://", 6, &flg1));
350: PetscCall(PetscStrncmp(url, "http://", 7, &flg2));
351: PetscCall(PetscStrncmp(url, "file://", 7, &flg3));
352: PetscCall(PetscStrncmp(url, "https://", 8, &flg4));
353: download = (PetscBool)(flg1 || flg2 || flg3 || flg4);
355: if (!download && !compressed) {
356: PetscCall(PetscStrncpy(localname, url, llen));
357: PetscCall(PetscTestFile(url, 'r', found));
358: if (*found) {
359: PetscCall(PetscInfo(NULL, "Found file %s\n", url));
360: } else {
361: PetscCall(PetscInfo(NULL, "Did not find file %s\n", url));
362: }
363: goto done;
364: }
366: /* look for uncompressed file in requested directory */
367: if (compressed) {
368: PetscCall(PetscStrncpy(localname, url, llen));
369: PetscCall(PetscStrstr(localname, ".gz", &par));
370: *par = 0; /* remove .gz extension */
371: PetscCall(PetscTestFile(localname, 'r', found));
372: if (*found) goto done;
373: }
375: /* look for file in current directory */
376: PetscCall(PetscStrrchr(url, '/', &tlocalname));
377: PetscCall(PetscStrncpy(localname, tlocalname, llen));
378: if (compressed) {
379: PetscCall(PetscStrstr(localname, ".gz", &par));
380: *par = 0; /* remove .gz extension */
381: }
382: PetscCall(PetscTestFile(localname, 'r', found));
383: if (*found) goto done;
385: if (download) {
386: /* local file is not already here so use curl to get it */
387: PetscCall(PetscStrncpy(localname, tlocalname, llen));
388: PetscCall(PetscStrncpy(buffer, "curl --fail --silent --show-error ", sizeof(buffer)));
389: PetscCall(PetscStrlcat(buffer, url, sizeof(buffer)));
390: PetscCall(PetscStrlcat(buffer, " > ", sizeof(buffer)));
391: PetscCall(PetscStrlcat(buffer, localname, sizeof(buffer)));
392: #if PetscDefined(HAVE_POPEN)
393: PetscCall(PetscPOpen(PETSC_COMM_SELF, NULL, buffer, "r", &fp));
394: PetscCall(PetscPClose(PETSC_COMM_SELF, fp));
395: #else
396: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP_SYS, "Cannot run external programs on this machine");
397: #endif
398: PetscCall(PetscTestFile(localname, 'r', found));
399: if (*found) {
400: FILE *fd;
401: char buf[1024], *str, *substring;
403: /* check if the file didn't exist so it downloaded an HTML message instead */
404: fd = fopen(localname, "r");
405: PetscCheck(fd, PETSC_COMM_SELF, PETSC_ERR_PLIB, "PetscTestFile() indicates %s exists but fopen() cannot open it", localname);
406: str = fgets(buf, sizeof(buf) - 1, fd);
407: while (str) {
408: PetscCall(PetscStrstr(buf, "<!DOCTYPE html>", &substring));
409: PetscCheck(!substring, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unable to download %s it does not appear to exist at this URL, dummy HTML file was downloaded", url);
410: PetscCall(PetscStrstr(buf, "Not Found", &substring));
411: PetscCheck(!substring, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unable to download %s it does not appear to exist at this URL, dummy HTML file was downloaded", url);
412: str = fgets(buf, sizeof(buf) - 1, fd);
413: }
414: fclose(fd);
415: }
416: } else if (compressed) {
417: PetscCall(PetscTestFile(url, 'r', found));
418: if (!*found) goto done;
419: PetscCall(PetscStrncpy(localname, url, llen));
420: }
421: if (compressed) {
422: PetscCall(PetscStrrchr(localname, '/', &tlocalname));
423: PetscCall(PetscStrncpy(name, tlocalname, PETSC_MAX_PATH_LEN));
424: PetscCall(PetscStrstr(name, ".gz", &par));
425: *par = 0; /* remove .gz extension */
426: /* uncompress file */
427: PetscCall(PetscStrncpy(buffer, "gzip -c -d ", sizeof(buffer)));
428: PetscCall(PetscStrlcat(buffer, localname, sizeof(buffer)));
429: PetscCall(PetscStrlcat(buffer, " > ", sizeof(buffer)));
430: PetscCall(PetscStrlcat(buffer, name, sizeof(buffer)));
431: #if PetscDefined(HAVE_POPEN)
432: PetscCall(PetscPOpen(PETSC_COMM_SELF, NULL, buffer, "r", &fp));
433: PetscCall(PetscPClose(PETSC_COMM_SELF, fp));
434: #else
435: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP_SYS, "Cannot run external programs on this machine");
436: #endif
437: PetscCall(PetscStrncpy(localname, name, llen));
438: PetscCall(PetscTestFile(localname, 'r', found));
439: }
440: }
441: done:
442: PetscCallMPI(MPI_Bcast(found, 1, MPI_C_BOOL, 0, comm));
443: PetscCallMPI(MPI_Bcast(localname, (PetscMPIInt)llen, MPI_CHAR, 0, comm));
444: PetscFunctionReturn(PETSC_SUCCESS);
445: }