Actual source code: ex9.c

  1: static const char help[] = "Tests PetscDeviceContextQueryIdle() and PetscDeviceContextDelay().\n\n";

  3: #include "petscdevicetestcommon.h"

  5: static PetscErrorCode CheckIdle(PetscDeviceContext dctx, const char operation[])
  6: {
  7:   PetscBool idle = PETSC_FALSE;

  9:   PetscFunctionBegin;
 10:   PetscCall(PetscDeviceContextQueryIdle(dctx, &idle));
 11:   if (!idle) {
 12:     PetscCall(PetscDeviceContextView(dctx, NULL));
 13:     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_PLIB, "PetscDeviceContext was not idle after %s!", operation);
 14:   }
 15:   PetscFunctionReturn(PETSC_SUCCESS);
 16: }

 18: static PetscErrorCode TestQueryIdle(PetscDeviceContext dctx)
 19: {
 20:   PetscDeviceContext other = NULL;

 22:   PetscFunctionBegin;
 23:   // Should of course be idle after synchronization
 24:   PetscCall(PetscDeviceContextSynchronize(dctx));
 25:   PetscCall(CheckIdle(dctx, "synchronization"));

 27:   // Creating an unrelated device context should leave it idle
 28:   PetscCall(PetscDeviceContextCreate(&other));
 29:   PetscCall(CheckIdle(dctx, "creating unrelated dctx"));

 31:   // Destroying an unrelated device context shouldn't change things either
 32:   PetscCall(PetscDeviceContextDestroy(&other));
 33:   PetscCall(CheckIdle(dctx, "destroying unrelated dctx"));

 35:   // Duplicating shouldn't change it either
 36:   PetscCall(PetscDeviceContextDuplicate(dctx, &other));
 37:   PetscCall(CheckIdle(dctx, "duplication"));

 39:   // Another ctx waiting on it (which may make the other ctx non-idle) should not make the
 40:   // current one non-idle.
 41:   PetscCall(PetscDeviceContextWaitForContext(other, dctx));
 42:   PetscCall(PetscDeviceContextSynchronize(other));
 43:   PetscCall(CheckIdle(dctx, "other context waited on it"));
 44:   PetscCall(CheckIdle(other, "synchronizing the waiting context"));

 46:   PetscCall(PetscDeviceContextDestroy(&other));
 47:   PetscFunctionReturn(PETSC_SUCCESS);
 48: }

 50: static PetscErrorCode TestDelay(PetscDeviceContext dctx)
 51: {
 52:   PetscDeviceContext other;
 53:   PetscDeviceType    type;
 54:   PetscLogDouble     start, end;
 55:   PetscBool          idle, synchronous;

 57:   PetscFunctionBeginUser;
 58:   PetscCall(PetscDeviceContextDuplicate(dctx, &other));
 59:   PetscCall(PetscDeviceContextSetStreamType(other, PETSC_STREAM_NONBLOCKING));
 60:   PetscCall(PetscDeviceContextSetUp(other));
 61:   PetscCall(PetscDeviceContextGetDeviceType(other, &type));
 62:   synchronous = (PetscBool)(type == PETSC_DEVICE_HOST);
 63:   PetscCall(PetscDeviceContextDelay(other, 0));
 64:   PetscCall(PetscDeviceContextDelay(other, 0.001));
 65:   PetscCall(PetscDeviceContextSynchronize(other));
 66:   PetscCall(PetscTime(&start));
 67:   PetscCall(PetscDeviceContextDelay(other, 0.1));
 68:   PetscCall(PetscTime(&end));
 69:   PetscCall(PetscDeviceContextQueryIdle(other, &idle));
 70:   if (synchronous) {
 71:     PetscCheck(idle, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Context is not idle after a synchronous delay");
 72:     PetscCheck(end - start >= 0.09, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Synchronous delay returned too early: %g seconds", (double)(end - start));
 73:   } else PetscCheck(!idle, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Delayed device context is unexpectedly idle");
 74:   PetscCall(PetscDeviceContextSynchronize(other));
 75:   PetscCall(CheckIdle(other, "synchronizing a delayed context"));
 76:   PetscCall(PetscDeviceContextDestroy(&other));
 77:   PetscFunctionReturn(PETSC_SUCCESS);
 78: }

 80: static PetscErrorCode TestDefaultContext(void)
 81: {
 82:   PetscDeviceContext saved, standard, work, resolved, host;
 83:   PetscDevice        device;
 84:   PetscDeviceType    type;
 85:   PetscStreamType    streamtype;
 86:   PetscBool          initialized[PETSC_DEVICE_MAX];
 87:   PetscInt           value  = 0;
 88:   PetscInt          *empty  = NULL;
 89:   const PetscInt    *source = &value;
 90:   void              *stream;

 92:   PetscFunctionBeginUser;
 93:   PetscCheck(PetscDeviceContextDefault, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Default placeholder is null");
 94:   // No-work calls must not initialize a device just to resolve the placeholder.
 95:   for (PetscInt i = 0; i < PETSC_DEVICE_MAX; ++i) initialized[i] = PetscDeviceInitialized((PetscDeviceType)i);
 96:   PetscCall(PetscDeviceMalloc(PetscDeviceContextDefault, PETSC_MEMTYPE_HOST, 0, PETSC_DECIDE, &empty));
 97:   PetscCheck(!empty, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Zero-length allocation returned a pointer");
 98:   PetscCall(PetscDeviceCalloc(PetscDeviceContextDefault, PETSC_MEMTYPE_HOST, 0, PETSC_DECIDE, &empty));
 99:   PetscCheck(!empty, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Zero-length allocation returned a pointer");
100:   PetscCall(PetscDeviceMemcpy(PetscDeviceContextDefault, NULL, NULL, 0));
101:   PetscCall(PetscDeviceMemcpy(PetscDeviceContextDefault, &value, source, sizeof(value)));
102:   PetscCall(PetscDeviceMemset(PetscDeviceContextDefault, NULL, 0, 0));
103:   PetscCall(PetscDeviceFree(PetscDeviceContextDefault, empty));
104:   PetscCall(PetscDeviceContextDelay(PetscDeviceContextDefault, 0));
105:   PetscCall(PetscDeviceContextWaitForContext(PetscDeviceContextDefault, PetscDeviceContextDefault));
106:   for (PetscInt i = 0; i < PETSC_DEVICE_MAX; ++i) PetscCheck(PetscDeviceInitialized((PetscDeviceType)i) == initialized[i], PETSC_COMM_SELF, PETSC_ERR_PLIB, "No-work call initialized device type %s", PetscDeviceTypes[i]);
107:   // The placeholder must work before any explicit context creation, including lazy initialization.
108:   PetscCall(PetscDeviceContextSetUp(PetscDeviceContextDefault));
109:   PetscCall(PetscDeviceContextGetStreamType(PetscDeviceContextDefault, &streamtype));
110:   PetscCheck(streamtype == PETSC_STREAM_DEFAULT, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Incorrect default stream type");
111:   PetscCall(PetscDeviceContextGetDefaultContext(&standard));
112:   PetscCheck(standard != PetscDeviceContextDefault, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Getter returned the placeholder");
113:   PetscCall(PetscDeviceContextGetCurrentContext(&saved));
114:   PetscCall(PetscDeviceContextDuplicate(PetscDeviceContextDefault, &work));
115:   PetscCall(PetscDeviceContextSetStreamType(work, PETSC_STREAM_NONBLOCKING));
116:   PetscCall(PetscDeviceContextSetUp(work));
117:   PetscCall(PetscDeviceContextSetCurrentContext(work));
118:   PetscCall(PetscDeviceContextGetDefaultContext(&resolved));
119:   PetscCheck(resolved == standard, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Changing streams changed the default context");
120:   PetscCall(PetscDeviceContextGetStreamType(PetscDeviceContextDefault, &streamtype));
121:   PetscCheck(streamtype == PETSC_STREAM_DEFAULT, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Placeholder selected the current stream");
122:   PetscCall(PetscDeviceContextGetDeviceType(PetscDeviceContextDefault, &type));
123:   if (type != PETSC_DEVICE_HOST) PetscCall(PetscDeviceContextGetStreamHandle(PetscDeviceContextDefault, &stream));
124:   PetscCall(PetscDeviceContextDelay(PetscDeviceContextDefault, 0));
125:   PetscCall(PetscDeviceContextWaitForContext(work, PetscDeviceContextDefault));
126:   PetscCall(PetscDeviceContextWaitForContext(PetscDeviceContextDefault, work));
127:   PetscCall(PetscDeviceContextSynchronize(PetscDeviceContextDefault));
128:   PetscCall(CheckIdle(PetscDeviceContextDefault, "default context synchronization"));
129:   PetscCall(PetscDeviceContextViewFromOptions(PetscDeviceContextDefault, NULL, "-default_context_view"));

131:   PetscCall(PetscDeviceCreate(PETSC_DEVICE_HOST, PETSC_DECIDE, &device));
132:   PetscCall(PetscDeviceConfigure(device));
133:   PetscCall(PetscDeviceContextCreate(&host));
134:   PetscCall(PetscDeviceContextSetDevice(host, device));
135:   PetscCall(PetscDeviceDestroy(&device));
136:   PetscCall(PetscDeviceContextSetUp(host));
137:   PetscCall(PetscDeviceContextSetCurrentContext(host));
138:   PetscCall(PetscDeviceContextGetDeviceType(PetscDeviceContextDefault, &type));
139:   PetscCheck(type == PETSC_DEVICE_HOST, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Placeholder did not follow the current device");
140:   PetscCall(PetscDeviceContextGetDefaultContext(&resolved));
141:   PetscCall(PetscDeviceContextGetDeviceType(standard, &type));
142:   PetscCheck((resolved == standard) == (type == PETSC_DEVICE_HOST), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Default contexts were not cached per device");
143:   PetscCall(PetscDeviceContextSetCurrentContext(work));
144:   PetscCall(PetscDeviceContextSetCurrentContext(PetscDeviceContextDefault));
145:   PetscCall(PetscDeviceContextGetCurrentContext(&resolved));
146:   PetscCheck(resolved == standard, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Setting current did not resolve the placeholder");
147:   PetscCall(PetscDeviceContextSetCurrentContext(saved));
148:   PetscCall(PetscDeviceContextDestroy(&host));
149:   PetscCall(PetscDeviceContextDestroy(&work));
150:   PetscFunctionReturn(PETSC_SUCCESS);
151: }

153: int main(int argc, char *argv[])
154: {
155:   PetscDeviceContext dctx = NULL;
156:   PetscDeviceType    type;

158:   PetscFunctionBeginUser;
159:   PetscCall(PetscInitialize(&argc, &argv, NULL, help));
160:   PetscCall(TestDefaultContext());

162:   PetscCall(PetscDeviceContextCreate(&dctx));
163:   PetscCall(PetscDeviceContextSetStreamType(dctx, PETSC_STREAM_DEFAULT));
164:   PetscCall(PetscDeviceContextSetUp(dctx));
165:   PetscCall(TestQueryIdle(dctx));
166:   PetscCall(PetscDeviceContextGetDeviceType(dctx, &type));
167:   if (type == PETSC_DEVICE_HOST || type == PETSC_DEVICE_CUDA || type == PETSC_DEVICE_HIP) PetscCall(TestDelay(dctx));
168:   PetscCall(PetscDeviceContextDestroy(&dctx));

170:   PetscCall(PetscDeviceContextGetCurrentContext(&dctx));
171:   PetscCall(TestQueryIdle(dctx));

173:   PetscCall(PetscPrintf(PETSC_COMM_WORLD, "EXIT_SUCCESS\n"));
174:   PetscCall(PetscDeviceContextSetCurrentContext(PetscDeviceContextDefault));
175:   PetscCall(PetscFinalize());
176:   return 0;
177: }

179: /*TEST

181:   testset:
182:     requires: defined(PETSC_DEVICELANGUAGE_CXX)
183:     output_file: output/ExitSuccess.out
184:     args: -device_enable {{lazy eager}}
185:     test:
186:       requires: !device
187:       suffix: host_no_device
188:     test:
189:       requires: device
190:       args: -default_device_type host
191:       suffix: host_with_device
192:     test:
193:       requires: cuda
194:       args: -default_device_type cuda
195:       suffix: cuda
196:     test:
197:       requires: hip
198:       args: -default_device_type hip
199:       suffix: hip
200:     test:
201:       requires: sycl
202:       TODO: unclear if it is needed
203:       args: -default_device_type sycl
204:       suffix: sycl

206:   test:
207:     requires: !defined(PETSC_DEVICELANGUAGE_CXX)
208:     output_file: output/ExitSuccess.out
209:     suffix: no_cxx

211: TEST*/