Actual source code: ex23.c
1: static const char help[] = "Test PetscSF with integers and MPIU_2INT \n\n";
3: #include <petscvec.h>
4: #include <petscsf.h>
5: #include <petscdevice.h>
7: int main(int argc, char *argv[])
8: {
9: PetscInt n, n2, N = 12;
10: PetscInt *indices;
11: IS ix, iy;
12: VecScatter vscat;
13: Vec x, y;
14: PetscInt rstart, rend;
15: PetscInt *xh, *yh, *xd, *yd;
16: PetscDeviceContext dctx;
18: PetscFunctionBeginUser;
19: PetscCall(PetscInitialize(&argc, &argv, NULL, help));
20: PetscCall(VecCreateFromOptions(PETSC_COMM_WORLD, NULL, 1, PETSC_DECIDE, N, &x));
21: PetscCall(VecDuplicate(x, &y));
22: PetscCall(VecGetLocalSize(x, &n));
24: PetscCall(VecGetOwnershipRange(x, &rstart, &rend));
25: PetscCall(ISCreateStride(PETSC_COMM_WORLD, n, rstart, 1, &ix));
26: PetscCall(PetscMalloc1(n, &indices));
27: for (PetscInt i = rstart; i < rend; i++) indices[i - rstart] = i / 2;
28: PetscCall(ISCreateGeneral(PETSC_COMM_WORLD, n, indices, PETSC_OWN_POINTER, &iy));
29: // connect y[0] to x[0..1], y[1] to x[2..3], etc
30: PetscCall(VecScatterCreate(y, iy, x, ix, &vscat)); // y has roots, x has leaves
32: PetscCall(PetscDeviceContextGetCurrentContext(&dctx));
34: // double the allocation since we will use MPIU_2INT later
35: n2 = 2 * n;
36: PetscCall(PetscDeviceMalloc(dctx, PETSC_MEMTYPE_HOST, n2, PETSC_DECIDE, &xh));
37: PetscCall(PetscDeviceMalloc(dctx, PETSC_MEMTYPE_HOST, n2, PETSC_DECIDE, &yh));
38: PetscCall(PetscDeviceMalloc(dctx, PETSC_MEMTYPE_DEVICE, n2, PETSC_DECIDE, &xd));
39: PetscCall(PetscDeviceMalloc(dctx, PETSC_MEMTYPE_DEVICE, n2, PETSC_DECIDE, &yd));
40: PetscCall(PetscDeviceContextSynchronize(dctx));
42: for (PetscInt i = 0; i < n; i++) {
43: xh[i] = xh[i + n] = i + rstart;
44: yh[i] = yh[i + n] = i + rstart;
45: }
46: PetscCall(PetscDeviceMemcpy(dctx, xd, xh, sizeof(PetscInt) * n2));
47: PetscCall(PetscDeviceMemcpy(dctx, yd, yh, sizeof(PetscInt) * n2));
49: PetscCall(PetscSFReduceWithMemTypeBegin(vscat, MPIU_INT, PETSC_MEMTYPE_DEVICE, xd, PETSC_MEMTYPE_DEVICE, yd, MPI_SUM));
50: PetscCall(PetscSFReduceEnd(vscat, MPIU_INT, xd, yd, MPI_SUM));
51: PetscCall(PetscDeviceMemcpy(dctx, yh, yd, sizeof(PetscInt) * n));
52: PetscCall(PetscDeviceContextSynchronize(dctx)); // finish the async memcpy
53: PetscCall(PetscIntView(n, yh, PETSC_VIEWER_STDOUT_WORLD));
55: PetscCall(PetscSFBcastWithMemTypeBegin(vscat, MPIU_2INT, PETSC_MEMTYPE_DEVICE, yd, PETSC_MEMTYPE_DEVICE, xd, MPI_MINLOC));
56: PetscCall(PetscSFBcastEnd(vscat, MPIU_2INT, yd, xd, MPI_MINLOC));
57: PetscCall(PetscDeviceMemcpy(dctx, xh, xd, sizeof(PetscInt) * n2));
58: PetscCall(PetscDeviceContextSynchronize(dctx)); // finish the async memcpy
59: PetscCall(PetscIntView(n2, xh, PETSC_VIEWER_STDOUT_WORLD));
61: PetscCall(PetscDeviceFree(dctx, xh));
62: PetscCall(PetscDeviceFree(dctx, yh));
63: PetscCall(PetscDeviceFree(dctx, xd));
64: PetscCall(PetscDeviceFree(dctx, yd));
65: PetscCall(ISDestroy(&ix));
66: PetscCall(ISDestroy(&iy));
67: PetscCall(VecDestroy(&x));
68: PetscCall(VecDestroy(&y));
69: PetscCall(VecScatterDestroy(&vscat));
70: PetscCall(PetscFinalize());
71: }
73: /*TEST
74: testset:
75: output_file: output/ex23.out
76: nsize: 3
78: test:
79: suffix: 1
80: requires: cuda
82: test:
83: suffix: 2
84: requires: hip
86: test:
87: suffix: 3
88: TODO: unclear if it is needed
89: requires: sycl
91: TEST*/