15   Vec                x, y;  in main()  local
26   /* Create two CUDA vectors x, y. Though we only care y's memory on host, we make y a CUDA vector,  in main()
27 … since we want to have y's memory on host pinned (i.e.,non-pagable), to really trigger asynchronous  in main()
32   PetscCall(VecCreateSeq(PETSC_COMM_WORLD, n, &y));  in main()
33   PetscCall(VecSetFromOptions(y));  in main()
35   /* Init x, y, and push them to GPU (their offloadmask = PETSC_OFFLOAD_GPU) */  in main()
40   PetscCall(VecSet(y, 314));  in main()
42   /* Pull y to CPU (make its offloadmask = PETSC_OFFLOAD_CPU) */  in main()
43   PetscCall(VecGetArray(y, &val));  in main()
44   PetscCall(VecRestoreArray(y, &val));  in main()
49   PetscCall(VecScatterCreate(x, ix, y, iy, &vscat));  in main()
51 …/* Do device to host vecscatter and then immediately use y on host. VecScat/SF may use asynchronous  in main()
52 …cudaMemcpy or kernels, but it must guarantee y is ready to use on host. Otherwise, wrong data will…  in main()
54   PetscCall(VecScatterBegin(vscat, x, y, INSERT_VALUES, SCATTER_FORWARD));  in main()
55   PetscCall(VecScatterEnd(vscat, x, y, INSERT_VALUES, SCATTER_FORWARD));  in main()
56   PetscCall(VecGetArrayRead(y, &yval));  in main()
57   /* Display the first and the last entries of y to see if it is valid on host */  in main()
58 …PetscCall(PetscPrintf(PETSC_COMM_SELF, "y[0]=%.f, y[%" PetscInt_FMT "] = %.f\n", (float)PetscRealP…  in main()
59   PetscCall(VecRestoreArrayRead(y, &yval));  in main()
62   PetscCall(VecDestroy(&y));  in main()