17def3d77Svaleriabarra# Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at 27def3d77Svaleriabarra# the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights 37def3d77Svaleriabarra# reserved. See files LICENSE and NOTICE for details. 47def3d77Svaleriabarra# 57def3d77Svaleriabarra# This file is part of CEED, a collection of benchmarks, miniapps, software 67def3d77Svaleriabarra# libraries and APIs for efficient high-order finite element and spectral 77def3d77Svaleriabarra# element discretizations for exascale applications. For more information and 87def3d77Svaleriabarra# source code availability see http://github.com/ceed. 97def3d77Svaleriabarra# 107def3d77Svaleriabarra# The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, 117def3d77Svaleriabarra# a collaborative effort of two U.S. Department of Energy organizations (Office 127def3d77Svaleriabarra# of Science and the National Nuclear Security Administration) responsible for 137def3d77Svaleriabarra# the planning and preparation of a capable exascale ecosystem, including 147def3d77Svaleriabarra# software, applications, hardware, advanced system engineering and early 157def3d77Svaleriabarra# testbed platforms, in support of the nation's exascale computing imperative. 167def3d77Svaleriabarra 177def3d77Svaleriabarrafrom _ceed_cffi import ffi, lib 187def3d77Svaleriabarraimport tempfile 197def3d77Svaleriabarraimport numpy as np 20e259ae81SJed Brownimport contextlib 21187168c7SJeremy L Thompsonfrom .ceed_constants import MEM_HOST, USE_POINTER, COPY_VALUES, NORM_2 227def3d77Svaleriabarra 237def3d77Svaleriabarra# ------------------------------------------------------------------------------ 247a7b0fa3SJed Brown 257a7b0fa3SJed Brown 267def3d77Svaleriabarraclass Vector(): 277def3d77Svaleriabarra """Ceed Vector: storing and manipulating vectors.""" 287def3d77Svaleriabarra 297def3d77Svaleriabarra # Constructor 307def3d77Svaleriabarra def __init__(self, ceed, size): 317def3d77Svaleriabarra # CeedVector object 327def3d77Svaleriabarra self._pointer = ffi.new("CeedVector *") 337def3d77Svaleriabarra 347def3d77Svaleriabarra # Reference to Ceed 357def3d77Svaleriabarra self._ceed = ceed 367def3d77Svaleriabarra 377def3d77Svaleriabarra # libCEED call 38477729cfSJeremy L Thompson err_code = lib.CeedVectorCreate( 39477729cfSJeremy L Thompson self._ceed._pointer[0], size, self._pointer) 40477729cfSJeremy L Thompson self._ceed._check_error(err_code) 417def3d77Svaleriabarra 427def3d77Svaleriabarra # Destructor 437def3d77Svaleriabarra def __del__(self): 447def3d77Svaleriabarra # libCEED call 45477729cfSJeremy L Thompson err_code = lib.CeedVectorDestroy(self._pointer) 46477729cfSJeremy L Thompson self._ceed._check_error(err_code) 477def3d77Svaleriabarra 487def3d77Svaleriabarra # Representation 497def3d77Svaleriabarra def __repr__(self): 507def3d77Svaleriabarra return "<CeedVector instance at " + hex(id(self)) + ">" 517def3d77Svaleriabarra 527def3d77Svaleriabarra # String conversion for print() to stdout 537def3d77Svaleriabarra def __str__(self): 547def3d77Svaleriabarra """View a Vector via print().""" 557def3d77Svaleriabarra 567def3d77Svaleriabarra # libCEED call 577def3d77Svaleriabarra fmt = ffi.new("char[]", "%f".encode('ascii')) 587def3d77Svaleriabarra with tempfile.NamedTemporaryFile() as key_file: 597def3d77Svaleriabarra with open(key_file.name, 'r+') as stream_file: 607def3d77Svaleriabarra stream = ffi.cast("FILE *", stream_file) 617def3d77Svaleriabarra 62477729cfSJeremy L Thompson err_code = lib.CeedVectorView(self._pointer[0], fmt, stream) 63477729cfSJeremy L Thompson self._ceed._check_error(err_code) 647def3d77Svaleriabarra 657def3d77Svaleriabarra stream_file.seek(0) 667def3d77Svaleriabarra out_string = stream_file.read() 677def3d77Svaleriabarra 687def3d77Svaleriabarra return out_string 697def3d77Svaleriabarra 707def3d77Svaleriabarra # Set Vector's data array 717def3d77Svaleriabarra def set_array(self, array, memtype=MEM_HOST, cmode=COPY_VALUES): 727def3d77Svaleriabarra """Set the array used by a Vector, freeing any previously allocated 737def3d77Svaleriabarra array if applicable. 747def3d77Svaleriabarra 757def3d77Svaleriabarra Args: 767f1dc7b9SJeremy L Thompson *array: Numpy or Numba array to be used 777def3d77Svaleriabarra **memtype: memory type of the array being passed, default CEED_MEM_HOST 787def3d77Svaleriabarra **cmode: copy mode for the array, default CEED_COPY_VALUES""" 797def3d77Svaleriabarra 80187168c7SJeremy L Thompson # Store array reference if needed 81187168c7SJeremy L Thompson if cmode == USE_POINTER: 82187168c7SJeremy L Thompson self._array_reference = array 83187168c7SJeremy L Thompson else: 84187168c7SJeremy L Thompson self._array_reference = None 85187168c7SJeremy L Thompson 867def3d77Svaleriabarra # Setup the numpy array for the libCEED call 877def3d77Svaleriabarra array_pointer = ffi.new("CeedScalar *") 887f1dc7b9SJeremy L Thompson if memtype == MEM_HOST: 897a7b0fa3SJed Brown array_pointer = ffi.cast( 907a7b0fa3SJed Brown "CeedScalar *", 917a7b0fa3SJed Brown array.__array_interface__['data'][0]) 927f1dc7b9SJeremy L Thompson else: 937f1dc7b9SJeremy L Thompson array_pointer = ffi.cast( 947f1dc7b9SJeremy L Thompson "CeedScalar *", 957f1dc7b9SJeremy L Thompson array.__cuda_array_interface__['data'][0]) 967def3d77Svaleriabarra 977def3d77Svaleriabarra # libCEED call 98477729cfSJeremy L Thompson err_code = lib.CeedVectorSetArray( 99477729cfSJeremy L Thompson self._pointer[0], memtype, cmode, array_pointer) 100477729cfSJeremy L Thompson self._ceed._check_error(err_code) 1017def3d77Svaleriabarra 1027def3d77Svaleriabarra # Get Vector's data array 1037def3d77Svaleriabarra def get_array(self, memtype=MEM_HOST): 1047def3d77Svaleriabarra """Get read/write access to a Vector via the specified memory type. 1057def3d77Svaleriabarra 1067def3d77Svaleriabarra Args: 1077def3d77Svaleriabarra **memtype: memory type of the array being passed, default CEED_MEM_HOST 1087def3d77Svaleriabarra 1097def3d77Svaleriabarra Returns: 1107f1dc7b9SJeremy L Thompson *array: Numpy or Numba array""" 1117def3d77Svaleriabarra 1127def3d77Svaleriabarra # Retrieve the length of the array 1137def3d77Svaleriabarra length_pointer = ffi.new("CeedInt *") 114477729cfSJeremy L Thompson err_code = lib.CeedVectorGetLength(self._pointer[0], length_pointer) 115477729cfSJeremy L Thompson self._ceed._check_error(err_code) 1167def3d77Svaleriabarra 1177def3d77Svaleriabarra # Setup the pointer's pointer 1187def3d77Svaleriabarra array_pointer = ffi.new("CeedScalar **") 1197def3d77Svaleriabarra 1207def3d77Svaleriabarra # libCEED call 121477729cfSJeremy L Thompson err_code = lib.CeedVectorGetArray( 122477729cfSJeremy L Thompson self._pointer[0], memtype, array_pointer) 123477729cfSJeremy L Thompson self._ceed._check_error(err_code) 1247def3d77Svaleriabarra 1257f1dc7b9SJeremy L Thompson # Return array created from buffer 1267f1dc7b9SJeremy L Thompson if memtype == MEM_HOST: 1277def3d77Svaleriabarra # Create buffer object from returned pointer 1287a7b0fa3SJed Brown buff = ffi.buffer( 1297a7b0fa3SJed Brown array_pointer[0], 1307a7b0fa3SJed Brown ffi.sizeof("CeedScalar") * 1317a7b0fa3SJed Brown length_pointer[0]) 1327f1dc7b9SJeremy L Thompson # return Numpy array 1337def3d77Svaleriabarra return np.frombuffer(buff, dtype="float64") 1347f1dc7b9SJeremy L Thompson else: 1357f1dc7b9SJeremy L Thompson # CUDA array interface 1367f1dc7b9SJeremy L Thompson # https://numba.pydata.org/numba-doc/latest/cuda/cuda_array_interface.html 137962dc42dSJeremy L Thompson import numba.cuda as nbcuda 1387f1dc7b9SJeremy L Thompson desc = { 1397f1dc7b9SJeremy L Thompson 'shape': (length_pointer[0]), 1407f1dc7b9SJeremy L Thompson 'typestr': '>f8', 1417f1dc7b9SJeremy L Thompson 'data': (int(ffi.cast("intptr_t", array_pointer[0])), False), 1427f1dc7b9SJeremy L Thompson 'version': 2 1437f1dc7b9SJeremy L Thompson } 1447f1dc7b9SJeremy L Thompson # return Numba array 1457f1dc7b9SJeremy L Thompson return nbcuda.from_cuda_array_interface(desc) 1467def3d77Svaleriabarra 1477def3d77Svaleriabarra # Get Vector's data array in read-only mode 1487def3d77Svaleriabarra def get_array_read(self, memtype=MEM_HOST): 1497def3d77Svaleriabarra """Get read-only access to a Vector via the specified memory type. 1507def3d77Svaleriabarra 1517def3d77Svaleriabarra Args: 1527def3d77Svaleriabarra **memtype: memory type of the array being passed, default CEED_MEM_HOST 1537def3d77Svaleriabarra 1547def3d77Svaleriabarra Returns: 1557f1dc7b9SJeremy L Thompson *array: Numpy or Numba array""" 1567def3d77Svaleriabarra 1577def3d77Svaleriabarra # Retrieve the length of the array 1587def3d77Svaleriabarra length_pointer = ffi.new("CeedInt *") 159477729cfSJeremy L Thompson err_code = lib.CeedVectorGetLength(self._pointer[0], length_pointer) 160477729cfSJeremy L Thompson self._ceed._check_error(err_code) 1617def3d77Svaleriabarra 1627def3d77Svaleriabarra # Setup the pointer's pointer 1637def3d77Svaleriabarra array_pointer = ffi.new("CeedScalar **") 1647def3d77Svaleriabarra 1657def3d77Svaleriabarra # libCEED call 166477729cfSJeremy L Thompson err_code = lib.CeedVectorGetArrayRead( 167477729cfSJeremy L Thompson self._pointer[0], memtype, array_pointer) 168477729cfSJeremy L Thompson self._ceed._check_error(err_code) 1697def3d77Svaleriabarra 1707f1dc7b9SJeremy L Thompson # Return array created from buffer 1717f1dc7b9SJeremy L Thompson if memtype == MEM_HOST: 1727def3d77Svaleriabarra # Create buffer object from returned pointer 1737a7b0fa3SJed Brown buff = ffi.buffer( 1747a7b0fa3SJed Brown array_pointer[0], 1757a7b0fa3SJed Brown ffi.sizeof("CeedScalar") * 1767a7b0fa3SJed Brown length_pointer[0]) 1777f1dc7b9SJeremy L Thompson # return read only Numpy array 1787def3d77Svaleriabarra ret = np.frombuffer(buff, dtype="float64") 1797def3d77Svaleriabarra ret.flags['WRITEABLE'] = False 1807def3d77Svaleriabarra return ret 1817f1dc7b9SJeremy L Thompson else: 1827f1dc7b9SJeremy L Thompson # CUDA array interface 1837f1dc7b9SJeremy L Thompson # https://numba.pydata.org/numba-doc/latest/cuda/cuda_array_interface.html 184962dc42dSJeremy L Thompson import numba.cuda as nbcuda 1857f1dc7b9SJeremy L Thompson desc = { 1867f1dc7b9SJeremy L Thompson 'shape': (length_pointer[0]), 1877f1dc7b9SJeremy L Thompson 'typestr': '>f8', 1887f1dc7b9SJeremy L Thompson 'data': (int(ffi.cast("intptr_t", array_pointer[0])), False), 1897f1dc7b9SJeremy L Thompson 'version': 2 1907f1dc7b9SJeremy L Thompson } 1917f1dc7b9SJeremy L Thompson # return read only Numba array 1927f1dc7b9SJeremy L Thompson return nbcuda.from_cuda_array_interface(desc) 1937def3d77Svaleriabarra 1947def3d77Svaleriabarra # Restore the Vector's data array 1957def3d77Svaleriabarra def restore_array(self): 1967def3d77Svaleriabarra """Restore an array obtained using get_array().""" 1977def3d77Svaleriabarra 1987def3d77Svaleriabarra # Setup the pointer's pointer 1997def3d77Svaleriabarra array_pointer = ffi.new("CeedScalar **") 2007def3d77Svaleriabarra 2017def3d77Svaleriabarra # libCEED call 202477729cfSJeremy L Thompson err_code = lib.CeedVectorRestoreArray(self._pointer[0], array_pointer) 203477729cfSJeremy L Thompson self._ceed._check_error(err_code) 2047def3d77Svaleriabarra 2057def3d77Svaleriabarra # Restore an array obtained using getArrayRead 2067def3d77Svaleriabarra def restore_array_read(self): 2077def3d77Svaleriabarra """Restore an array obtained using get_array_read().""" 2087def3d77Svaleriabarra 2097def3d77Svaleriabarra # Setup the pointer's pointer 2107def3d77Svaleriabarra array_pointer = ffi.new("CeedScalar **") 2117def3d77Svaleriabarra 2127def3d77Svaleriabarra # libCEED call 213477729cfSJeremy L Thompson err_code = lib.CeedVectorRestoreArrayRead( 214477729cfSJeremy L Thompson self._pointer[0], array_pointer) 215477729cfSJeremy L Thompson self._ceed._check_error(err_code) 2167def3d77Svaleriabarra 217e259ae81SJed Brown @contextlib.contextmanager 2187f1dc7b9SJeremy L Thompson def array(self, *shape, memtype=MEM_HOST): 219e259ae81SJed Brown """Context manager for array access. 220e259ae81SJed Brown 221e259ae81SJed Brown Args: 222e259ae81SJed Brown shape (tuple): shape of returned numpy.array 2237f1dc7b9SJeremy L Thompson **memtype: memory type of the array being passed, default CEED_MEM_HOST 2247f1dc7b9SJeremy L Thompson 225e259ae81SJed Brown 226e259ae81SJed Brown Returns: 227e259ae81SJed Brown np.array: writable view of vector 228e259ae81SJed Brown 229e259ae81SJed Brown Examples: 230e259ae81SJed Brown Constructing the identity inside a libceed.Vector: 231e259ae81SJed Brown 232e259ae81SJed Brown >>> vec = ceed.Vector(16) 233e259ae81SJed Brown >>> with vec.array(4, 4) as x: 234e259ae81SJed Brown >>> x[...] = np.eye(4) 235e259ae81SJed Brown """ 2367f1dc7b9SJeremy L Thompson x = self.get_array(memtype=memtype) 237e259ae81SJed Brown if shape: 238e259ae81SJed Brown x = x.reshape(shape) 239e259ae81SJed Brown yield x 240e259ae81SJed Brown self.restore_array() 241e259ae81SJed Brown 242e259ae81SJed Brown @contextlib.contextmanager 2437f1dc7b9SJeremy L Thompson def array_read(self, *shape, memtype=MEM_HOST): 244e259ae81SJed Brown """Context manager for read-only array access. 245e259ae81SJed Brown 246e259ae81SJed Brown Args: 247e259ae81SJed Brown shape (tuple): shape of returned numpy.array 2487f1dc7b9SJeremy L Thompson **memtype: memory type of the array being passed, default CEED_MEM_HOST 249e259ae81SJed Brown 250e259ae81SJed Brown Returns: 251e259ae81SJed Brown np.array: read-only view of vector 252e259ae81SJed Brown 253e259ae81SJed Brown Examples: 254187168c7SJeremy L Thompson Viewing contents of a reshaped libceed.Vector view: 255e259ae81SJed Brown 256e259ae81SJed Brown >>> vec = ceed.Vector(6) 257e259ae81SJed Brown >>> vec.set_value(1.3) 258e259ae81SJed Brown >>> with vec.array_read(2, 3) as x: 259e259ae81SJed Brown >>> print(x) 260e259ae81SJed Brown """ 2617f1dc7b9SJeremy L Thompson x = self.get_array_read(memtype=memtype) 262e259ae81SJed Brown if shape: 263e259ae81SJed Brown x = x.reshape(shape) 264e259ae81SJed Brown yield x 265e259ae81SJed Brown self.restore_array_read() 266e259ae81SJed Brown 2677def3d77Svaleriabarra # Get the length of a Vector 2687def3d77Svaleriabarra def get_length(self): 2697def3d77Svaleriabarra """Get the length of a Vector. 2707def3d77Svaleriabarra 2717def3d77Svaleriabarra Returns: 2727def3d77Svaleriabarra length: length of the Vector""" 2737def3d77Svaleriabarra 2747def3d77Svaleriabarra length_pointer = ffi.new("CeedInt *") 2757def3d77Svaleriabarra 2767def3d77Svaleriabarra # libCEED call 277477729cfSJeremy L Thompson err_code = lib.CeedVectorGetLength(self._pointer[0], length_pointer) 278477729cfSJeremy L Thompson self._ceed._check_error(err_code) 2797def3d77Svaleriabarra 2807def3d77Svaleriabarra return length_pointer[0] 2817def3d77Svaleriabarra 2827def3d77Svaleriabarra # Get the length of a Vector 2837def3d77Svaleriabarra def __len__(self): 2847def3d77Svaleriabarra """Get the length of a Vector. 2857def3d77Svaleriabarra 2867def3d77Svaleriabarra Returns: 2877def3d77Svaleriabarra length: length of the Vector""" 2887def3d77Svaleriabarra 2897def3d77Svaleriabarra length_pointer = ffi.new("CeedInt *") 2907def3d77Svaleriabarra 2917def3d77Svaleriabarra # libCEED call 292477729cfSJeremy L Thompson err_code = lib.CeedVectorGetLength(self._pointer[0], length_pointer) 293477729cfSJeremy L Thompson self._ceed._check_error(err_code) 2947def3d77Svaleriabarra 2957def3d77Svaleriabarra return length_pointer[0] 2967def3d77Svaleriabarra 2977def3d77Svaleriabarra # Set the Vector to a given constant value 2987def3d77Svaleriabarra def set_value(self, value): 2997def3d77Svaleriabarra """Set the Vector to a constant value. 3007def3d77Svaleriabarra 3017def3d77Svaleriabarra Args: 3027def3d77Svaleriabarra value: value to be used""" 3037def3d77Svaleriabarra 3047def3d77Svaleriabarra # libCEED call 305477729cfSJeremy L Thompson err_code = lib.CeedVectorSetValue(self._pointer[0], value) 306477729cfSJeremy L Thompson self._ceed._check_error(err_code) 3077def3d77Svaleriabarra 3087def3d77Svaleriabarra # Sync the Vector to a specified memtype 309547d9b97Sjeremylt def sync_array(self, memtype=MEM_HOST): 3107def3d77Svaleriabarra """Sync the Vector to a specified memtype. 3117def3d77Svaleriabarra 3127def3d77Svaleriabarra Args: 3137def3d77Svaleriabarra **memtype: memtype to be synced""" 3147def3d77Svaleriabarra 3157def3d77Svaleriabarra # libCEED call 316477729cfSJeremy L Thompson err_code = lib.CeedVectorSyncArray(self._pointer[0], memtype) 317477729cfSJeremy L Thompson self._ceed._check_error(err_code) 3187def3d77Svaleriabarra 319547d9b97Sjeremylt # Compute the norm of a vector 320547d9b97Sjeremylt def norm(self, normtype=NORM_2): 321547d9b97Sjeremylt """Get the norm of a Vector. 322547d9b97Sjeremylt 323547d9b97Sjeremylt Args: 324547d9b97Sjeremylt **normtype: type of norm to be computed""" 325547d9b97Sjeremylt 326547d9b97Sjeremylt norm_pointer = ffi.new("CeedScalar *") 327547d9b97Sjeremylt 328547d9b97Sjeremylt # libCEED call 329477729cfSJeremy L Thompson err_code = lib.CeedVectorNorm(self._pointer[0], normtype, norm_pointer) 330477729cfSJeremy L Thompson self._ceed._check_error(err_code) 331547d9b97Sjeremylt 332547d9b97Sjeremylt return norm_pointer[0] 333547d9b97Sjeremylt 334d99fa3c5SJeremy L Thompson # Take the reciprocal of a vector 335d99fa3c5SJeremy L Thompson def reciprocal(self): 336d99fa3c5SJeremy L Thompson """Take the reciprocal of a Vector.""" 337d99fa3c5SJeremy L Thompson 338d99fa3c5SJeremy L Thompson # libCEED call 339d99fa3c5SJeremy L Thompson err_code = lib.CeedVectorReciprocal(self._pointer[0]) 340d99fa3c5SJeremy L Thompson self._ceed._check_error(err_code) 341d99fa3c5SJeremy L Thompson 342d99fa3c5SJeremy L Thompson return self 343d99fa3c5SJeremy L Thompson 344*0c1bc3c2Sjeremylt # Compute self = alpha x + self 345*0c1bc3c2Sjeremylt def axpy(self, alpha, x): 346*0c1bc3c2Sjeremylt """Compute self = alpha x + self.""" 347*0c1bc3c2Sjeremylt 348*0c1bc3c2Sjeremylt # libCEED call 349*0c1bc3c2Sjeremylt err_code = lib.CeedVectorAXPY(self._pointer[0], alpha, x._pointer[0]) 350*0c1bc3c2Sjeremylt self._ceed._check_error(err_code) 351*0c1bc3c2Sjeremylt 352*0c1bc3c2Sjeremylt return self 353*0c1bc3c2Sjeremylt 354*0c1bc3c2Sjeremylt # Compute the pointwise multiplication self = x .* y 355*0c1bc3c2Sjeremylt def pointwise_mult(self, x, y): 356*0c1bc3c2Sjeremylt """Compute the pointwise multiplication self = x .* y.""" 357*0c1bc3c2Sjeremylt 358*0c1bc3c2Sjeremylt # libCEED call 359*0c1bc3c2Sjeremylt err_code = lib.CeedVectorPointwiseMult( 360*0c1bc3c2Sjeremylt self._pointer[0], x._pointer[0], y._pointer[0] 361*0c1bc3c2Sjeremylt ) 362*0c1bc3c2Sjeremylt self._ceed._check_error(err_code) 363*0c1bc3c2Sjeremylt 364*0c1bc3c2Sjeremylt return self 365*0c1bc3c2Sjeremylt 36619798369SJed Brown def _state(self): 36719798369SJed Brown """Return the modification state of the Vector. 36819798369SJed Brown 36919798369SJed Brown State is incremented each time the Vector is mutated, and is odd whenever a 37019798369SJed Brown mutable reference has not been returned. 37119798369SJed Brown """ 37219798369SJed Brown 37319798369SJed Brown state_pointer = ffi.new("uint64_t *") 37419798369SJed Brown err_code = lib.CeedVectorGetState(self._pointer[0], state_pointer) 37519798369SJed Brown self._ceed._check_error(err_code) 37619798369SJed Brown return state_pointer[0] 37719798369SJed Brown 3787def3d77Svaleriabarra# ------------------------------------------------------------------------------ 3797a7b0fa3SJed Brown 3807a7b0fa3SJed Brown 3817def3d77Svaleriabarraclass _VectorWrap(Vector): 3827def3d77Svaleriabarra """Wrap a CeedVector pointer in a Vector object.""" 3837def3d77Svaleriabarra 3847def3d77Svaleriabarra # Constructor 3857def3d77Svaleriabarra def __init__(self, ceed, pointer): 3867def3d77Svaleriabarra # CeedVector object 3877def3d77Svaleriabarra self._pointer = pointer 3887def3d77Svaleriabarra 3897def3d77Svaleriabarra # Reference to Ceed 3907def3d77Svaleriabarra self._ceed = ceed 3917def3d77Svaleriabarra 3927def3d77Svaleriabarra# ------------------------------------------------------------------------------ 393