17def3d77Svaleriabarra# Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at 27def3d77Svaleriabarra# the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights 37def3d77Svaleriabarra# reserved. See files LICENSE and NOTICE for details. 47def3d77Svaleriabarra# 57def3d77Svaleriabarra# This file is part of CEED, a collection of benchmarks, miniapps, software 67def3d77Svaleriabarra# libraries and APIs for efficient high-order finite element and spectral 77def3d77Svaleriabarra# element discretizations for exascale applications. For more information and 87def3d77Svaleriabarra# source code availability see http://github.com/ceed. 97def3d77Svaleriabarra# 107def3d77Svaleriabarra# The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, 117def3d77Svaleriabarra# a collaborative effort of two U.S. Department of Energy organizations (Office 127def3d77Svaleriabarra# of Science and the National Nuclear Security Administration) responsible for 137def3d77Svaleriabarra# the planning and preparation of a capable exascale ecosystem, including 147def3d77Svaleriabarra# software, applications, hardware, advanced system engineering and early 157def3d77Svaleriabarra# testbed platforms, in support of the nation's exascale computing imperative. 167def3d77Svaleriabarra 177def3d77Svaleriabarrafrom _ceed_cffi import ffi, lib 187def3d77Svaleriabarraimport tempfile 197def3d77Svaleriabarraimport numpy as np 20e259ae81SJed Brownimport contextlib 21*187168c7SJeremy L Thompsonfrom .ceed_constants import MEM_HOST, USE_POINTER, COPY_VALUES, NORM_2 227def3d77Svaleriabarra 237def3d77Svaleriabarra# ------------------------------------------------------------------------------ 247a7b0fa3SJed Brown 257a7b0fa3SJed Brown 267def3d77Svaleriabarraclass Vector(): 277def3d77Svaleriabarra """Ceed Vector: storing and manipulating vectors.""" 287def3d77Svaleriabarra 297def3d77Svaleriabarra # Attributes 30*187168c7SJeremy L Thompson _ceed = None 317def3d77Svaleriabarra _pointer = ffi.NULL 32*187168c7SJeremy L Thompson _array_reference = None 337def3d77Svaleriabarra 347def3d77Svaleriabarra # Constructor 357def3d77Svaleriabarra def __init__(self, ceed, size): 367def3d77Svaleriabarra # CeedVector object 377def3d77Svaleriabarra self._pointer = ffi.new("CeedVector *") 387def3d77Svaleriabarra 397def3d77Svaleriabarra # Reference to Ceed 407def3d77Svaleriabarra self._ceed = ceed 417def3d77Svaleriabarra 427def3d77Svaleriabarra # libCEED call 43477729cfSJeremy L Thompson err_code = lib.CeedVectorCreate( 44477729cfSJeremy L Thompson self._ceed._pointer[0], size, self._pointer) 45477729cfSJeremy L Thompson self._ceed._check_error(err_code) 467def3d77Svaleriabarra 477def3d77Svaleriabarra # Destructor 487def3d77Svaleriabarra def __del__(self): 497def3d77Svaleriabarra # libCEED call 50477729cfSJeremy L Thompson err_code = lib.CeedVectorDestroy(self._pointer) 51477729cfSJeremy L Thompson self._ceed._check_error(err_code) 527def3d77Svaleriabarra 537def3d77Svaleriabarra # Representation 547def3d77Svaleriabarra def __repr__(self): 557def3d77Svaleriabarra return "<CeedVector instance at " + hex(id(self)) + ">" 567def3d77Svaleriabarra 577def3d77Svaleriabarra # String conversion for print() to stdout 587def3d77Svaleriabarra def __str__(self): 597def3d77Svaleriabarra """View a Vector via print().""" 607def3d77Svaleriabarra 617def3d77Svaleriabarra # libCEED call 627def3d77Svaleriabarra fmt = ffi.new("char[]", "%f".encode('ascii')) 637def3d77Svaleriabarra with tempfile.NamedTemporaryFile() as key_file: 647def3d77Svaleriabarra with open(key_file.name, 'r+') as stream_file: 657def3d77Svaleriabarra stream = ffi.cast("FILE *", stream_file) 667def3d77Svaleriabarra 67477729cfSJeremy L Thompson err_code = lib.CeedVectorView(self._pointer[0], fmt, stream) 68477729cfSJeremy L Thompson self._ceed._check_error(err_code) 697def3d77Svaleriabarra 707def3d77Svaleriabarra stream_file.seek(0) 717def3d77Svaleriabarra out_string = stream_file.read() 727def3d77Svaleriabarra 737def3d77Svaleriabarra return out_string 747def3d77Svaleriabarra 757def3d77Svaleriabarra # Set Vector's data array 767def3d77Svaleriabarra def set_array(self, array, memtype=MEM_HOST, cmode=COPY_VALUES): 777def3d77Svaleriabarra """Set the array used by a Vector, freeing any previously allocated 787def3d77Svaleriabarra array if applicable. 797def3d77Svaleriabarra 807def3d77Svaleriabarra Args: 817f1dc7b9SJeremy L Thompson *array: Numpy or Numba array to be used 827def3d77Svaleriabarra **memtype: memory type of the array being passed, default CEED_MEM_HOST 837def3d77Svaleriabarra **cmode: copy mode for the array, default CEED_COPY_VALUES""" 847def3d77Svaleriabarra 85*187168c7SJeremy L Thompson # Store array reference if needed 86*187168c7SJeremy L Thompson if cmode == USE_POINTER: 87*187168c7SJeremy L Thompson self._array_reference = array 88*187168c7SJeremy L Thompson else: 89*187168c7SJeremy L Thompson self._array_reference = None 90*187168c7SJeremy L Thompson 917def3d77Svaleriabarra # Setup the numpy array for the libCEED call 927def3d77Svaleriabarra array_pointer = ffi.new("CeedScalar *") 937f1dc7b9SJeremy L Thompson if memtype == MEM_HOST: 947a7b0fa3SJed Brown array_pointer = ffi.cast( 957a7b0fa3SJed Brown "CeedScalar *", 967a7b0fa3SJed Brown array.__array_interface__['data'][0]) 977f1dc7b9SJeremy L Thompson else: 987f1dc7b9SJeremy L Thompson array_pointer = ffi.cast( 997f1dc7b9SJeremy L Thompson "CeedScalar *", 1007f1dc7b9SJeremy L Thompson array.__cuda_array_interface__['data'][0]) 1017def3d77Svaleriabarra 1027def3d77Svaleriabarra # libCEED call 103477729cfSJeremy L Thompson err_code = lib.CeedVectorSetArray( 104477729cfSJeremy L Thompson self._pointer[0], memtype, cmode, array_pointer) 105477729cfSJeremy L Thompson self._ceed._check_error(err_code) 1067def3d77Svaleriabarra 1077def3d77Svaleriabarra # Get Vector's data array 1087def3d77Svaleriabarra def get_array(self, memtype=MEM_HOST): 1097def3d77Svaleriabarra """Get read/write access to a Vector via the specified memory type. 1107def3d77Svaleriabarra 1117def3d77Svaleriabarra Args: 1127def3d77Svaleriabarra **memtype: memory type of the array being passed, default CEED_MEM_HOST 1137def3d77Svaleriabarra 1147def3d77Svaleriabarra Returns: 1157f1dc7b9SJeremy L Thompson *array: Numpy or Numba array""" 1167def3d77Svaleriabarra 1177def3d77Svaleriabarra # Retrieve the length of the array 1187def3d77Svaleriabarra length_pointer = ffi.new("CeedInt *") 119477729cfSJeremy L Thompson err_code = lib.CeedVectorGetLength(self._pointer[0], length_pointer) 120477729cfSJeremy L Thompson self._ceed._check_error(err_code) 1217def3d77Svaleriabarra 1227def3d77Svaleriabarra # Setup the pointer's pointer 1237def3d77Svaleriabarra array_pointer = ffi.new("CeedScalar **") 1247def3d77Svaleriabarra 1257def3d77Svaleriabarra # libCEED call 126477729cfSJeremy L Thompson err_code = lib.CeedVectorGetArray( 127477729cfSJeremy L Thompson self._pointer[0], memtype, array_pointer) 128477729cfSJeremy L Thompson self._ceed._check_error(err_code) 1297def3d77Svaleriabarra 1307f1dc7b9SJeremy L Thompson # Return array created from buffer 1317f1dc7b9SJeremy L Thompson if memtype == MEM_HOST: 1327def3d77Svaleriabarra # Create buffer object from returned pointer 1337a7b0fa3SJed Brown buff = ffi.buffer( 1347a7b0fa3SJed Brown array_pointer[0], 1357a7b0fa3SJed Brown ffi.sizeof("CeedScalar") * 1367a7b0fa3SJed Brown length_pointer[0]) 1377f1dc7b9SJeremy L Thompson # return Numpy array 1387def3d77Svaleriabarra return np.frombuffer(buff, dtype="float64") 1397f1dc7b9SJeremy L Thompson else: 1407f1dc7b9SJeremy L Thompson # CUDA array interface 1417f1dc7b9SJeremy L Thompson # https://numba.pydata.org/numba-doc/latest/cuda/cuda_array_interface.html 142962dc42dSJeremy L Thompson import numba.cuda as nbcuda 1437f1dc7b9SJeremy L Thompson desc = { 1447f1dc7b9SJeremy L Thompson 'shape': (length_pointer[0]), 1457f1dc7b9SJeremy L Thompson 'typestr': '>f8', 1467f1dc7b9SJeremy L Thompson 'data': (int(ffi.cast("intptr_t", array_pointer[0])), False), 1477f1dc7b9SJeremy L Thompson 'version': 2 1487f1dc7b9SJeremy L Thompson } 1497f1dc7b9SJeremy L Thompson # return Numba array 1507f1dc7b9SJeremy L Thompson return nbcuda.from_cuda_array_interface(desc) 1517def3d77Svaleriabarra 1527def3d77Svaleriabarra # Get Vector's data array in read-only mode 1537def3d77Svaleriabarra def get_array_read(self, memtype=MEM_HOST): 1547def3d77Svaleriabarra """Get read-only access to a Vector via the specified memory type. 1557def3d77Svaleriabarra 1567def3d77Svaleriabarra Args: 1577def3d77Svaleriabarra **memtype: memory type of the array being passed, default CEED_MEM_HOST 1587def3d77Svaleriabarra 1597def3d77Svaleriabarra Returns: 1607f1dc7b9SJeremy L Thompson *array: Numpy or Numba array""" 1617def3d77Svaleriabarra 1627def3d77Svaleriabarra # Retrieve the length of the array 1637def3d77Svaleriabarra length_pointer = ffi.new("CeedInt *") 164477729cfSJeremy L Thompson err_code = lib.CeedVectorGetLength(self._pointer[0], length_pointer) 165477729cfSJeremy L Thompson self._ceed._check_error(err_code) 1667def3d77Svaleriabarra 1677def3d77Svaleriabarra # Setup the pointer's pointer 1687def3d77Svaleriabarra array_pointer = ffi.new("CeedScalar **") 1697def3d77Svaleriabarra 1707def3d77Svaleriabarra # libCEED call 171477729cfSJeremy L Thompson err_code = lib.CeedVectorGetArrayRead( 172477729cfSJeremy L Thompson self._pointer[0], memtype, array_pointer) 173477729cfSJeremy L Thompson self._ceed._check_error(err_code) 1747def3d77Svaleriabarra 1757f1dc7b9SJeremy L Thompson # Return array created from buffer 1767f1dc7b9SJeremy L Thompson if memtype == MEM_HOST: 1777def3d77Svaleriabarra # Create buffer object from returned pointer 1787a7b0fa3SJed Brown buff = ffi.buffer( 1797a7b0fa3SJed Brown array_pointer[0], 1807a7b0fa3SJed Brown ffi.sizeof("CeedScalar") * 1817a7b0fa3SJed Brown length_pointer[0]) 1827f1dc7b9SJeremy L Thompson # return read only Numpy array 1837def3d77Svaleriabarra ret = np.frombuffer(buff, dtype="float64") 1847def3d77Svaleriabarra ret.flags['WRITEABLE'] = False 1857def3d77Svaleriabarra return ret 1867f1dc7b9SJeremy L Thompson else: 1877f1dc7b9SJeremy L Thompson # CUDA array interface 1887f1dc7b9SJeremy L Thompson # https://numba.pydata.org/numba-doc/latest/cuda/cuda_array_interface.html 189962dc42dSJeremy L Thompson import numba.cuda as nbcuda 1907f1dc7b9SJeremy L Thompson desc = { 1917f1dc7b9SJeremy L Thompson 'shape': (length_pointer[0]), 1927f1dc7b9SJeremy L Thompson 'typestr': '>f8', 1937f1dc7b9SJeremy L Thompson 'data': (int(ffi.cast("intptr_t", array_pointer[0])), False), 1947f1dc7b9SJeremy L Thompson 'version': 2 1957f1dc7b9SJeremy L Thompson } 1967f1dc7b9SJeremy L Thompson # return read only Numba array 1977f1dc7b9SJeremy L Thompson return nbcuda.from_cuda_array_interface(desc) 1987def3d77Svaleriabarra 1997def3d77Svaleriabarra # Restore the Vector's data array 2007def3d77Svaleriabarra def restore_array(self): 2017def3d77Svaleriabarra """Restore an array obtained using get_array().""" 2027def3d77Svaleriabarra 2037def3d77Svaleriabarra # Setup the pointer's pointer 2047def3d77Svaleriabarra array_pointer = ffi.new("CeedScalar **") 2057def3d77Svaleriabarra 2067def3d77Svaleriabarra # libCEED call 207477729cfSJeremy L Thompson err_code = lib.CeedVectorRestoreArray(self._pointer[0], array_pointer) 208477729cfSJeremy L Thompson self._ceed._check_error(err_code) 2097def3d77Svaleriabarra 2107def3d77Svaleriabarra # Restore an array obtained using getArrayRead 2117def3d77Svaleriabarra def restore_array_read(self): 2127def3d77Svaleriabarra """Restore an array obtained using get_array_read().""" 2137def3d77Svaleriabarra 2147def3d77Svaleriabarra # Setup the pointer's pointer 2157def3d77Svaleriabarra array_pointer = ffi.new("CeedScalar **") 2167def3d77Svaleriabarra 2177def3d77Svaleriabarra # libCEED call 218477729cfSJeremy L Thompson err_code = lib.CeedVectorRestoreArrayRead( 219477729cfSJeremy L Thompson self._pointer[0], array_pointer) 220477729cfSJeremy L Thompson self._ceed._check_error(err_code) 2217def3d77Svaleriabarra 222e259ae81SJed Brown @contextlib.contextmanager 2237f1dc7b9SJeremy L Thompson def array(self, *shape, memtype=MEM_HOST): 224e259ae81SJed Brown """Context manager for array access. 225e259ae81SJed Brown 226e259ae81SJed Brown Args: 227e259ae81SJed Brown shape (tuple): shape of returned numpy.array 2287f1dc7b9SJeremy L Thompson **memtype: memory type of the array being passed, default CEED_MEM_HOST 2297f1dc7b9SJeremy L Thompson 230e259ae81SJed Brown 231e259ae81SJed Brown Returns: 232e259ae81SJed Brown np.array: writable view of vector 233e259ae81SJed Brown 234e259ae81SJed Brown Examples: 235e259ae81SJed Brown Constructing the identity inside a libceed.Vector: 236e259ae81SJed Brown 237e259ae81SJed Brown >>> vec = ceed.Vector(16) 238e259ae81SJed Brown >>> with vec.array(4, 4) as x: 239e259ae81SJed Brown >>> x[...] = np.eye(4) 240e259ae81SJed Brown """ 2417f1dc7b9SJeremy L Thompson x = self.get_array(memtype=memtype) 242e259ae81SJed Brown if shape: 243e259ae81SJed Brown x = x.reshape(shape) 244e259ae81SJed Brown yield x 245e259ae81SJed Brown self.restore_array() 246e259ae81SJed Brown 247e259ae81SJed Brown @contextlib.contextmanager 2487f1dc7b9SJeremy L Thompson def array_read(self, *shape, memtype=MEM_HOST): 249e259ae81SJed Brown """Context manager for read-only array access. 250e259ae81SJed Brown 251e259ae81SJed Brown Args: 252e259ae81SJed Brown shape (tuple): shape of returned numpy.array 2537f1dc7b9SJeremy L Thompson **memtype: memory type of the array being passed, default CEED_MEM_HOST 254e259ae81SJed Brown 255e259ae81SJed Brown Returns: 256e259ae81SJed Brown np.array: read-only view of vector 257e259ae81SJed Brown 258e259ae81SJed Brown Examples: 259*187168c7SJeremy L Thompson Viewing contents of a reshaped libceed.Vector view: 260e259ae81SJed Brown 261e259ae81SJed Brown >>> vec = ceed.Vector(6) 262e259ae81SJed Brown >>> vec.set_value(1.3) 263e259ae81SJed Brown >>> with vec.array_read(2, 3) as x: 264e259ae81SJed Brown >>> print(x) 265e259ae81SJed Brown """ 2667f1dc7b9SJeremy L Thompson x = self.get_array_read(memtype=memtype) 267e259ae81SJed Brown if shape: 268e259ae81SJed Brown x = x.reshape(shape) 269e259ae81SJed Brown yield x 270e259ae81SJed Brown self.restore_array_read() 271e259ae81SJed Brown 2727def3d77Svaleriabarra # Get the length of a Vector 2737def3d77Svaleriabarra def get_length(self): 2747def3d77Svaleriabarra """Get the length of a Vector. 2757def3d77Svaleriabarra 2767def3d77Svaleriabarra Returns: 2777def3d77Svaleriabarra length: length of the Vector""" 2787def3d77Svaleriabarra 2797def3d77Svaleriabarra length_pointer = ffi.new("CeedInt *") 2807def3d77Svaleriabarra 2817def3d77Svaleriabarra # libCEED call 282477729cfSJeremy L Thompson err_code = lib.CeedVectorGetLength(self._pointer[0], length_pointer) 283477729cfSJeremy L Thompson self._ceed._check_error(err_code) 2847def3d77Svaleriabarra 2857def3d77Svaleriabarra return length_pointer[0] 2867def3d77Svaleriabarra 2877def3d77Svaleriabarra # Get the length of a Vector 2887def3d77Svaleriabarra def __len__(self): 2897def3d77Svaleriabarra """Get the length of a Vector. 2907def3d77Svaleriabarra 2917def3d77Svaleriabarra Returns: 2927def3d77Svaleriabarra length: length of the Vector""" 2937def3d77Svaleriabarra 2947def3d77Svaleriabarra length_pointer = ffi.new("CeedInt *") 2957def3d77Svaleriabarra 2967def3d77Svaleriabarra # libCEED call 297477729cfSJeremy L Thompson err_code = lib.CeedVectorGetLength(self._pointer[0], length_pointer) 298477729cfSJeremy L Thompson self._ceed._check_error(err_code) 2997def3d77Svaleriabarra 3007def3d77Svaleriabarra return length_pointer[0] 3017def3d77Svaleriabarra 3027def3d77Svaleriabarra # Set the Vector to a given constant value 3037def3d77Svaleriabarra def set_value(self, value): 3047def3d77Svaleriabarra """Set the Vector to a constant value. 3057def3d77Svaleriabarra 3067def3d77Svaleriabarra Args: 3077def3d77Svaleriabarra value: value to be used""" 3087def3d77Svaleriabarra 3097def3d77Svaleriabarra # libCEED call 310477729cfSJeremy L Thompson err_code = lib.CeedVectorSetValue(self._pointer[0], value) 311477729cfSJeremy L Thompson self._ceed._check_error(err_code) 3127def3d77Svaleriabarra 3137def3d77Svaleriabarra # Sync the Vector to a specified memtype 314547d9b97Sjeremylt def sync_array(self, memtype=MEM_HOST): 3157def3d77Svaleriabarra """Sync the Vector to a specified memtype. 3167def3d77Svaleriabarra 3177def3d77Svaleriabarra Args: 3187def3d77Svaleriabarra **memtype: memtype to be synced""" 3197def3d77Svaleriabarra 3207def3d77Svaleriabarra # libCEED call 321477729cfSJeremy L Thompson err_code = lib.CeedVectorSyncArray(self._pointer[0], memtype) 322477729cfSJeremy L Thompson self._ceed._check_error(err_code) 3237def3d77Svaleriabarra 324547d9b97Sjeremylt # Compute the norm of a vector 325547d9b97Sjeremylt def norm(self, normtype=NORM_2): 326547d9b97Sjeremylt """Get the norm of a Vector. 327547d9b97Sjeremylt 328547d9b97Sjeremylt Args: 329547d9b97Sjeremylt **normtype: type of norm to be computed""" 330547d9b97Sjeremylt 331547d9b97Sjeremylt norm_pointer = ffi.new("CeedScalar *") 332547d9b97Sjeremylt 333547d9b97Sjeremylt # libCEED call 334477729cfSJeremy L Thompson err_code = lib.CeedVectorNorm(self._pointer[0], normtype, norm_pointer) 335477729cfSJeremy L Thompson self._ceed._check_error(err_code) 336547d9b97Sjeremylt 337547d9b97Sjeremylt return norm_pointer[0] 338547d9b97Sjeremylt 339d99fa3c5SJeremy L Thompson # Take the reciprocal of a vector 340d99fa3c5SJeremy L Thompson def reciprocal(self): 341d99fa3c5SJeremy L Thompson """Take the reciprocal of a Vector.""" 342d99fa3c5SJeremy L Thompson 343d99fa3c5SJeremy L Thompson # libCEED call 344d99fa3c5SJeremy L Thompson err_code = lib.CeedVectorReciprocal(self._pointer[0]) 345d99fa3c5SJeremy L Thompson self._ceed._check_error(err_code) 346d99fa3c5SJeremy L Thompson 347d99fa3c5SJeremy L Thompson return self 348d99fa3c5SJeremy L Thompson 34919798369SJed Brown def _state(self): 35019798369SJed Brown """Return the modification state of the Vector. 35119798369SJed Brown 35219798369SJed Brown State is incremented each time the Vector is mutated, and is odd whenever a 35319798369SJed Brown mutable reference has not been returned. 35419798369SJed Brown """ 35519798369SJed Brown 35619798369SJed Brown state_pointer = ffi.new("uint64_t *") 35719798369SJed Brown err_code = lib.CeedVectorGetState(self._pointer[0], state_pointer) 35819798369SJed Brown self._ceed._check_error(err_code) 35919798369SJed Brown return state_pointer[0] 36019798369SJed Brown 3617def3d77Svaleriabarra# ------------------------------------------------------------------------------ 3627a7b0fa3SJed Brown 3637a7b0fa3SJed Brown 3647def3d77Svaleriabarraclass _VectorWrap(Vector): 3657def3d77Svaleriabarra """Wrap a CeedVector pointer in a Vector object.""" 3667def3d77Svaleriabarra 3677def3d77Svaleriabarra # Constructor 3687def3d77Svaleriabarra def __init__(self, ceed, pointer): 3697def3d77Svaleriabarra # CeedVector object 3707def3d77Svaleriabarra self._pointer = pointer 3717def3d77Svaleriabarra 3727def3d77Svaleriabarra # Reference to Ceed 3737def3d77Svaleriabarra self._ceed = ceed 3747def3d77Svaleriabarra 3757def3d77Svaleriabarra# ------------------------------------------------------------------------------ 376