Lines Matching refs:mmdata
2623 MatProductCtx_MatMatCusparse *mmdata = *(MatProductCtx_MatMatCusparse **)data; in MatProductCtxDestroy_MatMatCusparse() local
2626 PetscCallCUDA(cudaFree(mmdata->Bt)); in MatProductCtxDestroy_MatMatCusparse()
2627 delete mmdata->Bcsr; in MatProductCtxDestroy_MatMatCusparse()
2629 if (mmdata->matSpBDescr) PetscCallCUSPARSE(cusparseDestroySpMat(mmdata->matSpBDescr)); in MatProductCtxDestroy_MatMatCusparse()
2630 if (mmdata->matBDescr) PetscCallCUSPARSE(cusparseDestroyDnMat(mmdata->matBDescr)); in MatProductCtxDestroy_MatMatCusparse()
2631 if (mmdata->matCDescr) PetscCallCUSPARSE(cusparseDestroyDnMat(mmdata->matCDescr)); in MatProductCtxDestroy_MatMatCusparse()
2632 if (mmdata->spgemmDesc) PetscCallCUSPARSE(cusparseSpGEMM_destroyDescr(mmdata->spgemmDesc)); in MatProductCtxDestroy_MatMatCusparse()
2634 if (mmdata->dBuffer4) PetscCallCUDA(cudaFree(mmdata->dBuffer4)); in MatProductCtxDestroy_MatMatCusparse()
2635 if (mmdata->dBuffer5) PetscCallCUDA(cudaFree(mmdata->dBuffer5)); in MatProductCtxDestroy_MatMatCusparse()
2637 if (mmdata->mmBuffer) PetscCallCUDA(cudaFree(mmdata->mmBuffer)); in MatProductCtxDestroy_MatMatCusparse()
2638 if (mmdata->mmBuffer2) PetscCallCUDA(cudaFree(mmdata->mmBuffer2)); in MatProductCtxDestroy_MatMatCusparse()
2640 PetscCall(MatDestroy(&mmdata->X)); in MatProductCtxDestroy_MatMatCusparse()
2641 PetscCall(PetscFree(mmdata)); in MatProductCtxDestroy_MatMatCusparse()
2658 MatProductCtx_MatMatCusparse *mmdata; in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA() local
2665 mmdata = (MatProductCtx_MatMatCusparse *)product->data; in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2714 PetscCall(MatDenseGetArrayWriteAndMemType(mmdata->X, &carray, nullptr)); in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2715 PetscCall(MatDenseGetLDA(mmdata->X, &clda)); in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2731 if (!mmdata->initialized || mmdata->Blda != blda || mmdata->Clda != clda) { in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2733 if (mmdata->initialized && mmdata->Blda != blda) { in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2734 PetscCallCUSPARSE(cusparseDestroyDnMat(mmdata->matBDescr)); in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2735 mmdata->matBDescr = NULL; in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2737 if (!mmdata->matBDescr) { in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2738 …PetscCallCUSPARSE(cusparseCreateDnMat(&mmdata->matBDescr, B->rmap->n, B->cmap->n, blda, (void *)ba… in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2739 mmdata->Blda = blda; in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2742 if (mmdata->initialized && mmdata->Clda != clda) { in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2743 PetscCallCUSPARSE(cusparseDestroyDnMat(mmdata->matCDescr)); in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2744 mmdata->matCDescr = NULL; in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2746 if (!mmdata->matCDescr) { /* matCDescr is for C or mmdata->X */ in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2747 …PetscCallCUSPARSE(cusparseCreateDnMat(&mmdata->matCDescr, m, n, clda, (void *)carray, cusparse_sca… in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2748 mmdata->Clda = clda; in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2764 …Size(cusp->handle, opA, opB, mat->alpha_one, matADescr, mmdata->matBDescr, mat->beta_zero, mmdata-… in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2766 if ((mmdata->mmBuffer && mmdata->mmBufferSize < mmBufferSize) || !mmdata->mmBuffer) { in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2767 PetscCallCUDA(cudaFree(mmdata->mmBuffer)); in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2768 PetscCallCUDA(cudaMalloc(&mmdata->mmBuffer, mmBufferSize)); in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2769 mmdata->mmBufferSize = mmBufferSize; in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2773 … mat->alpha_one, matADescr, mmdata->matBDescr, mat->beta_zero, mmdata->matCDescr, cusparse_scalart… in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2776 mmdata->initialized = PETSC_TRUE; in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2780 PetscCallCUSPARSE(cusparseDnMatSetValues(mmdata->matBDescr, (void *)barray)); in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2781 PetscCallCUSPARSE(cusparseDnMatSetValues(mmdata->matCDescr, (void *)carray)); in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2785 … mat->alpha_one, matADescr, mmdata->matBDescr, mat->beta_zero, mmdata->matCDescr, cusparse_scalart… in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2794 …>n, &PETSC_CUSPARSE_ONE, barray, blda, &PETSC_CUSPARSE_ZERO, barray, blda, mmdata->Bt, B->cmap->n); in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2803 …->row_offsets->data().get(), csrmat->column_indices->data().get(), mmdata->Bt ? mmdata->Bt : barra… in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2810 PetscCall(MatDenseRestoreArrayWriteAndMemType(mmdata->X, &carray)); in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2811 …PetscCall(MatMatMultNumeric_SeqDenseCUDA_SeqDenseCUDA_Internal(B, mmdata->X, C, PETSC_FALSE, PETSC… in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2813 PetscCall(MatDenseRestoreArrayWriteAndMemType(mmdata->X, &carray)); in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2814 …PetscCall(MatMatMultNumeric_SeqDenseCUDA_SeqDenseCUDA_Internal(B, mmdata->X, C, PETSC_TRUE, PETSC_… in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2818 if (mmdata->cisdense) PetscCall(MatConvert(C, MATSEQDENSE, MAT_INPLACE_MATRIX, &C)); in MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA()
2829 MatProductCtx_MatMatCusparse *mmdata; in MatProductSymbolic_SeqAIJCUSPARSE_SeqDENSECUDA() local
2880 PetscCall(PetscNew(&mmdata)); in MatProductSymbolic_SeqAIJCUSPARSE_SeqDENSECUDA()
2881 mmdata->cisdense = cisdense; in MatProductSymbolic_SeqAIJCUSPARSE_SeqDENSECUDA()
2884 … || product->type == MATPRODUCT_RARt) PetscCallCUDA(cudaMalloc((void **)&mmdata->Bt, (size_t)B->rm… in MatProductSymbolic_SeqAIJCUSPARSE_SeqDENSECUDA()
2888 PetscCall(MatCreate(PetscObjectComm((PetscObject)C), &mmdata->X)); in MatProductSymbolic_SeqAIJCUSPARSE_SeqDENSECUDA()
2889 PetscCall(MatSetType(mmdata->X, MATSEQDENSECUDA)); in MatProductSymbolic_SeqAIJCUSPARSE_SeqDENSECUDA()
2891 PetscCall(MatSetSizes(mmdata->X, A->rmap->n, B->rmap->n, A->rmap->n, B->rmap->n)); in MatProductSymbolic_SeqAIJCUSPARSE_SeqDENSECUDA()
2893 PetscCall(MatSetSizes(mmdata->X, A->rmap->n, B->cmap->n, A->rmap->n, B->cmap->n)); in MatProductSymbolic_SeqAIJCUSPARSE_SeqDENSECUDA()
2896 C->product->data = mmdata; in MatProductSymbolic_SeqAIJCUSPARSE_SeqDENSECUDA()
2914 MatProductCtx_MatMatCusparse *mmdata; in MatProductNumeric_SeqAIJCUSPARSE_SeqAIJCUSPARSE() local
2925 mmdata = (MatProductCtx_MatMatCusparse *)C->product->data; in MatProductNumeric_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
2928 …if (mmdata->reusesym) { /* this happens when api_user is true, meaning that the matrix values have… in MatProductNumeric_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
2929 mmdata->reusesym = PETSC_FALSE; in MatProductNumeric_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
2984 …Bcsr = mmdata->Bcsr ? mmdata->Bcsr : (CsrMatrix *)Bmat->mat; /* B may be in compressed row storage… in MatProductNumeric_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
2991 …BmatSpDescr = mmdata->Bcsr ? mmdata->matSpBDescr : Bmat->matDescr; /* B may be in compressed row s… in MatProductNumeric_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
2994 …Cmat->beta_zero, Cmat->matDescr, cusparse_scalartype, CUSPARSE_SPGEMM_DEFAULT, mmdata->spgemmDesc); in MatProductNumeric_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
2997 …escr, cusparse_scalartype, CUSPARSE_SPGEMM_DEFAULT, mmdata->spgemmDesc, &mmdata->mmBufferSize, mmd… in MatProductNumeric_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
2999 …Cmat->beta_zero, Cmat->matDescr, cusparse_scalartype, CUSPARSE_SPGEMM_DEFAULT, mmdata->spgemmDesc); in MatProductNumeric_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
3007 PetscCall(PetscLogGpuFlops(mmdata->flops)); in MatProductNumeric_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
3036 MatProductCtx_MatMatCusparse *mmdata; in MatProductSymbolic_SeqAIJCUSPARSE_SeqAIJCUSPARSE() local
3059 PetscCall(PetscNew(&mmdata)); in MatProductSymbolic_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
3060 C->product->data = mmdata; in MatProductSymbolic_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
3181 mmdata->Bcsr = Bcsr; in MatProductSymbolic_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
3184 …stat = cusparseCreateCsr(&mmdata->matSpBDescr, Bcsr->num_rows, Bcsr->num_cols, Bcsr->num_entries, … in MatProductSymbolic_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
3187 BmatSpDescr = mmdata->matSpBDescr; in MatProductSymbolic_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
3212 mmdata->flops = flops; in MatProductSymbolic_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
3220 PetscCallCUSPARSE(cusparseSpGEMM_createDescr(&mmdata->spgemmDesc)); in MatProductSymbolic_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
3237 …B, Amat->matDescr, BmatSpDescr, Cmat->matDescr, CUSPARSE_SPGEMM_DEFAULT, mmdata->spgemmDesc, &buff… in MatProductSymbolic_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
3241 …B, Amat->matDescr, BmatSpDescr, Cmat->matDescr, CUSPARSE_SPGEMM_DEFAULT, mmdata->spgemmDesc, &buff… in MatProductSymbolic_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
3244 …B, Amat->matDescr, BmatSpDescr, Cmat->matDescr, CUSPARSE_SPGEMM_DEFAULT, mmdata->spgemmDesc, &buff… in MatProductSymbolic_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
3248 PetscCallCUDA(cudaMalloc((void **)&mmdata->dBuffer4, bufferSize4)); in MatProductSymbolic_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
3249 …Descr, CUSPARSE_SPGEMM_DEFAULT, mmdata->spgemmDesc, &bufferSize2, dBuffer2, &bufferSize3, dBuffer3… in MatProductSymbolic_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
3266 …B, Amat->matDescr, BmatSpDescr, Cmat->matDescr, CUSPARSE_SPGEMM_DEFAULT, mmdata->spgemmDesc, &buff… in MatProductSymbolic_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
3268 PetscCallCUDA(cudaMalloc((void **)&mmdata->dBuffer5, bufferSize5)); in MatProductSymbolic_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
3269 …, BmatSpDescr, Cmat->matDescr, CUSPARSE_SPGEMM_DEFAULT, mmdata->spgemmDesc, &bufferSize5, mmdata->… in MatProductSymbolic_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
3272 …Cmat->beta_zero, Cmat->matDescr, cusparse_scalartype, CUSPARSE_SPGEMM_DEFAULT, mmdata->spgemmDesc); in MatProductSymbolic_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
3279 …beta_zero, Cmat->matDescr, cusparse_scalartype, CUSPARSE_SPGEMM_DEFAULT, mmdata->spgemmDesc, &bufS… in MatProductSymbolic_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
3281 PetscCallCUDA(cudaMalloc((void **)&mmdata->mmBuffer2, bufSize2)); in MatProductSymbolic_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
3283 …->matDescr, cusparse_scalartype, CUSPARSE_SPGEMM_DEFAULT, mmdata->spgemmDesc, &bufSize2, mmdata->m… in MatProductSymbolic_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
3286 … Cmat->matDescr, cusparse_scalartype, CUSPARSE_SPGEMM_DEFAULT, mmdata->spgemmDesc, &mmdata->mmBuff… in MatProductSymbolic_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
3293 PetscCallCUDA(cudaMalloc((void **)&mmdata->mmBuffer, mmdata->mmBufferSize)); in MatProductSymbolic_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
3295 …escr, cusparse_scalartype, CUSPARSE_SPGEMM_DEFAULT, mmdata->spgemmDesc, &mmdata->mmBufferSize, mmd… in MatProductSymbolic_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
3301 mmdata->mmBufferSize / 1024)); in MatProductSymbolic_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
3308 …Cmat->beta_zero, Cmat->matDescr, cusparse_scalartype, CUSPARSE_SPGEMM_DEFAULT, mmdata->spgemmDesc); in MatProductSymbolic_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
3330 PetscCall(PetscLogGpuFlops(mmdata->flops)); in MatProductSymbolic_SeqAIJCUSPARSE_SeqAIJCUSPARSE()
3386 mmdata->reusesym = PETSC_TRUE; in MatProductSymbolic_SeqAIJCUSPARSE_SeqAIJCUSPARSE()