Actual source code: redundant.c
petsc-3.10.5 2019-03-28
Report Typos and Errors

      2: /*
      3:   This file defines a "solve the problem redundantly on each subgroup of processor" preconditioner.
      4: */
      5:  #include <petsc/private/pcimpl.h>
      6:  #include <petscksp.h>

      8: typedef struct {
      9:   KSP                ksp;
     10:   PC                 pc;                   /* actual preconditioner used on each processor */
     11:   Vec                xsub,ysub;            /* vectors of a subcommunicator to hold parallel vectors of PetscObjectComm((PetscObject)pc) */
     12:   Vec                xdup,ydup;            /* parallel vector that congregates xsub or ysub facilitating vector scattering */
     13:   Mat                pmats;                /* matrix and optional preconditioner matrix belong to a subcommunicator */
     14:   VecScatter         scatterin,scatterout; /* scatter used to move all values to each processor group (subcommunicator) */
     15:   PetscBool          useparallelmat;
     16:   PetscSubcomm       psubcomm;
     17:   PetscInt           nsubcomm;             /* num of data structure PetscSubcomm */
     18:   PetscBool          shifttypeset;
     19:   MatFactorShiftType shifttype;
     20: } PC_Redundant;

     22: PetscErrorCode  PCFactorSetShiftType_Redundant(PC pc,MatFactorShiftType shifttype)
     23: {
     24:   PC_Redundant   *red = (PC_Redundant*)pc->data;

     28:   if (red->ksp) {
     29:     PC pc;
     30:     KSPGetPC(red->ksp,&pc);
     31:     PCFactorSetShiftType(pc,shifttype);
     32:   } else {
     33:     red->shifttypeset = PETSC_TRUE;
     34:     red->shifttype    = shifttype;
     35:   }
     36:   return(0);
     37: }

     39: static PetscErrorCode PCView_Redundant(PC pc,PetscViewer viewer)
     40: {
     41:   PC_Redundant   *red = (PC_Redundant*)pc->data;
     43:   PetscBool      iascii,isstring;
     44:   PetscViewer    subviewer;

     47:   PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERASCII,&iascii);
     48:   PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERSTRING,&isstring);
     49:   if (iascii) {
     50:     if (!red->psubcomm) {
     51:       PetscViewerASCIIPrintf(viewer,"  Not yet setup\n");
     52:     } else {
     53:       PetscViewerASCIIPrintf(viewer,"  First (color=0) of %D PCs follows\n",red->nsubcomm);
     54:       PetscViewerGetSubViewer(viewer,((PetscObject)red->pc)->comm,&subviewer);
     55:       if (!red->psubcomm->color) { /* only view first redundant pc */
     56:         PetscViewerASCIIPushTab(subviewer);
     57:         KSPView(red->ksp,subviewer);
     58:         PetscViewerASCIIPopTab(subviewer);
     59:       }
     60:       PetscViewerRestoreSubViewer(viewer,((PetscObject)red->pc)->comm,&subviewer);
     61:     }
     62:   } else if (isstring) {
     63:     PetscViewerStringSPrintf(viewer," Redundant solver preconditioner");
     64:   }
     65:   return(0);
     66: }

     68:  #include <../src/mat/impls/aij/mpi/mpiaij.h>
     69: static PetscErrorCode PCSetUp_Redundant(PC pc)
     70: {
     71:   PC_Redundant   *red = (PC_Redundant*)pc->data;
     73:   PetscInt       mstart,mend,mlocal,M;
     74:   PetscMPIInt    size;
     75:   MPI_Comm       comm,subcomm;
     76:   Vec            x;

     79:   PetscObjectGetComm((PetscObject)pc,&comm);

     81:   /* if pmatrix set by user is sequential then we do not need to gather the parallel matrix */
     82:   MPI_Comm_size(comm,&size);
     83:   if (size == 1) red->useparallelmat = PETSC_FALSE;

     85:   if (!pc->setupcalled) {
     86:     PetscInt mloc_sub;
     87:     if (!red->psubcomm) { /* create red->psubcomm, new ksp and pc over subcomm */
     88:       KSP ksp;
     89:       PCRedundantGetKSP(pc,&ksp);
     90:     }
     91:     subcomm = PetscSubcommChild(red->psubcomm);

     93:     if (red->useparallelmat) {
     94:       /* grab the parallel matrix and put it into processors of a subcomminicator */
     95:       MatCreateRedundantMatrix(pc->pmat,red->psubcomm->n,subcomm,MAT_INITIAL_MATRIX,&red->pmats);

     97:       MPI_Comm_size(subcomm,&size);
     98:       if (size > 1) {
     99:         PetscBool foundpack;
    100:         MatGetFactorAvailable(red->pmats,NULL,MAT_FACTOR_LU,&foundpack);
    101:         if (!foundpack) { /* reset default ksp and pc */
    102:           KSPSetType(red->ksp,KSPGMRES);
    103:           PCSetType(red->pc,PCBJACOBI);
    104:         } else {
    105:           PCFactorSetMatSolverType(red->pc,NULL);
    106:         }
    107:       }

    109:       KSPSetOperators(red->ksp,red->pmats,red->pmats);

    111:       /* get working vectors xsub and ysub */
    112:       MatCreateVecs(red->pmats,&red->xsub,&red->ysub);

    114:       /* create working vectors xdup and ydup.
    115:        xdup concatenates all xsub's contigously to form a mpi vector over dupcomm  (see PetscSubcommCreate_interlaced())
    116:        ydup concatenates all ysub and has empty local arrays because ysub's arrays will be place into it.
    117:        Note: we use communicator dupcomm, not PetscObjectComm((PetscObject)pc)! */
    118:       MatGetLocalSize(red->pmats,&mloc_sub,NULL);
    119:       VecCreateMPI(PetscSubcommContiguousParent(red->psubcomm),mloc_sub,PETSC_DECIDE,&red->xdup);
    120:       VecCreateMPIWithArray(PetscSubcommContiguousParent(red->psubcomm),1,mloc_sub,PETSC_DECIDE,NULL,&red->ydup);

    122:       /* create vecscatters */
    123:       if (!red->scatterin) { /* efficiency of scatterin is independent from psubcomm_type! */
    124:         IS       is1,is2;
    125:         PetscInt *idx1,*idx2,i,j,k;

    127:         MatCreateVecs(pc->pmat,&x,0);
    128:         VecGetSize(x,&M);
    129:         VecGetOwnershipRange(x,&mstart,&mend);
    130:         mlocal = mend - mstart;
    131:         PetscMalloc2(red->psubcomm->n*mlocal,&idx1,red->psubcomm->n*mlocal,&idx2);
    132:         j    = 0;
    133:         for (k=0; k<red->psubcomm->n; k++) {
    134:           for (i=mstart; i<mend; i++) {
    135:             idx1[j]   = i;
    136:             idx2[j++] = i + M*k;
    137:           }
    138:         }
    139:         ISCreateGeneral(comm,red->psubcomm->n*mlocal,idx1,PETSC_COPY_VALUES,&is1);
    140:         ISCreateGeneral(comm,red->psubcomm->n*mlocal,idx2,PETSC_COPY_VALUES,&is2);
    141:         VecScatterCreate(x,is1,red->xdup,is2,&red->scatterin);
    142:         ISDestroy(&is1);
    143:         ISDestroy(&is2);

    145:         /* Impl below is good for PETSC_SUBCOMM_INTERLACED (no inter-process communication) and PETSC_SUBCOMM_CONTIGUOUS (communication within subcomm) */
    146:         ISCreateStride(comm,mlocal,mstart+ red->psubcomm->color*M,1,&is1);
    147:         ISCreateStride(comm,mlocal,mstart,1,&is2);
    148:         VecScatterCreate(red->xdup,is1,x,is2,&red->scatterout);
    149:         ISDestroy(&is1);
    150:         ISDestroy(&is2);
    151:         PetscFree2(idx1,idx2);
    152:         VecDestroy(&x);
    153:       }
    154:     } else { /* !red->useparallelmat */
    155:       KSPSetOperators(red->ksp,pc->mat,pc->pmat);
    156:     }
    157:   } else { /* pc->setupcalled */
    158:     if (red->useparallelmat) {
    159:       MatReuse       reuse;
    160:       /* grab the parallel matrix and put it into processors of a subcomminicator */
    161:       /*--------------------------------------------------------------------------*/
    162:       if (pc->flag == DIFFERENT_NONZERO_PATTERN) {
    163:         /* destroy old matrices */
    164:         MatDestroy(&red->pmats);
    165:         reuse = MAT_INITIAL_MATRIX;
    166:       } else {
    167:         reuse = MAT_REUSE_MATRIX;
    168:       }
    169:       MatCreateRedundantMatrix(pc->pmat,red->psubcomm->n,PetscSubcommChild(red->psubcomm),reuse,&red->pmats);
    170:       KSPSetOperators(red->ksp,red->pmats,red->pmats);
    171:     } else { /* !red->useparallelmat */
    172:       KSPSetOperators(red->ksp,pc->mat,pc->pmat);
    173:     }
    174:   }

    176:   if (pc->setfromoptionscalled) {
    177:     KSPSetFromOptions(red->ksp);
    178:   }
    179:   KSPSetUp(red->ksp);
    180:   return(0);
    181: }

    183: static PetscErrorCode PCApply_Redundant(PC pc,Vec x,Vec y)
    184: {
    185:   PC_Redundant   *red = (PC_Redundant*)pc->data;
    187:   PetscScalar    *array;

    190:   if (!red->useparallelmat) {
    191:     KSPSolve(red->ksp,x,y);
    192:     return(0);
    193:   }

    195:   /* scatter x to xdup */
    196:   VecScatterBegin(red->scatterin,x,red->xdup,INSERT_VALUES,SCATTER_FORWARD);
    197:   VecScatterEnd(red->scatterin,x,red->xdup,INSERT_VALUES,SCATTER_FORWARD);

    199:   /* place xdup's local array into xsub */
    200:   VecGetArray(red->xdup,&array);
    201:   VecPlaceArray(red->xsub,(const PetscScalar*)array);

    203:   /* apply preconditioner on each processor */
    204:   KSPSolve(red->ksp,red->xsub,red->ysub);
    205:   VecResetArray(red->xsub);
    206:   VecRestoreArray(red->xdup,&array);

    208:   /* place ysub's local array into ydup */
    209:   VecGetArray(red->ysub,&array);
    210:   VecPlaceArray(red->ydup,(const PetscScalar*)array);

    212:   /* scatter ydup to y */
    213:   VecScatterBegin(red->scatterout,red->ydup,y,INSERT_VALUES,SCATTER_FORWARD);
    214:   VecScatterEnd(red->scatterout,red->ydup,y,INSERT_VALUES,SCATTER_FORWARD);
    215:   VecResetArray(red->ydup);
    216:   VecRestoreArray(red->ysub,&array);
    217:   return(0);
    218: }

    220: static PetscErrorCode PCApplyTranspose_Redundant(PC pc,Vec x,Vec y)
    221: {
    222:   PC_Redundant   *red = (PC_Redundant*)pc->data;
    224:   PetscScalar    *array;

    227:   if (!red->useparallelmat) {
    228:     KSPSolveTranspose(red->ksp,x,y);
    229:     return(0);
    230:   }

    232:   /* scatter x to xdup */
    233:   VecScatterBegin(red->scatterin,x,red->xdup,INSERT_VALUES,SCATTER_FORWARD);
    234:   VecScatterEnd(red->scatterin,x,red->xdup,INSERT_VALUES,SCATTER_FORWARD);

    236:   /* place xdup's local array into xsub */
    237:   VecGetArray(red->xdup,&array);
    238:   VecPlaceArray(red->xsub,(const PetscScalar*)array);

    240:   /* apply preconditioner on each processor */
    241:   KSPSolveTranspose(red->ksp,red->xsub,red->ysub);
    242:   VecResetArray(red->xsub);
    243:   VecRestoreArray(red->xdup,&array);

    245:   /* place ysub's local array into ydup */
    246:   VecGetArray(red->ysub,&array);
    247:   VecPlaceArray(red->ydup,(const PetscScalar*)array);

    249:   /* scatter ydup to y */
    250:   VecScatterBegin(red->scatterout,red->ydup,y,INSERT_VALUES,SCATTER_FORWARD);
    251:   VecScatterEnd(red->scatterout,red->ydup,y,INSERT_VALUES,SCATTER_FORWARD);
    252:   VecResetArray(red->ydup);
    253:   VecRestoreArray(red->ysub,&array);
    254:   return(0);
    255: }

    257: static PetscErrorCode PCReset_Redundant(PC pc)
    258: {
    259:   PC_Redundant   *red = (PC_Redundant*)pc->data;

    263:   if (red->useparallelmat) {
    264:     VecScatterDestroy(&red->scatterin);
    265:     VecScatterDestroy(&red->scatterout);
    266:     VecDestroy(&red->ysub);
    267:     VecDestroy(&red->xsub);
    268:     VecDestroy(&red->xdup);
    269:     VecDestroy(&red->ydup);
    270:   }
    271:   MatDestroy(&red->pmats);
    272:   KSPReset(red->ksp);
    273:   return(0);
    274: }

    276: static PetscErrorCode PCDestroy_Redundant(PC pc)
    277: {
    278:   PC_Redundant   *red = (PC_Redundant*)pc->data;

    282:   PCReset_Redundant(pc);
    283:   KSPDestroy(&red->ksp);
    284:   PetscSubcommDestroy(&red->psubcomm);
    285:   PetscFree(pc->data);
    286:   return(0);
    287: }

    289: static PetscErrorCode PCSetFromOptions_Redundant(PetscOptionItems *PetscOptionsObject,PC pc)
    290: {
    292:   PC_Redundant   *red = (PC_Redundant*)pc->data;

    295:   PetscOptionsHead(PetscOptionsObject,"Redundant options");
    296:   PetscOptionsInt("-pc_redundant_number","Number of redundant pc","PCRedundantSetNumber",red->nsubcomm,&red->nsubcomm,0);
    297:   PetscOptionsTail();
    298:   return(0);
    299: }

    301: static PetscErrorCode PCRedundantSetNumber_Redundant(PC pc,PetscInt nreds)
    302: {
    303:   PC_Redundant *red = (PC_Redundant*)pc->data;

    306:   red->nsubcomm = nreds;
    307:   return(0);
    308: }

    310: /*@
    311:    PCRedundantSetNumber - Sets the number of redundant preconditioner contexts.

    313:    Logically Collective on PC

    315:    Input Parameters:
    316: +  pc - the preconditioner context
    317: -  nredundant - number of redundant preconditioner contexts; for example if you are using 64 MPI processes and
    318:                               use an nredundant of 4 there will be 4 parallel solves each on 16 = 64/4 processes.

    320:    Level: advanced

    322: .keywords: PC, redundant solve
    323: @*/
    324: PetscErrorCode PCRedundantSetNumber(PC pc,PetscInt nredundant)
    325: {

    330:   if (nredundant <= 0) SETERRQ1(PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_WRONG, "num of redundant pc %D must be positive",nredundant);
    331:   PetscTryMethod(pc,"PCRedundantSetNumber_C",(PC,PetscInt),(pc,nredundant));
    332:   return(0);
    333: }

    335: static PetscErrorCode PCRedundantSetScatter_Redundant(PC pc,VecScatter in,VecScatter out)
    336: {
    337:   PC_Redundant   *red = (PC_Redundant*)pc->data;

    341:   PetscObjectReference((PetscObject)in);
    342:   VecScatterDestroy(&red->scatterin);

    344:   red->scatterin  = in;

    346:   PetscObjectReference((PetscObject)out);
    347:   VecScatterDestroy(&red->scatterout);
    348:   red->scatterout = out;
    349:   return(0);
    350: }

    352: /*@
    353:    PCRedundantSetScatter - Sets the scatter used to copy values into the
    354:      redundant local solve and the scatter to move them back into the global
    355:      vector.

    357:    Logically Collective on PC

    359:    Input Parameters:
    360: +  pc - the preconditioner context
    361: .  in - the scatter to move the values in
    362: -  out - the scatter to move them out

    364:    Level: advanced

    366: .keywords: PC, redundant solve
    367: @*/
    368: PetscErrorCode PCRedundantSetScatter(PC pc,VecScatter in,VecScatter out)
    369: {

    376:   PetscTryMethod(pc,"PCRedundantSetScatter_C",(PC,VecScatter,VecScatter),(pc,in,out));
    377:   return(0);
    378: }

    380: static PetscErrorCode PCRedundantGetKSP_Redundant(PC pc,KSP *innerksp)
    381: {
    383:   PC_Redundant   *red = (PC_Redundant*)pc->data;
    384:   MPI_Comm       comm,subcomm;
    385:   const char     *prefix;

    388:   if (!red->psubcomm) {
    389:     PCGetOptionsPrefix(pc,&prefix);

    391:     PetscObjectGetComm((PetscObject)pc,&comm);
    392:     PetscSubcommCreate(comm,&red->psubcomm);
    393:     PetscSubcommSetNumber(red->psubcomm,red->nsubcomm);
    394:     PetscSubcommSetType(red->psubcomm,PETSC_SUBCOMM_CONTIGUOUS);

    396:     PetscSubcommSetOptionsPrefix(red->psubcomm,prefix);
    397:     PetscSubcommSetFromOptions(red->psubcomm);
    398:     PetscLogObjectMemory((PetscObject)pc,sizeof(PetscSubcomm));

    400:     /* create a new PC that processors in each subcomm have copy of */
    401:     subcomm = PetscSubcommChild(red->psubcomm);

    403:     KSPCreate(subcomm,&red->ksp);
    404:     KSPSetErrorIfNotConverged(red->ksp,pc->erroriffailure);
    405:     PetscObjectIncrementTabLevel((PetscObject)red->ksp,(PetscObject)pc,1);
    406:     PetscLogObjectParent((PetscObject)pc,(PetscObject)red->ksp);
    407:     KSPSetType(red->ksp,KSPPREONLY);
    408:     KSPGetPC(red->ksp,&red->pc);
    409:     PCSetType(red->pc,PCLU);
    410:     if (red->shifttypeset) {
    411:       PCFactorSetShiftType(red->pc,red->shifttype);
    412:       red->shifttypeset = PETSC_FALSE;
    413:     }
    414:     KSPSetOptionsPrefix(red->ksp,prefix);
    415:     KSPAppendOptionsPrefix(red->ksp,"redundant_");
    416:   }
    417:   *innerksp = red->ksp;
    418:   return(0);
    419: }

    421: /*@
    422:    PCRedundantGetKSP - Gets the less parallel KSP created by the redundant PC.

    424:    Not Collective

    426:    Input Parameter:
    427: .  pc - the preconditioner context

    429:    Output Parameter:
    430: .  innerksp - the KSP on the smaller set of processes

    432:    Level: advanced

    434: .keywords: PC, redundant solve
    435: @*/
    436: PetscErrorCode PCRedundantGetKSP(PC pc,KSP *innerksp)
    437: {

    443:   PetscUseMethod(pc,"PCRedundantGetKSP_C",(PC,KSP*),(pc,innerksp));
    444:   return(0);
    445: }

    447: static PetscErrorCode PCRedundantGetOperators_Redundant(PC pc,Mat *mat,Mat *pmat)
    448: {
    449:   PC_Redundant *red = (PC_Redundant*)pc->data;

    452:   if (mat)  *mat  = red->pmats;
    453:   if (pmat) *pmat = red->pmats;
    454:   return(0);
    455: }

    457: /*@
    458:    PCRedundantGetOperators - gets the sequential matrix and preconditioner matrix

    460:    Not Collective

    462:    Input Parameter:
    463: .  pc - the preconditioner context

    465:    Output Parameters:
    466: +  mat - the matrix
    467: -  pmat - the (possibly different) preconditioner matrix

    469:    Level: advanced

    471: .keywords: PC, redundant solve
    472: @*/
    473: PetscErrorCode PCRedundantGetOperators(PC pc,Mat *mat,Mat *pmat)
    474: {

    481:   PetscUseMethod(pc,"PCRedundantGetOperators_C",(PC,Mat*,Mat*),(pc,mat,pmat));
    482:   return(0);
    483: }

    485: /* -------------------------------------------------------------------------------------*/
    486: /*MC
    487:      PCREDUNDANT - Runs a KSP solver with preconditioner for the entire problem on subgroups of processors

    489:      Options for the redundant preconditioners can be set with -redundant_pc_xxx for the redundant KSP with -redundant_ksp_xxx

    491:   Options Database:
    492: .  -pc_redundant_number <n> - number of redundant solves, for example if you are using 64 MPI processes and
    493:                               use an n of 4 there will be 4 parallel solves each on 16 = 64/4 processes.

    495:    Level: intermediate

    497:    Notes:
    498:     The default KSP is preonly and the default PC is LU.

    500:    PCFactorSetShiftType() applied to this PC will convey they shift type into the inner PC if it is factorization based.

    502:    Developer Notes:
    503:     Note that PCSetInitialGuessNonzero()  is not used by this class but likely should be.

    505: .seealso:  PCCreate(), PCSetType(), PCType (for list of available types), PCRedundantSetScatter(),
    506:            PCRedundantGetKSP(), PCRedundantGetOperators(), PCRedundantSetNumber()
    507: M*/

    509: PETSC_EXTERN PetscErrorCode PCCreate_Redundant(PC pc)
    510: {
    512:   PC_Redundant   *red;
    513:   PetscMPIInt    size;

    516:   PetscNewLog(pc,&red);
    517:   MPI_Comm_size(PetscObjectComm((PetscObject)pc),&size);

    519:   red->nsubcomm       = size;
    520:   red->useparallelmat = PETSC_TRUE;
    521:   pc->data            = (void*)red;

    523:   pc->ops->apply          = PCApply_Redundant;
    524:   pc->ops->applytranspose = PCApplyTranspose_Redundant;
    525:   pc->ops->setup          = PCSetUp_Redundant;
    526:   pc->ops->destroy        = PCDestroy_Redundant;
    527:   pc->ops->reset          = PCReset_Redundant;
    528:   pc->ops->setfromoptions = PCSetFromOptions_Redundant;
    529:   pc->ops->view           = PCView_Redundant;

    531:   PetscObjectComposeFunction((PetscObject)pc,"PCRedundantSetScatter_C",PCRedundantSetScatter_Redundant);
    532:   PetscObjectComposeFunction((PetscObject)pc,"PCRedundantSetNumber_C",PCRedundantSetNumber_Redundant);
    533:   PetscObjectComposeFunction((PetscObject)pc,"PCRedundantGetKSP_C",PCRedundantGetKSP_Redundant);
    534:   PetscObjectComposeFunction((PetscObject)pc,"PCRedundantGetOperators_C",PCRedundantGetOperators_Redundant);
    535:   PetscObjectComposeFunction((PetscObject)pc,"PCFactorSetShiftType_C",PCFactorSetShiftType_Redundant);
    536:   return(0);
    537: }