Actual source code: telescope.c
petsc-3.10.5 2019-03-28
Report Typos and Errors

      2:  #include <petsc/private/matimpl.h>
      3:  #include <petsc/private/pcimpl.h>
      4:  #include <petscksp.h>
      5:  #include <petscdm.h>
      6: #include "../src/ksp/pc/impls/telescope/telescope.h"

      8: static PetscBool  cited = PETSC_FALSE;
      9: static const char citation[] =
     10: "@inproceedings{MaySananRuppKnepleySmith2016,\n"
     11: "  title     = {Extreme-Scale Multigrid Components within PETSc},\n"
     12: "  author    = {Dave A. May and Patrick Sanan and Karl Rupp and Matthew G. Knepley and Barry F. Smith},\n"
     13: "  booktitle = {Proceedings of the Platform for Advanced Scientific Computing Conference},\n"
     14: "  series    = {PASC '16},\n"
     15: "  isbn      = {978-1-4503-4126-4},\n"
     16: "  location  = {Lausanne, Switzerland},\n"
     17: "  pages     = {5:1--5:12},\n"
     18: "  articleno = {5},\n"
     19: "  numpages  = {12},\n"
     20: "  url       = {http://doi.acm.org/10.1145/2929908.2929913},\n"
     21: "  doi       = {10.1145/2929908.2929913},\n"
     22: "  acmid     = {2929913},\n"
     23: "  publisher = {ACM},\n"
     24: "  address   = {New York, NY, USA},\n"
     25: "  keywords  = {GPU, HPC, agglomeration, coarse-level solver, multigrid, parallel computing, preconditioning},\n"
     26: "  year      = {2016}\n"
     27: "}\n";

     29: /*
     30:  PCTelescopeSetUp_default()
     31:  PCTelescopeMatCreate_default()

     33:  default

     35:  // scatter in
     36:  x(comm) -> xtmp(comm)

     38:  xred(subcomm) <- xtmp
     39:  yred(subcomm)

     41:  yred(subcomm) --> xtmp

     43:  // scatter out
     44:  xtmp(comm) -> y(comm)
     45: */

     47: PetscBool isActiveRank(PetscSubcomm scomm)
     48: {
     49:   if (scomm->color == 0) { return PETSC_TRUE; }
     50:   else { return PETSC_FALSE; }
     51: }

     53: DM private_PCTelescopeGetSubDM(PC_Telescope sred)
     54: {
     55:   DM subdm = NULL;

     57:   if (!isActiveRank(sred->psubcomm)) { subdm = NULL; }
     58:   else {
     59:     switch (sred->sr_type) {
     60:     case TELESCOPE_DEFAULT: subdm = NULL;
     61:       break;
     62:     case TELESCOPE_DMDA:    subdm = ((PC_Telescope_DMDACtx*)sred->dm_ctx)->dmrepart;
     63:       break;
     64:     case TELESCOPE_DMPLEX:  subdm = NULL;
     65:       break;
     66:     }
     67:   }
     68:   return(subdm);
     69: }

     71: PetscErrorCode PCTelescopeSetUp_default(PC pc,PC_Telescope sred)
     72: {
     74:   PetscInt       m,M,bs,st,ed;
     75:   Vec            x,xred,yred,xtmp;
     76:   Mat            B;
     77:   MPI_Comm       comm,subcomm;
     78:   VecScatter     scatter;
     79:   IS             isin;

     82:   PetscInfo(pc,"PCTelescope: setup (default)\n");
     83:   comm = PetscSubcommParent(sred->psubcomm);
     84:   subcomm = PetscSubcommChild(sred->psubcomm);

     86:   PCGetOperators(pc,NULL,&B);
     87:   MatGetSize(B,&M,NULL);
     88:   MatGetBlockSize(B,&bs);
     89:   MatCreateVecs(B,&x,NULL);

     91:   xred = NULL;
     92:   m    = 0;
     93:   if (isActiveRank(sred->psubcomm)) {
     94:     VecCreate(subcomm,&xred);
     95:     VecSetSizes(xred,PETSC_DECIDE,M);
     96:     VecSetBlockSize(xred,bs);
     97:     VecSetFromOptions(xred);
     98:     VecGetLocalSize(xred,&m);
     99:   }

    101:   yred = NULL;
    102:   if (isActiveRank(sred->psubcomm)) {
    103:     VecDuplicate(xred,&yred);
    104:   }

    106:   VecCreate(comm,&xtmp);
    107:   VecSetSizes(xtmp,m,PETSC_DECIDE);
    108:   VecSetBlockSize(xtmp,bs);
    109:   VecSetType(xtmp,((PetscObject)x)->type_name);

    111:   if (isActiveRank(sred->psubcomm)) {
    112:     VecGetOwnershipRange(xred,&st,&ed);
    113:     ISCreateStride(comm,(ed-st),st,1,&isin);
    114:   } else {
    115:     VecGetOwnershipRange(x,&st,&ed);
    116:     ISCreateStride(comm,0,st,1,&isin);
    117:   }
    118:   ISSetBlockSize(isin,bs);

    120:   VecScatterCreate(x,isin,xtmp,NULL,&scatter);

    122:   sred->isin    = isin;
    123:   sred->scatter = scatter;
    124:   sred->xred    = xred;
    125:   sred->yred    = yred;
    126:   sred->xtmp    = xtmp;
    127:   VecDestroy(&x);
    128:   return(0);
    129: }

    131: PetscErrorCode PCTelescopeMatCreate_default(PC pc,PC_Telescope sred,MatReuse reuse,Mat *A)
    132: {
    134:   MPI_Comm       comm,subcomm;
    135:   Mat            Bred,B;
    136:   PetscInt       nr,nc;
    137:   IS             isrow,iscol;
    138:   Mat            Blocal,*_Blocal;

    141:   PetscInfo(pc,"PCTelescope: updating the redundant preconditioned operator (default)\n");
    142:   PetscObjectGetComm((PetscObject)pc,&comm);
    143:   subcomm = PetscSubcommChild(sred->psubcomm);
    144:   PCGetOperators(pc,NULL,&B);
    145:   MatGetSize(B,&nr,&nc);
    146:   isrow = sred->isin;
    147:   ISCreateStride(comm,nc,0,1,&iscol);
    148:   MatCreateSubMatrices(B,1,&isrow,&iscol,MAT_INITIAL_MATRIX,&_Blocal);
    149:   Blocal = *_Blocal;
    150:   PetscFree(_Blocal);
    151:   Bred = NULL;
    152:   if (isActiveRank(sred->psubcomm)) {
    153:     PetscInt mm;

    155:     if (reuse != MAT_INITIAL_MATRIX) { Bred = *A; }

    157:     MatGetSize(Blocal,&mm,NULL);
    158:     MatCreateMPIMatConcatenateSeqMat(subcomm,Blocal,mm,reuse,&Bred);
    159:   }
    160:   *A = Bred;
    161:   ISDestroy(&iscol);
    162:   MatDestroy(&Blocal);
    163:   return(0);
    164: }

    166: static PetscErrorCode PCTelescopeSubNullSpaceCreate_Telescope(PC pc,PC_Telescope sred,MatNullSpace nullspace,MatNullSpace *sub_nullspace)
    167: {
    168:   PetscErrorCode   ierr;
    169:   PetscBool        has_const;
    170:   const Vec        *vecs;
    171:   Vec              *sub_vecs = NULL;
    172:   PetscInt         i,k,n = 0;
    173:   MPI_Comm         subcomm;

    176:   subcomm = PetscSubcommChild(sred->psubcomm);
    177:   MatNullSpaceGetVecs(nullspace,&has_const,&n,&vecs);

    179:   if (isActiveRank(sred->psubcomm)) {
    180:     if (n) {
    181:       VecDuplicateVecs(sred->xred,n,&sub_vecs);
    182:     }
    183:   }

    185:   /* copy entries */
    186:   for (k=0; k<n; k++) {
    187:     const PetscScalar *x_array;
    188:     PetscScalar       *LA_sub_vec;
    189:     PetscInt          st,ed;

    191:     /* pull in vector x->xtmp */
    192:     VecScatterBegin(sred->scatter,vecs[k],sred->xtmp,INSERT_VALUES,SCATTER_FORWARD);
    193:     VecScatterEnd(sred->scatter,vecs[k],sred->xtmp,INSERT_VALUES,SCATTER_FORWARD);
    194:     if (sub_vecs) {
    195:       /* copy vector entries into xred */
    196:       VecGetArrayRead(sred->xtmp,&x_array);
    197:       if (sub_vecs[k]) {
    198:         VecGetOwnershipRange(sub_vecs[k],&st,&ed);
    199:         VecGetArray(sub_vecs[k],&LA_sub_vec);
    200:         for (i=0; i<ed-st; i++) {
    201:           LA_sub_vec[i] = x_array[i];
    202:         }
    203:         VecRestoreArray(sub_vecs[k],&LA_sub_vec);
    204:       }
    205:       VecRestoreArrayRead(sred->xtmp,&x_array);
    206:     }
    207:   }

    209:   if (isActiveRank(sred->psubcomm)) {
    210:     /* create new (near) nullspace for redundant object */
    211:     MatNullSpaceCreate(subcomm,has_const,n,sub_vecs,sub_nullspace);
    212:     VecDestroyVecs(n,&sub_vecs);
    213:     if (nullspace->remove) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_SUP,"Propagation of custom remove callbacks not supported when propagating (near) nullspaces with PCTelescope");
    214:     if (nullspace->rmctx) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_SUP,"Propagation of custom remove callback context not supported when propagating (near) nullspaces with PCTelescope");
    215:   }
    216:   return(0);
    217: }

    219: static PetscErrorCode PCTelescopeMatNullSpaceCreate_default(PC pc,PC_Telescope sred,Mat sub_mat)
    220: {
    221:   PetscErrorCode   ierr;
    222:   Mat              B;

    225:   PCGetOperators(pc,NULL,&B);

    227:   /* Propagate the nullspace if it exists */
    228:   {
    229:     MatNullSpace nullspace,sub_nullspace;
    230:     MatGetNullSpace(B,&nullspace);
    231:     if (nullspace) {
    232:       PetscInfo(pc,"PCTelescope: generating nullspace (default)\n");
    233:       PCTelescopeSubNullSpaceCreate_Telescope(pc,sred,nullspace,&sub_nullspace);
    234:       if (isActiveRank(sred->psubcomm)) {
    235:         MatSetNullSpace(sub_mat,sub_nullspace);
    236:         MatNullSpaceDestroy(&sub_nullspace);
    237:       }
    238:     }
    239:   }

    241:   /* Propagate the near nullspace if it exists */
    242:   {
    243:     MatNullSpace nearnullspace,sub_nearnullspace;
    244:     MatGetNearNullSpace(B,&nearnullspace);
    245:     if (nearnullspace) {
    246:       PetscInfo(pc,"PCTelescope: generating near nullspace (default)\n");
    247:       PCTelescopeSubNullSpaceCreate_Telescope(pc,sred,nearnullspace,&sub_nearnullspace);
    248:       if (isActiveRank(sred->psubcomm)) {
    249:         MatSetNearNullSpace(sub_mat,sub_nearnullspace);
    250:         MatNullSpaceDestroy(&sub_nearnullspace);
    251:       }
    252:     }
    253:   }
    254:   return(0);
    255: }

    257: static PetscErrorCode PCView_Telescope(PC pc,PetscViewer viewer)
    258: {
    259:   PC_Telescope     sred = (PC_Telescope)pc->data;
    260:   PetscErrorCode   ierr;
    261:   PetscBool        iascii,isstring;
    262:   PetscViewer      subviewer;

    265:   PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERASCII,&iascii);
    266:   PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERSTRING,&isstring);
    267:   if (iascii) {
    268:     if (!sred->psubcomm) {
    269:       PetscViewerASCIIPrintf(viewer,"  preconditioner not yet setup\n");
    270:     } else {
    271:       MPI_Comm    comm,subcomm;
    272:       PetscMPIInt comm_size,subcomm_size;
    273:       DM          dm,subdm;

    275:       PCGetDM(pc,&dm);
    276:       subdm = private_PCTelescopeGetSubDM(sred);
    277:       comm = PetscSubcommParent(sred->psubcomm);
    278:       subcomm = PetscSubcommChild(sred->psubcomm);
    279:       MPI_Comm_size(comm,&comm_size);
    280:       MPI_Comm_size(subcomm,&subcomm_size);

    282:       PetscViewerASCIIPrintf(viewer,"  parent comm size reduction factor = %D\n",sred->redfactor);
    283:       PetscViewerASCIIPrintf(viewer,"  comm_size = %d , subcomm_size = %d\n",(int)comm_size,(int)subcomm_size);
    284:       switch (sred->subcommtype) {
    285:         case PETSC_SUBCOMM_INTERLACED :
    286:           PetscViewerASCIIPrintf(viewer,"  subcomm type: interlaced\n",sred->subcommtype);
    287:           break;
    288:         case PETSC_SUBCOMM_CONTIGUOUS :
    289:           PetscViewerASCIIPrintf(viewer,"  subcomm type: contiguous\n",sred->subcommtype);
    290:           break;
    291:         default :
    292:           SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_SUP,"General subcomm type not supported by PCTelescope");
    293:       }
    294:       PetscViewerGetSubViewer(viewer,subcomm,&subviewer);
    295:       if (isActiveRank(sred->psubcomm)) {
    296:         PetscViewerASCIIPushTab(subviewer);

    298:         if (dm && sred->ignore_dm) {
    299:           PetscViewerASCIIPrintf(subviewer,"  ignoring DM\n");
    300:         }
    301:         if (sred->ignore_kspcomputeoperators) {
    302:           PetscViewerASCIIPrintf(subviewer,"  ignoring KSPComputeOperators\n");
    303:         }
    304:         switch (sred->sr_type) {
    305:         case TELESCOPE_DEFAULT:
    306:           PetscViewerASCIIPrintf(subviewer,"  using default setup\n");
    307:           break;
    308:         case TELESCOPE_DMDA:
    309:           PetscViewerASCIIPrintf(subviewer,"  DMDA detected\n");
    310:           DMView_DA_Short(subdm,subviewer);
    311:           break;
    312:         case TELESCOPE_DMPLEX:
    313:           PetscViewerASCIIPrintf(subviewer,"  DMPLEX detected\n");
    314:           break;
    315:         }

    317:         KSPView(sred->ksp,subviewer);
    318:         PetscViewerASCIIPopTab(subviewer);
    319:       }
    320:       PetscViewerRestoreSubViewer(viewer,subcomm,&subviewer);
    321:     }
    322:   }
    323:   return(0);
    324: }

    326: static PetscErrorCode PCSetUp_Telescope(PC pc)
    327: {
    328:   PC_Telescope      sred = (PC_Telescope)pc->data;
    329:   PetscErrorCode    ierr;
    330:   MPI_Comm          comm,subcomm=0;
    331:   PCTelescopeType   sr_type;

    334:   PetscObjectGetComm((PetscObject)pc,&comm);

    336:   /* subcomm definition */
    337:   if (!pc->setupcalled) {
    338:     if (!sred->psubcomm) {
    339:       PetscSubcommCreate(comm,&sred->psubcomm);
    340:       PetscSubcommSetNumber(sred->psubcomm,sred->redfactor);
    341:       PetscSubcommSetType(sred->psubcomm,sred->subcommtype);
    342:       PetscLogObjectMemory((PetscObject)pc,sizeof(PetscSubcomm));
    343:     }
    344:   }
    345:   subcomm = PetscSubcommChild(sred->psubcomm);

    347:   /* internal KSP */
    348:   if (!pc->setupcalled) {
    349:     const char *prefix;

    351:     if (isActiveRank(sred->psubcomm)) {
    352:       KSPCreate(subcomm,&sred->ksp);
    353:       KSPSetErrorIfNotConverged(sred->ksp,pc->erroriffailure);
    354:       PetscObjectIncrementTabLevel((PetscObject)sred->ksp,(PetscObject)pc,1);
    355:       PetscLogObjectParent((PetscObject)pc,(PetscObject)sred->ksp);
    356:       KSPSetType(sred->ksp,KSPPREONLY);
    357:       PCGetOptionsPrefix(pc,&prefix);
    358:       KSPSetOptionsPrefix(sred->ksp,prefix);
    359:       KSPAppendOptionsPrefix(sred->ksp,"telescope_");
    360:     }
    361:   }
    362:   /* Determine type of setup/update */
    363:   if (!pc->setupcalled) {
    364:     PetscBool has_dm,same;
    365:     DM        dm;

    367:     sr_type = TELESCOPE_DEFAULT;
    368:     has_dm = PETSC_FALSE;
    369:     PCGetDM(pc,&dm);
    370:     if (dm) { has_dm = PETSC_TRUE; }
    371:     if (has_dm) {
    372:       /* check for dmda */
    373:       PetscObjectTypeCompare((PetscObject)dm,DMDA,&same);
    374:       if (same) {
    375:         PetscInfo(pc,"PCTelescope: found DMDA\n");
    376:         sr_type = TELESCOPE_DMDA;
    377:       }
    378:       /* check for dmplex */
    379:       PetscObjectTypeCompare((PetscObject)dm,DMPLEX,&same);
    380:       if (same) {
    381:         PetscInfo(pc,"PCTelescope: found DMPLEX\n");
    382:         sr_type = TELESCOPE_DMPLEX;
    383:       }
    384:     }

    386:     if (sred->ignore_dm) {
    387:       PetscInfo(pc,"PCTelescope: ignore DM\n");
    388:       sr_type = TELESCOPE_DEFAULT;
    389:     }
    390:     sred->sr_type = sr_type;
    391:   } else {
    392:     sr_type = sred->sr_type;
    393:   }

    395:   /* set function pointers for repartition setup, matrix creation/update, matrix (near) nullspace, and reset functionality */
    396:   switch (sr_type) {
    397:   case TELESCOPE_DEFAULT:
    398:     sred->pctelescope_setup_type              = PCTelescopeSetUp_default;
    399:     sred->pctelescope_matcreate_type          = PCTelescopeMatCreate_default;
    400:     sred->pctelescope_matnullspacecreate_type = PCTelescopeMatNullSpaceCreate_default;
    401:     sred->pctelescope_reset_type              = NULL;
    402:     break;
    403:   case TELESCOPE_DMDA:
    404:     pc->ops->apply                            = PCApply_Telescope_dmda;
    405:     pc->ops->applyrichardson                  = PCApplyRichardson_Telescope_dmda;
    406:     sred->pctelescope_setup_type              = PCTelescopeSetUp_dmda;
    407:     sred->pctelescope_matcreate_type          = PCTelescopeMatCreate_dmda;
    408:     sred->pctelescope_matnullspacecreate_type = PCTelescopeMatNullSpaceCreate_dmda;
    409:     sred->pctelescope_reset_type              = PCReset_Telescope_dmda;
    410:     break;
    411:   case TELESCOPE_DMPLEX: SETERRQ(comm,PETSC_ERR_SUP,"Support for DMPLEX is currently not available");
    412:     break;
    413:   default: SETERRQ(comm,PETSC_ERR_SUP,"Only support for repartitioning DMDA is provided");
    414:     break;
    415:   }

    417:   /* setup */
    418:   if (sred->pctelescope_setup_type) {
    419:     sred->pctelescope_setup_type(pc,sred);
    420:   }
    421:   /* update */
    422:   if (!pc->setupcalled) {
    423:     if (sred->pctelescope_matcreate_type) {
    424:       sred->pctelescope_matcreate_type(pc,sred,MAT_INITIAL_MATRIX,&sred->Bred);
    425:     }
    426:     if (sred->pctelescope_matnullspacecreate_type) {
    427:       sred->pctelescope_matnullspacecreate_type(pc,sred,sred->Bred);
    428:     }
    429:   } else {
    430:     if (sred->pctelescope_matcreate_type) {
    431:       sred->pctelescope_matcreate_type(pc,sred,MAT_REUSE_MATRIX,&sred->Bred);
    432:     }
    433:   }

    435:   /* common - no construction */
    436:   if (isActiveRank(sred->psubcomm)) {
    437:     KSPSetOperators(sred->ksp,sred->Bred,sred->Bred);
    438:     if (pc->setfromoptionscalled && !pc->setupcalled){
    439:       KSPSetFromOptions(sred->ksp);
    440:     }
    441:   }
    442:   return(0);
    443: }

    445: static PetscErrorCode PCApply_Telescope(PC pc,Vec x,Vec y)
    446: {
    447:   PC_Telescope      sred = (PC_Telescope)pc->data;
    448:   PetscErrorCode    ierr;
    449:   Vec               xtmp,xred,yred;
    450:   PetscInt          i,st,ed;
    451:   VecScatter        scatter;
    452:   PetscScalar       *array;
    453:   const PetscScalar *x_array;

    456:   PetscCitationsRegister(citation,&cited);

    458:   xtmp    = sred->xtmp;
    459:   scatter = sred->scatter;
    460:   xred    = sred->xred;
    461:   yred    = sred->yred;

    463:   /* pull in vector x->xtmp */
    464:   VecScatterBegin(scatter,x,xtmp,INSERT_VALUES,SCATTER_FORWARD);
    465:   VecScatterEnd(scatter,x,xtmp,INSERT_VALUES,SCATTER_FORWARD);

    467:   /* copy vector entries into xred */
    468:   VecGetArrayRead(xtmp,&x_array);
    469:   if (xred) {
    470:     PetscScalar *LA_xred;
    471:     VecGetOwnershipRange(xred,&st,&ed);
    472:     VecGetArray(xred,&LA_xred);
    473:     for (i=0; i<ed-st; i++) {
    474:       LA_xred[i] = x_array[i];
    475:     }
    476:     VecRestoreArray(xred,&LA_xred);
    477:   }
    478:   VecRestoreArrayRead(xtmp,&x_array);
    479:   /* solve */
    480:   if (isActiveRank(sred->psubcomm)) {
    481:     KSPSolve(sred->ksp,xred,yred);
    482:   }
    483:   /* return vector */
    484:   VecGetArray(xtmp,&array);
    485:   if (yred) {
    486:     const PetscScalar *LA_yred;
    487:     VecGetOwnershipRange(yred,&st,&ed);
    488:     VecGetArrayRead(yred,&LA_yred);
    489:     for (i=0; i<ed-st; i++) {
    490:       array[i] = LA_yred[i];
    491:     }
    492:     VecRestoreArrayRead(yred,&LA_yred);
    493:   }
    494:   VecRestoreArray(xtmp,&array);
    495:   VecScatterBegin(scatter,xtmp,y,INSERT_VALUES,SCATTER_REVERSE);
    496:   VecScatterEnd(scatter,xtmp,y,INSERT_VALUES,SCATTER_REVERSE);
    497:   return(0);
    498: }

    500: static PetscErrorCode PCApplyRichardson_Telescope(PC pc,Vec x,Vec y,Vec w,PetscReal rtol,PetscReal abstol, PetscReal dtol,PetscInt its,PetscBool zeroguess,PetscInt *outits,PCRichardsonConvergedReason *reason)
    501: {
    502:   PC_Telescope      sred = (PC_Telescope)pc->data;
    503:   PetscErrorCode    ierr;
    504:   Vec               xtmp,yred;
    505:   PetscInt          i,st,ed;
    506:   VecScatter        scatter;
    507:   const PetscScalar *x_array;
    508:   PetscBool         default_init_guess_value;

    511:   xtmp    = sred->xtmp;
    512:   scatter = sred->scatter;
    513:   yred    = sred->yred;

    515:   if (its > 1) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_SUP,"PCApplyRichardson_Telescope only supports max_it = 1");
    516:   *reason = (PCRichardsonConvergedReason)0;

    518:   if (!zeroguess) {
    519:     PetscInfo(pc,"PCTelescope: Scattering y for non-zero initial guess\n");
    520:     /* pull in vector y->xtmp */
    521:     VecScatterBegin(scatter,y,xtmp,INSERT_VALUES,SCATTER_FORWARD);
    522:     VecScatterEnd(scatter,y,xtmp,INSERT_VALUES,SCATTER_FORWARD);

    524:     /* copy vector entries into xred */
    525:     VecGetArrayRead(xtmp,&x_array);
    526:     if (yred) {
    527:       PetscScalar *LA_yred;
    528:       VecGetOwnershipRange(yred,&st,&ed);
    529:       VecGetArray(yred,&LA_yred);
    530:       for (i=0; i<ed-st; i++) {
    531:         LA_yred[i] = x_array[i];
    532:       }
    533:       VecRestoreArray(yred,&LA_yred);
    534:     }
    535:     VecRestoreArrayRead(xtmp,&x_array);
    536:   }

    538:   if (isActiveRank(sred->psubcomm)) {
    539:     KSPGetInitialGuessNonzero(sred->ksp,&default_init_guess_value);
    540:     if (!zeroguess) KSPSetInitialGuessNonzero(sred->ksp,PETSC_TRUE);
    541:   }

    543:   PCApply_Telescope(pc,x,y);

    545:   if (isActiveRank(sred->psubcomm)) {
    546:     KSPSetInitialGuessNonzero(sred->ksp,default_init_guess_value);
    547:   }

    549:   if (!*reason) *reason = PCRICHARDSON_CONVERGED_ITS;
    550:   *outits = 1;
    551:   return(0);
    552: }

    554: static PetscErrorCode PCReset_Telescope(PC pc)
    555: {
    556:   PC_Telescope   sred = (PC_Telescope)pc->data;

    559:   ISDestroy(&sred->isin);
    560:   VecScatterDestroy(&sred->scatter);
    561:   VecDestroy(&sred->xred);
    562:   VecDestroy(&sred->yred);
    563:   VecDestroy(&sred->xtmp);
    564:   MatDestroy(&sred->Bred);
    565:   KSPReset(sred->ksp);
    566:   if (sred->pctelescope_reset_type) {
    567:     sred->pctelescope_reset_type(pc);
    568:   }
    569:   return(0);
    570: }

    572: static PetscErrorCode PCDestroy_Telescope(PC pc)
    573: {
    574:   PC_Telescope     sred = (PC_Telescope)pc->data;
    575:   PetscErrorCode   ierr;

    578:   PCReset_Telescope(pc);
    579:   KSPDestroy(&sred->ksp);
    580:   PetscSubcommDestroy(&sred->psubcomm);
    581:   PetscFree(sred->dm_ctx);
    582:   PetscFree(pc->data);
    583:   return(0);
    584: }

    586: static PetscErrorCode PCSetFromOptions_Telescope(PetscOptionItems *PetscOptionsObject,PC pc)
    587: {
    588:   PC_Telescope     sred = (PC_Telescope)pc->data;
    589:   PetscErrorCode   ierr;
    590:   MPI_Comm         comm;
    591:   PetscMPIInt      size;
    592:   PetscBool        flg;
    593:   PetscSubcommType subcommtype;

    596:   PetscObjectGetComm((PetscObject)pc,&comm);
    597:   MPI_Comm_size(comm,&size);
    598:   PetscOptionsHead(PetscOptionsObject,"Telescope options");
    599:   PetscOptionsEnum("-pc_telescope_subcomm_type","Subcomm type (interlaced or contiguous)","PCTelescopeSetSubcommType",PetscSubcommTypes,(PetscEnum)sred->subcommtype,(PetscEnum*)&subcommtype,&flg);
    600:   if (flg) {
    601:     PCTelescopeSetSubcommType(pc,subcommtype);
    602:   }
    603:   PetscOptionsInt("-pc_telescope_reduction_factor","Factor to reduce comm size by","PCTelescopeSetReductionFactor",sred->redfactor,&sred->redfactor,0);
    604:   if (sred->redfactor > size) SETERRQ(comm,PETSC_ERR_ARG_WRONG,"-pc_telescope_reduction_factor <= comm size");
    605:   PetscOptionsBool("-pc_telescope_ignore_dm","Ignore any DM attached to the PC","PCTelescopeSetIgnoreDM",sred->ignore_dm,&sred->ignore_dm,0);
    606:   PetscOptionsBool("-pc_telescope_ignore_kspcomputeoperators","Ignore method used to compute A","PCTelescopeSetIgnoreKSPComputeOperators",sred->ignore_kspcomputeoperators,&sred->ignore_kspcomputeoperators,0);
    607:   PetscOptionsTail();
    608:   return(0);
    609: }

    611: /* PC simplementation specific API's */

    613: static PetscErrorCode PCTelescopeGetKSP_Telescope(PC pc,KSP *ksp)
    614: {
    615:   PC_Telescope red = (PC_Telescope)pc->data;
    617:   if (ksp) *ksp = red->ksp;
    618:   return(0);
    619: }

    621: static PetscErrorCode PCTelescopeGetSubcommType_Telescope(PC pc,PetscSubcommType *subcommtype)
    622: {
    623:   PC_Telescope red = (PC_Telescope)pc->data;
    625:   if (subcommtype) *subcommtype = red->subcommtype;
    626:   return(0);
    627: }

    629: static PetscErrorCode PCTelescopeSetSubcommType_Telescope(PC pc,PetscSubcommType subcommtype)
    630: {
    631:   PC_Telescope     red = (PC_Telescope)pc->data;

    634:   if (pc->setupcalled) SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_WRONGSTATE,"You cannot change the subcommunicator type for PCTelescope after it has been set up.");
    635:   red->subcommtype = subcommtype;
    636:   return(0);
    637: }

    639: static PetscErrorCode PCTelescopeGetReductionFactor_Telescope(PC pc,PetscInt *fact)
    640: {
    641:   PC_Telescope red = (PC_Telescope)pc->data;
    643:   if (fact) *fact = red->redfactor;
    644:   return(0);
    645: }

    647: static PetscErrorCode PCTelescopeSetReductionFactor_Telescope(PC pc,PetscInt fact)
    648: {
    649:   PC_Telescope     red = (PC_Telescope)pc->data;
    650:   PetscMPIInt      size;
    651:   PetscErrorCode   ierr;

    654:   MPI_Comm_size(PetscObjectComm((PetscObject)pc),&size);
    655:   if (fact <= 0) SETERRQ1(PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_WRONG,"Reduction factor of telescoping PC %D must be positive",fact);
    656:   if (fact > size) SETERRQ1(PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_WRONG,"Reduction factor of telescoping PC %D must be <= comm.size",fact);
    657:   red->redfactor = fact;
    658:   return(0);
    659: }

    661: static PetscErrorCode PCTelescopeGetIgnoreDM_Telescope(PC pc,PetscBool *v)
    662: {
    663:   PC_Telescope red = (PC_Telescope)pc->data;
    665:   if (v) *v = red->ignore_dm;
    666:   return(0);
    667: }

    669: static PetscErrorCode PCTelescopeSetIgnoreDM_Telescope(PC pc,PetscBool v)
    670: {
    671:   PC_Telescope red = (PC_Telescope)pc->data;
    673:   red->ignore_dm = v;
    674:   return(0);
    675: }

    677: static PetscErrorCode PCTelescopeGetIgnoreKSPComputeOperators_Telescope(PC pc,PetscBool *v)
    678: {
    679:   PC_Telescope red = (PC_Telescope)pc->data;
    681:   if (v) *v = red->ignore_kspcomputeoperators;
    682:   return(0);
    683: }

    685: static PetscErrorCode PCTelescopeSetIgnoreKSPComputeOperators_Telescope(PC pc,PetscBool v)
    686: {
    687:   PC_Telescope red = (PC_Telescope)pc->data;
    689:   red->ignore_kspcomputeoperators = v;
    690:   return(0);
    691: }

    693: static PetscErrorCode PCTelescopeGetDM_Telescope(PC pc,DM *dm)
    694: {
    695:   PC_Telescope red = (PC_Telescope)pc->data;
    697:   *dm = private_PCTelescopeGetSubDM(red);
    698:   return(0);
    699: }

    701: /*@
    702:  PCTelescopeGetKSP - Gets the KSP created by the telescoping PC.

    704:  Not Collective

    706:  Input Parameter:
    707: .  pc - the preconditioner context

    709:  Output Parameter:
    710: .  subksp - the KSP defined the smaller set of processes

    712:  Level: advanced

    714: .keywords: PC, telescopting solve
    715: @*/
    716: PetscErrorCode PCTelescopeGetKSP(PC pc,KSP *subksp)
    717: {
    720:   PetscUseMethod(pc,"PCTelescopeGetKSP_C",(PC,KSP*),(pc,subksp));
    721:   return(0);
    722: }

    724: /*@
    725:  PCTelescopeGetReductionFactor - Gets the factor by which the original number of processes has been reduced by.

    727:  Not Collective

    729:  Input Parameter:
    730: .  pc - the preconditioner context

    732:  Output Parameter:
    733: .  fact - the reduction factor

    735:  Level: advanced

    737: .keywords: PC, telescoping solve
    738: @*/
    739: PetscErrorCode PCTelescopeGetReductionFactor(PC pc,PetscInt *fact)
    740: {
    743:   PetscUseMethod(pc,"PCTelescopeGetReductionFactor_C",(PC,PetscInt*),(pc,fact));
    744:   return(0);
    745: }

    747: /*@
    748:  PCTelescopeSetReductionFactor - Sets the factor by which the original number of processes has been reduced by.

    750:  Not Collective

    752:  Input Parameter:
    753: .  pc - the preconditioner context

    755:  Output Parameter:
    756: .  fact - the reduction factor

    758:  Level: advanced

    760: .keywords: PC, telescoping solve
    761: @*/
    762: PetscErrorCode PCTelescopeSetReductionFactor(PC pc,PetscInt fact)
    763: {
    766:   PetscTryMethod(pc,"PCTelescopeSetReductionFactor_C",(PC,PetscInt),(pc,fact));
    767:   return(0);
    768: }

    770: /*@
    771:  PCTelescopeGetIgnoreDM - Get the flag indicating if any DM attached to the PC will be used.

    773:  Not Collective

    775:  Input Parameter:
    776: .  pc - the preconditioner context

    778:  Output Parameter:
    779: .  v - the flag

    781:  Level: advanced

    783: .keywords: PC, telescoping solve
    784: @*/
    785: PetscErrorCode PCTelescopeGetIgnoreDM(PC pc,PetscBool *v)
    786: {
    789:   PetscUseMethod(pc,"PCTelescopeGetIgnoreDM_C",(PC,PetscBool*),(pc,v));
    790:   return(0);
    791: }

    793: /*@
    794:  PCTelescopeSetIgnoreDM - Set a flag to ignore any DM attached to the PC.

    796:  Not Collective

    798:  Input Parameter:
    799: .  pc - the preconditioner context

    801:  Output Parameter:
    802: .  v - Use PETSC_TRUE to ignore any DM

    804:  Level: advanced

    806: .keywords: PC, telescoping solve
    807: @*/
    808: PetscErrorCode PCTelescopeSetIgnoreDM(PC pc,PetscBool v)
    809: {
    812:   PetscTryMethod(pc,"PCTelescopeSetIgnoreDM_C",(PC,PetscBool),(pc,v));
    813:   return(0);
    814: }

    816: /*@
    817:  PCTelescopeGetIgnoreKSPComputeOperators - Get the flag indicating if KSPComputeOperators will be used.

    819:  Not Collective

    821:  Input Parameter:
    822: .  pc - the preconditioner context

    824:  Output Parameter:
    825: .  v - the flag

    827:  Level: advanced

    829: .keywords: PC, telescoping solve
    830: @*/
    831: PetscErrorCode PCTelescopeGetIgnoreKSPComputeOperators(PC pc,PetscBool *v)
    832: {
    835:   PetscUseMethod(pc,"PCTelescopeGetIgnoreKSPComputeOperators_C",(PC,PetscBool*),(pc,v));
    836:   return(0);
    837: }

    839: /*@
    840:  PCTelescopeSetIgnoreKSPComputeOperators - Set a flag to ignore KSPComputeOperators.

    842:  Not Collective

    844:  Input Parameter:
    845: .  pc - the preconditioner context

    847:  Output Parameter:
    848: .  v - Use PETSC_TRUE to ignore the method (if defined) set via KSPSetComputeOperators on pc

    850:  Level: advanced

    852: .keywords: PC, telescoping solve
    853: @*/
    854: PetscErrorCode PCTelescopeSetIgnoreKSPComputeOperators(PC pc,PetscBool v)
    855: {
    858:   PetscTryMethod(pc,"PCTelescopeSetIgnoreKSPComputeOperators_C",(PC,PetscBool),(pc,v));
    859:   return(0);
    860: }

    862: /*@
    863:  PCTelescopeGetDM - Get the re-partitioned DM attached to the sub KSP.

    865:  Not Collective

    867:  Input Parameter:
    868: .  pc - the preconditioner context

    870:  Output Parameter:
    871: .  subdm - The re-partitioned DM

    873:  Level: advanced

    875: .keywords: PC, telescoping solve
    876: @*/
    877: PetscErrorCode PCTelescopeGetDM(PC pc,DM *subdm)
    878: {
    881:   PetscUseMethod(pc,"PCTelescopeGetDM_C",(PC,DM*),(pc,subdm));
    882:   return(0);
    883: }

    885: /*@
    886:  PCTelescopeSetSubcommType - set subcommunicator type (interlaced or contiguous)

    888:  Logically Collective

    890:  Input Parameter:
    891: +  pc - the preconditioner context
    892: -  subcommtype - the subcommunicator type (see PetscSubcommType)

    894:  Level: advanced

    896: .keywords: PC, telescoping solve

    898: .seealso: PetscSubcommType, PetscSubcomm, PCTELESCOPE
    899: @*/
    900: PetscErrorCode PCTelescopeSetSubcommType(PC pc, PetscSubcommType subcommtype)
    901: {
    904:   PetscTryMethod(pc,"PCTelescopeSetSubcommType_C",(PC,PetscSubcommType),(pc,subcommtype));
    905:   return(0);
    906: }

    908: /*@
    909:  PCTelescopeGetSubcommType - Get the subcommunicator type (interlaced or contiguous)

    911:  Not Collective

    913:  Input Parameter:
    914: .  pc - the preconditioner context

    916:  Output Parameter:
    917: .  subcommtype - the subcommunicator type (see PetscSubcommType)

    919:  Level: advanced

    921: .keywords: PC, telescoping solve

    923: .seealso: PetscSubcomm, PetscSubcommType, PCTELESCOPE
    924: @*/
    925: PetscErrorCode PCTelescopeGetSubcommType(PC pc, PetscSubcommType *subcommtype)
    926: {
    929:   PetscUseMethod(pc,"PCTelescopeGetSubcommType_C",(PC,PetscSubcommType*),(pc,subcommtype));
    930:   return(0);
    931: }

    933: /* -------------------------------------------------------------------------------------*/
    934: /*MC
    935:    PCTELESCOPE - Runs a KSP solver on a sub-group of processors. MPI processes not in the sub-communicator are idle during the solve.

    937:    Options Database:
    938: +  -pc_telescope_reduction_factor <r> - factor to use communicator size by. e.g. with 64 MPI processes and r=4, the new sub-communicator will have 64/4 = 16 ranks.
    939: -  -pc_telescope_ignore_dm  - flag to indicate whether an attached DM should be ignored
    940: -  -pc_telescope_subcomm_type <interlaced,contiguous> - how to define the reduced communicator. see PetscSubcomm for more.

    942:    Level: advanced

    944:    Notes:
    945:    The preconditioner is deemed telescopic as it only calls KSPSolve() on a single
    946:    sub-communicator, in contrast with PCREDUNDANT which calls KSPSolve() on N sub-communicators.
    947:    This means there will be MPI processes which will be idle during the application of this preconditioner.

    949:    The default KSP is PREONLY. If a DM is attached to the PC, it is re-partitioned on the sub-communicator.
    950:    Both the Bmat operator and the right hand side vector are permuted into the new DOF ordering defined by the re-partitioned DM.
    951:    Currently only support for re-partitioning a DMDA is provided.
    952:    Any nullspace attached to the original Bmat operator is extracted, re-partitioned and set on the repartitioned Bmat operator.
    953:    KSPSetComputeOperators() is not propagated to the sub KSP.
    954:    Currently there is no support for the flag -pc_use_amat

    956:    Assuming that the parent preconditioner (PC) is defined on a communicator c, this implementation
    957:    creates a child sub-communicator (c') containing fewer MPI processes than the original parent preconditioner (PC).

    959:   Developer Notes:
    960:    During PCSetup, the B operator is scattered onto c'.
    961:    Within PCApply, the RHS vector (x) is scattered into a redundant vector, xred (defined on c').
    962:    Then, KSPSolve() is executed on the c' communicator.

    964:    The communicator used within the telescoping preconditioner is defined by a PetscSubcomm using the INTERLACED
    965:    creation routine by default (this can be changed with -pc_telescope_subcomm_type). We run the sub KSP on only the ranks within the communicator which have a color equal to zero.

    967:    The telescoping preconditioner is aware of nullspaces and near nullspaces which are attached to the B operator.
    968:    In the case where B has a (near) nullspace attached, the (near) nullspace vectors are extracted from B and mapped into
    969:    a new (near) nullspace, defined on the sub-communicator, which is attached to B' (the B operator which was scattered to c')

    971:    The telescoping preconditioner is aware of an attached DM. In the event that the DM is of type DMDA (2D or 3D -
    972:    1D support for 1D DMDAs is not provided), a new DMDA is created on c' (e.g. it is re-partitioned), and this new DM
    973:    is attached the sub KSPSolve(). The design of telescope is such that it should be possible to extend support
    974:    for re-partitioning other to DM's (e.g. DMPLEX). The user can supply a flag to ignore attached DMs.

    976:    By default, B' is defined by simply fusing rows from different MPI processes

    978:    When a DMDA is attached to the parent preconditioner, B' is defined by: (i) performing a symmetric permutation of B
    979:    into the ordering defined by the DMDA on c', (ii) extracting the local chunks via MatCreateSubMatrices(), (iii) fusing the
    980:    locally (sequential) matrices defined on the ranks common to c and c' into B' using MatCreateMPIMatConcatenateSeqMat()

    982:    Limitations/improvements include the following.
    983:    VecPlaceArray() could be used within PCApply() to improve efficiency and reduce memory usage.

    985:    The symmetric permutation used when a DMDA is encountered is performed via explicitly assmbleming a permutation matrix P,
    986:    and performing P^T.A.P. Possibly it might be more efficient to use MatPermute(). We opted to use P^T.A.P as it appears
    987:    VecPermute() does not supported for the use case required here. By computing P, one can permute both the operator and RHS in a
    988:    consistent manner.

    990:    Mapping of vectors is performed in the following way.
    991:    Suppose the parent comm size was 4, and we set a reduction factor of 2; this would give a comm size on c' of 2.
    992:    Using the interlaced creation routine, the ranks in c with color = 0 will be rank 0 and 2.
    993:    We perform the scatter to the sub-comm in the following way.
    994:    [1] Given a vector x defined on comm c

    996:    rank(c) : _________ 0 ______  ________ 1 _______  ________ 2 _____________ ___________ 3 __________
    997:          x : [0, 1, 2, 3, 4, 5] [6, 7, 8, 9, 10, 11] [12, 13, 14, 15, 16, 17] [18, 19, 20, 21, 22, 23]

    999:    scatter to xtmp defined also on comm c so that we have the following values

    1001:    rank(c) : ___________________ 0 ________________  _1_  ______________________ 2 _______________________  __3_
    1002:       xtmp : [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] [  ] [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23] [  ]

    1004:    The entries on rank 1 and 3 (ranks which do not have a color = 0 in c') have no values

    1007:    [2] Copy the value from rank 0, 2 (indices with respect to comm c) into the vector xred which is defined on communicator c'.
    1008:    Ranks 0 and 2 are the only ranks in the subcomm which have a color = 0.

    1010:     rank(c') : ___________________ 0 _______________  ______________________ 1 _____________________
    1011:       xred : [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]

    1014:   Contributed by Dave May

    1016:   Reference:
    1017:   Dave A. May, Patrick Sanan, Karl Rupp, Matthew G. Knepley, and Barry F. Smith, "Extreme-Scale Multigrid Components within PETSc". 2016. In Proceedings of the Platform for Advanced Scientific Computing Conference (PASC '16). DOI: 10.1145/2929908.2929913

    1019: .seealso:  PCTelescopeGetKSP(), PCTelescopeGetDM(), PCTelescopeGetReductionFactor(), PCTelescopeSetReductionFactor(), PCTelescopeGetIgnoreDM(), PCTelescopeSetIgnoreDM(), PCREDUNDANT
    1020: M*/
    1021: PETSC_EXTERN PetscErrorCode PCCreate_Telescope(PC pc)
    1022: {
    1023:   PetscErrorCode       ierr;
    1024:   struct _PC_Telescope *sred;

    1027:   PetscNewLog(pc,&sred);
    1028:   sred->subcommtype    = PETSC_SUBCOMM_INTERLACED;
    1029:   sred->redfactor      = 1;
    1030:   sred->ignore_dm      = PETSC_FALSE;
    1031:   sred->ignore_kspcomputeoperators = PETSC_FALSE;
    1032:   pc->data             = (void*)sred;

    1034:   pc->ops->apply           = PCApply_Telescope;
    1035:   pc->ops->applytranspose  = NULL;
    1036:   pc->ops->applyrichardson = PCApplyRichardson_Telescope;
    1037:   pc->ops->setup           = PCSetUp_Telescope;
    1038:   pc->ops->destroy         = PCDestroy_Telescope;
    1039:   pc->ops->reset           = PCReset_Telescope;
    1040:   pc->ops->setfromoptions  = PCSetFromOptions_Telescope;
    1041:   pc->ops->view            = PCView_Telescope;

    1043:   sred->pctelescope_setup_type              = PCTelescopeSetUp_default;
    1044:   sred->pctelescope_matcreate_type          = PCTelescopeMatCreate_default;
    1045:   sred->pctelescope_matnullspacecreate_type = PCTelescopeMatNullSpaceCreate_default;
    1046:   sred->pctelescope_reset_type              = NULL;

    1048:   PetscObjectComposeFunction((PetscObject)pc,"PCTelescopeGetKSP_C",PCTelescopeGetKSP_Telescope);
    1049:   PetscObjectComposeFunction((PetscObject)pc,"PCTelescopeGetSubcommType_C",PCTelescopeGetSubcommType_Telescope);
    1050:   PetscObjectComposeFunction((PetscObject)pc,"PCTelescopeSetSubcommType_C",PCTelescopeSetSubcommType_Telescope);
    1051:   PetscObjectComposeFunction((PetscObject)pc,"PCTelescopeGetReductionFactor_C",PCTelescopeGetReductionFactor_Telescope);
    1052:   PetscObjectComposeFunction((PetscObject)pc,"PCTelescopeSetReductionFactor_C",PCTelescopeSetReductionFactor_Telescope);
    1053:   PetscObjectComposeFunction((PetscObject)pc,"PCTelescopeGetIgnoreDM_C",PCTelescopeGetIgnoreDM_Telescope);
    1054:   PetscObjectComposeFunction((PetscObject)pc,"PCTelescopeSetIgnoreDM_C",PCTelescopeSetIgnoreDM_Telescope);
    1055:   PetscObjectComposeFunction((PetscObject)pc,"PCTelescopeGetIgnoreKSPComputeOperators_C",PCTelescopeGetIgnoreKSPComputeOperators_Telescope);
    1056:   PetscObjectComposeFunction((PetscObject)pc,"PCTelescopeSetIgnoreKSPComputeOperators_C",PCTelescopeSetIgnoreKSPComputeOperators_Telescope);
    1057:   PetscObjectComposeFunction((PetscObject)pc,"PCTelescopeGetDM_C",PCTelescopeGetDM_Telescope);
    1058:   return(0);
    1059: }