Actual source code: baijov.c
petsc-3.10.5 2019-03-28
Report Typos and Errors

      2: /*
      3:    Routines to compute overlapping regions of a parallel MPI matrix
      4:   and to find submatrices that were shared across processors.
      5: */
      6:  #include <../src/mat/impls/baij/mpi/mpibaij.h>
      7:  #include <petscbt.h>

      9: static PetscErrorCode MatIncreaseOverlap_MPIBAIJ_Local(Mat,PetscInt,char**,PetscInt*,PetscInt**);
     10: static PetscErrorCode MatIncreaseOverlap_MPIBAIJ_Receive(Mat,PetscInt,PetscInt**,PetscInt**,PetscInt*);
     11: extern PetscErrorCode MatGetRow_MPIBAIJ(Mat,PetscInt,PetscInt*,PetscInt**,PetscScalar**);
     12: extern PetscErrorCode MatRestoreRow_MPIBAIJ(Mat,PetscInt,PetscInt*,PetscInt**,PetscScalar**);

     14: PetscErrorCode MatIncreaseOverlap_MPIBAIJ(Mat C,PetscInt imax,IS is[],PetscInt ov)
     15: {
     17:   PetscInt       i,N=C->cmap->N, bs=C->rmap->bs;
     18:   IS             *is_new;

     21:   PetscMalloc1(imax,&is_new);
     22:   /* Convert the indices into block format */
     23:   ISCompressIndicesGeneral(N,C->rmap->n,bs,imax,is,is_new);
     24:   if (ov < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Negative overlap specified\n");
     25:   for (i=0; i<ov; ++i) {
     26:     MatIncreaseOverlap_MPIBAIJ_Once(C,imax,is_new);
     27:   }
     28:   for (i=0; i<imax; i++) {ISDestroy(&is[i]);}
     29:   ISExpandIndicesGeneral(N,N,bs,imax,is_new,is);
     30:   for (i=0; i<imax; i++) {ISDestroy(&is_new[i]);}
     31:   PetscFree(is_new);
     32:   return(0);
     33: }

     35: /*
     36:   Sample message format:
     37:   If a processor A wants processor B to process some elements corresponding
     38:   to index sets is[1], is[5]
     39:   mesg [0] = 2   (no of index sets in the mesg)
     40:   -----------
     41:   mesg [1] = 1 => is[1]
     42:   mesg [2] = sizeof(is[1]);
     43:   -----------
     44:   mesg [5] = 5  => is[5]
     45:   mesg [6] = sizeof(is[5]);
     46:   -----------
     47:   mesg [7]
     48:   mesg [n]  data(is[1])
     49:   -----------
     50:   mesg[n+1]
     51:   mesg[m]  data(is[5])
     52:   -----------

     54:   Notes:
     55:   nrqs - no of requests sent (or to be sent out)
     56:   nrqr - no of requests recieved (which have to be or which have been processed
     57: */
     58: PetscErrorCode MatIncreaseOverlap_MPIBAIJ_Once(Mat C,PetscInt imax,IS is[])
     59: {
     60:   Mat_MPIBAIJ    *c = (Mat_MPIBAIJ*)C->data;
     61:   const PetscInt **idx,*idx_i;
     62:   PetscInt       *n,*w3,*w4,**data,len;
     64:   PetscMPIInt    size,rank,tag1,tag2,*w2,*w1,nrqr;
     65:   PetscInt       Mbs,i,j,k,**rbuf,row,proc=-1,nrqs,msz,**outdat,**ptr;
     66:   PetscInt       *ctr,*pa,*tmp,*isz,*isz1,**xdata,**rbuf2,*d_p;
     67:   PetscMPIInt    *onodes1,*olengths1,*onodes2,*olengths2;
     68:   PetscBT        *table;
     69:   MPI_Comm       comm,*iscomms;
     70:   MPI_Request    *s_waits1,*r_waits1,*s_waits2,*r_waits2;
     71:   MPI_Status     *s_status,*recv_status;
     72:   char           *t_p;

     75:   PetscObjectGetComm((PetscObject)C,&comm);
     76:   size = c->size;
     77:   rank = c->rank;
     78:   Mbs  = c->Mbs;

     80:   PetscObjectGetNewTag((PetscObject)C,&tag1);
     81:   PetscObjectGetNewTag((PetscObject)C,&tag2);

     83:   PetscMalloc2(imax+1,(PetscInt***)&idx,imax,&n);

     85:   for (i=0; i<imax; i++) {
     86:     ISGetIndices(is[i],&idx[i]);
     87:     ISGetLocalSize(is[i],&n[i]);
     88:   }

     90:   /* evaluate communication - mesg to who,length of mesg, and buffer space
     91:      required. Based on this, buffers are allocated, and data copied into them*/
     92:   PetscCalloc4(size,&w1,size,&w2,size,&w3,size,&w4);
     93:   for (i=0; i<imax; i++) {
     94:     PetscMemzero(w4,size*sizeof(PetscInt)); /* initialise work vector*/
     95:     idx_i = idx[i];
     96:     len   = n[i];
     97:     for (j=0; j<len; j++) {
     98:       row = idx_i[j];
     99:       if (row < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Index set cannot have negative entries");
    100:       PetscLayoutFindOwner(C->rmap,row*C->rmap->bs,&proc);
    101:       w4[proc]++;
    102:     }
    103:     for (j=0; j<size; j++) {
    104:       if (w4[j]) { w1[j] += w4[j]; w3[j]++;}
    105:     }
    106:   }

    108:   nrqs     = 0;              /* no of outgoing messages */
    109:   msz      = 0;              /* total mesg length (for all proc */
    110:   w1[rank] = 0;              /* no mesg sent to itself */
    111:   w3[rank] = 0;
    112:   for (i=0; i<size; i++) {
    113:     if (w1[i])  {w2[i] = 1; nrqs++;} /* there exists a message to proc i */
    114:   }
    115:   /* pa - is list of processors to communicate with */
    116:   PetscMalloc1(nrqs+1,&pa);
    117:   for (i=0,j=0; i<size; i++) {
    118:     if (w1[i]) {pa[j] = i; j++;}
    119:   }

    121:   /* Each message would have a header = 1 + 2*(no of IS) + data */
    122:   for (i=0; i<nrqs; i++) {
    123:     j      = pa[i];
    124:     w1[j] += w2[j] + 2*w3[j];
    125:     msz   += w1[j];
    126:   }

    128:   /* Determine the number of messages to expect, their lengths, from from-ids */
    129:   PetscGatherNumberOfMessages(comm,w2,w1,&nrqr);
    130:   PetscGatherMessageLengths(comm,nrqs,nrqr,w1,&onodes1,&olengths1);

    132:   /* Now post the Irecvs corresponding to these messages */
    133:   PetscPostIrecvInt(comm,tag1,nrqr,onodes1,olengths1,&rbuf,&r_waits1);

    135:   /* Allocate Memory for outgoing messages */
    136:   PetscMalloc4(size,&outdat,size,&ptr,msz,&tmp,size,&ctr);
    137:   PetscMemzero(outdat,size*sizeof(PetscInt*));
    138:   PetscMemzero(ptr,size*sizeof(PetscInt*));
    139:   {
    140:     PetscInt *iptr = tmp,ict  = 0;
    141:     for (i=0; i<nrqs; i++) {
    142:       j         = pa[i];
    143:       iptr     +=  ict;
    144:       outdat[j] = iptr;
    145:       ict       = w1[j];
    146:     }
    147:   }

    149:   /* Form the outgoing messages */
    150:   /*plug in the headers*/
    151:   for (i=0; i<nrqs; i++) {
    152:     j            = pa[i];
    153:     outdat[j][0] = 0;
    154:     PetscMemzero(outdat[j]+1,2*w3[j]*sizeof(PetscInt));
    155:     ptr[j]       = outdat[j] + 2*w3[j] + 1;
    156:   }

    158:   /* Memory for doing local proc's work*/
    159:   {
    160:     PetscCalloc5(imax,&table, imax,&data, imax,&isz, Mbs*imax,&d_p, (Mbs/PETSC_BITS_PER_BYTE+1)*imax,&t_p);

    162:     for (i=0; i<imax; i++) {
    163:       table[i] = t_p + (Mbs/PETSC_BITS_PER_BYTE+1)*i;
    164:       data[i]  = d_p + (Mbs)*i;
    165:     }
    166:   }

    168:   /* Parse the IS and update local tables and the outgoing buf with the data*/
    169:   {
    170:     PetscInt n_i,*data_i,isz_i,*outdat_j,ctr_j;
    171:     PetscBT  table_i;

    173:     for (i=0; i<imax; i++) {
    174:       PetscMemzero(ctr,size*sizeof(PetscInt));
    175:       n_i     = n[i];
    176:       table_i = table[i];
    177:       idx_i   = idx[i];
    178:       data_i  = data[i];
    179:       isz_i   = isz[i];
    180:       for (j=0; j<n_i; j++) {   /* parse the indices of each IS */
    181:         row  = idx_i[j];
    182:         PetscLayoutFindOwner(C->rmap,row*C->rmap->bs,&proc);
    183:         if (proc != rank) { /* copy to the outgoing buffer */
    184:           ctr[proc]++;
    185:           *ptr[proc] = row;
    186:           ptr[proc]++;
    187:         } else { /* Update the local table */
    188:           if (!PetscBTLookupSet(table_i,row)) data_i[isz_i++] = row;
    189:         }
    190:       }
    191:       /* Update the headers for the current IS */
    192:       for (j=0; j<size; j++) { /* Can Optimise this loop by using pa[] */
    193:         if ((ctr_j = ctr[j])) {
    194:           outdat_j        = outdat[j];
    195:           k               = ++outdat_j[0];
    196:           outdat_j[2*k]   = ctr_j;
    197:           outdat_j[2*k-1] = i;
    198:         }
    199:       }
    200:       isz[i] = isz_i;
    201:     }
    202:   }

    204:   /*  Now  post the sends */
    205:   PetscMalloc1(nrqs+1,&s_waits1);
    206:   for (i=0; i<nrqs; ++i) {
    207:     j    = pa[i];
    208:     MPI_Isend(outdat[j],w1[j],MPIU_INT,j,tag1,comm,s_waits1+i);
    209:   }

    211:   /* No longer need the original indices*/
    212:   for (i=0; i<imax; ++i) {
    213:     ISRestoreIndices(is[i],idx+i);
    214:   }
    215:   PetscFree2(*(PetscInt***)&idx,n);

    217:   PetscMalloc1(imax,&iscomms);
    218:   for (i=0; i<imax; ++i) {
    219:     PetscCommDuplicate(PetscObjectComm((PetscObject)is[i]),&iscomms[i],NULL);
    220:     ISDestroy(&is[i]);
    221:   }

    223:   /* Do Local work*/
    224:   MatIncreaseOverlap_MPIBAIJ_Local(C,imax,table,isz,data);

    226:   /* Receive messages*/
    227:   PetscMalloc1(nrqr+1,&recv_status);
    228:   if (nrqr) {MPI_Waitall(nrqr,r_waits1,recv_status);}

    230:   PetscMalloc1(nrqs+1,&s_status);
    231:   if (nrqs) {MPI_Waitall(nrqs,s_waits1,s_status);}

    233:   /* Phase 1 sends are complete - deallocate buffers */
    234:   PetscFree4(outdat,ptr,tmp,ctr);
    235:   PetscFree4(w1,w2,w3,w4);

    237:   PetscMalloc1(nrqr+1,&xdata);
    238:   PetscMalloc1(nrqr+1,&isz1);
    239:   MatIncreaseOverlap_MPIBAIJ_Receive(C,nrqr,rbuf,xdata,isz1);
    240:   PetscFree(rbuf[0]);
    241:   PetscFree(rbuf);

    243:   /* Send the data back*/
    244:   /* Do a global reduction to know the buffer space req for incoming messages*/
    245:   {
    246:     PetscMPIInt *rw1;

    248:     PetscCalloc1(size,&rw1);

    250:     for (i=0; i<nrqr; ++i) {
    251:       proc = recv_status[i].MPI_SOURCE;
    252:       if (proc != onodes1[i]) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"MPI_SOURCE mismatch");
    253:       rw1[proc] = isz1[i];
    254:     }

    256:     PetscFree(onodes1);
    257:     PetscFree(olengths1);

    259:     /* Determine the number of messages to expect, their lengths, from from-ids */
    260:     PetscGatherMessageLengths(comm,nrqr,nrqs,rw1,&onodes2,&olengths2);
    261:     PetscFree(rw1);
    262:   }
    263:   /* Now post the Irecvs corresponding to these messages */
    264:   PetscPostIrecvInt(comm,tag2,nrqs,onodes2,olengths2,&rbuf2,&r_waits2);

    266:   /*  Now  post the sends */
    267:   PetscMalloc1(nrqr+1,&s_waits2);
    268:   for (i=0; i<nrqr; ++i) {
    269:     j    = recv_status[i].MPI_SOURCE;
    270:     MPI_Isend(xdata[i],isz1[i],MPIU_INT,j,tag2,comm,s_waits2+i);
    271:   }

    273:   /* receive work done on other processors*/
    274:   {
    275:     PetscMPIInt idex;
    276:     PetscInt    is_no,ct1,max,*rbuf2_i,isz_i,*data_i,jmax;
    277:     PetscBT     table_i;
    278:     MPI_Status  *status2;

    280:     PetscMalloc1(PetscMax(nrqr,nrqs)+1,&status2);
    281:     for (i=0; i<nrqs; ++i) {
    282:       MPI_Waitany(nrqs,r_waits2,&idex,status2+i);
    283:       /* Process the message*/
    284:       rbuf2_i = rbuf2[idex];
    285:       ct1     = 2*rbuf2_i[0]+1;
    286:       jmax    = rbuf2[idex][0];
    287:       for (j=1; j<=jmax; j++) {
    288:         max     = rbuf2_i[2*j];
    289:         is_no   = rbuf2_i[2*j-1];
    290:         isz_i   = isz[is_no];
    291:         data_i  = data[is_no];
    292:         table_i = table[is_no];
    293:         for (k=0; k<max; k++,ct1++) {
    294:           row = rbuf2_i[ct1];
    295:           if (!PetscBTLookupSet(table_i,row)) data_i[isz_i++] = row;
    296:         }
    297:         isz[is_no] = isz_i;
    298:       }
    299:     }
    300:     if (nrqr) {MPI_Waitall(nrqr,s_waits2,status2);}
    301:     PetscFree(status2);
    302:   }

    304:   for (i=0; i<imax; ++i) {
    305:     ISCreateGeneral(iscomms[i],isz[i],data[i],PETSC_COPY_VALUES,is+i);
    306:     PetscCommDestroy(&iscomms[i]);
    307:   }

    309:   PetscFree(iscomms);
    310:   PetscFree(onodes2);
    311:   PetscFree(olengths2);

    313:   PetscFree(pa);
    314:   PetscFree(rbuf2[0]);
    315:   PetscFree(rbuf2);
    316:   PetscFree(s_waits1);
    317:   PetscFree(r_waits1);
    318:   PetscFree(s_waits2);
    319:   PetscFree(r_waits2);
    320:   PetscFree5(table,data,isz,d_p,t_p);
    321:   PetscFree(s_status);
    322:   PetscFree(recv_status);
    323:   PetscFree(xdata[0]);
    324:   PetscFree(xdata);
    325:   PetscFree(isz1);
    326:   return(0);
    327: }

    329: /*
    330:    MatIncreaseOverlap_MPIBAIJ_Local - Called by MatincreaseOverlap, to do
    331:        the work on the local processor.

    333:      Inputs:
    334:       C      - MAT_MPIBAIJ;
    335:       imax - total no of index sets processed at a time;
    336:       table  - an array of char - size = Mbs bits.

    338:      Output:
    339:       isz    - array containing the count of the solution elements corresponding
    340:                to each index set;
    341:       data   - pointer to the solutions
    342: */
    343: static PetscErrorCode MatIncreaseOverlap_MPIBAIJ_Local(Mat C,PetscInt imax,PetscBT *table,PetscInt *isz,PetscInt **data)
    344: {
    345:   Mat_MPIBAIJ *c = (Mat_MPIBAIJ*)C->data;
    346:   Mat         A  = c->A,B = c->B;
    347:   Mat_SeqBAIJ *a = (Mat_SeqBAIJ*)A->data,*b = (Mat_SeqBAIJ*)B->data;
    348:   PetscInt    start,end,val,max,rstart,cstart,*ai,*aj;
    349:   PetscInt    *bi,*bj,*garray,i,j,k,row,*data_i,isz_i;
    350:   PetscBT     table_i;

    353:   rstart = c->rstartbs;
    354:   cstart = c->cstartbs;
    355:   ai     = a->i;
    356:   aj     = a->j;
    357:   bi     = b->i;
    358:   bj     = b->j;
    359:   garray = c->garray;

    362:   for (i=0; i<imax; i++) {
    363:     data_i  = data[i];
    364:     table_i = table[i];
    365:     isz_i   = isz[i];
    366:     for (j=0,max=isz[i]; j<max; j++) {
    367:       row   = data_i[j] - rstart;
    368:       start = ai[row];
    369:       end   = ai[row+1];
    370:       for (k=start; k<end; k++) { /* Amat */
    371:         val = aj[k] + cstart;
    372:         if (!PetscBTLookupSet(table_i,val)) data_i[isz_i++] = val;
    373:       }
    374:       start = bi[row];
    375:       end   = bi[row+1];
    376:       for (k=start; k<end; k++) { /* Bmat */
    377:         val = garray[bj[k]];
    378:         if (!PetscBTLookupSet(table_i,val)) data_i[isz_i++] = val;
    379:       }
    380:     }
    381:     isz[i] = isz_i;
    382:   }
    383:   return(0);
    384: }
    385: /*
    386:       MatIncreaseOverlap_MPIBAIJ_Receive - Process the recieved messages,
    387:          and return the output

    389:          Input:
    390:            C    - the matrix
    391:            nrqr - no of messages being processed.
    392:            rbuf - an array of pointers to the recieved requests

    394:          Output:
    395:            xdata - array of messages to be sent back
    396:            isz1  - size of each message

    398:   For better efficiency perhaps we should malloc separately each xdata[i],
    399: then if a remalloc is required we need only copy the data for that one row
    400: rather than all previous rows as it is now where a single large chunck of
    401: memory is used.

    403: */
    404: static PetscErrorCode MatIncreaseOverlap_MPIBAIJ_Receive(Mat C,PetscInt nrqr,PetscInt **rbuf,PetscInt **xdata,PetscInt * isz1)
    405: {
    406:   Mat_MPIBAIJ    *c = (Mat_MPIBAIJ*)C->data;
    407:   Mat            A  = c->A,B = c->B;
    408:   Mat_SeqBAIJ    *a = (Mat_SeqBAIJ*)A->data,*b = (Mat_SeqBAIJ*)B->data;
    410:   PetscInt       rstart,cstart,*ai,*aj,*bi,*bj,*garray,i,j,k;
    411:   PetscInt       row,total_sz,ct,ct1,ct2,ct3,mem_estimate,oct2,l,start,end;
    412:   PetscInt       val,max1,max2,Mbs,no_malloc =0,*tmp,new_estimate,ctr;
    413:   PetscInt       *rbuf_i,kmax,rbuf_0;
    414:   PetscBT        xtable;

    417:   Mbs    = c->Mbs;
    418:   rstart = c->rstartbs;
    419:   cstart = c->cstartbs;
    420:   ai     = a->i;
    421:   aj     = a->j;
    422:   bi     = b->i;
    423:   bj     = b->j;
    424:   garray = c->garray;

    427:   for (i=0,ct=0,total_sz=0; i<nrqr; ++i) {
    428:     rbuf_i =  rbuf[i];
    429:     rbuf_0 =  rbuf_i[0];
    430:     ct    += rbuf_0;
    431:     for (j=1; j<=rbuf_0; j++) total_sz += rbuf_i[2*j];
    432:   }

    434:   if (c->Mbs) max1 = ct*(a->nz +b->nz)/c->Mbs;
    435:   else        max1 = 1;
    436:   mem_estimate = 3*((total_sz > max1 ? total_sz : max1)+1);
    437:   PetscMalloc1(mem_estimate,&xdata[0]);
    438:   ++no_malloc;
    439:   PetscBTCreate(Mbs,&xtable);
    440:   PetscMemzero(isz1,nrqr*sizeof(PetscInt));

    442:   ct3 = 0;
    443:   for (i=0; i<nrqr; i++) { /* for easch mesg from proc i */
    444:     rbuf_i =  rbuf[i];
    445:     rbuf_0 =  rbuf_i[0];
    446:     ct1    =  2*rbuf_0+1;
    447:     ct2    =  ct1;
    448:     ct3   += ct1;
    449:     for (j=1; j<=rbuf_0; j++) { /* for each IS from proc i*/
    450:       PetscBTMemzero(Mbs,xtable);
    451:       oct2 = ct2;
    452:       kmax = rbuf_i[2*j];
    453:       for (k=0; k<kmax; k++,ct1++) {
    454:         row = rbuf_i[ct1];
    455:         if (!PetscBTLookupSet(xtable,row)) {
    456:           if (!(ct3 < mem_estimate)) {
    457:             new_estimate = (PetscInt)(1.5*mem_estimate)+1;
    458:             PetscMalloc1(new_estimate,&tmp);
    459:             PetscMemcpy(tmp,xdata[0],mem_estimate*sizeof(PetscInt));
    460:             PetscFree(xdata[0]);
    461:             xdata[0]     = tmp;
    462:             mem_estimate = new_estimate; ++no_malloc;
    463:             for (ctr=1; ctr<=i; ctr++) xdata[ctr] = xdata[ctr-1] + isz1[ctr-1];
    464:           }
    465:           xdata[i][ct2++] = row;
    466:           ct3++;
    467:         }
    468:       }
    469:       for (k=oct2,max2=ct2; k<max2; k++)  {
    470:         row   = xdata[i][k] - rstart;
    471:         start = ai[row];
    472:         end   = ai[row+1];
    473:         for (l=start; l<end; l++) {
    474:           val = aj[l] + cstart;
    475:           if (!PetscBTLookupSet(xtable,val)) {
    476:             if (!(ct3 < mem_estimate)) {
    477:               new_estimate = (PetscInt)(1.5*mem_estimate)+1;
    478:               PetscMalloc1(new_estimate,&tmp);
    479:               PetscMemcpy(tmp,xdata[0],mem_estimate*sizeof(PetscInt));
    480:               PetscFree(xdata[0]);
    481:               xdata[0]     = tmp;
    482:               mem_estimate = new_estimate; ++no_malloc;
    483:               for (ctr=1; ctr<=i; ctr++) xdata[ctr] = xdata[ctr-1] + isz1[ctr-1];
    484:             }
    485:             xdata[i][ct2++] = val;
    486:             ct3++;
    487:           }
    488:         }
    489:         start = bi[row];
    490:         end   = bi[row+1];
    491:         for (l=start; l<end; l++) {
    492:           val = garray[bj[l]];
    493:           if (!PetscBTLookupSet(xtable,val)) {
    494:             if (!(ct3 < mem_estimate)) {
    495:               new_estimate = (PetscInt)(1.5*mem_estimate)+1;
    496:               PetscMalloc1(new_estimate,&tmp);
    497:               PetscMemcpy(tmp,xdata[0],mem_estimate*sizeof(PetscInt));
    498:               PetscFree(xdata[0]);
    499:               xdata[0]     = tmp;
    500:               mem_estimate = new_estimate; ++no_malloc;
    501:               for (ctr =1; ctr <=i; ctr++) xdata[ctr] = xdata[ctr-1] + isz1[ctr-1];
    502:             }
    503:             xdata[i][ct2++] = val;
    504:             ct3++;
    505:           }
    506:         }
    507:       }
    508:       /* Update the header*/
    509:       xdata[i][2*j]   = ct2 - oct2; /* Undo the vector isz1 and use only a var*/
    510:       xdata[i][2*j-1] = rbuf_i[2*j-1];
    511:     }
    512:     xdata[i][0] = rbuf_0;
    513:     xdata[i+1]  = xdata[i] + ct2;
    514:     isz1[i]     = ct2; /* size of each message */
    515:   }
    516:   PetscBTDestroy(&xtable);
    517:   PetscInfo3(C,"Allocated %D bytes, required %D, no of mallocs = %D\n",mem_estimate,ct3,no_malloc);
    518:   return(0);
    519: }

    521: PetscErrorCode MatCreateSubMatrices_MPIBAIJ(Mat C,PetscInt ismax,const IS isrow[],const IS iscol[],MatReuse scall,Mat *submat[])
    522: {
    523:   IS             *isrow_block,*iscol_block;
    524:   Mat_MPIBAIJ    *c = (Mat_MPIBAIJ*)C->data;
    526:   PetscInt       nmax,nstages_local,nstages,i,pos,max_no,N=C->cmap->N,bs=C->rmap->bs;
    527:   Mat_SeqBAIJ    *subc;
    528:   Mat_SubSppt    *smat;

    531:   /* The compression and expansion should be avoided. Doesn't point
    532:      out errors, might change the indices, hence buggey */
    533:   PetscMalloc2(ismax+1,&isrow_block,ismax+1,&iscol_block);
    534:   ISCompressIndicesGeneral(N,C->rmap->n,bs,ismax,isrow,isrow_block);
    535:   ISCompressIndicesGeneral(N,C->cmap->n,bs,ismax,iscol,iscol_block);

    537:   /* Determine the number of stages through which submatrices are done */
    538:   if (!C->cmap->N) nmax=20*1000000/sizeof(PetscInt);
    539:   else nmax = 20*1000000 / (c->Nbs * sizeof(PetscInt));
    540:   if (!nmax) nmax = 1;

    542:   if (scall == MAT_INITIAL_MATRIX) {
    543:     nstages_local = ismax/nmax + ((ismax % nmax) ? 1 : 0); /* local nstages */

    545:     /* Make sure every processor loops through the nstages */
    546:     MPIU_Allreduce(&nstages_local,&nstages,1,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)C));

    548:     /* Allocate memory to hold all the submatrices and dummy submatrices */
    549:     PetscCalloc1(ismax+nstages,submat);
    550:   } else { /* MAT_REUSE_MATRIX */
    551:     if (ismax) {
    552:       subc = (Mat_SeqBAIJ*)((*submat)[0]->data);
    553:       smat   = subc->submatis1;
    554:     } else { /* (*submat)[0] is a dummy matrix */
    555:       smat = (Mat_SubSppt*)(*submat)[0]->data;
    556:     }
    557:     if (!smat) {
    558:       SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NULL,"MatCreateSubMatrices(...,MAT_REUSE_MATRIX,...) requires submat");
    559:     }
    560:     nstages = smat->nstages;
    561:   }

    563:   for (i=0,pos=0; i<nstages; i++) {
    564:     if (pos+nmax <= ismax) max_no = nmax;
    565:     else if (pos == ismax) max_no = 0;
    566:     else                   max_no = ismax-pos;

    568:     MatCreateSubMatrices_MPIBAIJ_local(C,max_no,isrow_block+pos,iscol_block+pos,scall,*submat+pos);
    569:     if (!max_no && scall == MAT_INITIAL_MATRIX) { /* submat[pos] is a dummy matrix */
    570:       smat = (Mat_SubSppt*)(*submat)[pos]->data;
    571:       smat->nstages = nstages;
    572:     }
    573:     pos += max_no;
    574:   }

    576:   if (scall == MAT_INITIAL_MATRIX && ismax) {
    577:     /* save nstages for reuse */
    578:     subc = (Mat_SeqBAIJ*)((*submat)[0]->data);
    579:     smat = subc->submatis1;
    580:     smat->nstages = nstages;
    581:   }

    583:   for (i=0; i<ismax; i++) {
    584:     ISDestroy(&isrow_block[i]);
    585:     ISDestroy(&iscol_block[i]);
    586:   }
    587:   PetscFree2(isrow_block,iscol_block);
    588:   return(0);
    589: }

    591: #if defined(PETSC_USE_CTABLE)
    592: PetscErrorCode PetscGetProc(const PetscInt row, const PetscMPIInt size, const PetscInt proc_gnode[], PetscMPIInt *rank)
    593: {
    594:   PetscInt       nGlobalNd = proc_gnode[size];
    595:   PetscMPIInt    fproc;

    599:   PetscMPIIntCast((PetscInt)(((float)row * (float)size / (float)nGlobalNd + 0.5)),&fproc);
    600:   if (fproc > size) fproc = size;
    601:   while (row < proc_gnode[fproc] || row >= proc_gnode[fproc+1]) {
    602:     if (row < proc_gnode[fproc]) fproc--;
    603:     else                         fproc++;
    604:   }
    605:   *rank = fproc;
    606:   return(0);
    607: }
    608: #endif

    610: /* -------------------------------------------------------------------------*/
    611: /* This code is used for BAIJ and SBAIJ matrices (unfortunate dependency) */
    612: PetscErrorCode MatCreateSubMatrices_MPIBAIJ_local(Mat C,PetscInt ismax,const IS isrow[],const IS iscol[],MatReuse scall,Mat *submats)
    613: {
    614:   Mat_MPIBAIJ    *c = (Mat_MPIBAIJ*)C->data;
    615:   Mat            A  = c->A;
    616:   Mat_SeqBAIJ    *a = (Mat_SeqBAIJ*)A->data,*b = (Mat_SeqBAIJ*)c->B->data,*subc;
    617:   const PetscInt **icol,**irow;
    618:   PetscInt       *nrow,*ncol,start;
    620:   PetscMPIInt    rank,size,tag0,tag2,tag3,tag4,*w1,*w2,*w3,*w4,nrqr;
    621:   PetscInt       **sbuf1,**sbuf2,*sbuf2_i,i,j,k,l,ct1,ct2,**rbuf1,row,proc=-1;
    622:   PetscInt       nrqs=0,msz,**ptr=NULL,*req_size=NULL,*ctr=NULL,*pa,*tmp=NULL,tcol;
    623:   PetscInt       **rbuf3=NULL,*req_source1=NULL,*req_source2,**sbuf_aj,**rbuf2=NULL,max1,max2;
    624:   PetscInt       **lens,is_no,ncols,*cols,mat_i,*mat_j,tmp2,jmax;
    625: #if defined(PETSC_USE_CTABLE)
    626:   PetscTable     *cmap,cmap_i=NULL,*rmap,rmap_i;
    627: #else
    628:   PetscInt       **cmap,*cmap_i=NULL,**rmap,*rmap_i;
    629: #endif
    630:   const PetscInt *irow_i,*icol_i;
    631:   PetscInt       ctr_j,*sbuf1_j,*sbuf_aj_i,*rbuf1_i,kmax,*lens_i;
    632:   MPI_Request    *s_waits1,*r_waits1,*s_waits2,*r_waits2,*r_waits3;
    633:   MPI_Request    *r_waits4,*s_waits3,*s_waits4;
    634:   MPI_Status     *r_status1,*r_status2,*s_status1,*s_status3,*s_status2;
    635:   MPI_Status     *r_status3,*r_status4,*s_status4;
    636:   MPI_Comm       comm;
    637:   PetscScalar    **rbuf4,*rbuf4_i=NULL,**sbuf_aa,*vals,*mat_a=NULL,*imat_a,*sbuf_aa_i;
    638:   PetscMPIInt    *onodes1,*olengths1,end;
    639:   PetscInt       **row2proc,*row2proc_i,*imat_ilen,*imat_j,*imat_i;
    640:   Mat_SubSppt    *smat_i;
    641:   PetscBool      *issorted,colflag,iscsorted=PETSC_TRUE;
    642:   PetscInt       *sbuf1_i,*rbuf2_i,*rbuf3_i,ilen;
    643:   PetscInt       bs=C->rmap->bs,bs2=c->bs2,rstart = c->rstartbs;
    644:   PetscBool      ijonly=c->ijonly; /* private flag indicates only matrix data structures are requested */
    645:   PetscInt       nzA,nzB,*a_i=a->i,*b_i=b->i,*a_j = a->j,*b_j = b->j,ctmp,imark,*cworkA,*cworkB;
    646:   PetscScalar    *vworkA=NULL,*vworkB=NULL,*a_a = a->a,*b_a = b->a;
    647:   PetscInt       cstart = c->cstartbs,*bmap = c->garray;
    648:   PetscBool      *allrows,*allcolumns;

    651:   PetscObjectGetComm((PetscObject)C,&comm);
    652:   size = c->size;
    653:   rank = c->rank;

    655:   PetscMalloc5(ismax,&row2proc,ismax,&cmap,ismax,&rmap,ismax+1,&allcolumns,ismax,&allrows);
    656:   PetscMalloc5(ismax,(PetscInt***)&irow,ismax,(PetscInt***)&icol,ismax,&nrow,ismax,&ncol,ismax,&issorted);

    658:   for (i=0; i<ismax; i++) {
    659:     ISSorted(iscol[i],&issorted[i]);
    660:     if (!issorted[i]) iscsorted = issorted[i]; /* columns are not sorted! */
    661:     ISSorted(isrow[i],&issorted[i]);

    663:     /* Check for special case: allcolumns */
    664:     ISIdentity(iscol[i],&colflag);
    665:     ISGetLocalSize(iscol[i],&ncol[i]);

    667:     if (colflag && ncol[i] == c->Nbs) {
    668:       allcolumns[i] = PETSC_TRUE;
    669:       icol[i]       = NULL;
    670:     } else {
    671:       allcolumns[i] = PETSC_FALSE;
    672:       ISGetIndices(iscol[i],&icol[i]);
    673:     }

    675:     /* Check for special case: allrows */
    676:     ISIdentity(isrow[i],&colflag);
    677:     ISGetLocalSize(isrow[i],&nrow[i]);
    678:     if (colflag && nrow[i] == c->Mbs) {
    679:       allrows[i] = PETSC_TRUE;
    680:       irow[i]    = NULL;
    681:     } else {
    682:       allrows[i] = PETSC_FALSE;
    683:       ISGetIndices(isrow[i],&irow[i]);
    684:     }
    685:   }

    687:   if (scall == MAT_REUSE_MATRIX) {
    688:     /* Assumes new rows are same length as the old rows */
    689:     for (i=0; i<ismax; i++) {
    690:       subc = (Mat_SeqBAIJ*)(submats[i]->data);
    691:       if (subc->mbs != nrow[i] || subc->nbs != ncol[i]) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_SIZ,"Cannot reuse matrix. wrong size");

    693:       /* Initial matrix as if empty */
    694:       PetscMemzero(subc->ilen,subc->mbs*sizeof(PetscInt));

    696:       /* Initial matrix as if empty */
    697:       submats[i]->factortype = C->factortype;

    699:       smat_i   = subc->submatis1;

    701:       nrqs        = smat_i->nrqs;
    702:       nrqr        = smat_i->nrqr;
    703:       rbuf1       = smat_i->rbuf1;
    704:       rbuf2       = smat_i->rbuf2;
    705:       rbuf3       = smat_i->rbuf3;
    706:       req_source2 = smat_i->req_source2;

    708:       sbuf1     = smat_i->sbuf1;
    709:       sbuf2     = smat_i->sbuf2;
    710:       ptr       = smat_i->ptr;
    711:       tmp       = smat_i->tmp;
    712:       ctr       = smat_i->ctr;

    714:       pa          = smat_i->pa;
    715:       req_size    = smat_i->req_size;
    716:       req_source1 = smat_i->req_source1;

    718:       allcolumns[i] = smat_i->allcolumns;
    719:       allrows[i]    = smat_i->allrows;
    720:       row2proc[i]   = smat_i->row2proc;
    721:       rmap[i]       = smat_i->rmap;
    722:       cmap[i]       = smat_i->cmap;
    723:     }

    725:     if (!ismax){ /* Get dummy submatrices and retrieve struct submatis1 */
    726:       if (!submats[0]) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_NULL,"submats are null, cannot reuse");
    727:       smat_i = (Mat_SubSppt*)submats[0]->data;

    729:       nrqs        = smat_i->nrqs;
    730:       nrqr        = smat_i->nrqr;
    731:       rbuf1       = smat_i->rbuf1;
    732:       rbuf2       = smat_i->rbuf2;
    733:       rbuf3       = smat_i->rbuf3;
    734:       req_source2 = smat_i->req_source2;

    736:       sbuf1       = smat_i->sbuf1;
    737:       sbuf2       = smat_i->sbuf2;
    738:       ptr         = smat_i->ptr;
    739:       tmp         = smat_i->tmp;
    740:       ctr         = smat_i->ctr;

    742:       pa          = smat_i->pa;
    743:       req_size    = smat_i->req_size;
    744:       req_source1 = smat_i->req_source1;

    746:       allcolumns[0] = PETSC_FALSE;
    747:     }
    748:   } else { /* scall == MAT_INITIAL_MATRIX */
    749:     /* Get some new tags to keep the communication clean */
    750:     PetscObjectGetNewTag((PetscObject)C,&tag2);
    751:     PetscObjectGetNewTag((PetscObject)C,&tag3);

    753:     /* evaluate communication - mesg to who, length of mesg, and buffer space
    754:      required. Based on this, buffers are allocated, and data copied into them*/
    755:     PetscCalloc4(size,&w1,size,&w2,size,&w3,size,&w4);   /* mesg size, initialize work vectors */

    757:     for (i=0; i<ismax; i++) {
    758:       jmax   = nrow[i];
    759:       irow_i = irow[i];

    761:       PetscMalloc1(jmax,&row2proc_i);
    762:       row2proc[i] = row2proc_i;

    764:       if (issorted[i]) proc = 0;
    765:       for (j=0; j<jmax; j++) {
    766:         if (!issorted[i]) proc = 0;
    767:         if (allrows[i]) row = j;
    768:         else row = irow_i[j];

    770:         while (row >= c->rangebs[proc+1]) proc++;
    771:         w4[proc]++;
    772:         row2proc_i[j] = proc; /* map row index to proc */
    773:       }
    774:       for (j=0; j<size; j++) {
    775:         if (w4[j]) { w1[j] += w4[j];  w3[j]++; w4[j] = 0;}
    776:       }
    777:     }

    779:     nrqs     = 0;              /* no of outgoing messages */
    780:     msz      = 0;              /* total mesg length (for all procs) */
    781:     w1[rank] = 0;              /* no mesg sent to self */
    782:     w3[rank] = 0;
    783:     for (i=0; i<size; i++) {
    784:       if (w1[i])  { w2[i] = 1; nrqs++;} /* there exists a message to proc i */
    785:     }
    786:     PetscMalloc1(nrqs+1,&pa); /*(proc -array)*/
    787:     for (i=0,j=0; i<size; i++) {
    788:       if (w1[i]) { pa[j] = i; j++; }
    789:     }

    791:     /* Each message would have a header = 1 + 2*(no of IS) + data */
    792:     for (i=0; i<nrqs; i++) {
    793:       j      = pa[i];
    794:       w1[j] += w2[j] + 2* w3[j];
    795:       msz   += w1[j];
    796:     }
    797:     PetscInfo2(0,"Number of outgoing messages %D Total message length %D\n",nrqs,msz);

    799:     /* Determine the number of messages to expect, their lengths, from from-ids */
    800:     PetscGatherNumberOfMessages(comm,w2,w1,&nrqr);
    801:     PetscGatherMessageLengths(comm,nrqs,nrqr,w1,&onodes1,&olengths1);

    803:     /* Now post the Irecvs corresponding to these messages */
    804:     tag0 = ((PetscObject)C)->tag;
    805:     PetscPostIrecvInt(comm,tag0,nrqr,onodes1,olengths1,&rbuf1,&r_waits1);

    807:     PetscFree(onodes1);
    808:     PetscFree(olengths1);

    810:     /* Allocate Memory for outgoing messages */
    811:     PetscMalloc4(size,&sbuf1,size,&ptr,2*msz,&tmp,size,&ctr);
    812:     PetscMemzero(sbuf1,size*sizeof(PetscInt*));
    813:     PetscMemzero(ptr,size*sizeof(PetscInt*));

    815:     {
    816:       PetscInt *iptr = tmp;
    817:       k    = 0;
    818:       for (i=0; i<nrqs; i++) {
    819:         j        = pa[i];
    820:         iptr    += k;
    821:         sbuf1[j] = iptr;
    822:         k        = w1[j];
    823:       }
    824:     }

    826:     /* Form the outgoing messages. Initialize the header space */
    827:     for (i=0; i<nrqs; i++) {
    828:       j           = pa[i];
    829:       sbuf1[j][0] = 0;
    830:       PetscMemzero(sbuf1[j]+1,2*w3[j]*sizeof(PetscInt));
    831:       ptr[j]      = sbuf1[j] + 2*w3[j] + 1;
    832:     }

    834:     /* Parse the isrow and copy data into outbuf */
    835:     for (i=0; i<ismax; i++) {
    836:       row2proc_i = row2proc[i];
    837:       PetscMemzero(ctr,size*sizeof(PetscInt));
    838:       irow_i = irow[i];
    839:       jmax   = nrow[i];
    840:       for (j=0; j<jmax; j++) {  /* parse the indices of each IS */
    841:         proc = row2proc_i[j];
    842:         if (allrows[i]) row = j;
    843:         else row = irow_i[j];

    845:         if (proc != rank) { /* copy to the outgoing buf*/
    846:           ctr[proc]++;
    847:           *ptr[proc] = row;
    848:           ptr[proc]++;
    849:         }
    850:       }
    851:       /* Update the headers for the current IS */
    852:       for (j=0; j<size; j++) { /* Can Optimise this loop too */
    853:         if ((ctr_j = ctr[j])) {
    854:           sbuf1_j        = sbuf1[j];
    855:           k              = ++sbuf1_j[0];
    856:           sbuf1_j[2*k]   = ctr_j;
    857:           sbuf1_j[2*k-1] = i;
    858:         }
    859:       }
    860:     }

    862:     /*  Now  post the sends */
    863:     PetscMalloc1(nrqs+1,&s_waits1);
    864:     for (i=0; i<nrqs; ++i) {
    865:       j    = pa[i];
    866:       MPI_Isend(sbuf1[j],w1[j],MPIU_INT,j,tag0,comm,s_waits1+i);
    867:     }

    869:     /* Post Receives to capture the buffer size */
    870:     PetscMalloc1(nrqs+1,&r_waits2);
    871:     PetscMalloc3(nrqs+1,&req_source2,nrqs+1,&rbuf2,nrqs+1,&rbuf3);
    872:     rbuf2[0] = tmp + msz;
    873:     for (i=1; i<nrqs; ++i) {
    874:       rbuf2[i] = rbuf2[i-1]+w1[pa[i-1]];
    875:     }
    876:     for (i=0; i<nrqs; ++i) {
    877:       j    = pa[i];
    878:       MPI_Irecv(rbuf2[i],w1[j],MPIU_INT,j,tag2,comm,r_waits2+i);
    879:     }

    881:     /* Send to other procs the buf size they should allocate */
    882:     /* Receive messages*/
    883:     PetscMalloc1(nrqr+1,&s_waits2);
    884:     PetscMalloc1(nrqr+1,&r_status1);
    885:     PetscMalloc3(nrqr,&sbuf2,nrqr,&req_size,nrqr,&req_source1);

    887:     MPI_Waitall(nrqr,r_waits1,r_status1);
    888:     for (i=0; i<nrqr; ++i) {
    889:       req_size[i] = 0;
    890:       rbuf1_i        = rbuf1[i];
    891:       start          = 2*rbuf1_i[0] + 1;
    892:       MPI_Get_count(r_status1+i,MPIU_INT,&end);
    893:       PetscMalloc1(end+1,&sbuf2[i]);
    894:       sbuf2_i        = sbuf2[i];
    895:       for (j=start; j<end; j++) {
    896:         row             = rbuf1_i[j] - rstart;
    897:         ncols           = a_i[row+1] - a_i[row] + b_i[row+1] - b_i[row];
    898:         sbuf2_i[j]      = ncols;
    899:         req_size[i] += ncols;
    900:       }
    901:       req_source1[i] = r_status1[i].MPI_SOURCE;
    902:       /* form the header */
    903:       sbuf2_i[0] = req_size[i];
    904:       for (j=1; j<start; j++) sbuf2_i[j] = rbuf1_i[j];

    906:       MPI_Isend(sbuf2_i,end,MPIU_INT,req_source1[i],tag2,comm,s_waits2+i);
    907:     }

    909:     PetscFree(r_status1);
    910:     PetscFree(r_waits1);
    911:     PetscFree4(w1,w2,w3,w4);

    913:     /* Receive messages*/
    914:     PetscMalloc1(nrqs+1,&r_waits3);
    915:     PetscMalloc1(nrqs+1,&r_status2);

    917:     MPI_Waitall(nrqs,r_waits2,r_status2);
    918:     for (i=0; i<nrqs; ++i) {
    919:       PetscMalloc1(rbuf2[i][0]+1,&rbuf3[i]);
    920:       req_source2[i] = r_status2[i].MPI_SOURCE;
    921:       MPI_Irecv(rbuf3[i],rbuf2[i][0],MPIU_INT,req_source2[i],tag3,comm,r_waits3+i);
    922:     }
    923:     PetscFree(r_status2);
    924:     PetscFree(r_waits2);

    926:     /* Wait on sends1 and sends2 */
    927:     PetscMalloc1(nrqs+1,&s_status1);
    928:     PetscMalloc1(nrqr+1,&s_status2);

    930:     if (nrqs) {MPI_Waitall(nrqs,s_waits1,s_status1);}
    931:     if (nrqr) {MPI_Waitall(nrqr,s_waits2,s_status2);}
    932:     PetscFree(s_status1);
    933:     PetscFree(s_status2);
    934:     PetscFree(s_waits1);
    935:     PetscFree(s_waits2);

    937:     /* Now allocate sending buffers for a->j, and send them off */
    938:     PetscMalloc1(nrqr+1,&sbuf_aj);
    939:     for (i=0,j=0; i<nrqr; i++) j += req_size[i];
    940:     PetscMalloc1(j+1,&sbuf_aj[0]);
    941:     for (i=1; i<nrqr; i++) sbuf_aj[i] = sbuf_aj[i-1] + req_size[i-1];

    943:     PetscMalloc1(nrqr+1,&s_waits3);
    944:     {

    946:       for (i=0; i<nrqr; i++) {
    947:         rbuf1_i   = rbuf1[i];
    948:         sbuf_aj_i = sbuf_aj[i];
    949:         ct1       = 2*rbuf1_i[0] + 1;
    950:         ct2       = 0;
    951:         for (j=1,max1=rbuf1_i[0]; j<=max1; j++) {
    952:           kmax = rbuf1[i][2*j];
    953:           for (k=0; k<kmax; k++,ct1++) {
    954:             row    = rbuf1_i[ct1] - rstart;
    955:             nzA    = a_i[row+1] - a_i[row]; nzB = b_i[row+1] - b_i[row];
    956:             ncols  = nzA + nzB;
    957:             cworkA = a_j + a_i[row]; cworkB = b_j + b_i[row];

    959:             /* load the column indices for this row into cols */
    960:             cols = sbuf_aj_i + ct2;
    961:             for (l=0; l<nzB; l++) {
    962:               if ((ctmp = bmap[cworkB[l]]) < cstart) cols[l] = ctmp;
    963:               else break;
    964:             }
    965:             imark = l;
    966:             for (l=0; l<nzA; l++) {cols[imark+l] = cstart + cworkA[l];}
    967:             for (l=imark; l<nzB; l++) cols[nzA+l] = bmap[cworkB[l]];
    968:             ct2 += ncols;
    969:           }
    970:         }
    971:         MPI_Isend(sbuf_aj_i,req_size[i],MPIU_INT,req_source1[i],tag3,comm,s_waits3+i);
    972:       }
    973:     }
    974:     PetscMalloc2(nrqs+1,&r_status3,nrqr+1,&s_status3);

    976:     /* create col map: global col of C -> local col of submatrices */
    977: #if defined(PETSC_USE_CTABLE)
    978:     for (i=0; i<ismax; i++) {
    979:       if (!allcolumns[i]) {
    980:         PetscTableCreate(ncol[i]+1,c->Nbs+1,&cmap[i]);

    982:         jmax   = ncol[i];
    983:         icol_i = icol[i];
    984:         cmap_i = cmap[i];
    985:         for (j=0; j<jmax; j++) {
    986:           PetscTableAdd(cmap[i],icol_i[j]+1,j+1,INSERT_VALUES);
    987:         }
    988:       } else cmap[i] = NULL;
    989:     }
    990: #else
    991:     for (i=0; i<ismax; i++) {
    992:       if (!allcolumns[i]) {
    993:         PetscCalloc1(c->Nbs,&cmap[i]);
    994:         jmax   = ncol[i];
    995:         icol_i = icol[i];
    996:         cmap_i = cmap[i];
    997:         for (j=0; j<jmax; j++) cmap_i[icol_i[j]] = j+1;
    998:       } else cmap[i] = NULL;
    999:     }
    1000: #endif

    1002:     /* Create lens which is required for MatCreate... */
    1003:     for (i=0,j=0; i<ismax; i++) j += nrow[i];
    1004:     PetscMalloc1(ismax,&lens);

    1006:     if (ismax) {
    1007:       PetscCalloc1(j,&lens[0]);
    1008:     }
    1009:     for (i=1; i<ismax; i++) lens[i] = lens[i-1] + nrow[i-1];

    1011:     /* Update lens from local data */
    1012:     for (i=0; i<ismax; i++) {
    1013:       row2proc_i = row2proc[i];
    1014:       jmax = nrow[i];
    1015:       if (!allcolumns[i]) cmap_i = cmap[i];
    1016:       irow_i = irow[i];
    1017:       lens_i = lens[i];
    1018:       for (j=0; j<jmax; j++) {
    1019:         if (allrows[i]) row = j;
    1020:         else row = irow_i[j]; /* global blocked row of C */

    1022:         proc = row2proc_i[j];
    1023:         if (proc == rank) {
    1024:           /* Get indices from matA and then from matB */
    1025: #if defined(PETSC_USE_CTABLE)
    1026:           PetscInt   tt;
    1027: #endif
    1028:           row    = row - rstart;
    1029:           nzA    = a_i[row+1] - a_i[row];
    1030:           nzB    = b_i[row+1] - b_i[row];
    1031:           cworkA =  a_j + a_i[row];
    1032:           cworkB = b_j + b_i[row];

    1034:           if (!allcolumns[i]) {
    1035: #if defined(PETSC_USE_CTABLE)
    1036:             for (k=0; k<nzA; k++) {
    1037:               PetscTableFind(cmap_i,cstart+cworkA[k]+1,&tt);
    1038:               if (tt) lens_i[j]++;
    1039:             }
    1040:             for (k=0; k<nzB; k++) {
    1041:               PetscTableFind(cmap_i,bmap[cworkB[k]]+1,&tt);
    1042:               if (tt) lens_i[j]++;
    1043:             }

    1045: #else
    1046:             for (k=0; k<nzA; k++) {
    1047:               if (cmap_i[cstart + cworkA[k]]) lens_i[j]++;
    1048:             }
    1049:             for (k=0; k<nzB; k++) {
    1050:               if (cmap_i[bmap[cworkB[k]]]) lens_i[j]++;
    1051:             }
    1052: #endif
    1053:           } else { /* allcolumns */
    1054:             lens_i[j] = nzA + nzB;
    1055:           }
    1056:         }
    1057:       }
    1058:     }

    1060:     /* Create row map: global row of C -> local row of submatrices */
    1061:     for (i=0; i<ismax; i++) {
    1062:       if (!allrows[i]) {
    1063: #if defined(PETSC_USE_CTABLE)
    1064:         PetscTableCreate(nrow[i]+1,c->Mbs+1,&rmap[i]);
    1065:         irow_i = irow[i];
    1066:         jmax   = nrow[i];
    1067:         for (j=0; j<jmax; j++) {
    1068:           if (allrows[i]) {
    1069:             PetscTableAdd(rmap[i],j+1,j+1,INSERT_VALUES);
    1070:           } else {
    1071:             PetscTableAdd(rmap[i],irow_i[j]+1,j+1,INSERT_VALUES);
    1072:           }
    1073:         }
    1074: #else
    1075:         PetscCalloc1(c->Mbs,&rmap[i]);
    1076:         rmap_i = rmap[i];
    1077:         irow_i = irow[i];
    1078:         jmax   = nrow[i];
    1079:         for (j=0; j<jmax; j++) {
    1080:           if (allrows[i]) rmap_i[j] = j;
    1081:           else rmap_i[irow_i[j]] = j;
    1082:         }
    1083: #endif
    1084:       } else rmap[i] = NULL;
    1085:     }

    1087:     /* Update lens from offproc data */
    1088:     {
    1089:       PetscInt *rbuf2_i,*rbuf3_i,*sbuf1_i;

    1091:       MPI_Waitall(nrqs,r_waits3,r_status3);
    1092:       for (tmp2=0; tmp2<nrqs; tmp2++) {
    1093:         sbuf1_i = sbuf1[pa[tmp2]];
    1094:         jmax    = sbuf1_i[0];
    1095:         ct1     = 2*jmax+1;
    1096:         ct2     = 0;
    1097:         rbuf2_i = rbuf2[tmp2];
    1098:         rbuf3_i = rbuf3[tmp2];
    1099:         for (j=1; j<=jmax; j++) {
    1100:           is_no  = sbuf1_i[2*j-1];
    1101:           max1   = sbuf1_i[2*j];
    1102:           lens_i = lens[is_no];
    1103:           if (!allcolumns[is_no]) cmap_i = cmap[is_no];
    1104:           rmap_i = rmap[is_no];
    1105:           for (k=0; k<max1; k++,ct1++) {
    1106:             if (allrows[is_no]) {
    1107:               row = sbuf1_i[ct1];
    1108:             } else {
    1109: #if defined(PETSC_USE_CTABLE)
    1110:               PetscTableFind(rmap_i,sbuf1_i[ct1]+1,&row);
    1111:               row--;
    1112:               if (row < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"row not found in table");
    1113: #else
    1114:               row = rmap_i[sbuf1_i[ct1]]; /* the val in the new matrix to be */
    1115: #endif
    1116:             }
    1117:             max2 = rbuf2_i[ct1];
    1118:             for (l=0; l<max2; l++,ct2++) {
    1119:               if (!allcolumns[is_no]) {
    1120: #if defined(PETSC_USE_CTABLE)
    1121:                 PetscTableFind(cmap_i,rbuf3_i[ct2]+1,&tcol);
    1122: #else
    1123:                 tcol = cmap_i[rbuf3_i[ct2]];
    1124: #endif
    1125:                 if (tcol) lens_i[row]++;
    1126:               } else { /* allcolumns */
    1127:                 lens_i[row]++; /* lens_i[row] += max2 ? */
    1128:               }
    1129:             }
    1130:           }
    1131:         }
    1132:       }
    1133:     }
    1134:     PetscFree(r_waits3);
    1135:     if (nrqr) {MPI_Waitall(nrqr,s_waits3,s_status3);}
    1136:     PetscFree2(r_status3,s_status3);
    1137:     PetscFree(s_waits3);

    1139:     /* Create the submatrices */
    1140:     for (i=0; i<ismax; i++) {
    1141:       PetscInt bs_tmp;
    1142:       if (ijonly) bs_tmp = 1;
    1143:       else        bs_tmp = bs;

    1145:       MatCreate(PETSC_COMM_SELF,submats+i);
    1146:       MatSetSizes(submats[i],nrow[i]*bs_tmp,ncol[i]*bs_tmp,PETSC_DETERMINE,PETSC_DETERMINE);

    1148:       MatSetType(submats[i],((PetscObject)A)->type_name);
    1149:       MatSeqBAIJSetPreallocation(submats[i],bs_tmp,0,lens[i]);
    1150:       MatSeqSBAIJSetPreallocation(submats[i],bs_tmp,0,lens[i]); /* this subroutine is used by SBAIJ routines */

    1152:       /* create struct Mat_SubSppt and attached it to submat */
    1153:       PetscNew(&smat_i);
    1154:       subc = (Mat_SeqBAIJ*)submats[i]->data;
    1155:       subc->submatis1 = smat_i;

    1157:       smat_i->destroy          = submats[i]->ops->destroy;
    1158:       submats[i]->ops->destroy = MatDestroySubMatrix_SeqBAIJ;
    1159:       submats[i]->factortype   = C->factortype;

    1161:       smat_i->id          = i;
    1162:       smat_i->nrqs        = nrqs;
    1163:       smat_i->nrqr        = nrqr;
    1164:       smat_i->rbuf1       = rbuf1;
    1165:       smat_i->rbuf2       = rbuf2;
    1166:       smat_i->rbuf3       = rbuf3;
    1167:       smat_i->sbuf2       = sbuf2;
    1168:       smat_i->req_source2 = req_source2;

    1170:       smat_i->sbuf1       = sbuf1;
    1171:       smat_i->ptr         = ptr;
    1172:       smat_i->tmp         = tmp;
    1173:       smat_i->ctr         = ctr;

    1175:       smat_i->pa           = pa;
    1176:       smat_i->req_size     = req_size;
    1177:       smat_i->req_source1  = req_source1;

    1179:       smat_i->allcolumns  = allcolumns[i];
    1180:       smat_i->allrows     = allrows[i];
    1181:       smat_i->singleis    = PETSC_FALSE;
    1182:       smat_i->row2proc    = row2proc[i];
    1183:       smat_i->rmap        = rmap[i];
    1184:       smat_i->cmap        = cmap[i];
    1185:     }

    1187:     if (!ismax) { /* Create dummy submats[0] for reuse struct subc */
    1188:       MatCreate(PETSC_COMM_SELF,&submats[0]);
    1189:       MatSetSizes(submats[0],0,0,PETSC_DETERMINE,PETSC_DETERMINE);
    1190:       MatSetType(submats[0],MATDUMMY);

    1192:       /* create struct Mat_SubSppt and attached it to submat */
    1193:       PetscNewLog(submats[0],&smat_i);
    1194:       submats[0]->data = (void*)smat_i;

    1196:       smat_i->destroy          = submats[0]->ops->destroy;
    1197:       submats[0]->ops->destroy = MatDestroySubMatrix_Dummy;
    1198:       submats[0]->factortype   = C->factortype;

    1200:       smat_i->id          = 0;
    1201:       smat_i->nrqs        = nrqs;
    1202:       smat_i->nrqr        = nrqr;
    1203:       smat_i->rbuf1       = rbuf1;
    1204:       smat_i->rbuf2       = rbuf2;
    1205:       smat_i->rbuf3       = rbuf3;
    1206:       smat_i->sbuf2       = sbuf2;
    1207:       smat_i->req_source2 = req_source2;

    1209:       smat_i->sbuf1       = sbuf1;
    1210:       smat_i->ptr         = ptr;
    1211:       smat_i->tmp         = tmp;
    1212:       smat_i->ctr         = ctr;

    1214:       smat_i->pa           = pa;
    1215:       smat_i->req_size     = req_size;
    1216:       smat_i->req_source1  = req_source1;

    1218:       smat_i->allcolumns  = PETSC_FALSE;
    1219:       smat_i->singleis    = PETSC_FALSE;
    1220:       smat_i->row2proc    = NULL;
    1221:       smat_i->rmap        = NULL;
    1222:       smat_i->cmap        = NULL;
    1223:     }

    1226:     if (ismax) {PetscFree(lens[0]);}
    1227:     PetscFree(lens);
    1228:     PetscFree(sbuf_aj[0]);
    1229:     PetscFree(sbuf_aj);

    1231:   } /* endof scall == MAT_INITIAL_MATRIX */

    1233:   /* Post recv matrix values */
    1234:   if (!ijonly) {
    1235:     PetscObjectGetNewTag((PetscObject)C,&tag4);
    1236:     PetscMalloc1(nrqs+1,&rbuf4);
    1237:     PetscMalloc1(nrqs+1,&r_waits4);
    1238:     PetscMalloc1(nrqs+1,&r_status4);
    1239:     PetscMalloc1(nrqr+1,&s_status4);
    1240:     for (i=0; i<nrqs; ++i) {
    1241:       PetscMalloc1(rbuf2[i][0]*bs2,&rbuf4[i]);
    1242:       MPI_Irecv(rbuf4[i],rbuf2[i][0]*bs2,MPIU_SCALAR,req_source2[i],tag4,comm,r_waits4+i);
    1243:     }

    1245:     /* Allocate sending buffers for a->a, and send them off */
    1246:     PetscMalloc1(nrqr+1,&sbuf_aa);
    1247:     for (i=0,j=0; i<nrqr; i++) j += req_size[i];

    1249:     PetscMalloc1((j+1)*bs2,&sbuf_aa[0]);
    1250:     for (i=1; i<nrqr; i++) sbuf_aa[i] = sbuf_aa[i-1] + req_size[i-1]*bs2;

    1252:     PetscMalloc1(nrqr+1,&s_waits4);

    1254:     for (i=0; i<nrqr; i++) {
    1255:       rbuf1_i   = rbuf1[i];
    1256:       sbuf_aa_i = sbuf_aa[i];
    1257:       ct1       = 2*rbuf1_i[0]+1;
    1258:       ct2       = 0;
    1259:       for (j=1,max1=rbuf1_i[0]; j<=max1; j++) {
    1260:         kmax = rbuf1_i[2*j];
    1261:         for (k=0; k<kmax; k++,ct1++) {
    1262:           row    = rbuf1_i[ct1] - rstart;
    1263:           nzA    = a_i[row+1] - a_i[row];
    1264:           nzB    = b_i[row+1] - b_i[row];
    1265:           ncols  = nzA + nzB;
    1266:           cworkB = b_j + b_i[row];
    1267:           vworkA = a_a + a_i[row]*bs2;
    1268:           vworkB = b_a + b_i[row]*bs2;

    1270:           /* load the column values for this row into vals*/
    1271:           vals = sbuf_aa_i+ct2*bs2;
    1272:           for (l=0; l<nzB; l++) {
    1273:             if ((bmap[cworkB[l]]) < cstart) {
    1274:               PetscMemcpy(vals+l*bs2,vworkB+l*bs2,bs2*sizeof(MatScalar));
    1275:             } else break;
    1276:           }
    1277:           imark = l;
    1278:           for (l=0; l<nzA; l++) {
    1279:             PetscMemcpy(vals+(imark+l)*bs2,vworkA+l*bs2,bs2*sizeof(MatScalar));
    1280:           }
    1281:           for (l=imark; l<nzB; l++) {
    1282:             PetscMemcpy(vals+(nzA+l)*bs2,vworkB+l*bs2,bs2*sizeof(MatScalar));
    1283:           }

    1285:           ct2 += ncols;
    1286:         }
    1287:       }
    1288:       MPI_Isend(sbuf_aa_i,req_size[i]*bs2,MPIU_SCALAR,req_source1[i],tag4,comm,s_waits4+i);
    1289:     }
    1290:   }

    1292:   /* Assemble the matrices */
    1293:   /* First assemble the local rows */
    1294:   for (i=0; i<ismax; i++) {
    1295:     row2proc_i = row2proc[i];
    1296:     subc      = (Mat_SeqBAIJ*)submats[i]->data;
    1297:     imat_ilen = subc->ilen;
    1298:     imat_j    = subc->j;
    1299:     imat_i    = subc->i;
    1300:     imat_a    = subc->a;

    1302:     if (!allcolumns[i]) cmap_i = cmap[i];
    1303:     rmap_i = rmap[i];
    1304:     irow_i = irow[i];
    1305:     jmax   = nrow[i];
    1306:     for (j=0; j<jmax; j++) {
    1307:       if (allrows[i]) row = j;
    1308:       else row  = irow_i[j];
    1309:       proc = row2proc_i[j];

    1311:       if (proc == rank) {

    1313:         row    = row - rstart;
    1314:         nzA    = a_i[row+1] - a_i[row];
    1315:         nzB    = b_i[row+1] - b_i[row];
    1316:         cworkA = a_j + a_i[row];
    1317:         cworkB = b_j + b_i[row];
    1318:         if (!ijonly) {
    1319:           vworkA = a_a + a_i[row]*bs2;
    1320:           vworkB = b_a + b_i[row]*bs2;
    1321:         }

    1323:         if (allrows[i]) {
    1324:           row = row+rstart;
    1325:         } else {
    1326: #if defined(PETSC_USE_CTABLE)
    1327:           PetscTableFind(rmap_i,row+rstart+1,&row);
    1328:           row--;

    1330:           if (row < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"row not found in table");
    1331: #else
    1332:           row = rmap_i[row + rstart];
    1333: #endif
    1334:         }
    1335:         mat_i = imat_i[row];
    1336:         if (!ijonly) mat_a = imat_a + mat_i*bs2;
    1337:         mat_j    = imat_j + mat_i;
    1338:         ilen = imat_ilen[row];

    1340:         /* load the column indices for this row into cols*/
    1341:         if (!allcolumns[i]) {
    1342:           for (l=0; l<nzB; l++) {
    1343:             if ((ctmp = bmap[cworkB[l]]) < cstart) {
    1344: #if defined(PETSC_USE_CTABLE)
    1345:               PetscTableFind(cmap_i,ctmp+1,&tcol);
    1346:               if (tcol) {
    1347: #else
    1348:               if ((tcol = cmap_i[ctmp])) {
    1349: #endif
    1350:                 *mat_j++ = tcol - 1;
    1351:                 PetscMemcpy(mat_a,vworkB+l*bs2,bs2*sizeof(MatScalar));
    1352:                 mat_a   += bs2;
    1353:                 ilen++;
    1354:               }
    1355:             } else break;
    1356:           }
    1357:           imark = l;
    1358:           for (l=0; l<nzA; l++) {
    1359: #if defined(PETSC_USE_CTABLE)
    1360:             PetscTableFind(cmap_i,cstart+cworkA[l]+1,&tcol);
    1361:             if (tcol) {
    1362: #else
    1363:             if ((tcol = cmap_i[cstart + cworkA[l]])) {
    1364: #endif
    1365:               *mat_j++ = tcol - 1;
    1366:               if (!ijonly) {
    1367:                 PetscMemcpy(mat_a,vworkA+l*bs2,bs2*sizeof(MatScalar));
    1368:                 mat_a += bs2;
    1369:               }
    1370:               ilen++;
    1371:             }
    1372:           }
    1373:           for (l=imark; l<nzB; l++) {
    1374: #if defined(PETSC_USE_CTABLE)
    1375:             PetscTableFind(cmap_i,bmap[cworkB[l]]+1,&tcol);
    1376:             if (tcol) {
    1377: #else
    1378:             if ((tcol = cmap_i[bmap[cworkB[l]]])) {
    1379: #endif
    1380:               *mat_j++ = tcol - 1;
    1381:               if (!ijonly) {
    1382:                 PetscMemcpy(mat_a,vworkB+l*bs2,bs2*sizeof(MatScalar));
    1383:                 mat_a += bs2;
    1384:               }
    1385:               ilen++;
    1386:             }
    1387:           }
    1388:         } else { /* allcolumns */
    1389:           for (l=0; l<nzB; l++) {
    1390:             if ((ctmp = bmap[cworkB[l]]) < cstart) {
    1391:               *mat_j++ = ctmp;
    1392:               PetscMemcpy(mat_a,vworkB+l*bs2,bs2*sizeof(MatScalar));
    1393:               mat_a   += bs2;
    1394:               ilen++;
    1395:             } else break;
    1396:           }
    1397:           imark = l;
    1398:           for (l=0; l<nzA; l++) {
    1399:             *mat_j++ = cstart+cworkA[l];
    1400:             if (!ijonly) {
    1401:               PetscMemcpy(mat_a,vworkA+l*bs2,bs2*sizeof(MatScalar));
    1402:               mat_a += bs2;
    1403:             }
    1404:             ilen++;
    1405:           }
    1406:           for (l=imark; l<nzB; l++) {
    1407:             *mat_j++ = bmap[cworkB[l]];
    1408:             if (!ijonly) {
    1409:               PetscMemcpy(mat_a,vworkB+l*bs2,bs2*sizeof(MatScalar));
    1410:               mat_a += bs2;
    1411:             }
    1412:             ilen++;
    1413:           }
    1414:         }
    1415:         imat_ilen[row] = ilen;
    1416:       }
    1417:     }
    1418:   }

    1420:   /* Now assemble the off proc rows */
    1421:   if (!ijonly) {
    1422:     MPI_Waitall(nrqs,r_waits4,r_status4);
    1423:   }
    1424:   for (tmp2=0; tmp2<nrqs; tmp2++) {
    1425:     sbuf1_i = sbuf1[pa[tmp2]];
    1426:     jmax    = sbuf1_i[0];
    1427:     ct1     = 2*jmax + 1;
    1428:     ct2     = 0;
    1429:     rbuf2_i = rbuf2[tmp2];
    1430:     rbuf3_i = rbuf3[tmp2];
    1431:     if (!ijonly) rbuf4_i = rbuf4[tmp2];
    1432:     for (j=1; j<=jmax; j++) {
    1433:       is_no     = sbuf1_i[2*j-1];
    1434:       rmap_i    = rmap[is_no];
    1435:       if (!allcolumns[is_no]) cmap_i = cmap[is_no];
    1436:       subc      = (Mat_SeqBAIJ*)submats[is_no]->data;
    1437:       imat_ilen = subc->ilen;
    1438:       imat_j    = subc->j;
    1439:       imat_i    = subc->i;
    1440:       if (!ijonly) imat_a    = subc->a;
    1441:       max1      = sbuf1_i[2*j];
    1442:       for (k=0; k<max1; k++,ct1++) { /* for each recved block row */
    1443:         row = sbuf1_i[ct1];

    1445:         if (allrows[is_no]) {
    1446:           row = sbuf1_i[ct1];
    1447:         } else {
    1448: #if defined(PETSC_USE_CTABLE)
    1449:           PetscTableFind(rmap_i,row+1,&row);
    1450:           row--;
    1451:           if (row < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"row not found in table");
    1452: #else
    1453:           row = rmap_i[row];
    1454: #endif
    1455:         }
    1456:         ilen  = imat_ilen[row];
    1457:         mat_i = imat_i[row];
    1458:         if (!ijonly) mat_a = imat_a + mat_i*bs2;
    1459:         mat_j = imat_j + mat_i;
    1460:         max2  = rbuf2_i[ct1];
    1461:         if (!allcolumns[is_no]) {
    1462:           for (l=0; l<max2; l++,ct2++) {
    1463: #if defined(PETSC_USE_CTABLE)
    1464:             PetscTableFind(cmap_i,rbuf3_i[ct2]+1,&tcol);
    1465: #else
    1466:             tcol = cmap_i[rbuf3_i[ct2]];
    1467: #endif
    1468:             if (tcol) {
    1469:               *mat_j++ = tcol - 1;
    1470:               if (!ijonly) {
    1471:                 PetscMemcpy(mat_a,rbuf4_i+ct2*bs2,bs2*sizeof(MatScalar));
    1472:                 mat_a += bs2;
    1473:               }
    1474:               ilen++;
    1475:             }
    1476:           }
    1477:         } else { /* allcolumns */
    1478:           for (l=0; l<max2; l++,ct2++) {
    1479:             *mat_j++ = rbuf3_i[ct2]; /* same global column index of C */
    1480:             if (!ijonly) {
    1481:               PetscMemcpy(mat_a,rbuf4_i+ct2*bs2,bs2*sizeof(MatScalar));
    1482:               mat_a += bs2;
    1483:             }
    1484:             ilen++;
    1485:           }
    1486:         }
    1487:         imat_ilen[row] = ilen;
    1488:       }
    1489:     }
    1490:   }

    1492:   if (!iscsorted) { /* sort column indices of the rows */
    1493:     MatScalar *work;

    1495:     PetscMalloc1(bs2,&work);
    1496:     for (i=0; i<ismax; i++) {
    1497:       subc      = (Mat_SeqBAIJ*)submats[i]->data;
    1498:       imat_ilen = subc->ilen;
    1499:       imat_j    = subc->j;
    1500:       imat_i    = subc->i;
    1501:       if (!ijonly) imat_a = subc->a;
    1502:       if (allcolumns[i]) continue;

    1504:       jmax = nrow[i];
    1505:       for (j=0; j<jmax; j++) {
    1506:         mat_i = imat_i[j];
    1507:         mat_j = imat_j + mat_i;
    1508:         ilen  = imat_ilen[j];
    1509:         if (ijonly) {
    1510:           PetscSortInt(ilen,mat_j);
    1511:         } else {
    1512:           mat_a = imat_a + mat_i*bs2;
    1513:           PetscSortIntWithDataArray(ilen,mat_j,mat_a,bs2*sizeof(MatScalar),work);
    1514:         }
    1515:       }
    1516:     }
    1517:     PetscFree(work);
    1518:   }

    1520:   if (!ijonly) {
    1521:     PetscFree(r_status4);
    1522:     PetscFree(r_waits4);
    1523:     if (nrqr) {MPI_Waitall(nrqr,s_waits4,s_status4);}
    1524:     PetscFree(s_waits4);
    1525:     PetscFree(s_status4);
    1526:   }

    1528:   /* Restore the indices */
    1529:   for (i=0; i<ismax; i++) {
    1530:     if (!allrows[i]) {
    1531:       ISRestoreIndices(isrow[i],irow+i);
    1532:     }
    1533:     if (!allcolumns[i]) {
    1534:       ISRestoreIndices(iscol[i],icol+i);
    1535:     }
    1536:   }

    1538:   for (i=0; i<ismax; i++) {
    1539:     MatAssemblyBegin(submats[i],MAT_FINAL_ASSEMBLY);
    1540:     MatAssemblyEnd(submats[i],MAT_FINAL_ASSEMBLY);
    1541:   }

    1543:   PetscFree5(*(PetscInt***)&irow,*(PetscInt***)&icol,nrow,ncol,issorted);
    1544:   PetscFree5(row2proc,cmap,rmap,allcolumns,allrows);

    1546:   if (!ijonly) {
    1547:     PetscFree(sbuf_aa[0]);
    1548:     PetscFree(sbuf_aa);

    1550:     for (i=0; i<nrqs; ++i) {
    1551:       PetscFree(rbuf4[i]);
    1552:     }
    1553:     PetscFree(rbuf4);
    1554:   }
    1555:   c->ijonly = PETSC_FALSE; /* set back to the default */
    1556:   return(0);
    1557: }