Actual source code: dmmoab.cxx
petsc-3.10.5 2019-03-28
Report Typos and Errors

      1:  #include <petsc/private/dmmbimpl.h>

      3:  #include <petscdmmoab.h>
      4: #include <MBTagConventions.hpp>
      5: #include <moab/NestedRefine.hpp>
      6: #include <moab/Skinner.hpp>

      8: /*MC
      9:   DMMOAB = "moab" - A DM object that encapsulates an unstructured mesh described by the MOAB mesh database.
     10:                     Direct access to the MOAB Interface and other mesh manipulation related objects are available
     11:                     through public API. Ability to create global and local representation of Vecs containing all
     12:                     unknowns in the interior and shared boundary via a transparent tag-data wrapper is provided
     13:                     along with utility functions to traverse the mesh and assemble a discrete system via
     14:                     field-based/blocked Vec(Get/Set) methods. Input from and output to different formats are
     15:                     available.

     17:   Reference: http://www.mcs.anl.gov/~fathom/moab-docs/html/contents.html

     19:   Level: intermediate

     21: .seealso: DMType, DMMoabCreate(), DMCreate(), DMSetType(), DMMoabCreateMoab()
     22: M*/

     24: /* External function declarations here */
     25: PETSC_EXTERN PetscErrorCode DMCreateInterpolation_Moab(DM dmCoarse, DM dmFine, Mat *interpolation, Vec *scaling);
     26: PETSC_EXTERN PetscErrorCode DMCreateDefaultConstraints_Moab(DM dm);
     27: PETSC_EXTERN PetscErrorCode DMCreateMatrix_Moab(DM dm,  Mat *J);
     28: PETSC_EXTERN PetscErrorCode DMCreateCoordinateDM_Moab(DM dm, DM *cdm);
     29: PETSC_EXTERN PetscErrorCode DMRefine_Moab(DM dm, MPI_Comm comm, DM *dmRefined);
     30: PETSC_EXTERN PetscErrorCode DMCoarsen_Moab(DM dm, MPI_Comm comm, DM *dmCoarsened);
     31: PETSC_EXTERN PetscErrorCode DMRefineHierarchy_Moab(DM dm, PetscInt nlevels, DM dmRefined[]);
     32: PETSC_EXTERN PetscErrorCode DMCoarsenHierarchy_Moab(DM dm, PetscInt nlevels, DM dmCoarsened[]);
     33: PETSC_EXTERN PetscErrorCode DMClone_Moab(DM dm, DM *newdm);
     34: PETSC_EXTERN PetscErrorCode DMCreateGlobalVector_Moab(DM, Vec *);
     35: PETSC_EXTERN PetscErrorCode DMCreateLocalVector_Moab(DM, Vec *);
     36: PETSC_EXTERN PetscErrorCode DMCreateMatrix_Moab(DM dm, Mat *J);
     37: PETSC_EXTERN PetscErrorCode DMGlobalToLocalBegin_Moab(DM, Vec, InsertMode, Vec);
     38: PETSC_EXTERN PetscErrorCode DMGlobalToLocalEnd_Moab(DM, Vec, InsertMode, Vec);
     39: PETSC_EXTERN PetscErrorCode DMLocalToGlobalBegin_Moab(DM, Vec, InsertMode, Vec);
     40: PETSC_EXTERN PetscErrorCode DMLocalToGlobalEnd_Moab(DM, Vec, InsertMode, Vec);

     43: /* Un-implemented routines */
     44: /*
     45: PETSC_EXTERN PetscErrorCode DMCreateDefaultSection_Moab(DM dm);
     46: PETSC_EXTERN PetscErrorCode DMCreateInjection_Moab(DM dmCoarse, DM dmFine, Mat *mat);
     47: PETSC_EXTERN PetscErrorCode DMLoad_Moab(DM dm, PetscViewer viewer);
     48: PETSC_EXTERN PetscErrorCode DMGetDimPoints_Moab(DM dm, PetscInt dim, PetscInt *pStart, PetscInt *pEnd);
     49: PETSC_EXTERN PetscErrorCode DMCreateSubDM_Moab(DM dm, PetscInt numFields, PetscInt fields[], IS *is, DM *subdm);
     50: PETSC_EXTERN PetscErrorCode DMLocatePoints_Moab(DM dm, Vec v, IS *cellIS);
     51: */

     53: /*@
     54:   DMMoabCreate - Creates a DMMoab object, which encapsulates a moab instance

     56:   Collective on MPI_Comm

     58:   Input Parameter:
     59: . comm - The communicator for the DMMoab object

     61:   Output Parameter:
     62: . dmb  - The DMMoab object

     64:   Level: beginner

     66: .keywords: DMMoab, create
     67: @*/
     68: PetscErrorCode DMMoabCreate(MPI_Comm comm, DM *dmb)
     69: {

     74:   DMCreate(comm, dmb);
     75:   DMSetType(*dmb, DMMOAB);
     76:   return(0);
     77: }

     79: /*@
     80:   DMMoabCreateMoab - Creates a DMMoab object, optionally from an instance and other data

     82:   Collective on MPI_Comm

     84:   Input Parameter:
     85: . comm - The communicator for the DMMoab object
     86: . mbiface - (ptr to) the MOAB Instance; if passed in NULL, MOAB instance is created inside PETSc, and destroyed
     87:          along with the DMMoab
     88: . pcomm - (ptr to) a ParallelComm; if NULL, creates one internally for the whole communicator
     89: . ltog_tag - A tag to use to retrieve global id for an entity; if 0, will use GLOBAL_ID_TAG_NAME/tag
     90: . range - If non-NULL, contains range of entities to which DOFs will be assigned

     92:   Output Parameter:
     93: . dmb  - The DMMoab object

     95:   Level: intermediate

     97: .keywords: DMMoab, create
     98: @*/
     99: PetscErrorCode DMMoabCreateMoab(MPI_Comm comm, moab::Interface *mbiface, moab::Tag *ltog_tag, moab::Range *range, DM *dmb)
    100: {
    102:   moab::ErrorCode merr;
    103:   DM             dmmb;
    104:   DM_Moab        *dmmoab;

    109:   DMMoabCreate(comm, &dmmb);
    110:   dmmoab = (DM_Moab*)(dmmb)->data;

    112:   if (!mbiface) {
    113:     dmmoab->mbiface = new moab::Core();
    114:     dmmoab->icreatedinstance = PETSC_TRUE;
    115:   }
    116:   else {
    117:     dmmoab->mbiface = mbiface;
    118:     dmmoab->icreatedinstance = PETSC_FALSE;
    119:   }

    121:   /* by default the fileset = root set. This set stores the hierarchy of entities belonging to current DM */
    122:   dmmoab->fileset = 0;
    123:   dmmoab->hlevel = 0;
    124:   dmmoab->nghostrings = 0;

    126: #ifdef MOAB_HAVE_MPI
    127:   moab::EntityHandle partnset;

    129:   /* Create root sets for each mesh.  Then pass these
    130:       to the load_file functions to be populated. */
    131:   merr = dmmoab->mbiface->create_meshset(moab::MESHSET_SET, partnset); MBERR("Creating partition set failed", merr);

    133:   /* Create the parallel communicator object with the partition handle associated with MOAB */
    134:   dmmoab->pcomm = moab::ParallelComm::get_pcomm(dmmoab->mbiface, partnset, &comm);
    135: #endif

    137:   /* do the remaining initializations for DMMoab */
    138:   dmmoab->bs = 1;
    139:   dmmoab->numFields = 1;
    140:   PetscMalloc(dmmoab->numFields * sizeof(char*), &dmmoab->fieldNames);
    141:   PetscStrallocpy("DEFAULT", (char**) &dmmoab->fieldNames[0]);
    142:   dmmoab->rw_dbglevel = 0;
    143:   dmmoab->partition_by_rank = PETSC_FALSE;
    144:   dmmoab->extra_read_options[0] = '\0';
    145:   dmmoab->extra_write_options[0] = '\0';
    146:   dmmoab->read_mode = READ_PART;
    147:   dmmoab->write_mode = WRITE_PART;

    149:   /* set global ID tag handle */
    150:   if (ltog_tag && *ltog_tag) {
    151:     DMMoabSetLocalToGlobalTag(dmmb, *ltog_tag);
    152:   }
    153:   else {
    154:     merr = dmmoab->mbiface->tag_get_handle(GLOBAL_ID_TAG_NAME, dmmoab->ltog_tag); MBERRNM(merr);
    155:     if (ltog_tag) *ltog_tag = dmmoab->ltog_tag;
    156:   }

    158:   merr = dmmoab->mbiface->tag_get_handle(MATERIAL_SET_TAG_NAME, dmmoab->material_tag); MBERRNM(merr);

    160:   /* set the local range of entities (vertices) of interest */
    161:   if (range) {
    162:     DMMoabSetLocalVertices(dmmb, range);
    163:   }
    164:   *dmb = dmmb;
    165:   return(0);
    166: }

    169: #ifdef MOAB_HAVE_MPI

    171: /*@
    172:   DMMoabGetParallelComm - Get the ParallelComm used with this DMMoab

    174:   Collective on MPI_Comm

    176:   Input Parameter:
    177: . dm    - The DMMoab object being set

    179:   Output Parameter:
    180: . pcomm - The ParallelComm for the DMMoab

    182:   Level: beginner

    184: .keywords: DMMoab, create
    185: @*/
    186: PetscErrorCode DMMoabGetParallelComm(DM dm, moab::ParallelComm **pcomm)
    187: {
    190:   *pcomm = ((DM_Moab*)(dm)->data)->pcomm;
    191:   return(0);
    192: }

    194: #endif /* MOAB_HAVE_MPI */

    197: /*@
    198:   DMMoabSetInterface - Set the MOAB instance used with this DMMoab

    200:   Collective on MPI_Comm

    202:   Input Parameter:
    203: . dm      - The DMMoab object being set
    204: . mbiface - The MOAB instance being set on this DMMoab

    206:   Level: beginner

    208: .keywords: DMMoab, create
    209: @*/
    210: PetscErrorCode DMMoabSetInterface(DM dm, moab::Interface *mbiface)
    211: {
    212:   DM_Moab        *dmmoab = (DM_Moab*)(dm)->data;

    217: #ifdef MOAB_HAVE_MPI
    218:   dmmoab->pcomm = NULL;
    219: #endif
    220:   dmmoab->mbiface = mbiface;
    221:   dmmoab->icreatedinstance = PETSC_FALSE;
    222:   return(0);
    223: }

    226: /*@
    227:   DMMoabGetInterface - Get the MOAB instance used with this DMMoab

    229:   Collective on MPI_Comm

    231:   Input Parameter:
    232: . dm      - The DMMoab object being set

    234:   Output Parameter:
    235: . mbiface - The MOAB instance set on this DMMoab

    237:   Level: beginner

    239: .keywords: DMMoab, create
    240: @*/
    241: PetscErrorCode DMMoabGetInterface(DM dm, moab::Interface **mbiface)
    242: {
    243:   PetscErrorCode   ierr;
    244:   static PetscBool cite = PETSC_FALSE;

    248:   PetscCitationsRegister("@techreport{tautges_moab:_2004,\n  type = {{SAND2004-1592}},\n  title = {{MOAB:} A Mesh-Oriented Database},  institution = {Sandia National Laboratories},\n  author = {Tautges, T. J. and Meyers, R. and Merkley, K. and Stimpson, C. and Ernst, C.},\n  year = {2004},  note = {Report}\n}\n", &cite);
    249:   *mbiface = ((DM_Moab*)dm->data)->mbiface;
    250:   return(0);
    251: }

    254: /*@
    255:   DMMoabSetLocalVertices - Set the entities having DOFs on this DMMoab

    257:   Collective on MPI_Comm

    259:   Input Parameter:
    260: . dm    - The DMMoab object being set
    261: . range - The entities treated by this DMMoab

    263:   Level: beginner

    265: .keywords: DMMoab, create
    266: @*/
    267: PetscErrorCode DMMoabSetLocalVertices(DM dm, moab::Range *range)
    268: {
    269:   moab::Range     tmpvtxs;
    270:   DM_Moab        *dmmoab = (DM_Moab*)(dm)->data;

    274:   dmmoab->vlocal->clear();
    275:   dmmoab->vowned->clear();

    277:   dmmoab->vlocal->insert(range->begin(), range->end());

    279: #ifdef MOAB_HAVE_MPI
    280:   moab::ErrorCode merr;
    281:   /* filter based on parallel status */
    282:   merr = dmmoab->pcomm->filter_pstatus(*dmmoab->vlocal, PSTATUS_NOT_OWNED, PSTATUS_NOT, -1, dmmoab->vowned); MBERRNM(merr);

    284:   /* filter all the non-owned and shared entities out of the list */
    285:   tmpvtxs = moab::subtract(*dmmoab->vlocal, *dmmoab->vowned);
    286:   merr = dmmoab->pcomm->filter_pstatus(tmpvtxs, PSTATUS_INTERFACE, PSTATUS_OR, -1, dmmoab->vghost); MBERRNM(merr);
    287:   tmpvtxs = moab::subtract(tmpvtxs, *dmmoab->vghost);
    288:   *dmmoab->vlocal = moab::subtract(*dmmoab->vlocal, tmpvtxs);
    289: #else
    290:   *dmmoab->vowned = *dmmoab->vlocal;
    291: #endif

    293:   /* compute and cache the sizes of local and ghosted entities */
    294:   dmmoab->nloc = dmmoab->vowned->size();
    295:   dmmoab->nghost = dmmoab->vghost->size();
    296: #ifdef MOAB_HAVE_MPI
    297:   PetscErrorCode  ierr;
    298:   MPIU_Allreduce(&dmmoab->nloc, &dmmoab->n, 1, MPI_INTEGER, MPI_SUM, ((PetscObject)dm)->comm);
    299: #else
    300:   dmmoab->n = dmmoab->nloc;
    301: #endif
    302:   return(0);
    303: }

    306: /*@
    307:   DMMoabGetAllVertices - Get the entities having DOFs on this DMMoab

    309:   Collective on MPI_Comm

    311:   Input Parameter:
    312: . dm    - The DMMoab object being set

    314:   Output Parameter:
    315: . owned - The local vertex entities in this DMMoab = (owned+ghosted)

    317:   Level: beginner

    319: .keywords: DMMoab, create
    320: @*/
    321: PetscErrorCode DMMoabGetAllVertices(DM dm, moab::Range *local)
    322: {
    325:   if (local) *local = *((DM_Moab*)dm->data)->vlocal;
    326:   return(0);
    327: }

    331: /*@
    332:   DMMoabGetLocalVertices - Get the entities having DOFs on this DMMoab

    334:   Collective on MPI_Comm

    336:   Input Parameter:
    337: . dm    - The DMMoab object being set

    339:   Output Parameter:
    340: . owned - The owned vertex entities in this DMMoab
    341: . ghost - The ghosted entities (non-owned) stored locally in this partition

    343:   Level: beginner

    345: .keywords: DMMoab, create
    346: @*/
    347: PetscErrorCode DMMoabGetLocalVertices(DM dm, const moab::Range **owned, const moab::Range **ghost)
    348: {
    351:   if (owned) *owned = ((DM_Moab*)dm->data)->vowned;
    352:   if (ghost) *ghost = ((DM_Moab*)dm->data)->vghost;
    353:   return(0);
    354: }

    356: /*@
    357:   DMMoabGetLocalElements - Get the higher-dimensional entities that are locally owned

    359:   Collective on MPI_Comm

    361:   Input Parameter:
    362: . dm    - The DMMoab object being set

    364:   Output Parameter:
    365: . range - The entities owned locally

    367:   Level: beginner

    369: .keywords: DMMoab, create
    370: @*/
    371: PetscErrorCode DMMoabGetLocalElements(DM dm, const moab::Range **range)
    372: {
    375:   if (range) *range = ((DM_Moab*)dm->data)->elocal;
    376:   return(0);
    377: }

    380: /*@
    381:   DMMoabSetLocalElements - Set the entities having DOFs on this DMMoab

    383:   Collective on MPI_Comm

    385:   Input Parameter:
    386: . dm    - The DMMoab object being set
    387: . range - The entities treated by this DMMoab

    389:   Level: beginner

    391: .keywords: DMMoab, create
    392: @*/
    393: PetscErrorCode DMMoabSetLocalElements(DM dm, moab::Range *range)
    394: {
    395:   DM_Moab        *dmmoab = (DM_Moab*)(dm)->data;

    399:   dmmoab->elocal->clear();
    400:   dmmoab->eghost->clear();
    401:   dmmoab->elocal->insert(range->begin(), range->end());
    402: #ifdef MOAB_HAVE_MPI
    403:   moab::ErrorCode merr;
    404:   merr = dmmoab->pcomm->filter_pstatus(*dmmoab->elocal, PSTATUS_NOT_OWNED, PSTATUS_NOT); MBERRNM(merr);
    405:   *dmmoab->eghost = moab::subtract(*range, *dmmoab->elocal);
    406: #endif
    407:   dmmoab->neleloc = dmmoab->elocal->size();
    408:   dmmoab->neleghost = dmmoab->eghost->size();
    409: #ifdef MOAB_HAVE_MPI
    410:   PetscErrorCode  ierr;
    411:   MPIU_Allreduce(&dmmoab->neleloc, &dmmoab->nele, 1, MPI_INTEGER, MPI_SUM, ((PetscObject)dm)->comm);
    412:   PetscInfo2(dm, "Created %D local and %D global elements.\n", dmmoab->neleloc, dmmoab->nele);
    413: #else
    414:   dmmoab->nele = dmmoab->neleloc;
    415: #endif
    416:   return(0);
    417: }

    420: /*@
    421:   DMMoabSetLocalToGlobalTag - Set the tag used for local to global numbering

    423:   Collective on MPI_Comm

    425:   Input Parameter:
    426: . dm      - The DMMoab object being set
    427: . ltogtag - The MOAB tag used for local to global ids

    429:   Level: beginner

    431: .keywords: DMMoab, create
    432: @*/
    433: PetscErrorCode DMMoabSetLocalToGlobalTag(DM dm, moab::Tag ltogtag)
    434: {
    437:   ((DM_Moab*)dm->data)->ltog_tag = ltogtag;
    438:   return(0);
    439: }

    442: /*@
    443:   DMMoabGetLocalToGlobalTag - Get the tag used for local to global numbering

    445:   Collective on MPI_Comm

    447:   Input Parameter:
    448: . dm      - The DMMoab object being set

    450:   Output Parameter:
    451: . ltogtag - The MOAB tag used for local to global ids

    453:   Level: beginner

    455: .keywords: DMMoab, create
    456: @*/
    457: PetscErrorCode DMMoabGetLocalToGlobalTag(DM dm, moab::Tag *ltog_tag)
    458: {
    461:   *ltog_tag = ((DM_Moab*)dm->data)->ltog_tag;
    462:   return(0);
    463: }

    466: /*@
    467:   DMMoabSetBlockSize - Set the block size used with this DMMoab

    469:   Collective on MPI_Comm

    471:   Input Parameter:
    472: . dm - The DMMoab object being set
    473: . bs - The block size used with this DMMoab

    475:   Level: beginner

    477: .keywords: DMMoab, create
    478: @*/
    479: PetscErrorCode DMMoabSetBlockSize(DM dm, PetscInt bs)
    480: {
    483:   ((DM_Moab*)dm->data)->bs = bs;
    484:   return(0);
    485: }

    488: /*@
    489:   DMMoabGetBlockSize - Get the block size used with this DMMoab

    491:   Collective on MPI_Comm

    493:   Input Parameter:
    494: . dm - The DMMoab object being set

    496:   Output Parameter:
    497: . bs - The block size used with this DMMoab

    499:   Level: beginner

    501: .keywords: DMMoab, create
    502: @*/
    503: PetscErrorCode DMMoabGetBlockSize(DM dm, PetscInt *bs)
    504: {
    507:   *bs = ((DM_Moab*)dm->data)->bs;
    508:   return(0);
    509: }

    512: /*@
    513:   DMMoabGetSize - Get the global vertex size used with this DMMoab

    515:   Collective on DM

    517:   Input Parameter:
    518: . dm - The DMMoab object being set

    520:   Output Parameter:
    521: . neg - The number of global elements in the DMMoab instance
    522: . nvg - The number of global vertices in the DMMoab instance

    524:   Level: beginner

    526: .keywords: DMMoab, create
    527: @*/
    528: PetscErrorCode DMMoabGetSize(DM dm, PetscInt *neg, PetscInt *nvg)
    529: {
    532:   if (neg) *neg = ((DM_Moab*)dm->data)->nele;
    533:   if (nvg) *nvg = ((DM_Moab*)dm->data)->n;
    534:   return(0);
    535: }

    538: /*@
    539:   DMMoabGetLocalSize - Get the local and ghosted vertex size used with this DMMoab

    541:   Collective on DM

    543:   Input Parameter:
    544: . dm - The DMMoab object being set

    546:   Output Parameter:
    547: + nel - The number of owned elements in this processor
    548: . neg - The number of ghosted elements in this processor
    549: . nvl - The number of owned vertices in this processor
    550: . nvg - The number of ghosted vertices in this processor

    552:   Level: beginner

    554: .keywords: DMMoab, create
    555: @*/
    556: PetscErrorCode DMMoabGetLocalSize(DM dm, PetscInt *nel, PetscInt *neg, PetscInt *nvl, PetscInt *nvg)
    557: {
    560:   if (nel) *nel = ((DM_Moab*)dm->data)->neleloc;
    561:   if (neg) *neg = ((DM_Moab*)dm->data)->neleghost;
    562:   if (nvl) *nvl = ((DM_Moab*)dm->data)->nloc;
    563:   if (nvg) *nvg = ((DM_Moab*)dm->data)->nghost;
    564:   return(0);
    565: }

    568: /*@
    569:   DMMoabGetOffset - Get the local offset for the global vector

    571:   Collective on MPI_Comm

    573:   Input Parameter:
    574: . dm - The DMMoab object being set

    576:   Output Parameter:
    577: . offset - The local offset for the global vector

    579:   Level: beginner

    581: .keywords: DMMoab, create
    582: @*/
    583: PetscErrorCode DMMoabGetOffset(DM dm, PetscInt *offset)
    584: {
    587:   *offset = ((DM_Moab*)dm->data)->vstart;
    588:   return(0);
    589: }

    592: /*@
    593:   DMMoabGetDimension - Get the dimension of the DM Mesh

    595:   Collective on MPI_Comm

    597:   Input Parameter:
    598: . dm - The DMMoab object

    600:   Output Parameter:
    601: . dim - The dimension of DM

    603:   Level: beginner

    605: .keywords: DMMoab, create
    606: @*/
    607: PetscErrorCode DMMoabGetDimension(DM dm, PetscInt *dim)
    608: {
    611:   *dim = ((DM_Moab*)dm->data)->dim;
    612:   return(0);
    613: }

    616: /*@
    617:   DMMoabGetHierarchyLevel - Get the current level of the mesh hierarchy
    618:   generated through uniform refinement.

    620:   Collective on DM

    622:   Input Parameter:
    623: . dm - The DMMoab object being set

    625:   Output Parameter:
    626: . nvg - The current mesh hierarchy level

    628:   Level: beginner

    630: .keywords: DMMoab, multigrid
    631: @*/
    632: PetscErrorCode DMMoabGetHierarchyLevel(DM dm, PetscInt *nlevel)
    633: {
    636:   if (nlevel) *nlevel = ((DM_Moab*)dm->data)->hlevel;
    637:   return(0);
    638: }

    641: /*@
    642:   DMMoabGetMaterialBlock - Get the material ID corresponding to the current entity of the DM Mesh

    644:   Collective on MPI_Comm

    646:   Input Parameter:
    647: . dm - The DMMoab object
    648: . ehandle - The element entity handle

    650:   Output Parameter:
    651: . mat - The material ID for the current entity

    653:   Level: beginner

    655: .keywords: DMMoab, create
    656: @*/
    657: PetscErrorCode DMMoabGetMaterialBlock(DM dm, const moab::EntityHandle ehandle, PetscInt *mat)
    658: {
    659:   DM_Moab         *dmmoab;

    663:   if (*mat) {
    664:     dmmoab = (DM_Moab*)(dm)->data;
    665:     *mat = dmmoab->materials[dmmoab->elocal->index(ehandle)];
    666:   }
    667:   return(0);
    668: }

    671: /*@
    672:   DMMoabGetVertexCoordinates - Get the coordinates corresponding to the requested vertex entities

    674:   Collective on MPI_Comm

    676:   Input Parameter:
    677: . dm - The DMMoab object
    678: . nconn - Number of entities whose coordinates are needed
    679: . conn - The vertex entity handles

    681:   Output Parameter:
    682: . vpos - The coordinates of the requested vertex entities

    684:   Level: beginner

    686: .seealso: DMMoabGetVertexConnectivity()
    687: @*/
    688: PetscErrorCode DMMoabGetVertexCoordinates(DM dm, PetscInt nconn, const moab::EntityHandle *conn, PetscReal *vpos)
    689: {
    690:   DM_Moab         *dmmoab;
    691:   moab::ErrorCode merr;

    697:   dmmoab = (DM_Moab*)(dm)->data;

    699:   /* Get connectivity information in MOAB canonical ordering */
    700:   if (dmmoab->hlevel) {
    701:     merr = dmmoab->hierarchy->get_coordinates(const_cast<moab::EntityHandle*>(conn), nconn, dmmoab->hlevel, vpos);MBERRNM(merr);
    702:   }
    703:   else {
    704:     merr = dmmoab->mbiface->get_coords(conn, nconn, vpos);MBERRNM(merr);
    705:   }
    706:   return(0);
    707: }

    710: /*@
    711:   DMMoabGetVertexConnectivity - Get the vertex adjacency for the given entity

    713:   Collective on MPI_Comm

    715:   Input Parameter:
    716: . dm - The DMMoab object
    717: . vhandle - Vertex entity handle

    719:   Output Parameter:
    720: . nconn - Number of entities whose coordinates are needed
    721: . conn - The vertex entity handles

    723:   Level: beginner

    725: .seealso: DMMoabGetVertexCoordinates(), DMMoabRestoreVertexConnectivity()
    726: @*/
    727: PetscErrorCode DMMoabGetVertexConnectivity(DM dm, moab::EntityHandle vhandle, PetscInt* nconn, moab::EntityHandle **conn)
    728: {
    729:   DM_Moab        *dmmoab;
    730:   std::vector<moab::EntityHandle> adj_entities, connect;
    731:   PetscErrorCode  ierr;
    732:   moab::ErrorCode merr;

    737:   dmmoab = (DM_Moab*)(dm)->data;

    739:   /* Get connectivity information in MOAB canonical ordering */
    740:   merr = dmmoab->mbiface->get_adjacencies(&vhandle, 1, 1, true, adj_entities, moab::Interface::UNION); MBERRNM(merr);
    741:   merr = dmmoab->mbiface->get_connectivity(&adj_entities[0], adj_entities.size(), connect); MBERRNM(merr);

    743:   if (conn) {
    744:     PetscMalloc(sizeof(moab::EntityHandle) * connect.size(), conn);
    745:     PetscMemcpy(*conn, &connect[0], sizeof(moab::EntityHandle) * connect.size());
    746:   }
    747:   if (nconn) *nconn = connect.size();
    748:   return(0);
    749: }

    752: /*@
    753:   DMMoabRestoreVertexConnectivity - Restore the vertex connectivity for the given entity

    755:   Collective on MPI_Comm

    757:   Input Parameter:
    758: . dm - The DMMoab object
    759: . vhandle - Vertex entity handle
    760: . nconn - Number of entities whose coordinates are needed
    761: . conn - The vertex entity handles

    763:   Level: beginner

    765: .seealso: DMMoabGetVertexCoordinates(), DMMoabGetVertexConnectivity()
    766: @*/
    767: PetscErrorCode DMMoabRestoreVertexConnectivity(DM dm, moab::EntityHandle ehandle, PetscInt* nconn, moab::EntityHandle **conn)
    768: {
    769:   PetscErrorCode  ierr;

    775:   if (conn) {
    776:     PetscFree(*conn);
    777:   }
    778:   if (nconn) *nconn = 0;
    779:   return(0);
    780: }

    783: /*@
    784:   DMMoabGetElementConnectivity - Get the vertex adjacency for the given entity

    786:   Collective on MPI_Comm

    788:   Input Parameter:
    789: . dm - The DMMoab object
    790: . ehandle - Vertex entity handle

    792:   Output Parameter:
    793: . nconn - Number of entities whose coordinates are needed
    794: . conn - The vertex entity handles

    796:   Level: beginner

    798: .seealso: DMMoabGetVertexCoordinates(), DMMoabGetVertexConnectivity(), DMMoabRestoreVertexConnectivity()
    799: @*/
    800: PetscErrorCode DMMoabGetElementConnectivity(DM dm, moab::EntityHandle ehandle, PetscInt* nconn, const moab::EntityHandle **conn)
    801: {
    802:   DM_Moab        *dmmoab;
    803:   const moab::EntityHandle *connect;
    804:   std::vector<moab::EntityHandle> vconn;
    805:   moab::ErrorCode merr;
    806:   PetscInt nnodes;

    811:   dmmoab = (DM_Moab*)(dm)->data;

    813:   /* Get connectivity information in MOAB canonical ordering */
    814:   merr = dmmoab->mbiface->get_connectivity(ehandle, connect, nnodes); MBERRNM(merr);
    815:   if (conn) *conn = connect;
    816:   if (nconn) *nconn = nnodes;
    817:   return(0);
    818: }

    821: /*@
    822:   DMMoabIsEntityOnBoundary - Check whether a given entity is on the boundary (vertex, edge, face, element)

    824:   Collective on MPI_Comm

    826:   Input Parameter:
    827: . dm - The DMMoab object
    828: . ent - Entity handle

    830:   Output Parameter:
    831: . ent_on_boundary - PETSC_TRUE if entity on boundary; PETSC_FALSE otherwise

    833:   Level: beginner

    835: .seealso: DMMoabCheckBoundaryVertices()
    836: @*/
    837: PetscErrorCode DMMoabIsEntityOnBoundary(DM dm, const moab::EntityHandle ent, PetscBool* ent_on_boundary)
    838: {
    839:   moab::EntityType etype;
    840:   DM_Moab         *dmmoab;
    841:   PetscInt         edim;

    846:   dmmoab = (DM_Moab*)(dm)->data;

    848:   /* get the entity type and handle accordingly */
    849:   etype = dmmoab->mbiface->type_from_handle(ent);
    850:   if (etype >= moab::MBPOLYHEDRON) SETERRQ1(PETSC_COMM_WORLD, PETSC_ERR_ARG_OUTOFRANGE, "Entity type on the boundary skin is invalid. EntityType = %D\n", etype);

    852:   /* get the entity dimension */
    853:   edim = dmmoab->mbiface->dimension_from_handle(ent);

    855:   *ent_on_boundary = PETSC_FALSE;
    856:   if (etype == moab::MBVERTEX && edim == 0) {
    857:     *ent_on_boundary = ((dmmoab->bndyvtx->index(ent) >= 0) ? PETSC_TRUE : PETSC_FALSE);
    858:   }
    859:   else {
    860:     if (edim == dmmoab->dim) {  /* check the higher-dimensional elements first */
    861:       if (dmmoab->bndyelems->index(ent) >= 0) *ent_on_boundary = PETSC_TRUE;
    862:     }
    863:     else {                      /* next check the lower-dimensional faces */
    864:       if (dmmoab->bndyfaces->index(ent) >= 0) *ent_on_boundary = PETSC_TRUE;
    865:     }
    866:   }
    867:   return(0);
    868: }

    871: /*@
    872:   DMMoabIsEntityOnBoundary - Check whether a given entity is on the boundary (vertex, edge, face, element)

    874:   Input Parameter:
    875: . dm - The DMMoab object
    876: . nconn - Number of handles
    877: . cnt - Array of entity handles

    879:   Output Parameter:
    880: . isbdvtx - Array of boundary markers - PETSC_TRUE if entity on boundary; PETSC_FALSE otherwise

    882:   Level: beginner

    884: .seealso: DMMoabIsEntityOnBoundary()
    885: @*/
    886: PetscErrorCode DMMoabCheckBoundaryVertices(DM dm, PetscInt nconn, const moab::EntityHandle *cnt, PetscBool* isbdvtx)
    887: {
    888:   DM_Moab        *dmmoab;
    889:   PetscInt       i;

    895:   dmmoab = (DM_Moab*)(dm)->data;

    897:   for (i = 0; i < nconn; ++i) {
    898:     isbdvtx[i] = (dmmoab->bndyvtx->index(cnt[i]) >= 0 ? PETSC_TRUE : PETSC_FALSE);
    899:   }
    900:   return(0);
    901: }

    904: /*@
    905:   DMMoabGetBoundaryMarkers - Return references to the vertices, faces, elements on the boundary

    907:   Input Parameter:
    908: . dm - The DMMoab object

    910:   Output Parameter:
    911: . bdvtx - Boundary vertices
    912: . bdelems - Boundary elements
    913: . bdfaces - Boundary faces

    915:   Level: beginner

    917: .seealso: DMMoabCheckBoundaryVertices(), DMMoabIsEntityOnBoundary()
    918: @*/
    919: PetscErrorCode DMMoabGetBoundaryMarkers(DM dm, const moab::Range **bdvtx, const moab::Range** bdelems, const moab::Range** bdfaces)
    920: {
    921:   DM_Moab        *dmmoab;

    925:   dmmoab = (DM_Moab*)(dm)->data;

    927:   if (bdvtx)  *bdvtx = dmmoab->bndyvtx;
    928:   if (bdfaces)  *bdfaces = dmmoab->bndyfaces;
    929:   if (bdelems)  *bdfaces = dmmoab->bndyelems;
    930:   return(0);
    931: }

    934: PETSC_EXTERN PetscErrorCode DMDestroy_Moab(DM dm)
    935: {
    936:   PetscErrorCode  ierr;
    937:   PetscInt        i;
    938:   moab::ErrorCode merr;
    939:   DM_Moab        *dmmoab = (DM_Moab*)dm->data;

    944:   dmmoab->refct--;
    945:   if (!dmmoab->refct) {
    946:     delete dmmoab->vlocal;
    947:     delete dmmoab->vowned;
    948:     delete dmmoab->vghost;
    949:     delete dmmoab->elocal;
    950:     delete dmmoab->eghost;
    951:     delete dmmoab->bndyvtx;
    952:     delete dmmoab->bndyfaces;
    953:     delete dmmoab->bndyelems;

    955:     PetscFree(dmmoab->gsindices);
    956:     PetscFree2(dmmoab->gidmap, dmmoab->lidmap);
    957:     PetscFree(dmmoab->dfill);
    958:     PetscFree(dmmoab->ofill);
    959:     PetscFree(dmmoab->materials);
    960:     if (dmmoab->fieldNames) {
    961:       for (i = 0; i < dmmoab->numFields; i++) {
    962:         PetscFree(dmmoab->fieldNames[i]);
    963:       }
    964:       PetscFree(dmmoab->fieldNames);
    965:     }

    967:     if (dmmoab->nhlevels) {
    968:       PetscFree(dmmoab->hsets);
    969:       dmmoab->nhlevels = 0;
    970:       if (!dmmoab->hlevel && dmmoab->icreatedinstance) delete dmmoab->hierarchy;
    971:       dmmoab->hierarchy = NULL;
    972:     }

    974:     if (dmmoab->icreatedinstance) {
    975:       delete dmmoab->pcomm;
    976:       merr = dmmoab->mbiface->delete_mesh(); MBERRNM(merr);
    977:       delete dmmoab->mbiface;
    978:     }
    979:     dmmoab->mbiface = NULL;
    980: #ifdef MOAB_HAVE_MPI
    981:     dmmoab->pcomm = NULL;
    982: #endif
    983:     VecScatterDestroy(&dmmoab->ltog_sendrecv);
    984:     ISLocalToGlobalMappingDestroy(&dmmoab->ltog_map);
    985:     PetscFree(dm->data);
    986:   }
    987:   return(0);
    988: }

    991: PETSC_EXTERN PetscErrorCode DMSetFromOptions_Moab(PetscOptionItems *PetscOptionsObject, DM dm)
    992: {
    994:   DM_Moab        *dmmoab = (DM_Moab*)dm->data;

    998:   PetscOptionsHead(PetscOptionsObject, "DMMoab Options");
    999:   PetscOptionsInt("-dm_moab_rw_dbg", "The verbosity level for reading and writing MOAB meshes", "DMView", dmmoab->rw_dbglevel, &dmmoab->rw_dbglevel, NULL);
    1000:   PetscOptionsBool("-dm_moab_partiton_by_rank", "Use partition by rank when reading MOAB meshes from file", "DMView", dmmoab->partition_by_rank, &dmmoab->partition_by_rank, NULL);
    1001:   /* TODO: typically, the read options are needed before a DM is completely created and available in which case, the options wont be available ?? */
    1002:   PetscOptionsString("-dm_moab_read_opts", "Extra options to enable MOAB reader to load DM from file", "DMView", dmmoab->extra_read_options, dmmoab->extra_read_options, PETSC_MAX_PATH_LEN, NULL);
    1003:   PetscOptionsString("-dm_moab_write_opts", "Extra options to enable MOAB writer to serialize DM to file", "DMView", dmmoab->extra_write_options, dmmoab->extra_write_options, PETSC_MAX_PATH_LEN, NULL);
    1004:   PetscOptionsEnum("-dm_moab_read_mode", "MOAB parallel read mode", "DMView", MoabReadModes, (PetscEnum)dmmoab->read_mode, (PetscEnum*)&dmmoab->read_mode, NULL);
    1005:   PetscOptionsEnum("-dm_moab_write_mode", "MOAB parallel write mode", "DMView", MoabWriteModes, (PetscEnum)dmmoab->write_mode, (PetscEnum*)&dmmoab->write_mode, NULL);
    1006:   return(0);
    1007: }

    1010: PETSC_EXTERN PetscErrorCode DMSetUp_Moab(DM dm)
    1011: {
    1012:   PetscErrorCode          ierr;
    1013:   moab::ErrorCode         merr;
    1014:   Vec                     local, global;
    1015:   IS                      from, to;
    1016:   moab::Range::iterator   iter;
    1017:   PetscInt                i, j, f, bs, vent, totsize, *lgmap;
    1018:   DM_Moab                *dmmoab = (DM_Moab*)dm->data;
    1019:   moab::Range             adjs;

    1023:   /* Get the local and shared vertices and cache it */
    1024:   if (dmmoab->mbiface == NULL) SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ORDER, "Set the MOAB Interface before calling SetUp.");
    1025: #ifdef MOAB_HAVE_MPI
    1026:   if (dmmoab->pcomm == NULL) SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ORDER, "Set the MOAB ParallelComm object before calling SetUp.");
    1027: #endif

    1029:   /* Get the entities recursively in the current part of the mesh, if user did not set the local vertices explicitly */
    1030:   if (dmmoab->vlocal->empty())
    1031:   {
    1032:     //merr = dmmoab->mbiface->get_entities_by_type(dmmoab->fileset,moab::MBVERTEX,*dmmoab->vlocal,true);MBERRNM(merr);
    1033:     merr = dmmoab->mbiface->get_entities_by_dimension(dmmoab->fileset, 0, *dmmoab->vlocal, false); MBERRNM(merr);

    1035: #ifdef MOAB_HAVE_MPI
    1036:     /* filter based on parallel status */
    1037:     merr = dmmoab->pcomm->filter_pstatus(*dmmoab->vlocal, PSTATUS_NOT_OWNED, PSTATUS_NOT, -1, dmmoab->vowned); MBERRNM(merr);

    1039:     /* filter all the non-owned and shared entities out of the list */
    1040:     // *dmmoab->vghost = moab::subtract(*dmmoab->vlocal, *dmmoab->vowned);
    1041:     adjs = moab::subtract(*dmmoab->vlocal, *dmmoab->vowned);
    1042:     merr = dmmoab->pcomm->filter_pstatus(adjs, PSTATUS_GHOST | PSTATUS_INTERFACE, PSTATUS_OR, -1, dmmoab->vghost); MBERRNM(merr);
    1043:     adjs = moab::subtract(adjs, *dmmoab->vghost);
    1044:     *dmmoab->vlocal = moab::subtract(*dmmoab->vlocal, adjs);
    1045: #else
    1046:     *dmmoab->vowned = *dmmoab->vlocal;
    1047: #endif

    1049:     /* compute and cache the sizes of local and ghosted entities */
    1050:     dmmoab->nloc = dmmoab->vowned->size();
    1051:     dmmoab->nghost = dmmoab->vghost->size();

    1053: #ifdef MOAB_HAVE_MPI
    1054:     MPIU_Allreduce(&dmmoab->nloc, &dmmoab->n, 1, MPI_INTEGER, MPI_SUM, ((PetscObject)dm)->comm);
    1055:     PetscInfo4(NULL, "Filset ID: %u, Vertices: local - %D, owned - %D, ghosted - %D.\n", dmmoab->fileset, dmmoab->vlocal->size(), dmmoab->nloc, dmmoab->nghost);
    1056: #else
    1057:     dmmoab->n = dmmoab->nloc;
    1058: #endif
    1059:   }

    1061:   {
    1062:     /* get the information about the local elements in the mesh */
    1063:     dmmoab->eghost->clear();

    1065:     /* first decipher the leading dimension */
    1066:     for (i = 3; i > 0; i--) {
    1067:       dmmoab->elocal->clear();
    1068:       merr = dmmoab->mbiface->get_entities_by_dimension(dmmoab->fileset, i, *dmmoab->elocal, false); MBERRNM(merr);

    1070:       /* store the current mesh dimension */
    1071:       if (dmmoab->elocal->size()) {
    1072:         dmmoab->dim = i;
    1073:         break;
    1074:       }
    1075:     }

    1077:     DMSetDimension(dm, dmmoab->dim);

    1079: #ifdef MOAB_HAVE_MPI
    1080:     /* filter the ghosted and owned element list */
    1081:     *dmmoab->eghost = *dmmoab->elocal;
    1082:     merr = dmmoab->pcomm->filter_pstatus(*dmmoab->elocal, PSTATUS_NOT_OWNED, PSTATUS_NOT); MBERRNM(merr);
    1083:     *dmmoab->eghost = moab::subtract(*dmmoab->eghost, *dmmoab->elocal);
    1084: #endif

    1086:     dmmoab->neleloc = dmmoab->elocal->size();
    1087:     dmmoab->neleghost = dmmoab->eghost->size();

    1089: #ifdef MOAB_HAVE_MPI
    1090:     MPIU_Allreduce(&dmmoab->neleloc, &dmmoab->nele, 1, MPI_INTEGER, MPI_SUM, ((PetscObject)dm)->comm);
    1091:     PetscInfo3(NULL, "%d-dim elements: owned - %D, ghosted - %D.\n", dmmoab->dim, dmmoab->neleloc, dmmoab->neleghost);
    1092: #else
    1093:     dmmoab->nele = dmmoab->neleloc;
    1094: #endif
    1095:   }

    1097:   bs = dmmoab->bs;
    1098:   if (!dmmoab->ltog_tag) {
    1099:     /* Get the global ID tag. The global ID tag is applied to each
    1100:        vertex. It acts as an global identifier which MOAB uses to
    1101:        assemble the individual pieces of the mesh */
    1102:     merr = dmmoab->mbiface->tag_get_handle(GLOBAL_ID_TAG_NAME, dmmoab->ltog_tag); MBERRNM(merr);
    1103:   }

    1105:   totsize = dmmoab->vlocal->size();
    1106:   if (totsize != dmmoab->nloc + dmmoab->nghost) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Mismatch between local and owned+ghost vertices. %D != %D.", totsize, dmmoab->nloc + dmmoab->nghost);
    1107:   PetscCalloc1(totsize, &dmmoab->gsindices);
    1108:   {
    1109:     /* first get the local indices */
    1110:     merr = dmmoab->mbiface->tag_get_data(dmmoab->ltog_tag, *dmmoab->vowned, &dmmoab->gsindices[0]); MBERRNM(merr);
    1111:     if (dmmoab->nghost) {  /* next get the ghosted indices */
    1112:       merr = dmmoab->mbiface->tag_get_data(dmmoab->ltog_tag, *dmmoab->vghost, &dmmoab->gsindices[dmmoab->nloc]); MBERRNM(merr);
    1113:     }

    1115:     /* find out the local and global minima of GLOBAL_ID */
    1116:     dmmoab->lminmax[0] = dmmoab->lminmax[1] = dmmoab->gsindices[0];
    1117:     for (i = 0; i < totsize; ++i) {
    1118:       if (dmmoab->lminmax[0] > dmmoab->gsindices[i]) dmmoab->lminmax[0] = dmmoab->gsindices[i];
    1119:       if (dmmoab->lminmax[1] < dmmoab->gsindices[i]) dmmoab->lminmax[1] = dmmoab->gsindices[i];
    1120:     }

    1122:     MPIU_Allreduce(&dmmoab->lminmax[0], &dmmoab->gminmax[0], 1, MPI_INT, MPI_MIN, ((PetscObject)dm)->comm);
    1123:     MPIU_Allreduce(&dmmoab->lminmax[1], &dmmoab->gminmax[1], 1, MPI_INT, MPI_MAX, ((PetscObject)dm)->comm);

    1125:     /* set the GID map */
    1126:     for (i = 0; i < totsize; ++i) {
    1127:       dmmoab->gsindices[i] -= dmmoab->gminmax[0]; /* zero based index needed for IS */

    1129:     }
    1130:     dmmoab->lminmax[0] -= dmmoab->gminmax[0];
    1131:     dmmoab->lminmax[1] -= dmmoab->gminmax[0];

    1133:     PetscInfo4(NULL, "GLOBAL_ID: Local [min, max] - [%D, %D], Global [min, max] - [%D, %D]\n", dmmoab->lminmax[0], dmmoab->lminmax[1], dmmoab->gminmax[0], dmmoab->gminmax[1]);
    1134:   }
    1135:   if (!(dmmoab->bs == dmmoab->numFields || dmmoab->bs == 1)) SETERRQ3(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mismatch between block size and number of component fields. %D != 1 OR %D != %D.", dmmoab->bs, dmmoab->bs, dmmoab->numFields);

    1137:   {
    1138:     dmmoab->seqstart = dmmoab->mbiface->id_from_handle(dmmoab->vlocal->front());
    1139:     dmmoab->seqend = dmmoab->mbiface->id_from_handle(dmmoab->vlocal->back());
    1140:     PetscInfo2(NULL, "SEQUENCE: Local [min, max] - [%D, %D]\n", dmmoab->seqstart, dmmoab->seqend);

    1142:     PetscMalloc2(dmmoab->seqend - dmmoab->seqstart + 1, &dmmoab->gidmap, dmmoab->seqend - dmmoab->seqstart + 1, &dmmoab->lidmap);
    1143:     PetscMalloc1(totsize * dmmoab->numFields, &lgmap);

    1145:     i = j = 0;
    1146:     /* set the owned vertex data first */
    1147:     for (moab::Range::iterator iter = dmmoab->vowned->begin(); iter != dmmoab->vowned->end(); iter++, i++) {
    1148:       vent = dmmoab->mbiface->id_from_handle(*iter) - dmmoab->seqstart;
    1149:       dmmoab->gidmap[vent] = dmmoab->gsindices[i];
    1150:       dmmoab->lidmap[vent] = i;
    1151:       for (f = 0; f < dmmoab->numFields; f++, j++) {
    1152:         lgmap[j] = (bs > 1 ? dmmoab->gsindices[i] * dmmoab->numFields + f : totsize * f + dmmoab->gsindices[i]);
    1153:       }
    1154:     }
    1155:     /* next arrange all the ghosted data information */
    1156:     for (moab::Range::iterator iter = dmmoab->vghost->begin(); iter != dmmoab->vghost->end(); iter++, i++) {
    1157:       vent = dmmoab->mbiface->id_from_handle(*iter) - dmmoab->seqstart;
    1158:       dmmoab->gidmap[vent] = dmmoab->gsindices[i];
    1159:       dmmoab->lidmap[vent] = i;
    1160:       for (f = 0; f < dmmoab->numFields; f++, j++) {
    1161:         lgmap[j] = (bs > 1 ? dmmoab->gsindices[i] * dmmoab->numFields + f : totsize * f + dmmoab->gsindices[i]);
    1162:       }
    1163:     }

    1165:     /* We need to create the Global to Local Vector Scatter Contexts
    1166:        1) First create a local and global vector
    1167:        2) Create a local and global IS
    1168:        3) Create VecScatter and LtoGMapping objects
    1169:        4) Cleanup the IS and Vec objects
    1170:     */
    1171:     DMCreateGlobalVector(dm, &global);
    1172:     DMCreateLocalVector(dm, &local);

    1174:     VecGetOwnershipRange(global, &dmmoab->vstart, &dmmoab->vend);

    1176:     /* global to local must retrieve ghost points */
    1177:     ISCreateStride(((PetscObject)dm)->comm, dmmoab->nloc * dmmoab->numFields, dmmoab->vstart, 1, &from);
    1178:     ISSetBlockSize(from, bs);

    1180:     ISCreateGeneral(((PetscObject)dm)->comm, dmmoab->nloc * dmmoab->numFields, &lgmap[0], PETSC_COPY_VALUES, &to);
    1181:     ISSetBlockSize(to, bs);

    1183:     if (!dmmoab->ltog_map) {
    1184:       /* create to the local to global mapping for vectors in order to use VecSetValuesLocal */
    1185:       ISLocalToGlobalMappingCreate(((PetscObject)dm)->comm, dmmoab->bs, totsize * dmmoab->numFields, lgmap,
    1186:                                           PETSC_COPY_VALUES, &dmmoab->ltog_map);
    1187:     }

    1189:     /* now create the scatter object from local to global vector */
    1190:     VecScatterCreate(local, from, global, to, &dmmoab->ltog_sendrecv);

    1192:     /* clean up IS, Vec */
    1193:     PetscFree(lgmap);
    1194:     ISDestroy(&from);
    1195:     ISDestroy(&to);
    1196:     VecDestroy(&local);
    1197:     VecDestroy(&global);
    1198:   }

    1200:   dmmoab->bndyvtx = new moab::Range();
    1201:   dmmoab->bndyfaces = new moab::Range();
    1202:   dmmoab->bndyelems = new moab::Range();
    1203:   /* skin the boundary and store nodes */
    1204:   if (!dmmoab->hlevel) {
    1205:     /* get the skin vertices of boundary faces for the current partition and then filter
    1206:        the local, boundary faces, vertices and elements alone via PSTATUS flags;
    1207:        this should not give us any ghosted boundary, but if user needs such a functionality
    1208:        it would be easy to add it based on the find_skin query below */
    1209:     moab::Skinner skinner(dmmoab->mbiface);

    1211:     /* get the entities on the skin - only the faces */
    1212:     merr = skinner.find_skin(dmmoab->fileset, *dmmoab->elocal, false, *dmmoab->bndyfaces, NULL, true, true, false); MBERRNM(merr); // 'false' param indicates we want faces back, not vertices

    1214: #ifdef MOAB_HAVE_MPI
    1215:     /* filter all the non-owned and shared entities out of the list */
    1216:     merr = dmmoab->pcomm->filter_pstatus(*dmmoab->bndyfaces, PSTATUS_NOT_OWNED, PSTATUS_NOT); MBERRNM(merr);
    1217:     merr = dmmoab->pcomm->filter_pstatus(*dmmoab->bndyfaces, PSTATUS_INTERFACE, PSTATUS_NOT); MBERRNM(merr);
    1218: #endif

    1220:     /* get all the nodes via connectivity and the parent elements via adjacency information */
    1221:     merr = dmmoab->mbiface->get_connectivity(*dmmoab->bndyfaces, *dmmoab->bndyvtx, false); MBERRNM(ierr);
    1222:     merr = dmmoab->mbiface->get_adjacencies(*dmmoab->bndyvtx, dmmoab->dim, false, *dmmoab->bndyelems, moab::Interface::UNION); MBERRNM(ierr);
    1223:   }
    1224:   else {
    1225:     /* Let us query the hierarchy manager and get the results directly for this level */
    1226:     for (moab::Range::iterator iter = dmmoab->elocal->begin(); iter != dmmoab->elocal->end(); iter++) {
    1227:       moab::EntityHandle elemHandle = *iter;
    1228:       if (dmmoab->hierarchy->is_entity_on_boundary(elemHandle)) {
    1229:         dmmoab->bndyelems->insert(elemHandle);
    1230:         /* For this boundary element, query the vertices and add them to the list */
    1231:         std::vector<moab::EntityHandle> connect;
    1232:         merr = dmmoab->hierarchy->get_connectivity(elemHandle, dmmoab->hlevel, connect); MBERRNM(ierr);
    1233:         for (unsigned iv=0; iv < connect.size(); ++iv)
    1234:           if (dmmoab->hierarchy->is_entity_on_boundary(connect[iv]))
    1235:             dmmoab->bndyvtx->insert(connect[iv]);
    1236:         /* Next, let us query the boundary faces and add them also to the list */
    1237:         std::vector<moab::EntityHandle> faces;
    1238:         merr = dmmoab->hierarchy->get_adjacencies(elemHandle, dmmoab->dim-1, faces); MBERRNM(ierr);
    1239:         for (unsigned ifa=0; ifa < faces.size(); ++ifa)
    1240:           if (dmmoab->hierarchy->is_entity_on_boundary(faces[ifa]))
    1241:             dmmoab->bndyfaces->insert(faces[ifa]);
    1242:       }
    1243:     }
    1244: #ifdef MOAB_HAVE_MPI
    1245:     /* filter all the non-owned and shared entities out of the list */
    1246:     merr = dmmoab->pcomm->filter_pstatus(*dmmoab->bndyvtx,   PSTATUS_NOT_OWNED, PSTATUS_NOT); MBERRNM(merr);
    1247:     merr = dmmoab->pcomm->filter_pstatus(*dmmoab->bndyfaces, PSTATUS_NOT_OWNED, PSTATUS_NOT); MBERRNM(merr);
    1248:     merr = dmmoab->pcomm->filter_pstatus(*dmmoab->bndyelems, PSTATUS_NOT_OWNED, PSTATUS_NOT); MBERRNM(merr);
    1249: #endif

    1251:   }
    1252:   PetscInfo3(NULL, "Found %D boundary vertices, %D boundary faces and %D boundary elements.\n", dmmoab->bndyvtx->size(), dmmoab->bndyfaces->size(), dmmoab->bndyelems->size());

    1254:   /* Get the material sets and populate the data for all locally owned elements */
    1255:   {
    1256:     PetscCalloc1(dmmoab->elocal->size(), &dmmoab->materials);
    1257:     /* Get the count of entities of particular type from dmmoab->elocal
    1258:        -- Then, for each non-zero type, loop through and query the fileset to get the material tag data */
    1259:     moab::Range msets;
    1260:     merr = dmmoab->mbiface->get_entities_by_type_and_tag(dmmoab->fileset, moab::MBENTITYSET, &dmmoab->material_tag, NULL, 1, msets, moab::Interface::UNION);MB_CHK_ERR(merr);
    1261:     if (msets.size() == 0) {
    1262:       PetscInfo(NULL, "No material sets found in the fileset.");
    1263:     }

    1265:     for (unsigned i=0; i < msets.size(); ++i) {
    1266:       moab::Range msetelems;
    1267:       merr = dmmoab->mbiface->get_entities_by_dimension(msets[i], dmmoab->dim, msetelems, true);MB_CHK_ERR(merr);
    1268: #ifdef MOAB_HAVE_MPI
    1269:       /* filter all the non-owned and shared entities out of the list */
    1270:       merr = dmmoab->pcomm->filter_pstatus(msetelems, PSTATUS_NOT_OWNED, PSTATUS_NOT); MBERRNM(merr);
    1271: #endif

    1273:       int partID;
    1274:       moab::EntityHandle mset=msets[i];
    1275:       merr = dmmoab->mbiface->tag_get_data(dmmoab->material_tag, &mset, 1, &partID);MB_CHK_ERR(merr);

    1277:       for (unsigned j=0; j < msetelems.size(); ++j)
    1278:         dmmoab->materials[dmmoab->elocal->index(msetelems[j])]=partID;
    1279:     }
    1280:   }

    1282:   return(0);
    1283: }

    1286: /*@
    1287:   DMMoabCreateVertices - Creates and adds several vertices to the primary set represented by the DM.

    1289:   Collective on MPI_Comm

    1291:   Input Parameters:
    1292: + dm - The DM object
    1293: . type - The type of element to create and add (Edge/Tri/Quad/Tet/Hex/Prism/Pyramid/Polygon/Polyhedra)
    1294: . conn - The connectivity of the element
    1295: . nverts - The number of vertices that form the element

    1297:   Output Parameter:
    1298: . overts  - The list of vertices that were created (can be NULL)

    1300:   Level: beginner

    1302: .keywords: DM, create vertices

    1304: .seealso: DMMoabCreateSubmesh(), DMMoabCreateElement()
    1305: @*/
    1306: PetscErrorCode DMMoabCreateVertices(DM dm, const PetscReal* coords, PetscInt nverts, moab::Range* overts)
    1307: {
    1308:   moab::ErrorCode     merr;
    1309:   DM_Moab            *dmmoab;
    1310:   moab::Range         verts;

    1316:   dmmoab = (DM_Moab*) dm->data;

    1318:   /* Insert new points */
    1319:   merr = dmmoab->mbiface->create_vertices(&coords[0], nverts, verts); MBERRNM(merr);
    1320:   merr = dmmoab->mbiface->add_entities(dmmoab->fileset, verts); MBERRNM(merr);

    1322:   if (overts) *overts = verts;
    1323:   return(0);
    1324: }

    1327: /*@
    1328:   DMMoabCreateElement - Adds an element of specified type to the primary set represented by the DM.

    1330:   Collective on MPI_Comm

    1332:   Input Parameters:
    1333: + dm - The DM object
    1334: . type - The type of element to create and add (Edge/Tri/Quad/Tet/Hex/Prism/Pyramid/Polygon/Polyhedra)
    1335: . conn - The connectivity of the element
    1336: . nverts - The number of vertices that form the element

    1338:   Output Parameter:
    1339: . oelem  - The handle to the element created and added to the DM object

    1341:   Level: beginner

    1343: .keywords: DM, create element

    1345: .seealso: DMMoabCreateSubmesh(), DMMoabCreateVertices()
    1346: @*/
    1347: PetscErrorCode DMMoabCreateElement(DM dm, const moab::EntityType type, const moab::EntityHandle* conn, PetscInt nverts, moab::EntityHandle* oelem)
    1348: {
    1349:   moab::ErrorCode     merr;
    1350:   DM_Moab            *dmmoab;
    1351:   moab::EntityHandle  elem;

    1357:   dmmoab = (DM_Moab*) dm->data;

    1359:   /* Insert new element */
    1360:   merr = dmmoab->mbiface->create_element(type, conn, nverts, elem); MBERRNM(merr);
    1361:   merr = dmmoab->mbiface->add_entities(dmmoab->fileset, &elem, 1); MBERRNM(merr);

    1363:   if (oelem) *oelem = elem;
    1364:   return(0);
    1365: }

    1368: /*@
    1369:   DMMoabCreateSubmesh - Creates a sub-DM object with a set that contains all vertices/elements of the parent
    1370:   in addition to providing support for dynamic mesh modifications. This is useful for AMR calculations to
    1371:   create a DM object on a refined level.

    1373:   Collective on MPI_Comm

    1375:   Input Parameters:
    1376: + dm - The DM object

    1378:   Output Parameter:
    1379: . newdm  - The sub DM object with updated set information

    1381:   Level: advanced

    1383: .keywords: DM, sub-DM

    1385: .seealso: DMCreate(), DMMoabCreateVertices(), DMMoabCreateElement()
    1386: @*/
    1387: PetscErrorCode DMMoabCreateSubmesh(DM dm, DM *newdm)
    1388: {
    1389:   DM_Moab            *dmmoab;
    1390:   DM_Moab            *ndmmoab;
    1391:   moab::ErrorCode    merr;
    1392:   PetscErrorCode     ierr;

    1397:   dmmoab = (DM_Moab*) dm->data;

    1399:   /* Create the basic DMMoab object and keep the default parameters created by DM impls */
    1400:   DMMoabCreateMoab(((PetscObject)dm)->comm, dmmoab->mbiface, &dmmoab->ltog_tag, PETSC_NULL, newdm);

    1402:   /* get all the necessary handles from the private DM object */
    1403:   ndmmoab = (DM_Moab*) (*newdm)->data;

    1405:   /* set the sub-mesh's parent DM reference */
    1406:   ndmmoab->parent = &dm;

    1408:   /* create a file set to associate all entities in current mesh */
    1409:   merr = ndmmoab->mbiface->create_meshset(moab::MESHSET_SET, ndmmoab->fileset); MBERR("Creating file set failed", merr);

    1411:   /* create a meshset and then add old fileset as child */
    1412:   merr = ndmmoab->mbiface->add_entities(ndmmoab->fileset, *dmmoab->vlocal); MBERR("Adding child vertices to parent failed", merr);
    1413:   merr = ndmmoab->mbiface->add_entities(ndmmoab->fileset, *dmmoab->elocal); MBERR("Adding child elements to parent failed", merr);

    1415:   /* preserve the field association between the parent and sub-mesh objects */
    1416:   DMMoabSetFieldNames(*newdm, dmmoab->numFields, dmmoab->fieldNames);
    1417:   return(0);
    1418: }

    1421: PETSC_EXTERN PetscErrorCode DMMoabView_Ascii(DM dm, PetscViewer viewer)
    1422: {
    1423:   DM_Moab          *dmmoab = (DM_Moab*)(dm)->data;
    1424:   const char       *name;
    1425:   MPI_Comm          comm;
    1426:   PetscMPIInt       size;
    1427:   PetscErrorCode    ierr;

    1430:   PetscObjectGetComm((PetscObject)dm, &comm);
    1431:   MPI_Comm_size(comm, &size);
    1432:   PetscObjectGetName((PetscObject) dm, &name);
    1433:   PetscViewerASCIIPushTab(viewer);
    1434:   if (name) {PetscViewerASCIIPrintf(viewer, "%s in %D dimensions:\n", name, dmmoab->dim);}
    1435:   else      {PetscViewerASCIIPrintf(viewer, "Mesh in %D dimensions:\n", dmmoab->dim);}
    1436:   /* print details about the global mesh */
    1437:   {
    1438:     PetscViewerASCIIPushTab(viewer);
    1439:     PetscViewerASCIIPrintf(viewer, "Sizes: cells=%D, vertices=%D, blocks=%D\n", dmmoab->nele, dmmoab->n, dmmoab->bs);
    1440:     /* print boundary data */
    1441:     PetscViewerASCIIPrintf(viewer, "Boundary trace:\n", dmmoab->bndyelems->size(), dmmoab->bndyfaces->size(), dmmoab->bndyvtx->size());
    1442:     {
    1443:       PetscViewerASCIIPushTab(viewer);
    1444:       PetscViewerASCIIPrintf(viewer, "cells=%D, faces=%D, vertices=%D\n", dmmoab->bndyelems->size(), dmmoab->bndyfaces->size(), dmmoab->bndyvtx->size());
    1445:       PetscViewerASCIIPopTab(viewer);
    1446:     }
    1447:     /* print field data */
    1448:     PetscViewerASCIIPrintf(viewer, "Fields: %D components\n", dmmoab->numFields);
    1449:     {
    1450:       PetscViewerASCIIPushTab(viewer);
    1451:       for (int i = 0; i < dmmoab->numFields; ++i) {
    1452:         PetscViewerASCIIPrintf(viewer, "[%D] - %s\n", i, dmmoab->fieldNames[i]);
    1453:       }
    1454:       PetscViewerASCIIPopTab(viewer);
    1455:     }
    1456:     PetscViewerASCIIPopTab(viewer);
    1457:   }
    1458:   PetscViewerASCIIPopTab(viewer);
    1459:   PetscViewerFlush(viewer);
    1460:   return(0);
    1461: }

    1463: PETSC_EXTERN PetscErrorCode DMMoabView_VTK(DM dm, PetscViewer v)
    1464: {
    1465:   return(0);
    1466: }

    1468: PETSC_EXTERN PetscErrorCode DMMoabView_HDF5(DM dm, PetscViewer v)
    1469: {
    1470:   return(0);
    1471: }

    1473: PETSC_EXTERN PetscErrorCode DMView_Moab(DM dm, PetscViewer viewer)
    1474: {
    1475:   PetscBool      iascii, ishdf5, isvtk;

    1481:   PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERASCII, &iascii);
    1482:   PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERVTK,   &isvtk);
    1483:   PetscObjectTypeCompare((PetscObject) viewer, PETSCVIEWERHDF5,  &ishdf5);
    1484:   if (iascii) {
    1485:     DMMoabView_Ascii(dm, viewer);
    1486:   } else if (ishdf5) {
    1487: #if defined(PETSC_HAVE_HDF5) && defined(MOAB_HAVE_HDF5)
    1488:     PetscViewerPushFormat(viewer, PETSC_VIEWER_HDF5_VIZ);
    1489:     DMMoabView_HDF5(dm, viewer);
    1490:     PetscViewerPopFormat(viewer);
    1491: #else
    1492:     SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "HDF5 not supported in this build.\nPlease reconfigure using --download-hdf5");
    1493: #endif
    1494:   }
    1495:   else if (isvtk) {
    1496:     DMMoabView_VTK(dm, viewer);
    1497:   }
    1498:   return(0);
    1499: }

    1502: PETSC_EXTERN PetscErrorCode DMInitialize_Moab(DM dm)
    1503: {
    1505:   dm->ops->view                            = DMView_Moab;
    1506:   dm->ops->load                            = NULL /* DMLoad_Moab */;
    1507:   dm->ops->setfromoptions                  = DMSetFromOptions_Moab;
    1508:   dm->ops->clone                           = DMClone_Moab;
    1509:   dm->ops->setup                           = DMSetUp_Moab;
    1510:   dm->ops->createdefaultsection            = NULL;
    1511:   dm->ops->createdefaultconstraints        = NULL;
    1512:   dm->ops->createglobalvector              = DMCreateGlobalVector_Moab;
    1513:   dm->ops->createlocalvector               = DMCreateLocalVector_Moab;
    1514:   dm->ops->getlocaltoglobalmapping         = NULL;
    1515:   dm->ops->createfieldis                   = NULL;
    1516:   dm->ops->createcoordinatedm              = NULL /* DMCreateCoordinateDM_Moab */;
    1517:   dm->ops->getcoloring                     = NULL;
    1518:   dm->ops->creatematrix                    = DMCreateMatrix_Moab;
    1519:   dm->ops->createinterpolation             = DMCreateInterpolation_Moab;
    1520:   dm->ops->getaggregates                   = NULL;
    1521:   dm->ops->getinjection                    = NULL /* DMCreateInjection_Moab */;
    1522:   dm->ops->refine                          = DMRefine_Moab;
    1523:   dm->ops->coarsen                         = DMCoarsen_Moab;
    1524:   dm->ops->refinehierarchy                 = DMRefineHierarchy_Moab;
    1525:   dm->ops->coarsenhierarchy                = DMCoarsenHierarchy_Moab;
    1526:   dm->ops->globaltolocalbegin              = DMGlobalToLocalBegin_Moab;
    1527:   dm->ops->globaltolocalend                = DMGlobalToLocalEnd_Moab;
    1528:   dm->ops->localtoglobalbegin              = DMLocalToGlobalBegin_Moab;
    1529:   dm->ops->localtoglobalend                = DMLocalToGlobalEnd_Moab;
    1530:   dm->ops->destroy                         = DMDestroy_Moab;
    1531:   dm->ops->createsubdm                     = NULL /* DMCreateSubDM_Moab */;
    1532:   dm->ops->getdimpoints                    = NULL /* DMGetDimPoints_Moab */;
    1533:   dm->ops->locatepoints                    = NULL /* DMLocatePoints_Moab */;
    1534:   return(0);
    1535: }

    1538: PETSC_EXTERN PetscErrorCode DMClone_Moab(DM dm, DM *newdm)
    1539: {
    1540:   PetscErrorCode     ierr;

    1543:   PetscObjectChangeTypeName((PetscObject) * newdm, DMMOAB);

    1545:   /* get all the necessary handles from the private DM object */
    1546:   (*newdm)->data = (DM_Moab*) dm->data;
    1547:   ((DM_Moab*)dm->data)->refct++;

    1549:   DMInitialize_Moab(*newdm);
    1550:   return(0);
    1551: }

    1554: PETSC_EXTERN PetscErrorCode DMCreate_Moab(DM dm)
    1555: {

    1560:   PetscNewLog(dm, (DM_Moab**)&dm->data);

    1562:   ((DM_Moab*)dm->data)->bs = 1;
    1563:   ((DM_Moab*)dm->data)->numFields = 1;
    1564:   ((DM_Moab*)dm->data)->n = 0;
    1565:   ((DM_Moab*)dm->data)->nloc = 0;
    1566:   ((DM_Moab*)dm->data)->nghost = 0;
    1567:   ((DM_Moab*)dm->data)->nele = 0;
    1568:   ((DM_Moab*)dm->data)->neleloc = 0;
    1569:   ((DM_Moab*)dm->data)->neleghost = 0;
    1570:   ((DM_Moab*)dm->data)->ltog_map = NULL;
    1571:   ((DM_Moab*)dm->data)->ltog_sendrecv = NULL;

    1573:   ((DM_Moab*)dm->data)->refct = 1;
    1574:   ((DM_Moab*)dm->data)->parent = NULL;
    1575:   ((DM_Moab*)dm->data)->vlocal = new moab::Range();
    1576:   ((DM_Moab*)dm->data)->vowned = new moab::Range();
    1577:   ((DM_Moab*)dm->data)->vghost = new moab::Range();
    1578:   ((DM_Moab*)dm->data)->elocal = new moab::Range();
    1579:   ((DM_Moab*)dm->data)->eghost = new moab::Range();

    1581:   DMInitialize_Moab(dm);
    1582:   return(0);
    1583: }