Actual source code: fretrieve.c
petsc-3.10.5 2019-03-28
Report Typos and Errors

      2: /*
      3:       Code for opening and closing files.
      4: */
      5:  #include <petscsys.h>
      6: #if defined(PETSC_HAVE_PWD_H)
      7: #include <pwd.h>
      8: #endif
      9: #include <ctype.h>
     10: #include <sys/stat.h>
     11: #if defined(PETSC_HAVE_UNISTD_H)
     12: #include <unistd.h>
     13: #endif
     14: #if defined(PETSC_HAVE_SYS_UTSNAME_H)
     15: #include <sys/utsname.h>
     16: #endif
     17: #include <fcntl.h>
     18: #include <time.h>
     19: #if defined(PETSC_HAVE_SYS_SYSTEMINFO_H)
     20: #include <sys/systeminfo.h>
     21: #endif

     23: /*
     24:    Private routine to delete tmp/shared storage

     26:    This is called by MPI, not by users.

     28:    Note: this is declared extern "C" because it is passed to MPI_Comm_create_keyval()

     30: */
     31: PETSC_EXTERN PetscMPIInt MPIAPI Petsc_DelTmpShared(MPI_Comm comm,PetscMPIInt keyval,void *count_val,void *extra_state)
     32: {

     36:   PetscInfo1(0,"Deleting tmp/shared data in an MPI_Comm %ld\n",(long)comm);CHKERRMPI(ierr);
     37:   PetscFree(count_val);CHKERRMPI(ierr);
     38:   PetscFunctionReturn(MPI_SUCCESS);
     39: }

     41: /*@C
     42:    PetscGetTmp - Gets the name of the tmp directory

     44:    Collective on MPI_Comm

     46:    Input Parameters:
     47: +  comm - MPI_Communicator that may share /tmp
     48: -  len - length of string to hold name

     50:    Output Parameters:
     51: .  dir - directory name

     53:    Options Database Keys:
     54: +    -shared_tmp
     55: .    -not_shared_tmp
     56: -    -tmp tmpdir

     58:    Environmental Variables:
     59: +     PETSC_SHARED_TMP
     60: .     PETSC_NOT_SHARED_TMP
     61: -     PETSC_TMP

     63:    Level: developer

     66:    If the environmental variable PETSC_TMP is set it will use this directory
     67:   as the "/tmp" directory.

     69: @*/
     70: PetscErrorCode  PetscGetTmp(MPI_Comm comm,char dir[],size_t len)
     71: {
     73:   PetscBool      flg;

     76:   PetscOptionsGetenv(comm,"PETSC_TMP",dir,len,&flg);
     77:   if (!flg) {
     78:     PetscStrncpy(dir,"/tmp",len);
     79:   }
     80:   return(0);
     81: }

     83: /*@C
     84:    PetscSharedTmp - Determines if all processors in a communicator share a
     85:          /tmp or have different ones.

     87:    Collective on MPI_Comm

     89:    Input Parameters:
     90: .  comm - MPI_Communicator that may share /tmp

     92:    Output Parameters:
     93: .  shared - PETSC_TRUE or PETSC_FALSE

     95:    Options Database Keys:
     96: +    -shared_tmp
     97: .    -not_shared_tmp
     98: -    -tmp tmpdir

    100:    Environmental Variables:
    101: +     PETSC_SHARED_TMP
    102: .     PETSC_NOT_SHARED_TMP
    103: -     PETSC_TMP

    105:    Level: developer

    107:    Notes:
    108:    Stores the status as a MPI attribute so it does not have
    109:     to be redetermined each time.

    111:       Assumes that all processors in a communicator either
    112:        1) have a common /tmp or
    113:        2) each has a separate /tmp
    114:       eventually we can write a fancier one that determines which processors
    115:       share a common /tmp.

    117:    This will be very slow on runs with a large number of processors since
    118:    it requires O(p*p) file opens.

    120:    If the environmental variable PETSC_TMP is set it will use this directory
    121:   as the "/tmp" directory.

    123: @*/
    124: PetscErrorCode  PetscSharedTmp(MPI_Comm comm,PetscBool  *shared)
    125: {
    126:   PetscErrorCode     ierr;
    127:   PetscMPIInt        size,rank,*tagvalp,sum,cnt,i;
    128:   PetscBool          flg,iflg;
    129:   FILE               *fd;
    130:   static PetscMPIInt Petsc_Tmp_keyval = MPI_KEYVAL_INVALID;
    131:   int                err;

    134:   MPI_Comm_size(comm,&size);
    135:   if (size == 1) {
    136:     *shared = PETSC_TRUE;
    137:     return(0);
    138:   }

    140:   PetscOptionsGetenv(comm,"PETSC_SHARED_TMP",NULL,0,&flg);
    141:   if (flg) {
    142:     *shared = PETSC_TRUE;
    143:     return(0);
    144:   }

    146:   PetscOptionsGetenv(comm,"PETSC_NOT_SHARED_TMP",NULL,0,&flg);
    147:   if (flg) {
    148:     *shared = PETSC_FALSE;
    149:     return(0);
    150:   }

    152:   if (Petsc_Tmp_keyval == MPI_KEYVAL_INVALID) {
    153:     MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN,Petsc_DelTmpShared,&Petsc_Tmp_keyval,0);
    154:   }

    156:   MPI_Comm_get_attr(comm,Petsc_Tmp_keyval,(void**)&tagvalp,(int*)&iflg);
    157:   if (!iflg) {
    158:     char filename[PETSC_MAX_PATH_LEN],tmpname[PETSC_MAX_PATH_LEN];

    160:     /* This communicator does not yet have a shared tmp attribute */
    161:     PetscMalloc1(1,&tagvalp);
    162:     MPI_Comm_set_attr(comm,Petsc_Tmp_keyval,tagvalp);

    164:     PetscOptionsGetenv(comm,"PETSC_TMP",tmpname,238,&iflg);
    165:     if (!iflg) {
    166:       PetscStrcpy(filename,"/tmp");
    167:     } else {
    168:       PetscStrcpy(filename,tmpname);
    169:     }

    171:     PetscStrcat(filename,"/petsctestshared");
    172:     MPI_Comm_rank(comm,&rank);

    174:     /* each processor creates a /tmp file and all the later ones check */
    175:     /* this makes sure no subset of processors is shared */
    176:     *shared = PETSC_FALSE;
    177:     for (i=0; i<size-1; i++) {
    178:       if (rank == i) {
    179:         fd = fopen(filename,"w");
    180:         if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open test file %s",filename);
    181:         err = fclose(fd);
    182:         if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
    183:       }
    184:       MPI_Barrier(comm);
    185:       if (rank >= i) {
    186:         fd = fopen(filename,"r");
    187:         if (fd) cnt = 1;
    188:         else cnt = 0;
    189:         if (fd) {
    190:           err = fclose(fd);
    191:           if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
    192:         }
    193:       } else cnt = 0;

    195:       MPIU_Allreduce(&cnt,&sum,1,MPI_INT,MPI_SUM,comm);
    196:       if (rank == i) unlink(filename);

    198:       if (sum == size) {
    199:         *shared = PETSC_TRUE;
    200:         break;
    201:       } else if (sum != 1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP_SYS,"Subset of processes share /tmp ");
    202:     }
    203:     *tagvalp = (int)*shared;
    204:     PetscInfo2(0,"processors %s %s\n",(*shared) ? "share":"do NOT share",(iflg ? tmpname:"/tmp"));
    205:   } else *shared = (PetscBool) *tagvalp;
    206:   return(0);
    207: }

    209: /*@C
    210:    PetscSharedWorkingDirectory - Determines if all processors in a communicator share a
    211:          working directory or have different ones.

    213:    Collective on MPI_Comm

    215:    Input Parameters:
    216: .  comm - MPI_Communicator that may share working directory

    218:    Output Parameters:
    219: .  shared - PETSC_TRUE or PETSC_FALSE

    221:    Options Database Keys:
    222: +    -shared_working_directory
    223: .    -not_shared_working_directory

    225:    Environmental Variables:
    226: +     PETSC_SHARED_WORKING_DIRECTORY
    227: .     PETSC_NOT_SHARED_WORKING_DIRECTORY

    229:    Level: developer

    231:    Notes:
    232:    Stores the status as a MPI attribute so it does not have
    233:     to be redetermined each time.

    235:       Assumes that all processors in a communicator either
    236:        1) have a common working directory or
    237:        2) each has a separate working directory
    238:       eventually we can write a fancier one that determines which processors
    239:       share a common working directory.

    241:    This will be very slow on runs with a large number of processors since
    242:    it requires O(p*p) file opens.

    244: @*/
    245: PetscErrorCode  PetscSharedWorkingDirectory(MPI_Comm comm,PetscBool  *shared)
    246: {
    247:   PetscErrorCode     ierr;
    248:   PetscMPIInt        size,rank,*tagvalp,sum,cnt,i;
    249:   PetscBool          flg,iflg;
    250:   FILE               *fd;
    251:   static PetscMPIInt Petsc_WD_keyval = MPI_KEYVAL_INVALID;
    252:   int                err;

    255:   MPI_Comm_size(comm,&size);
    256:   if (size == 1) {
    257:     *shared = PETSC_TRUE;
    258:     return(0);
    259:   }

    261:   PetscOptionsGetenv(comm,"PETSC_SHARED_WORKING_DIRECTORY",NULL,0,&flg);
    262:   if (flg) {
    263:     *shared = PETSC_TRUE;
    264:     return(0);
    265:   }

    267:   PetscOptionsGetenv(comm,"PETSC_NOT_SHARED_WORKING_DIRECTORY",NULL,0,&flg);
    268:   if (flg) {
    269:     *shared = PETSC_FALSE;
    270:     return(0);
    271:   }

    273:   if (Petsc_WD_keyval == MPI_KEYVAL_INVALID) {
    274:     MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN,Petsc_DelTmpShared,&Petsc_WD_keyval,0);
    275:   }

    277:   MPI_Comm_get_attr(comm,Petsc_WD_keyval,(void**)&tagvalp,(int*)&iflg);
    278:   if (!iflg) {
    279:     char filename[PETSC_MAX_PATH_LEN];

    281:     /* This communicator does not yet have a shared  attribute */
    282:     PetscMalloc1(1,&tagvalp);
    283:     MPI_Comm_set_attr(comm,Petsc_WD_keyval,tagvalp);

    285:     PetscGetWorkingDirectory(filename,240);
    286:     PetscStrcat(filename,"/petsctestshared");
    287:     MPI_Comm_rank(comm,&rank);

    289:     /* each processor creates a  file and all the later ones check */
    290:     /* this makes sure no subset of processors is shared */
    291:     *shared = PETSC_FALSE;
    292:     for (i=0; i<size-1; i++) {
    293:       if (rank == i) {
    294:         fd = fopen(filename,"w");
    295:         if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open test file %s",filename);
    296:         err = fclose(fd);
    297:         if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
    298:       }
    299:       MPI_Barrier(comm);
    300:       if (rank >= i) {
    301:         fd = fopen(filename,"r");
    302:         if (fd) cnt = 1;
    303:         else cnt = 0;
    304:         if (fd) {
    305:           err = fclose(fd);
    306:           if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
    307:         }
    308:       } else cnt = 0;

    310:       MPIU_Allreduce(&cnt,&sum,1,MPI_INT,MPI_SUM,comm);
    311:       if (rank == i) unlink(filename);

    313:       if (sum == size) {
    314:         *shared = PETSC_TRUE;
    315:         break;
    316:       } else if (sum != 1) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP_SYS,"Subset of processes share working directory");
    317:     }
    318:     *tagvalp = (int)*shared;
    319:   } else *shared = (PetscBool) *tagvalp;
    320:   PetscInfo1(0,"processors %s working directory\n",(*shared) ? "shared" : "do NOT share");
    321:   return(0);
    322: }

    325: /*@C
    326:     PetscFileRetrieve - Obtains a file from a URL or compressed
    327:         and copies into local disk space as uncompressed.

    329:     Collective on MPI_Comm

    331:     Input Parameter:
    332: +   comm     - processors accessing the file
    333: .   url      - name of file, including entire URL (with or without .gz)
    334: -   llen     - length of localname

    336:     Output Parameter:
    337: +   localname - name of local copy of file - valid on only process zero
    338: -   found - if found or retrieved the file - valid on all processes

    340:     Notes:
    341:     if the file already exists local this function just returns without downloading it.

    343:     Level: intermediate
    344: @*/
    345: PetscErrorCode  PetscFileRetrieve(MPI_Comm comm,const char url[],char localname[],size_t llen,PetscBool  *found)
    346: {
    347:   char           buffer[PETSC_MAX_PATH_LEN],*par,*tlocalname,name[PETSC_MAX_PATH_LEN];
    348:   FILE           *fp;
    350:   PetscMPIInt    rank;
    351:   size_t         len = 0;
    352:   PetscBool      flg1,flg2,flg3,flg4,download,compressed = PETSC_FALSE;

    355:   MPI_Comm_rank(comm,&rank);
    356:   if (!rank) {
    357:     *found = PETSC_FALSE;

    359:     PetscStrstr(url,".gz",&par);
    360:     if (par) {
    361:       PetscStrlen(par,&len);
    362:       if (len == 3) compressed = PETSC_TRUE;
    363:     }

    365:     PetscStrncmp(url,"ftp://",6,&flg1);
    366:     PetscStrncmp(url,"http://",7,&flg2);
    367:     PetscStrncmp(url,"file://",7,&flg3);
    368:     PetscStrncmp(url,"https://",8,&flg4);
    369:     download = (PetscBool) (flg1 || flg2 || flg3 || flg4);

    371:     if (!download && !compressed) {
    372:       PetscStrncpy(localname,url,llen);
    373:       PetscTestFile(url,'r',found);
    374:       if (*found) {
    375:         PetscInfo1(NULL,"Found file %s\n",url);
    376:       } else {
    377:         PetscInfo1(NULL,"Did not find file %s\n",url);
    378:       }
    379:       goto done;
    380:     }

    382:     /* look for uncompressed file in requested directory */
    383:     if (compressed) {
    384:       PetscStrncpy(localname,url,llen);
    385:       PetscStrstr(localname,".gz",&par);
    386:       *par = 0; /* remove .gz extension */
    387:       PetscTestFile(localname,'r',found);
    388:       if (*found) goto done;
    389:     }

    391:     /* look for file in current directory */
    392:     PetscStrrchr(url,'/',&tlocalname);
    393:     PetscStrncpy(localname,tlocalname,llen);
    394:     if (compressed) {
    395:       PetscStrstr(localname,".gz",&par);
    396:       *par = 0; /* remove .gz extension */
    397:     }
    398:     PetscTestFile(localname,'r',found);
    399:     if (*found) goto done;

    401:     if (download) {
    402:       /* local file is not already here so use curl to get it */
    403:       PetscStrncpy(localname,tlocalname,llen);
    404:       PetscStrcpy(buffer,"curl --fail --silent --show-error ");
    405:       PetscStrcat(buffer,url);
    406:       PetscStrcat(buffer," > ");
    407:       PetscStrcat(buffer,localname);
    408: #if defined(PETSC_HAVE_POPEN)
    409:       PetscPOpen(PETSC_COMM_SELF,NULL,buffer,"r",&fp);
    410:       PetscPClose(PETSC_COMM_SELF,fp);
    411: #else
    412:       SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP_SYS,"Cannot run external programs on this machine");
    413: #endif
    414:       PetscTestFile(localname,'r',found);
    415:       if (*found) {
    416:         FILE      *fd;
    417:         char      buf[1024],*str,*substring;

    419:         /* check if the file didn't exist so it downloaded an HTML message instead */
    420:         fd = fopen(localname,"r");
    421:         if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"PetscTestFile() indicates %s exists but fopen() cannot open it",localname);
    422:         str = fgets(buf,sizeof(buf)-1,fd);
    423:         while (str) {
    424:           PetscStrstr(buf,"<!DOCTYPE html>",&substring);
    425:           if (substring) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unable to download %s it does not appear to exist at this URL, dummy HTML file was downloaded",url);
    426:           PetscStrstr(buf,"Not Found",&substring);
    427:           if (substring) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Unable to download %s it does not appear to exist at this URL, dummy HTML file was downloaded",url);
    428:           str = fgets(buf,sizeof(buf)-1,fd);
    429:         }
    430:         fclose(fd);
    431:       }
    432:     } else if (compressed) {
    433:       PetscTestFile(url,'r',found);
    434:       if (!*found) goto done;
    435:       PetscStrncpy(localname,url,llen);
    436:     }
    437:     if (compressed) {
    438:       PetscStrrchr(localname,'/',&tlocalname);
    439:       PetscStrncpy(name,tlocalname,PETSC_MAX_PATH_LEN);
    440:       PetscStrstr(name,".gz",&par);
    441:       *par = 0; /* remove .gz extension */
    442:       /* uncompress file */
    443:       PetscStrcpy(buffer,"gzip -c -d ");
    444:       PetscStrcat(buffer,localname);
    445:       PetscStrcat(buffer," > ");
    446:       PetscStrcat(buffer,name);
    447: #if defined(PETSC_HAVE_POPEN)
    448:       PetscPOpen(PETSC_COMM_SELF,NULL,buffer,"r",&fp);
    449:       PetscPClose(PETSC_COMM_SELF,fp);
    450: #else
    451:       SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP_SYS,"Cannot run external programs on this machine");
    452: #endif
    453:       PetscStrncpy(localname,name,llen);
    454:       PetscTestFile(localname,'r',found);
    455:     }
    456:   }
    457:   done:
    458:   MPI_Bcast(found,1,MPIU_BOOL,0,comm);
    459:   MPI_Bcast(localname, llen, MPI_CHAR, 0, comm);
    460:   return(0);
    461: }