Actual source code: ntl.c
petsc-3.10.5 2019-03-28
Report Typos and Errors

      1:  #include <../src/tao/unconstrained/impls/ntl/ntlimpl.h>

      3:  #include <petscksp.h>

      5: #define NTL_INIT_CONSTANT         0
      6: #define NTL_INIT_DIRECTION        1
      7: #define NTL_INIT_INTERPOLATION    2
      8: #define NTL_INIT_TYPES            3

     10: #define NTL_UPDATE_REDUCTION      0
     11: #define NTL_UPDATE_INTERPOLATION  1
     12: #define NTL_UPDATE_TYPES          2

     14: static const char *NTL_INIT[64] = {"constant","direction","interpolation"};

     16: static const char *NTL_UPDATE[64] = {"reduction","interpolation"};

     18: /* Implements Newton's Method with a trust-region, line-search approach for
     19:    solving unconstrained minimization problems.  A More'-Thuente line search
     20:    is used to guarantee that the bfgs preconditioner remains positive
     21:    definite. */

     23: #define NTL_NEWTON              0
     24: #define NTL_BFGS                1
     25: #define NTL_SCALED_GRADIENT     2
     26: #define NTL_GRADIENT            3

     28: static PetscErrorCode TaoSolve_NTL(Tao tao)
     29: {
     30:   TAO_NTL                      *tl = (TAO_NTL *)tao->data;
     31:   KSPType                      ksp_type;
     32:   PetscBool                    is_nash,is_stcg,is_gltr,is_bfgs,is_jacobi,is_symmetric,sym_set;
     33:   KSPConvergedReason           ksp_reason;
     34:   PC                           pc;
     35:   TaoLineSearchConvergedReason ls_reason;

     37:   PetscReal                    fmin, ftrial, prered, actred, kappa, sigma;
     38:   PetscReal                    tau, tau_1, tau_2, tau_max, tau_min, max_radius;
     39:   PetscReal                    f, fold, gdx, gnorm;
     40:   PetscReal                    step = 1.0;

     42:   PetscReal                    norm_d = 0.0;
     43:   PetscErrorCode               ierr;
     44:   PetscInt                     stepType;
     45:   PetscInt                     its;

     47:   PetscInt                     bfgsUpdates = 0;
     48:   PetscInt                     needH;

     50:   PetscInt                     i_max = 5;
     51:   PetscInt                     j_max = 1;
     52:   PetscInt                     i, j, n, N;

     54:   PetscInt                     tr_reject;

     57:   if (tao->XL || tao->XU || tao->ops->computebounds) {
     58:     PetscPrintf(((PetscObject)tao)->comm,"WARNING: Variable bounds have been set but will be ignored by ntl algorithm\n");
     59:   }

     61:   KSPGetType(tao->ksp,&ksp_type);
     62:   PetscStrcmp(ksp_type,KSPCGNASH,&is_nash);
     63:   PetscStrcmp(ksp_type,KSPCGSTCG,&is_stcg);
     64:   PetscStrcmp(ksp_type,KSPCGGLTR,&is_gltr);
     65:   if (!is_nash && !is_stcg && !is_gltr) {
     66:     SETERRQ(PETSC_COMM_SELF,1,"TAO_NTR requires nash, stcg, or gltr for the KSP");
     67:   }

     69:   /* Initialize the radius and modify if it is too large or small */
     70:   tao->trust = tao->trust0;
     71:   tao->trust = PetscMax(tao->trust, tl->min_radius);
     72:   tao->trust = PetscMin(tao->trust, tl->max_radius);

     74:   /* Allocate the vectors needed for the BFGS approximation */
     75:   KSPGetPC(tao->ksp, &pc);
     76:   PetscObjectTypeCompare((PetscObject)pc, PCLMVM, &is_bfgs);
     77:   PetscObjectTypeCompare((PetscObject)pc, PCJACOBI, &is_jacobi);
     78:   if (is_bfgs) {
     79:     tl->bfgs_pre = pc;
     80:     PCLMVMGetMatLMVM(tl->bfgs_pre, &tl->M);
     81:     VecGetLocalSize(tao->solution, &n);
     82:     VecGetSize(tao->solution, &N);
     83:     MatSetSizes(tl->M, n, n, N, N);
     84:     MatLMVMAllocate(tl->M, tao->solution, tao->gradient);
     85:     MatIsSymmetricKnown(tl->M, &sym_set, &is_symmetric);
     86:     if (!sym_set || !is_symmetric) SETERRQ(PetscObjectComm((PetscObject)tao), PETSC_ERR_ARG_INCOMP, "LMVM matrix in the LMVM preconditioner must be symmetric.");
     87:   } else if (is_jacobi) {
     88:     PCJacobiSetUseAbs(pc,PETSC_TRUE);
     89:   }

     91:   /* Check convergence criteria */
     92:   TaoComputeObjectiveAndGradient(tao, tao->solution, &f, tao->gradient);
     93:   VecNorm(tao->gradient, NORM_2, &gnorm);
     94:   if (PetscIsInfOrNanReal(f) || PetscIsInfOrNanReal(gnorm)) SETERRQ(PETSC_COMM_SELF,1, "User provided compute function generated Inf or NaN");
     95:   needH = 1;

     97:   tao->reason = TAO_CONTINUE_ITERATING;
     98:   TaoLogConvergenceHistory(tao,f,gnorm,0.0,tao->ksp_its);
     99:   TaoMonitor(tao,tao->niter,f,gnorm,0.0,step);
    100:   (*tao->ops->convergencetest)(tao,tao->cnvP);
    101:   if (tao->reason != TAO_CONTINUE_ITERATING) return(0);

    103:   /* Initialize trust-region radius */
    104:   switch(tl->init_type) {
    105:   case NTL_INIT_CONSTANT:
    106:     /* Use the initial radius specified */
    107:     break;

    109:   case NTL_INIT_INTERPOLATION:
    110:     /* Use the initial radius specified */
    111:     max_radius = 0.0;

    113:     for (j = 0; j < j_max; ++j) {
    114:       fmin = f;
    115:       sigma = 0.0;

    117:       if (needH) {
    118:         TaoComputeHessian(tao,tao->solution,tao->hessian,tao->hessian_pre);
    119:         needH = 0;
    120:       }

    122:       for (i = 0; i < i_max; ++i) {
    123:         VecCopy(tao->solution, tl->W);
    124:         VecAXPY(tl->W, -tao->trust/gnorm, tao->gradient);

    126:         TaoComputeObjective(tao, tl->W, &ftrial);
    127:         if (PetscIsInfOrNanReal(ftrial)) {
    128:           tau = tl->gamma1_i;
    129:         } else {
    130:           if (ftrial < fmin) {
    131:             fmin = ftrial;
    132:             sigma = -tao->trust / gnorm;
    133:           }

    135:           MatMult(tao->hessian, tao->gradient, tao->stepdirection);
    136:           VecDot(tao->gradient, tao->stepdirection, &prered);

    138:           prered = tao->trust * (gnorm - 0.5 * tao->trust * prered / (gnorm * gnorm));
    139:           actred = f - ftrial;
    140:           if ((PetscAbsScalar(actred) <= tl->epsilon) && (PetscAbsScalar(prered) <= tl->epsilon)) {
    141:             kappa = 1.0;
    142:           } else {
    143:             kappa = actred / prered;
    144:           }

    146:           tau_1 = tl->theta_i * gnorm * tao->trust / (tl->theta_i * gnorm * tao->trust + (1.0 - tl->theta_i) * prered - actred);
    147:           tau_2 = tl->theta_i * gnorm * tao->trust / (tl->theta_i * gnorm * tao->trust - (1.0 + tl->theta_i) * prered + actred);
    148:           tau_min = PetscMin(tau_1, tau_2);
    149:           tau_max = PetscMax(tau_1, tau_2);

    151:           if (PetscAbsScalar(kappa - 1.0) <= tl->mu1_i) {
    152:             /* Great agreement */
    153:             max_radius = PetscMax(max_radius, tao->trust);

    155:             if (tau_max < 1.0) {
    156:               tau = tl->gamma3_i;
    157:             } else if (tau_max > tl->gamma4_i) {
    158:               tau = tl->gamma4_i;
    159:             } else if (tau_1 >= 1.0 && tau_1 <= tl->gamma4_i && tau_2 < 1.0) {
    160:               tau = tau_1;
    161:             } else if (tau_2 >= 1.0 && tau_2 <= tl->gamma4_i && tau_1 < 1.0) {
    162:               tau = tau_2;
    163:             } else {
    164:               tau = tau_max;
    165:             }
    166:           } else if (PetscAbsScalar(kappa - 1.0) <= tl->mu2_i) {
    167:             /* Good agreement */
    168:             max_radius = PetscMax(max_radius, tao->trust);

    170:             if (tau_max < tl->gamma2_i) {
    171:               tau = tl->gamma2_i;
    172:             } else if (tau_max > tl->gamma3_i) {
    173:               tau = tl->gamma3_i;
    174:             } else {
    175:               tau = tau_max;
    176:             }
    177:           } else {
    178:             /* Not good agreement */
    179:             if (tau_min > 1.0) {
    180:               tau = tl->gamma2_i;
    181:             } else if (tau_max < tl->gamma1_i) {
    182:               tau = tl->gamma1_i;
    183:             } else if ((tau_min < tl->gamma1_i) && (tau_max >= 1.0)) {
    184:               tau = tl->gamma1_i;
    185:             } else if ((tau_1 >= tl->gamma1_i) && (tau_1 < 1.0) &&  ((tau_2 < tl->gamma1_i) || (tau_2 >= 1.0))) {
    186:               tau = tau_1;
    187:             } else if ((tau_2 >= tl->gamma1_i) && (tau_2 < 1.0) &&  ((tau_1 < tl->gamma1_i) || (tau_2 >= 1.0))) {
    188:               tau = tau_2;
    189:             } else {
    190:               tau = tau_max;
    191:             }
    192:           }
    193:         }
    194:         tao->trust = tau * tao->trust;
    195:       }

    197:       if (fmin < f) {
    198:         f = fmin;
    199:         VecAXPY(tao->solution, sigma, tao->gradient);
    200:         TaoComputeGradient(tao, tao->solution, tao->gradient);

    202:         VecNorm(tao->gradient, NORM_2, &gnorm);
    203:         if (PetscIsInfOrNanReal(f) || PetscIsInfOrNanReal(gnorm)) SETERRQ(PETSC_COMM_SELF,1, "User provided compute function generated Inf or NaN");
    204:         needH = 1;

    206:         TaoLogConvergenceHistory(tao,f,gnorm,0.0,tao->ksp_its);
    207:         TaoMonitor(tao,tao->niter,f,gnorm,0.0,step);
    208:         (*tao->ops->convergencetest)(tao,tao->cnvP);
    209:         if (tao->reason != TAO_CONTINUE_ITERATING) return(0);
    210:       }
    211:     }
    212:     tao->trust = PetscMax(tao->trust, max_radius);

    214:     /* Modify the radius if it is too large or small */
    215:     tao->trust = PetscMax(tao->trust, tl->min_radius);
    216:     tao->trust = PetscMin(tao->trust, tl->max_radius);
    217:     break;

    219:   default:
    220:     /* Norm of the first direction will initialize radius */
    221:     tao->trust = 0.0;
    222:     break;
    223:   }

    225:   /* Set counter for gradient/reset steps */
    226:   tl->ntrust = 0;
    227:   tl->newt = 0;
    228:   tl->bfgs = 0;
    229:   tl->grad = 0;

    231:   /* Have not converged; continue with Newton method */
    232:   while (tao->reason == TAO_CONTINUE_ITERATING) {
    233:     ++tao->niter;
    234:     tao->ksp_its=0;
    235:     /* Compute the Hessian */
    236:     if (needH) {
    237:       TaoComputeHessian(tao,tao->solution,tao->hessian,tao->hessian_pre);
    238:     }

    240:     if (tl->bfgs_pre) {
    241:       /* Update the limited memory preconditioner */
    242:       MatLMVMUpdate(tl->M,tao->solution, tao->gradient);
    243:       ++bfgsUpdates;
    244:     }
    245:     KSPSetOperators(tao->ksp, tao->hessian, tao->hessian_pre);
    246:     /* Solve the Newton system of equations */
    247:     KSPCGSetRadius(tao->ksp,tl->max_radius);
    248:     KSPSolve(tao->ksp, tao->gradient, tao->stepdirection);
    249:     KSPGetIterationNumber(tao->ksp,&its);
    250:     tao->ksp_its+=its;
    251:     tao->ksp_tot_its+=its;
    252:     KSPCGGetNormD(tao->ksp, &norm_d);

    254:     if (0.0 == tao->trust) {
    255:       /* Radius was uninitialized; use the norm of the direction */
    256:       if (norm_d > 0.0) {
    257:         tao->trust = norm_d;

    259:         /* Modify the radius if it is too large or small */
    260:         tao->trust = PetscMax(tao->trust, tl->min_radius);
    261:         tao->trust = PetscMin(tao->trust, tl->max_radius);
    262:       } else {
    263:         /* The direction was bad; set radius to default value and re-solve
    264:            the trust-region subproblem to get a direction */
    265:         tao->trust = tao->trust0;

    267:         /* Modify the radius if it is too large or small */
    268:         tao->trust = PetscMax(tao->trust, tl->min_radius);
    269:         tao->trust = PetscMin(tao->trust, tl->max_radius);

    271:         KSPCGSetRadius(tao->ksp,tl->max_radius);
    272:         KSPSolve(tao->ksp, tao->gradient, tao->stepdirection);
    273:         KSPGetIterationNumber(tao->ksp,&its);
    274:         tao->ksp_its+=its;
    275:         tao->ksp_tot_its+=its;
    276:         KSPCGGetNormD(tao->ksp, &norm_d);

    278:         if (norm_d == 0.0) SETERRQ(PETSC_COMM_SELF,1, "Initial direction zero");
    279:       }
    280:     }

    282:     VecScale(tao->stepdirection, -1.0);
    283:     KSPGetConvergedReason(tao->ksp, &ksp_reason);
    284:     if ((KSP_DIVERGED_INDEFINITE_PC == ksp_reason) && (tl->bfgs_pre)) {
    285:       /* Preconditioner is numerically indefinite; reset the
    286:          approximate if using BFGS preconditioning. */
    287:       MatLMVMReset(tl->M, PETSC_FALSE);
    288:       MatLMVMUpdate(tl->M, tao->solution, tao->gradient);
    289:       bfgsUpdates = 1;
    290:     }

    292:     /* Check trust-region reduction conditions */
    293:     tr_reject = 0;
    294:     if (NTL_UPDATE_REDUCTION == tl->update_type) {
    295:       /* Get predicted reduction */
    296:       KSPCGGetObjFcn(tao->ksp,&prered);
    297:       if (prered >= 0.0) {
    298:         /* The predicted reduction has the wrong sign.  This cannot
    299:            happen in infinite precision arithmetic.  Step should
    300:            be rejected! */
    301:         tao->trust = tl->alpha1 * PetscMin(tao->trust, norm_d);
    302:         tr_reject = 1;
    303:       } else {
    304:         /* Compute trial step and function value */
    305:         VecCopy(tao->solution, tl->W);
    306:         VecAXPY(tl->W, 1.0, tao->stepdirection);
    307:         TaoComputeObjective(tao, tl->W, &ftrial);

    309:         if (PetscIsInfOrNanReal(ftrial)) {
    310:           tao->trust = tl->alpha1 * PetscMin(tao->trust, norm_d);
    311:           tr_reject = 1;
    312:         } else {
    313:           /* Compute and actual reduction */
    314:           actred = f - ftrial;
    315:           prered = -prered;
    316:           if ((PetscAbsScalar(actred) <= tl->epsilon) &&
    317:               (PetscAbsScalar(prered) <= tl->epsilon)) {
    318:             kappa = 1.0;
    319:           } else {
    320:             kappa = actred / prered;
    321:           }

    323:           /* Accept of reject the step and update radius */
    324:           if (kappa < tl->eta1) {
    325:             /* Reject the step */
    326:             tao->trust = tl->alpha1 * PetscMin(tao->trust, norm_d);
    327:             tr_reject = 1;
    328:           } else {
    329:             /* Accept the step */
    330:             if (kappa < tl->eta2) {
    331:               /* Marginal bad step */
    332:               tao->trust = tl->alpha2 * PetscMin(tao->trust, norm_d);
    333:             } else if (kappa < tl->eta3) {
    334:               /* Reasonable step */
    335:               tao->trust = tl->alpha3 * tao->trust;
    336:             } else if (kappa < tl->eta4) {
    337:               /* Good step */
    338:               tao->trust = PetscMax(tl->alpha4 * norm_d, tao->trust);
    339:             } else {
    340:               /* Very good step */
    341:               tao->trust = PetscMax(tl->alpha5 * norm_d, tao->trust);
    342:             }
    343:           }
    344:         }
    345:       }
    346:     } else {
    347:       /* Get predicted reduction */
    348:       KSPCGGetObjFcn(tao->ksp,&prered);
    349:       if (prered >= 0.0) {
    350:         /* The predicted reduction has the wrong sign.  This cannot
    351:            happen in infinite precision arithmetic.  Step should
    352:            be rejected! */
    353:         tao->trust = tl->gamma1 * PetscMin(tao->trust, norm_d);
    354:         tr_reject = 1;
    355:       } else {
    356:         VecCopy(tao->solution, tl->W);
    357:         VecAXPY(tl->W, 1.0, tao->stepdirection);
    358:         TaoComputeObjective(tao, tl->W, &ftrial);
    359:         if (PetscIsInfOrNanReal(ftrial)) {
    360:           tao->trust = tl->gamma1 * PetscMin(tao->trust, norm_d);
    361:           tr_reject = 1;
    362:         } else {
    363:           VecDot(tao->gradient, tao->stepdirection, &gdx);

    365:           actred = f - ftrial;
    366:           prered = -prered;
    367:           if ((PetscAbsScalar(actred) <= tl->epsilon) &&
    368:               (PetscAbsScalar(prered) <= tl->epsilon)) {
    369:             kappa = 1.0;
    370:           } else {
    371:             kappa = actred / prered;
    372:           }

    374:           tau_1 = tl->theta * gdx / (tl->theta * gdx - (1.0 - tl->theta) * prered + actred);
    375:           tau_2 = tl->theta * gdx / (tl->theta * gdx + (1.0 + tl->theta) * prered - actred);
    376:           tau_min = PetscMin(tau_1, tau_2);
    377:           tau_max = PetscMax(tau_1, tau_2);

    379:           if (kappa >= 1.0 - tl->mu1) {
    380:             /* Great agreement; accept step and update radius */
    381:             if (tau_max < 1.0) {
    382:               tao->trust = PetscMax(tao->trust, tl->gamma3 * norm_d);
    383:             } else if (tau_max > tl->gamma4) {
    384:               tao->trust = PetscMax(tao->trust, tl->gamma4 * norm_d);
    385:             } else {
    386:               tao->trust = PetscMax(tao->trust, tau_max * norm_d);
    387:             }
    388:           } else if (kappa >= 1.0 - tl->mu2) {
    389:             /* Good agreement */

    391:             if (tau_max < tl->gamma2) {
    392:               tao->trust = tl->gamma2 * PetscMin(tao->trust, norm_d);
    393:             } else if (tau_max > tl->gamma3) {
    394:               tao->trust = PetscMax(tao->trust, tl->gamma3 * norm_d);
    395:             } else if (tau_max < 1.0) {
    396:               tao->trust = tau_max * PetscMin(tao->trust, norm_d);
    397:             } else {
    398:               tao->trust = PetscMax(tao->trust, tau_max * norm_d);
    399:             }
    400:           } else {
    401:             /* Not good agreement */
    402:             if (tau_min > 1.0) {
    403:               tao->trust = tl->gamma2 * PetscMin(tao->trust, norm_d);
    404:             } else if (tau_max < tl->gamma1) {
    405:               tao->trust = tl->gamma1 * PetscMin(tao->trust, norm_d);
    406:             } else if ((tau_min < tl->gamma1) && (tau_max >= 1.0)) {
    407:               tao->trust = tl->gamma1 * PetscMin(tao->trust, norm_d);
    408:             } else if ((tau_1 >= tl->gamma1) && (tau_1 < 1.0) && ((tau_2 < tl->gamma1) || (tau_2 >= 1.0))) {
    409:               tao->trust = tau_1 * PetscMin(tao->trust, norm_d);
    410:             } else if ((tau_2 >= tl->gamma1) && (tau_2 < 1.0) && ((tau_1 < tl->gamma1) || (tau_2 >= 1.0))) {
    411:               tao->trust = tau_2 * PetscMin(tao->trust, norm_d);
    412:             } else {
    413:               tao->trust = tau_max * PetscMin(tao->trust, norm_d);
    414:             }
    415:             tr_reject = 1;
    416:           }
    417:         }
    418:       }
    419:     }

    421:     if (tr_reject) {
    422:       /* The trust-region constraints rejected the step.  Apply a linesearch.
    423:          Check for descent direction. */
    424:       VecDot(tao->stepdirection, tao->gradient, &gdx);
    425:       if ((gdx >= 0.0) || PetscIsInfOrNanReal(gdx)) {
    426:         /* Newton step is not descent or direction produced Inf or NaN */

    428:         if (!tl->bfgs_pre) {
    429:           /* We don't have the bfgs matrix around and updated
    430:              Must use gradient direction in this case */
    431:           VecCopy(tao->gradient, tao->stepdirection);
    432:           VecScale(tao->stepdirection, -1.0);
    433:           ++tl->grad;
    434:           stepType = NTL_GRADIENT;
    435:         } else {
    436:           /* Attempt to use the BFGS direction */
    437:           MatSolve(tl->M, tao->gradient, tao->stepdirection);
    438:           VecScale(tao->stepdirection, -1.0);

    440:           /* Check for success (descent direction) */
    441:           VecDot(tao->stepdirection, tao->gradient, &gdx);
    442:           if ((gdx >= 0) || PetscIsInfOrNanReal(gdx)) {
    443:             /* BFGS direction is not descent or direction produced not a number
    444:                We can assert bfgsUpdates > 1 in this case because
    445:                the first solve produces the scaled gradient direction,
    446:                which is guaranteed to be descent */

    448:             /* Use steepest descent direction (scaled) */
    449:             MatLMVMReset(tl->M, PETSC_FALSE);
    450:             MatLMVMUpdate(tl->M, tao->solution, tao->gradient);
    451:             MatSolve(tl->M, tao->gradient, tao->stepdirection);
    452:             VecScale(tao->stepdirection, -1.0);

    454:             bfgsUpdates = 1;
    455:             ++tl->grad;
    456:             stepType = NTL_GRADIENT;
    457:           } else {
    458:             MatLMVMGetUpdateCount(tl->M, &bfgsUpdates);
    459:             if (1 == bfgsUpdates) {
    460:               /* The first BFGS direction is always the scaled gradient */
    461:               ++tl->grad;
    462:               stepType = NTL_GRADIENT;
    463:             } else {
    464:               ++tl->bfgs;
    465:               stepType = NTL_BFGS;
    466:             }
    467:           }
    468:         }
    469:       } else {
    470:         /* Computed Newton step is descent */
    471:         ++tl->newt;
    472:         stepType = NTL_NEWTON;
    473:       }

    475:       /* Perform the linesearch */
    476:       fold = f;
    477:       VecCopy(tao->solution, tl->Xold);
    478:       VecCopy(tao->gradient, tl->Gold);

    480:       step = 1.0;
    481:       TaoLineSearchApply(tao->linesearch, tao->solution, &f, tao->gradient, tao->stepdirection, &step, &ls_reason);
    482:       TaoAddLineSearchCounts(tao);

    484:       while (ls_reason != TAOLINESEARCH_SUCCESS && ls_reason != TAOLINESEARCH_SUCCESS_USER && stepType != NTL_GRADIENT) {      /* Linesearch failed */
    485:         /* Linesearch failed */
    486:         f = fold;
    487:         VecCopy(tl->Xold, tao->solution);
    488:         VecCopy(tl->Gold, tao->gradient);

    490:         switch(stepType) {
    491:         case NTL_NEWTON:
    492:           /* Failed to obtain acceptable iterate with Newton step */

    494:           if (tl->bfgs_pre) {
    495:             /* We don't have the bfgs matrix around and being updated
    496:                Must use gradient direction in this case */
    497:             VecCopy(tao->gradient, tao->stepdirection);
    498:             ++tl->grad;
    499:             stepType = NTL_GRADIENT;
    500:           } else {
    501:             /* Attempt to use the BFGS direction */
    502:             MatSolve(tl->M, tao->gradient, tao->stepdirection);

    505:             /* Check for success (descent direction) */
    506:             VecDot(tao->stepdirection, tao->gradient, &gdx);
    507:             if ((gdx <= 0) || PetscIsInfOrNanReal(gdx)) {
    508:               /* BFGS direction is not descent or direction produced
    509:                  not a number.  We can assert bfgsUpdates > 1 in this case
    510:                  Use steepest descent direction (scaled) */
    511:               MatLMVMReset(tl->M, PETSC_FALSE);
    512:               MatLMVMUpdate(tl->M, tao->solution, tao->gradient);
    513:               MatSolve(tl->M, tao->gradient, tao->stepdirection);

    515:               bfgsUpdates = 1;
    516:               ++tl->grad;
    517:               stepType = NTL_GRADIENT;
    518:             } else {
    519:               MatLMVMGetUpdateCount(tl->M, &bfgsUpdates);
    520:               if (1 == bfgsUpdates) {
    521:                 /* The first BFGS direction is always the scaled gradient */
    522:                 ++tl->grad;
    523:                 stepType = NTL_GRADIENT;
    524:               } else {
    525:                 ++tl->bfgs;
    526:                 stepType = NTL_BFGS;
    527:               }
    528:             }
    529:           }
    530:           break;

    532:         case NTL_BFGS:
    533:           /* Can only enter if pc_type == NTL_PC_BFGS
    534:              Failed to obtain acceptable iterate with BFGS step
    535:              Attempt to use the scaled gradient direction */
    536:           MatLMVMReset(tl->M, PETSC_FALSE);
    537:           MatLMVMUpdate(tl->M, tao->solution, tao->gradient);
    538:           MatSolve(tl->M, tao->gradient, tao->stepdirection);

    540:           bfgsUpdates = 1;
    541:           ++tl->grad;
    542:           stepType = NTL_GRADIENT;
    543:           break;
    544:         }
    545:         VecScale(tao->stepdirection, -1.0);

    547:         /* This may be incorrect; linesearch has values for stepmax and stepmin
    548:            that should be reset. */
    549:         step = 1.0;
    550:         TaoLineSearchApply(tao->linesearch, tao->solution, &f, tao->gradient, tao->stepdirection, &step, &ls_reason);
    551:         TaoAddLineSearchCounts(tao);
    552:       }

    554:       if (ls_reason != TAOLINESEARCH_SUCCESS && ls_reason != TAOLINESEARCH_SUCCESS_USER) {
    555:         /* Failed to find an improving point */
    556:         f = fold;
    557:         VecCopy(tl->Xold, tao->solution);
    558:         VecCopy(tl->Gold, tao->gradient);
    559:         tao->trust = 0.0;
    560:         step = 0.0;
    561:         tao->reason = TAO_DIVERGED_LS_FAILURE;
    562:         break;
    563:       } else if (stepType == NTL_NEWTON) {
    564:         if (step < tl->nu1) {
    565:           /* Very bad step taken; reduce radius */
    566:           tao->trust = tl->omega1 * PetscMin(norm_d, tao->trust);
    567:         } else if (step < tl->nu2) {
    568:           /* Reasonably bad step taken; reduce radius */
    569:           tao->trust = tl->omega2 * PetscMin(norm_d, tao->trust);
    570:         } else if (step < tl->nu3) {
    571:           /* Reasonable step was taken; leave radius alone */
    572:           if (tl->omega3 < 1.0) {
    573:             tao->trust = tl->omega3 * PetscMin(norm_d, tao->trust);
    574:           } else if (tl->omega3 > 1.0) {
    575:             tao->trust = PetscMax(tl->omega3 * norm_d, tao->trust);
    576:           }
    577:         } else if (step < tl->nu4) {
    578:           /* Full step taken; increase the radius */
    579:           tao->trust = PetscMax(tl->omega4 * norm_d, tao->trust);
    580:         } else {
    581:           /* More than full step taken; increase the radius */
    582:           tao->trust = PetscMax(tl->omega5 * norm_d, tao->trust);
    583:         }
    584:       } else {
    585:         /* Newton step was not good; reduce the radius */
    586:         tao->trust = tl->omega1 * PetscMin(norm_d, tao->trust);
    587:       }
    588:     } else {
    589:       /* Trust-region step is accepted */
    590:       VecCopy(tl->W, tao->solution);
    591:       f = ftrial;
    592:       TaoComputeGradient(tao, tao->solution, tao->gradient);
    593:       ++tl->ntrust;
    594:     }

    596:     /* The radius may have been increased; modify if it is too large */
    597:     tao->trust = PetscMin(tao->trust, tl->max_radius);

    599:     /* Check for converged */
    600:     VecNorm(tao->gradient, NORM_2, &gnorm);
    601:     if (PetscIsInfOrNanReal(f) || PetscIsInfOrNanReal(gnorm)) SETERRQ(PETSC_COMM_SELF,1,"User provided compute function generated Not-a-Number");
    602:     needH = 1;

    604:     TaoLogConvergenceHistory(tao,f,gnorm,0.0,tao->ksp_its);
    605:     TaoMonitor(tao,tao->niter,f,gnorm,0.0,step);
    606:     (*tao->ops->convergencetest)(tao,tao->cnvP);
    607:   }
    608:   return(0);
    609: }

    611: /* ---------------------------------------------------------- */
    612: static PetscErrorCode TaoSetUp_NTL(Tao tao)
    613: {
    614:   TAO_NTL        *tl = (TAO_NTL *)tao->data;

    618:   if (!tao->gradient) {VecDuplicate(tao->solution, &tao->gradient); }
    619:   if (!tao->stepdirection) {VecDuplicate(tao->solution, &tao->stepdirection);}
    620:   if (!tl->W) { VecDuplicate(tao->solution, &tl->W);}
    621:   if (!tl->Xold) { VecDuplicate(tao->solution, &tl->Xold);}
    622:   if (!tl->Gold) { VecDuplicate(tao->solution, &tl->Gold);}
    623:   tl->bfgs_pre = 0;
    624:   tl->M = 0;
    625:   return(0);
    626: }

    628: /*------------------------------------------------------------*/
    629: static PetscErrorCode TaoDestroy_NTL(Tao tao)
    630: {
    631:   TAO_NTL        *tl = (TAO_NTL *)tao->data;

    635:   if (tao->setupcalled) {
    636:     VecDestroy(&tl->W);
    637:     VecDestroy(&tl->Xold);
    638:     VecDestroy(&tl->Gold);
    639:   }
    640:   PetscFree(tao->data);
    641:   return(0);
    642: }

    644: /*------------------------------------------------------------*/
    645: static PetscErrorCode TaoSetFromOptions_NTL(PetscOptionItems *PetscOptionsObject,Tao tao)
    646: {
    647:   TAO_NTL        *tl = (TAO_NTL *)tao->data;

    651:   PetscOptionsHead(PetscOptionsObject,"Newton trust region with line search method for unconstrained optimization");
    652:   PetscOptionsEList("-tao_ntl_init_type", "radius initialization type", "", NTL_INIT, NTL_INIT_TYPES, NTL_INIT[tl->init_type], &tl->init_type,NULL);
    653:   PetscOptionsEList("-tao_ntl_update_type", "radius update type", "", NTL_UPDATE, NTL_UPDATE_TYPES, NTL_UPDATE[tl->update_type], &tl->update_type,NULL);
    654:   PetscOptionsReal("-tao_ntl_eta1", "poor steplength; reduce radius", "", tl->eta1, &tl->eta1,NULL);
    655:   PetscOptionsReal("-tao_ntl_eta2", "reasonable steplength; leave radius alone", "", tl->eta2, &tl->eta2,NULL);
    656:   PetscOptionsReal("-tao_ntl_eta3", "good steplength; increase radius", "", tl->eta3, &tl->eta3,NULL);
    657:   PetscOptionsReal("-tao_ntl_eta4", "excellent steplength; greatly increase radius", "", tl->eta4, &tl->eta4,NULL);
    658:   PetscOptionsReal("-tao_ntl_alpha1", "", "", tl->alpha1, &tl->alpha1,NULL);
    659:   PetscOptionsReal("-tao_ntl_alpha2", "", "", tl->alpha2, &tl->alpha2,NULL);
    660:   PetscOptionsReal("-tao_ntl_alpha3", "", "", tl->alpha3, &tl->alpha3,NULL);
    661:   PetscOptionsReal("-tao_ntl_alpha4", "", "", tl->alpha4, &tl->alpha4,NULL);
    662:   PetscOptionsReal("-tao_ntl_alpha5", "", "", tl->alpha5, &tl->alpha5,NULL);
    663:   PetscOptionsReal("-tao_ntl_nu1", "poor steplength; reduce radius", "", tl->nu1, &tl->nu1,NULL);
    664:   PetscOptionsReal("-tao_ntl_nu2", "reasonable steplength; leave radius alone", "", tl->nu2, &tl->nu2,NULL);
    665:   PetscOptionsReal("-tao_ntl_nu3", "good steplength; increase radius", "", tl->nu3, &tl->nu3,NULL);
    666:   PetscOptionsReal("-tao_ntl_nu4", "excellent steplength; greatly increase radius", "", tl->nu4, &tl->nu4,NULL);
    667:   PetscOptionsReal("-tao_ntl_omega1", "", "", tl->omega1, &tl->omega1,NULL);
    668:   PetscOptionsReal("-tao_ntl_omega2", "", "", tl->omega2, &tl->omega2,NULL);
    669:   PetscOptionsReal("-tao_ntl_omega3", "", "", tl->omega3, &tl->omega3,NULL);
    670:   PetscOptionsReal("-tao_ntl_omega4", "", "", tl->omega4, &tl->omega4,NULL);
    671:   PetscOptionsReal("-tao_ntl_omega5", "", "", tl->omega5, &tl->omega5,NULL);
    672:   PetscOptionsReal("-tao_ntl_mu1_i", "", "", tl->mu1_i, &tl->mu1_i,NULL);
    673:   PetscOptionsReal("-tao_ntl_mu2_i", "", "", tl->mu2_i, &tl->mu2_i,NULL);
    674:   PetscOptionsReal("-tao_ntl_gamma1_i", "", "", tl->gamma1_i, &tl->gamma1_i,NULL);
    675:   PetscOptionsReal("-tao_ntl_gamma2_i", "", "", tl->gamma2_i, &tl->gamma2_i,NULL);
    676:   PetscOptionsReal("-tao_ntl_gamma3_i", "", "", tl->gamma3_i, &tl->gamma3_i,NULL);
    677:   PetscOptionsReal("-tao_ntl_gamma4_i", "", "", tl->gamma4_i, &tl->gamma4_i,NULL);
    678:   PetscOptionsReal("-tao_ntl_theta_i", "", "", tl->theta_i, &tl->theta_i,NULL);
    679:   PetscOptionsReal("-tao_ntl_mu1", "", "", tl->mu1, &tl->mu1,NULL);
    680:   PetscOptionsReal("-tao_ntl_mu2", "", "", tl->mu2, &tl->mu2,NULL);
    681:   PetscOptionsReal("-tao_ntl_gamma1", "", "", tl->gamma1, &tl->gamma1,NULL);
    682:   PetscOptionsReal("-tao_ntl_gamma2", "", "", tl->gamma2, &tl->gamma2,NULL);
    683:   PetscOptionsReal("-tao_ntl_gamma3", "", "", tl->gamma3, &tl->gamma3,NULL);
    684:   PetscOptionsReal("-tao_ntl_gamma4", "", "", tl->gamma4, &tl->gamma4,NULL);
    685:   PetscOptionsReal("-tao_ntl_theta", "", "", tl->theta, &tl->theta,NULL);
    686:   PetscOptionsReal("-tao_ntl_min_radius", "lower bound on initial radius", "", tl->min_radius, &tl->min_radius,NULL);
    687:   PetscOptionsReal("-tao_ntl_max_radius", "upper bound on radius", "", tl->max_radius, &tl->max_radius,NULL);
    688:   PetscOptionsReal("-tao_ntl_epsilon", "tolerance used when computing actual and predicted reduction", "", tl->epsilon, &tl->epsilon,NULL);
    689:   PetscOptionsTail();
    690:   TaoLineSearchSetFromOptions(tao->linesearch);
    691:   KSPSetFromOptions(tao->ksp);
    692:   return(0);
    693: }

    695: /*------------------------------------------------------------*/
    696: static PetscErrorCode TaoView_NTL(Tao tao, PetscViewer viewer)
    697: {
    698:   TAO_NTL        *tl = (TAO_NTL *)tao->data;
    699:   PetscBool      isascii;

    703:   PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERASCII,&isascii);
    704:   if (isascii) {
    705:     PetscViewerASCIIPushTab(viewer);
    706:     PetscViewerASCIIPrintf(viewer, "Trust-region steps: %D\n", tl->ntrust);
    707:     PetscViewerASCIIPrintf(viewer, "Newton search steps: %D\n", tl->newt);
    708:     PetscViewerASCIIPrintf(viewer, "BFGS search steps: %D\n", tl->bfgs);
    709:     PetscViewerASCIIPrintf(viewer, "Gradient search steps: %D\n", tl->grad);
    710:     PetscViewerASCIIPopTab(viewer);
    711:   }
    712:   return(0);
    713: }

    715: /* ---------------------------------------------------------- */
    716: /*MC
    717:   TAONTR - Newton's method with trust region and linesearch
    718:   for unconstrained minimization.
    719:   At each iteration, the Newton trust region method solves the system for d
    720:   and performs a line search in the d direction:

    722:             min_d  .5 dT Hk d + gkT d,  s.t.   ||d|| < Delta_k

    724:   Options Database Keys:
    725: + -tao_ntl_init_type - "constant","direction","interpolation"
    726: . -tao_ntl_update_type - "reduction","interpolation"
    727: . -tao_ntl_min_radius - lower bound on trust region radius
    728: . -tao_ntl_max_radius - upper bound on trust region radius
    729: . -tao_ntl_epsilon - tolerance for accepting actual / predicted reduction
    730: . -tao_ntl_mu1_i - mu1 interpolation init factor
    731: . -tao_ntl_mu2_i - mu2 interpolation init factor
    732: . -tao_ntl_gamma1_i - gamma1 interpolation init factor
    733: . -tao_ntl_gamma2_i - gamma2 interpolation init factor
    734: . -tao_ntl_gamma3_i - gamma3 interpolation init factor
    735: . -tao_ntl_gamma4_i - gamma4 interpolation init factor
    736: . -tao_ntl_theta_i - thetha1 interpolation init factor
    737: . -tao_ntl_eta1 - eta1 reduction update factor
    738: . -tao_ntl_eta2 - eta2 reduction update factor
    739: . -tao_ntl_eta3 - eta3 reduction update factor
    740: . -tao_ntl_eta4 - eta4 reduction update factor
    741: . -tao_ntl_alpha1 - alpha1 reduction update factor
    742: . -tao_ntl_alpha2 - alpha2 reduction update factor
    743: . -tao_ntl_alpha3 - alpha3 reduction update factor
    744: . -tao_ntl_alpha4 - alpha4 reduction update factor
    745: . -tao_ntl_alpha4 - alpha4 reduction update factor
    746: . -tao_ntl_mu1 - mu1 interpolation update
    747: . -tao_ntl_mu2 - mu2 interpolation update
    748: . -tao_ntl_gamma1 - gamma1 interpolcation update
    749: . -tao_ntl_gamma2 - gamma2 interpolcation update
    750: . -tao_ntl_gamma3 - gamma3 interpolcation update
    751: . -tao_ntl_gamma4 - gamma4 interpolation update
    752: - -tao_ntl_theta - theta1 interpolation update

    754:   Level: beginner
    755: M*/

    757: PETSC_EXTERN PetscErrorCode TaoCreate_NTL(Tao tao)
    758: {
    759:   TAO_NTL        *tl;
    761:   const char     *morethuente_type = TAOLINESEARCHMT;

    764:   PetscNewLog(tao,&tl);
    765:   tao->ops->setup = TaoSetUp_NTL;
    766:   tao->ops->solve = TaoSolve_NTL;
    767:   tao->ops->view = TaoView_NTL;
    768:   tao->ops->setfromoptions = TaoSetFromOptions_NTL;
    769:   tao->ops->destroy = TaoDestroy_NTL;

    771:   /* Override default settings (unless already changed) */
    772:   if (!tao->max_it_changed) tao->max_it = 50;
    773:   if (!tao->trust0_changed) tao->trust0 = 100.0;

    775:   tao->data = (void*)tl;

    777:   /* Default values for trust-region radius update based on steplength */
    778:   tl->nu1 = 0.25;
    779:   tl->nu2 = 0.50;
    780:   tl->nu3 = 1.00;
    781:   tl->nu4 = 1.25;

    783:   tl->omega1 = 0.25;
    784:   tl->omega2 = 0.50;
    785:   tl->omega3 = 1.00;
    786:   tl->omega4 = 2.00;
    787:   tl->omega5 = 4.00;

    789:   /* Default values for trust-region radius update based on reduction */
    790:   tl->eta1 = 1.0e-4;
    791:   tl->eta2 = 0.25;
    792:   tl->eta3 = 0.50;
    793:   tl->eta4 = 0.90;

    795:   tl->alpha1 = 0.25;
    796:   tl->alpha2 = 0.50;
    797:   tl->alpha3 = 1.00;
    798:   tl->alpha4 = 2.00;
    799:   tl->alpha5 = 4.00;

    801:   /* Default values for trust-region radius update based on interpolation */
    802:   tl->mu1 = 0.10;
    803:   tl->mu2 = 0.50;

    805:   tl->gamma1 = 0.25;
    806:   tl->gamma2 = 0.50;
    807:   tl->gamma3 = 2.00;
    808:   tl->gamma4 = 4.00;

    810:   tl->theta = 0.05;

    812:   /* Default values for trust region initialization based on interpolation */
    813:   tl->mu1_i = 0.35;
    814:   tl->mu2_i = 0.50;

    816:   tl->gamma1_i = 0.0625;
    817:   tl->gamma2_i = 0.5;
    818:   tl->gamma3_i = 2.0;
    819:   tl->gamma4_i = 5.0;

    821:   tl->theta_i = 0.25;

    823:   /* Remaining parameters */
    824:   tl->min_radius = 1.0e-10;
    825:   tl->max_radius = 1.0e10;
    826:   tl->epsilon = 1.0e-6;

    828:   tl->init_type       = NTL_INIT_INTERPOLATION;
    829:   tl->update_type     = NTL_UPDATE_REDUCTION;

    831:   TaoLineSearchCreate(((PetscObject)tao)->comm, &tao->linesearch);
    832:   PetscObjectIncrementTabLevel((PetscObject)tao->linesearch,(PetscObject)tao,1);
    833:   TaoLineSearchSetType(tao->linesearch, morethuente_type);
    834:   TaoLineSearchUseTaoRoutines(tao->linesearch, tao);
    835:   TaoLineSearchSetOptionsPrefix(tao->linesearch,tao->hdr.prefix);
    836:   KSPCreate(((PetscObject)tao)->comm,&tao->ksp);
    837:   PetscObjectIncrementTabLevel((PetscObject)tao->ksp,(PetscObject)tao,1);
    838:   KSPSetOptionsPrefix(tao->ksp,tao->hdr.prefix);
    839:   KSPAppendOptionsPrefix(tao->ksp,"tao_ntl_");
    840:   KSPSetType(tao->ksp,KSPCGSTCG);
    841:   return(0);
    842: }