Lines Matching defs:prec
211 return ctx->emin - (ctx->prec - 1);
218 return ctx->emax - (ctx->prec - 1);
752 workctx->prec = ctx->prec;
791 _mpd_idiv_word(&len, &r, ctx->prec, MPD_RDIGITS);
799 result->digits = ctx->prec;
811 * Cut off the most significant digits so that the rest fits in ctx->prec.
820 if (result->len > 0 && result->digits > ctx->prec) {
821 _mpd_idiv_word(&len, &r, ctx->prec, MPD_RDIGITS);
841 * fits in ctx->prec - ctx->clamp. Cannot fail.
847 mpd_ssize_t prec;
850 prec = ctx->prec - ctx->clamp;
851 if (result->len > 0 && result->digits > prec) {
852 if (prec == 0) {
857 _mpd_idiv_word(&len, &r, prec, MPD_RDIGITS);
1740 /* We have a number with exactly ctx->prec digits. The increment
1742 * that case, the result is a power of ten with prec+1 digits.
1757 if (dec->digits > ctx->prec) {
1760 dec->digits = ctx->prec;
1805 if (dec->digits > ctx->prec) {
1828 dec->exp -= (ctx->prec-1);
1843 dec->exp = ctx->emax - ctx->prec + 1;
1848 dec->exp = ctx->emax - ctx->prec + 1;
1857 dec->exp = ctx->emax - ctx->prec + 1;
1871 /* At this point adjexp=exp+digits-1 <= emax and exp > etop=emax-prec+1:
1872 * (1) shift = exp -emax+prec-1 > 0
1873 * (2) digits+shift = exp+digits-1 - emax + prec <= prec */
1881 /* Underflow is impossible, since exp < etiny=emin-prec+1
1882 * and exp > etop=emax-prec+1 would imply emax < emin. */
1901 /* At this point adjexp=exp+digits-1 < emin and exp < etiny=emin-prec+1:
1902 * (1) shift = emin-prec+1 - exp > 0
1903 * (2) digits-shift = exp+digits-1 - emin + prec < prec */
1918 /* Case exp >= etiny=emin-prec+1:
1920 * (2) digits < emin-exp+1 <= prec */
1950 if (dec->digits > ctx->prec) {
1951 shift = dec->digits - ctx->prec;
2814 digits = (a->digits < ctx->prec) ? ctx->prec : a->digits;
2989 if (n > ctx->prec || n < -ctx->prec) {
3000 rshift = ctx->prec-n;
3003 lshift = ctx->prec+n;
3007 if (a->digits > ctx->prec) {
3036 * b must be an integer with exponent 0 and in the range +-2*(emax + prec).
3037 * XXX: In my opinion +-(2*emax + prec) would be more sensible.
3066 maxjump = 2 * (mpd_uint_t)(ctx->emax + ctx->prec);
3122 if (n >= 0 && n <= ctx->prec) {
3126 else if (n < 0 && n >= -ctx->prec) {
3164 if (n > ctx->prec || n < -ctx->prec) {
3357 exp += (big->digits > ctx->prec) ? 0 : big->digits-ctx->prec-1;
3365 * bdigits <= prec AND
3366 * bdigits+shift >= prec+2+sdigits AND
3367 * exp = bexp+bdigits-prec-2
3372 * |- prec -|
3376 * bdigits > prec AND
3383 * |- prec -|
3791 shift = (b->digits - a->digits) + ctx->prec + 1;
3925 workctx.prec = a->digits + b->digits * 4;
3926 if (workctx.prec >= ctx->prec) {
3987 if (expdiff > ctx->prec) {
3995 * (1) 0 <= a->exp + a->digits - b->exp - b->digits <= prec
3997 * (3) a->exp - b->exp <= prec + b->digits - a->digits
4002 /* by (3), after the shift a->digits <= prec + b->digits */
4067 if (q->digits > ctx->prec) {
4277 /* Pad the result with trailing zeros if it has fewer digits than prec. */
4282 result->digits < ctx->prec) {
4283 mpd_ssize_t shift = ctx->prec - result->digits;
4294 MPD_NEW_CONST(lim,0,-(ctx->prec+1),1,1,1,9);
4299 /* abs(a) <= 9 * 10**(-prec-1) */
4355 * 1) abs(a) <= 9 * 10**(-prec-1) ==> result == 1
4357 * Absolute error: abs(1 - e**x) < 10**(-prec)
4360 * 2) abs(a) > 9 * 10**(-prec-1)
4362 * Relative error: abs(result - e**x) < 0.5 * 10**(-prec) * e**x
4431 /* abs(a) <= 9 * 10**(-prec-1) */
4437 workctx.prec = ctx->prec + t + 2;
4438 workctx.prec = (workctx.prec < 10) ? 10 : workctx.prec;
4448 * 1) 9 * 10**(-prec-1) < abs(a)
4449 * 2) 9 * 10**(-prec-t-1) < abs(r)
4450 * 3) log10(9) - prec - t - 1 < log10(abs(r)) < adjexp(abs(r)) + 1
4451 * 4) - prec - t - 2 < adjexp(abs(r)) <= -1
4453 n = _mpd_get_exp_iterations(result, workctx.prec);
4520 mpd_ssize_t prec;
4533 prec = ctx->prec + 3;
4535 workctx.prec = prec;
4541 ulpexp = result->exp + result->digits - workctx.prec;
4550 * 1) abs(result - e**x) < 0.5 * 10**(-prec) * e**x
4558 * [1] If abs(a) <= 9 * 10**(-prec-1), use the absolute
4561 workctx.prec = ctx->prec;
4572 prec += MPD_RDIGITS;
4701 * Relative error: abs(result - log(10)) < 5 * 10**-prec * log(10)
4707 mpd_qln10(mpd_t *result, mpd_ssize_t prec, uint32_t *status)
4717 assert(prec >= 1);
4719 shift = MPD_MINALLOC_MAX*MPD_RDIGITS-prec;
4730 if (prec < MPD_MINALLOC_MAX*MPD_RDIGITS) {
4731 maxcontext.prec = prec;
4740 i = ln_schedule_prec(klist, prec+2, -result->exp);
4742 varcontext.prec = 2*klist[i]+3;
4755 maxcontext.prec = prec;
4839 * Relative error: abs(result - log(a)) < 0.1 * 10**-prec * abs(log(a))
4904 maxprec = ctx->prec + 2;
4938 * p = ctx->prec+2-adjexp(lower), then the relative error of
4942 * <= 10**(-ctx->prec-2)
4950 varcontext.prec = 2*klist[i]+3;
4955 if (v.digits > varcontext.prec) {
4956 shift = v.digits - varcontext.prec;
4976 * less than 10**(-ctx.prec-1).
4980 * p := maxprec = ctx->prec + 2
4994 * < 0.5 * 10**(-p + 1) = 0.5 * 10**(-ctx->prec-1)
5077 mpd_ssize_t prec;
5088 prec = ctx->prec + 3;
5090 workctx.prec = prec;
5093 result->exp + result->digits-workctx.prec);
5095 workctx.prec = ctx->prec;
5105 prec += MPD_RDIGITS;
5122 * Relative error: abs(result - log10(a)) < 0.1 * 10**-prec * abs(log10(a))
5135 workctx.prec = ctx->prec + 3;
5140 mpd_qln10(&ln10, workctx.prec, status);
5226 mpd_ssize_t prec;
5237 prec = ctx->prec + 3;
5239 workctx.prec = prec;
5242 result->exp + result->digits-workctx.prec);
5244 workctx.prec = ctx->prec;
5254 prec += MPD_RDIGITS;
6280 * abs(err) = 5 * 10**-prec
6322 * abs(err) = 5 * 10**-prec
6360 * final rounding to prec:
6361 * abs(result - base**exp) < 0.1 * 10**-prec * abs(base**exp)
6375 workctx.prec += (exp->digits + exp->exp + 2);
6380 workctx.prec += 1;
6420 * with a coefficient of length prec. Otherwise, result is undefined.
6431 shift = ctx->prec-1;
6472 if (shift > ctx->prec-1) {
6473 shift = ctx->prec-1;
6478 shift = ctx->prec-1;
6653 * Relative error: abs(result - e**y) < e**y * 1/5 * 10**(-prec - 1)
6668 workctx.prec = (base->digits > ctx->prec) ? base->digits : ctx->prec;
6669 workctx.prec += (4 + MPD_EXPDIGITS);
6675 * wp := prec + 4 + extra
6689 * 4) abs(y * (2*err + err**2)) < 1/5 * 10**(-prec - 2)
6693 * 6) abs(e**(y * (2*err + err**2)) - 1) < 10**(-prec - 2)
6695 * 7) abs(result - e**y) < e**y * 1/5 * 10**(-prec - 1)
6789 mpd_ssize_t shift = ctx->prec-1;
6863 if (mod->digits+mod->exp > ctx->prec) {
6994 if (a->digits + expdiff > ctx->prec) {
7007 /* At this point expdiff < 0 and a->digits+expdiff <= prec,
7008 * so the shift before an increment will fit in prec. */
7187 if (allnine && qdigits == ctx->prec) {
7188 /* abs(quotient) + 1 == 10**prec */
7488 maxcontext.prec = MPD_MAX_PREC + 100;
7490 maxprec = ctx->prec;
7499 varcontext.prec = 2*klist[i] + 5;
7500 if (v->digits > varcontext.prec) {
7502 mpd_qshiftr(&t, v, v->digits-varcontext.prec, status);
7503 t.exp = -varcontext.prec;
7576 /* Let prec := adigits - bdigits + 4 */
7577 workctx.prec = a->digits - b->digits + 1 + 3;
7578 if (a->digits > MPD_MAX_PREC || workctx.prec > MPD_MAX_PREC) {
7583 /* Let x := _mpd_qreciprocal(b, prec)
7585 * 1) 1/b - 10**(-prec - bdigits) < x < 1/b + 10**(-prec - bdigits)
7600 workctx.prec = aa.digits + 3;
7733 * 4) 0 <= 10**prec/a - 1/sqrt(v) < 10**-prec
7749 * Relative error: abs(result - 1/sqrt(a)) < 10**-prec * 1/sqrt(a)
7811 maxprec = ctx->prec + 1;
7816 varcontext.prec = 2*klist[i]+2;
7818 if (v->digits > varcontext.prec) {
7819 shift = v->digits - varcontext.prec;
7880 workctx.prec += 2;
7897 mpd_ssize_t prec, ideal_exp;
7926 prec = ctx->prec + 1;
7943 shift = prec - l;
7957 if (!mpd_qshiftl(result, &one, prec, status)) {
8052 workctx.prec = a->digits;
8053 if (workctx.prec >= ctx->prec) {
8773 .prec=MPD_MAX_PREC,