Lines Matching refs:NMSEDEC_FRACBITS
82 #define NMSEDEC_FRACBITS (NMSEDEC_BITS-1)
583 mask = ~((1<<NMSEDEC_FRACBITS)-1);
586 lut_nmsedec_sig[i] = FFMAX((3 * i << (13 - NMSEDEC_FRACBITS)) - (9 << 11), 0);
587 lut_nmsedec_sig0[i] = FFMAX((i*i + (1<<NMSEDEC_FRACBITS-1) & mask) << 1, 0);
590 lut_nmsedec_ref[i] = FFMAX((a - 2) * (i << (13 - NMSEDEC_FRACBITS)) +
592 lut_nmsedec_ref0[i] = FFMAX(((i * i - (i << NMSEDEC_BITS) + (1 << 2 * NMSEDEC_FRACBITS) + (1 << (NMSEDEC_FRACBITS - 1))) & mask)
601 if (bpno > NMSEDEC_FRACBITS)
602 return lut_nmsedec_sig[(x >> (bpno - NMSEDEC_FRACBITS)) & ((1 << NMSEDEC_BITS) - 1)];
608 if (bpno > NMSEDEC_FRACBITS)
609 return lut_nmsedec_ref[(x >> (bpno - NMSEDEC_FRACBITS)) & ((1 << NMSEDEC_BITS) - 1)];
615 int y0, x, y, mask = 1 << (bpno + NMSEDEC_FRACBITS);
627 *nmsedec += getnmsedec_sig(t1->data[(y) * t1->stride + x], bpno + NMSEDEC_FRACBITS);
637 int y0, x, y, mask = 1 << (bpno + NMSEDEC_FRACBITS);
643 *nmsedec += getnmsedec_ref(t1->data[(y) * t1->stride + x], bpno + NMSEDEC_FRACBITS);
651 int y0, x, y, mask = 1 << (bpno + NMSEDEC_FRACBITS);
678 *nmsedec += getnmsedec_sig(t1->data[(y) * t1->stride + x], bpno + NMSEDEC_FRACBITS);
693 *nmsedec += getnmsedec_sig(t1->data[(y) * t1->stride + x], bpno + NMSEDEC_FRACBITS);
725 cblk->nonzerobits = av_log2(max) + 1 - NMSEDEC_FRACBITS;
1448 *ptr++ = comp->i_data[(comp->coord[0][1] - comp->coord[0][0]) * y + x] * (1 << NMSEDEC_FRACBITS);
1456 *ptr = (int64_t)*ptr * (int64_t)(16384 * 65536 / band->i_stepsize) >> 15 - NMSEDEC_FRACBITS;