xref: /third_party/ffmpeg/libavcodec/agm.c (revision cabdff1a)
1/*
2 * Amuse Graphics Movie decoder
3 *
4 * Copyright (c) 2018 Paul B Mahol
5 *
6 * This file is part of FFmpeg.
7 *
8 * FFmpeg is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License as published by the Free Software Foundation; either
11 * version 2.1 of the License, or (at your option) any later version.
12 *
13 * FFmpeg is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
16 * Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public
19 * License along with FFmpeg; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 */
22
23#include <stdio.h>
24#include <stdlib.h>
25#include <string.h>
26
27#define BITSTREAM_READER_LE
28
29#include "libavutil/mem_internal.h"
30
31#include "avcodec.h"
32#include "bytestream.h"
33#include "codec_internal.h"
34#include "copy_block.h"
35#include "get_bits.h"
36#include "idctdsp.h"
37#include "internal.h"
38
39static const uint8_t unscaled_luma[64] = {
40    16, 11, 10, 16, 24, 40, 51, 61, 12, 12, 14, 19,
41    26, 58, 60, 55, 14, 13, 16, 24, 40, 57, 69, 56,
42    14, 17, 22, 29, 51, 87, 80, 62, 18, 22, 37, 56,
43    68,109,103, 77, 24, 35, 55, 64, 81,104,113, 92,
44    49, 64, 78, 87,103,121,120,101, 72, 92, 95, 98,
45    112,100,103,99
46};
47
48static const uint8_t unscaled_chroma[64] = {
49    17, 18, 24, 47, 99, 99, 99, 99, 18, 21, 26, 66,
50    99, 99, 99, 99, 24, 26, 56, 99, 99, 99, 99, 99,
51    47, 66, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99,
52    99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99,
53    99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99,
54    99, 99, 99, 99
55};
56
57typedef struct MotionVector {
58    int16_t x, y;
59} MotionVector;
60
61typedef struct AGMContext {
62    const AVClass  *class;
63    AVCodecContext *avctx;
64    GetBitContext   gb;
65    GetByteContext  gbyte;
66
67    int key_frame;
68    int bitstream_size;
69    int compression;
70    int blocks_w;
71    int blocks_h;
72    int size[3];
73    int plus;
74    int dct;
75    int rgb;
76    unsigned flags;
77    unsigned fflags;
78
79    uint8_t *output;
80    unsigned padded_output_size;
81    unsigned output_size;
82
83    MotionVector *mvectors;
84    unsigned      mvectors_size;
85
86    VLC vlc;
87
88    AVFrame *prev_frame;
89
90    int luma_quant_matrix[64];
91    int chroma_quant_matrix[64];
92
93    ScanTable scantable;
94    DECLARE_ALIGNED(32, int16_t, block)[64];
95
96    int16_t *wblocks;
97    unsigned wblocks_size;
98
99    int      *map;
100    unsigned  map_size;
101
102    IDCTDSPContext idsp;
103} AGMContext;
104
105static int read_code(GetBitContext *gb, int *oskip, int *level, int *map, int mode)
106{
107    int len = 0, skip = 0, max;
108
109    if (get_bits_left(gb) < 2)
110        return AVERROR_INVALIDDATA;
111
112    if (show_bits(gb, 2)) {
113        switch (show_bits(gb, 4)) {
114        case 1:
115        case 9:
116            len = 1;
117            skip = 3;
118            break;
119        case 2:
120            len = 3;
121            skip = 4;
122            break;
123        case 3:
124            len = 7;
125            skip = 4;
126            break;
127        case 5:
128        case 13:
129            len = 2;
130            skip = 3;
131            break;
132        case 6:
133            len = 4;
134            skip = 4;
135            break;
136        case 7:
137            len = 8;
138            skip = 4;
139            break;
140        case 10:
141            len = 5;
142            skip = 4;
143            break;
144        case 11:
145            len = 9;
146            skip = 4;
147            break;
148        case 14:
149            len = 6;
150            skip = 4;
151            break;
152        case 15:
153            len = ((show_bits(gb, 5) & 0x10) | 0xA0) >> 4;
154            skip = 5;
155            break;
156        default:
157            return AVERROR_INVALIDDATA;
158        }
159
160        skip_bits(gb, skip);
161        *level = get_bits(gb, len);
162        *map = 1;
163        *oskip = 0;
164        max = 1 << (len - 1);
165        if (*level < max)
166            *level = -(max + *level);
167    } else if (show_bits(gb, 3) & 4) {
168        skip_bits(gb, 3);
169        if (mode == 1) {
170            if (show_bits(gb, 4)) {
171                if (show_bits(gb, 4) == 1) {
172                    skip_bits(gb, 4);
173                    *oskip = get_bits(gb, 16);
174                } else {
175                    *oskip = get_bits(gb, 4);
176                }
177            } else {
178                skip_bits(gb, 4);
179                *oskip = get_bits(gb, 10);
180            }
181        } else if (mode == 0) {
182            *oskip = get_bits(gb, 10);
183        }
184        *level = 0;
185    } else {
186        skip_bits(gb, 3);
187        if (mode == 0)
188            *oskip = get_bits(gb, 4);
189        else if (mode == 1)
190            *oskip = 0;
191        *level = 0;
192    }
193
194    return 0;
195}
196
197static int decode_intra_blocks(AGMContext *s, GetBitContext *gb,
198                               const int *quant_matrix, int *skip, int *dc_level)
199{
200    const uint8_t *scantable = s->scantable.permutated;
201    int level, ret, map = 0;
202
203    memset(s->wblocks, 0, s->wblocks_size);
204
205    for (int i = 0; i < 64; i++) {
206        int16_t *block = s->wblocks + scantable[i];
207
208        for (int j = 0; j < s->blocks_w;) {
209            if (*skip > 0) {
210                int rskip;
211
212                rskip = FFMIN(*skip, s->blocks_w - j);
213                j += rskip;
214                if (i == 0) {
215                    for (int k = 0; k < rskip; k++)
216                        block[64 * k] = *dc_level * quant_matrix[0];
217                }
218                block += rskip * 64;
219                *skip -= rskip;
220            } else {
221                ret = read_code(gb, skip, &level, &map, s->flags & 1);
222                if (ret < 0)
223                    return ret;
224
225                if (i == 0)
226                    *dc_level += level;
227
228                block[0] = (i == 0 ? *dc_level : level) * quant_matrix[i];
229                block += 64;
230                j++;
231            }
232        }
233    }
234
235    return 0;
236}
237
238static int decode_inter_blocks(AGMContext *s, GetBitContext *gb,
239                               const int *quant_matrix, int *skip,
240                               int *map)
241{
242    const uint8_t *scantable = s->scantable.permutated;
243    int level, ret;
244
245    memset(s->wblocks, 0, s->wblocks_size);
246    memset(s->map, 0, s->map_size);
247
248    for (int i = 0; i < 64; i++) {
249        int16_t *block = s->wblocks + scantable[i];
250
251        for (int j = 0; j < s->blocks_w;) {
252            if (*skip > 0) {
253                int rskip;
254
255                rskip = FFMIN(*skip, s->blocks_w - j);
256                j += rskip;
257                block += rskip * 64;
258                *skip -= rskip;
259            } else {
260                ret = read_code(gb, skip, &level, &map[j], s->flags & 1);
261                if (ret < 0)
262                    return ret;
263
264                block[0] = level * quant_matrix[i];
265                block += 64;
266                j++;
267            }
268        }
269    }
270
271    return 0;
272}
273
274static int decode_intra_block(AGMContext *s, GetBitContext *gb,
275                              const int *quant_matrix, int *skip, int *dc_level)
276{
277    const uint8_t *scantable = s->scantable.permutated;
278    const int offset = s->plus ? 0 : 1024;
279    int16_t *block = s->block;
280    int level, ret, map = 0;
281
282    memset(block, 0, sizeof(s->block));
283
284    if (*skip > 0) {
285        (*skip)--;
286    } else {
287        ret = read_code(gb, skip, &level, &map, s->flags & 1);
288        if (ret < 0)
289            return ret;
290        *dc_level += level;
291    }
292    block[scantable[0]] = offset + *dc_level * quant_matrix[0];
293
294    for (int i = 1; i < 64;) {
295        if (*skip > 0) {
296            int rskip;
297
298            rskip = FFMIN(*skip, 64 - i);
299            i += rskip;
300            *skip -= rskip;
301        } else {
302            ret = read_code(gb, skip, &level, &map, s->flags & 1);
303            if (ret < 0)
304                return ret;
305
306            block[scantable[i]] = level * quant_matrix[i];
307            i++;
308        }
309    }
310
311    return 0;
312}
313
314static int decode_intra_plane(AGMContext *s, GetBitContext *gb, int size,
315                              const int *quant_matrix, AVFrame *frame,
316                              int plane)
317{
318    int ret, skip = 0, dc_level = 0;
319    const int offset = s->plus ? 0 : 1024;
320
321    if ((ret = init_get_bits8(gb, s->gbyte.buffer, size)) < 0)
322        return ret;
323
324    if (s->flags & 1) {
325        av_fast_padded_malloc(&s->wblocks, &s->wblocks_size,
326                              64 * s->blocks_w * sizeof(*s->wblocks));
327        if (!s->wblocks)
328            return AVERROR(ENOMEM);
329
330        for (int y = 0; y < s->blocks_h; y++) {
331            ret = decode_intra_blocks(s, gb, quant_matrix, &skip, &dc_level);
332            if (ret < 0)
333                return ret;
334
335            for (int x = 0; x < s->blocks_w; x++) {
336                s->wblocks[64 * x] += offset;
337                s->idsp.idct_put(frame->data[plane] + (s->blocks_h - 1 - y) * 8 * frame->linesize[plane] + x * 8,
338                                 frame->linesize[plane], s->wblocks + 64 * x);
339            }
340        }
341    } else {
342        for (int y = 0; y < s->blocks_h; y++) {
343            for (int x = 0; x < s->blocks_w; x++) {
344                ret = decode_intra_block(s, gb, quant_matrix, &skip, &dc_level);
345                if (ret < 0)
346                    return ret;
347
348                s->idsp.idct_put(frame->data[plane] + (s->blocks_h - 1 - y) * 8 * frame->linesize[plane] + x * 8,
349                                 frame->linesize[plane], s->block);
350            }
351        }
352    }
353
354    align_get_bits(gb);
355    if (get_bits_left(gb) < 0)
356        av_log(s->avctx, AV_LOG_WARNING, "overread\n");
357    if (get_bits_left(gb) > 0)
358        av_log(s->avctx, AV_LOG_WARNING, "underread: %d\n", get_bits_left(gb));
359
360    return 0;
361}
362
363static int decode_inter_block(AGMContext *s, GetBitContext *gb,
364                              const int *quant_matrix, int *skip,
365                              int *map)
366{
367    const uint8_t *scantable = s->scantable.permutated;
368    int16_t *block = s->block;
369    int level, ret;
370
371    memset(block, 0, sizeof(s->block));
372
373    for (int i = 0; i < 64;) {
374        if (*skip > 0) {
375            int rskip;
376
377            rskip = FFMIN(*skip, 64 - i);
378            i += rskip;
379            *skip -= rskip;
380        } else {
381            ret = read_code(gb, skip, &level, map, s->flags & 1);
382            if (ret < 0)
383                return ret;
384
385            block[scantable[i]] = level * quant_matrix[i];
386            i++;
387        }
388    }
389
390    return 0;
391}
392
393static int decode_inter_plane(AGMContext *s, GetBitContext *gb, int size,
394                              const int *quant_matrix, AVFrame *frame,
395                              AVFrame *prev, int plane)
396{
397    int ret, skip = 0;
398
399    if ((ret = init_get_bits8(gb, s->gbyte.buffer, size)) < 0)
400        return ret;
401
402    if (s->flags == 3) {
403        av_fast_padded_malloc(&s->wblocks, &s->wblocks_size,
404                              64 * s->blocks_w * sizeof(*s->wblocks));
405        if (!s->wblocks)
406            return AVERROR(ENOMEM);
407
408        av_fast_padded_malloc(&s->map, &s->map_size,
409                              s->blocks_w * sizeof(*s->map));
410        if (!s->map)
411            return AVERROR(ENOMEM);
412
413        for (int y = 0; y < s->blocks_h; y++) {
414            ret = decode_inter_blocks(s, gb, quant_matrix, &skip, s->map);
415            if (ret < 0)
416                return ret;
417
418            for (int x = 0; x < s->blocks_w; x++) {
419                int shift = plane == 0;
420                int mvpos = (y >> shift) * (s->blocks_w >> shift) + (x >> shift);
421                int orig_mv_x = s->mvectors[mvpos].x;
422                int mv_x = s->mvectors[mvpos].x / (1 + !shift);
423                int mv_y = s->mvectors[mvpos].y / (1 + !shift);
424                int h = s->avctx->coded_height >> !shift;
425                int w = s->avctx->coded_width  >> !shift;
426                int map = s->map[x];
427
428                if (orig_mv_x >= -32) {
429                    if (y * 8 + mv_y < 0 || y * 8 + mv_y + 8 > h ||
430                        x * 8 + mv_x < 0 || x * 8 + mv_x + 8 > w)
431                        return AVERROR_INVALIDDATA;
432
433                    copy_block8(frame->data[plane] + (s->blocks_h - 1 - y) * 8 * frame->linesize[plane] + x * 8,
434                                prev->data[plane] + ((s->blocks_h - 1 - y) * 8 - mv_y) * prev->linesize[plane] + (x * 8 + mv_x),
435                                frame->linesize[plane], prev->linesize[plane], 8);
436                    if (map) {
437                        s->idsp.idct(s->wblocks + x * 64);
438                        for (int i = 0; i < 64; i++)
439                            s->wblocks[i + x * 64] = (s->wblocks[i + x * 64] + 1) & 0xFFFC;
440                        s->idsp.add_pixels_clamped(&s->wblocks[x*64], frame->data[plane] + (s->blocks_h - 1 - y) * 8 * frame->linesize[plane] + x * 8,
441                                                   frame->linesize[plane]);
442                    }
443                } else if (map) {
444                    s->idsp.idct_put(frame->data[plane] + (s->blocks_h - 1 - y) * 8 * frame->linesize[plane] + x * 8,
445                                     frame->linesize[plane], s->wblocks + x * 64);
446                }
447            }
448        }
449    } else if (s->flags & 2) {
450        for (int y = 0; y < s->blocks_h; y++) {
451            for (int x = 0; x < s->blocks_w; x++) {
452                int shift = plane == 0;
453                int mvpos = (y >> shift) * (s->blocks_w >> shift) + (x >> shift);
454                int orig_mv_x = s->mvectors[mvpos].x;
455                int mv_x = s->mvectors[mvpos].x / (1 + !shift);
456                int mv_y = s->mvectors[mvpos].y / (1 + !shift);
457                int h = s->avctx->coded_height >> !shift;
458                int w = s->avctx->coded_width  >> !shift;
459                int map = 0;
460
461                ret = decode_inter_block(s, gb, quant_matrix, &skip, &map);
462                if (ret < 0)
463                    return ret;
464
465                if (orig_mv_x >= -32) {
466                    if (y * 8 + mv_y < 0 || y * 8 + mv_y + 8 > h ||
467                        x * 8 + mv_x < 0 || x * 8 + mv_x + 8 > w)
468                        return AVERROR_INVALIDDATA;
469
470                    copy_block8(frame->data[plane] + (s->blocks_h - 1 - y) * 8 * frame->linesize[plane] + x * 8,
471                                prev->data[plane] + ((s->blocks_h - 1 - y) * 8 - mv_y) * prev->linesize[plane] + (x * 8 + mv_x),
472                                frame->linesize[plane], prev->linesize[plane], 8);
473                    if (map) {
474                        s->idsp.idct(s->block);
475                        for (int i = 0; i < 64; i++)
476                            s->block[i] = (s->block[i] + 1) & 0xFFFC;
477                        s->idsp.add_pixels_clamped(s->block, frame->data[plane] + (s->blocks_h - 1 - y) * 8 * frame->linesize[plane] + x * 8,
478                                                   frame->linesize[plane]);
479                    }
480                } else if (map) {
481                    s->idsp.idct_put(frame->data[plane] + (s->blocks_h - 1 - y) * 8 * frame->linesize[plane] + x * 8,
482                                     frame->linesize[plane], s->block);
483                }
484            }
485        }
486    } else if (s->flags & 1) {
487        av_fast_padded_malloc(&s->wblocks, &s->wblocks_size,
488                              64 * s->blocks_w * sizeof(*s->wblocks));
489        if (!s->wblocks)
490            return AVERROR(ENOMEM);
491
492        av_fast_padded_malloc(&s->map, &s->map_size,
493                              s->blocks_w * sizeof(*s->map));
494        if (!s->map)
495            return AVERROR(ENOMEM);
496
497        for (int y = 0; y < s->blocks_h; y++) {
498            ret = decode_inter_blocks(s, gb, quant_matrix, &skip, s->map);
499            if (ret < 0)
500                return ret;
501
502            for (int x = 0; x < s->blocks_w; x++) {
503                if (!s->map[x])
504                    continue;
505                s->idsp.idct_add(frame->data[plane] + (s->blocks_h - 1 - y) * 8 * frame->linesize[plane] + x * 8,
506                                 frame->linesize[plane], s->wblocks + 64 * x);
507            }
508        }
509    } else {
510        for (int y = 0; y < s->blocks_h; y++) {
511            for (int x = 0; x < s->blocks_w; x++) {
512                int map = 0;
513
514                ret = decode_inter_block(s, gb, quant_matrix, &skip, &map);
515                if (ret < 0)
516                    return ret;
517
518                if (!map)
519                    continue;
520                s->idsp.idct_add(frame->data[plane] + (s->blocks_h - 1 - y) * 8 * frame->linesize[plane] + x * 8,
521                                 frame->linesize[plane], s->block);
522            }
523        }
524    }
525
526    align_get_bits(gb);
527    if (get_bits_left(gb) < 0)
528        av_log(s->avctx, AV_LOG_WARNING, "overread\n");
529    if (get_bits_left(gb) > 0)
530        av_log(s->avctx, AV_LOG_WARNING, "underread: %d\n", get_bits_left(gb));
531
532    return 0;
533}
534
535static void compute_quant_matrix(AGMContext *s, double qscale)
536{
537    int luma[64], chroma[64];
538    double f = 1.0 - fabs(qscale);
539
540    if (!s->key_frame && (s->flags & 2)) {
541        if (qscale >= 0.0) {
542            for (int i = 0; i < 64; i++) {
543                luma[i]   = FFMAX(1, 16 * f);
544                chroma[i] = FFMAX(1, 16 * f);
545            }
546        } else {
547            for (int i = 0; i < 64; i++) {
548                luma[i]   = FFMAX(1, 16 - qscale * 32);
549                chroma[i] = FFMAX(1, 16 - qscale * 32);
550            }
551        }
552    } else {
553        if (qscale >= 0.0) {
554            for (int i = 0; i < 64; i++) {
555                luma[i]   = FFMAX(1, unscaled_luma  [(i & 7) * 8 + (i >> 3)] * f);
556                chroma[i] = FFMAX(1, unscaled_chroma[(i & 7) * 8 + (i >> 3)] * f);
557            }
558        } else {
559            for (int i = 0; i < 64; i++) {
560                luma[i]   = FFMAX(1, 255.0 - (255 - unscaled_luma  [(i & 7) * 8 + (i >> 3)]) * f);
561                chroma[i] = FFMAX(1, 255.0 - (255 - unscaled_chroma[(i & 7) * 8 + (i >> 3)]) * f);
562            }
563        }
564    }
565
566    for (int i = 0; i < 64; i++) {
567        int pos = ff_zigzag_direct[i];
568
569        s->luma_quant_matrix[i]   = luma[pos]   * ((pos / 8) & 1 ? -1 : 1);
570        s->chroma_quant_matrix[i] = chroma[pos] * ((pos / 8) & 1 ? -1 : 1);
571    }
572}
573
574static int decode_raw_intra_rgb(AVCodecContext *avctx, GetByteContext *gbyte, AVFrame *frame)
575{
576    uint8_t *dst = frame->data[0] + (avctx->height - 1) * frame->linesize[0];
577    uint8_t r = 0, g = 0, b = 0;
578
579    if (bytestream2_get_bytes_left(gbyte) < 3 * avctx->width * avctx->height)
580        return AVERROR_INVALIDDATA;
581
582    for (int y = 0; y < avctx->height; y++) {
583        for (int x = 0; x < avctx->width; x++) {
584            dst[x*3+0] = bytestream2_get_byteu(gbyte) + r;
585            r = dst[x*3+0];
586            dst[x*3+1] = bytestream2_get_byteu(gbyte) + g;
587            g = dst[x*3+1];
588            dst[x*3+2] = bytestream2_get_byteu(gbyte) + b;
589            b = dst[x*3+2];
590        }
591        dst -= frame->linesize[0];
592    }
593
594    return 0;
595}
596
597av_always_inline static int fill_pixels(uint8_t **y0, uint8_t **y1,
598                       uint8_t **u, uint8_t **v,
599                       int ylinesize, int ulinesize, int vlinesize,
600                       uint8_t *fill,
601                       int *nx, int *ny, int *np, int w, int h)
602{
603    uint8_t *y0dst = *y0;
604    uint8_t *y1dst = *y1;
605    uint8_t *udst = *u;
606    uint8_t *vdst = *v;
607    int x = *nx, y = *ny, pos = *np;
608
609    if (pos == 0) {
610        y0dst[2*x+0] += fill[0];
611        y0dst[2*x+1] += fill[1];
612        y1dst[2*x+0] += fill[2];
613        y1dst[2*x+1] += fill[3];
614        pos++;
615    } else if (pos == 1) {
616        udst[x] += fill[0];
617        vdst[x] += fill[1];
618        x++;
619        if (x >= w) {
620            x = 0;
621            y++;
622            if (y >= h)
623                return 1;
624            y0dst -= 2*ylinesize;
625            y1dst -= 2*ylinesize;
626            udst  -=   ulinesize;
627            vdst  -=   vlinesize;
628        }
629        y0dst[2*x+0] += fill[2];
630        y0dst[2*x+1] += fill[3];
631        pos++;
632    } else if (pos == 2) {
633        y1dst[2*x+0] += fill[0];
634        y1dst[2*x+1] += fill[1];
635        udst[x]      += fill[2];
636        vdst[x]      += fill[3];
637        x++;
638        if (x >= w) {
639            x = 0;
640            y++;
641            if (y >= h)
642                return 1;
643            y0dst -= 2*ylinesize;
644            y1dst -= 2*ylinesize;
645            udst  -=   ulinesize;
646            vdst  -=   vlinesize;
647        }
648        pos = 0;
649    }
650
651    *y0 = y0dst;
652    *y1 = y1dst;
653    *u = udst;
654    *v = vdst;
655    *np = pos;
656    *nx = x;
657    *ny = y;
658
659    return 0;
660}
661
662static int decode_runlen_rgb(AVCodecContext *avctx, GetByteContext *gbyte, AVFrame *frame)
663{
664    uint8_t *dst = frame->data[0] + (avctx->height - 1) * frame->linesize[0];
665    int runlen, y = 0, x = 0;
666    uint8_t fill[4];
667    unsigned code;
668
669    while (bytestream2_get_bytes_left(gbyte) > 0) {
670        code = bytestream2_peek_le32(gbyte);
671        runlen = code & 0xFFFFFF;
672
673        if (code >> 24 == 0x77) {
674            bytestream2_skip(gbyte, 4);
675
676            for (int i = 0; i < 4; i++)
677                fill[i] = bytestream2_get_byte(gbyte);
678
679            while (runlen > 0) {
680                runlen--;
681
682                for (int i = 0; i < 4; i++) {
683                    dst[x] += fill[i];
684                    x++;
685                    if (x >= frame->width * 3) {
686                        x = 0;
687                        y++;
688                        dst -= frame->linesize[0];
689                        if (y >= frame->height)
690                            return 0;
691                    }
692                }
693            }
694        } else {
695            for (int i = 0; i < 4; i++)
696                fill[i] = bytestream2_get_byte(gbyte);
697
698            for (int i = 0; i < 4; i++) {
699                dst[x] += fill[i];
700                x++;
701                if (x >= frame->width * 3) {
702                    x = 0;
703                    y++;
704                    dst -= frame->linesize[0];
705                    if (y >= frame->height)
706                        return 0;
707                }
708            }
709        }
710    }
711
712    return 0;
713}
714
715static int decode_runlen(AVCodecContext *avctx, GetByteContext *gbyte, AVFrame *frame)
716{
717    uint8_t *y0dst = frame->data[0] + (avctx->height - 1) * frame->linesize[0];
718    uint8_t *y1dst = y0dst - frame->linesize[0];
719    uint8_t *udst = frame->data[1] + ((avctx->height >> 1) - 1) * frame->linesize[1];
720    uint8_t *vdst = frame->data[2] + ((avctx->height >> 1) - 1) * frame->linesize[2];
721    int runlen, y = 0, x = 0, pos = 0;
722    uint8_t fill[4];
723    unsigned code;
724
725    while (bytestream2_get_bytes_left(gbyte) > 0) {
726        code = bytestream2_peek_le32(gbyte);
727        runlen = code & 0xFFFFFF;
728
729        if (code >> 24 == 0x77) {
730            bytestream2_skip(gbyte, 4);
731
732            for (int i = 0; i < 4; i++)
733                fill[i] = bytestream2_get_byte(gbyte);
734
735            while (runlen > 0) {
736                runlen--;
737
738                if (fill_pixels(&y0dst, &y1dst, &udst, &vdst,
739                                frame->linesize[0],
740                                frame->linesize[1],
741                                frame->linesize[2],
742                                fill, &x, &y, &pos,
743                                avctx->width / 2,
744                                avctx->height / 2))
745                    return 0;
746            }
747        } else {
748            for (int i = 0; i < 4; i++)
749                fill[i] = bytestream2_get_byte(gbyte);
750
751            if (fill_pixels(&y0dst, &y1dst, &udst, &vdst,
752                            frame->linesize[0],
753                            frame->linesize[1],
754                            frame->linesize[2],
755                            fill, &x, &y, &pos,
756                            avctx->width / 2,
757                            avctx->height / 2))
758                return 0;
759        }
760    }
761
762    return 0;
763}
764
765static int decode_raw_intra(AVCodecContext *avctx, GetByteContext *gbyte, AVFrame *frame)
766{
767    uint8_t *y0dst = frame->data[0] + (avctx->height - 1) * frame->linesize[0];
768    uint8_t *y1dst = y0dst - frame->linesize[0];
769    uint8_t *udst = frame->data[1] + ((avctx->height >> 1) - 1) * frame->linesize[1];
770    uint8_t *vdst = frame->data[2] + ((avctx->height >> 1) - 1) * frame->linesize[2];
771    uint8_t ly0 = 0, ly1 = 0, ly2 = 0, ly3 = 0, lu = 0, lv = 0;
772
773    for (int y = 0; y < avctx->height / 2; y++) {
774        for (int x = 0; x < avctx->width / 2; x++) {
775            y0dst[x*2+0] = bytestream2_get_byte(gbyte) + ly0;
776            ly0 = y0dst[x*2+0];
777            y0dst[x*2+1] = bytestream2_get_byte(gbyte) + ly1;
778            ly1 = y0dst[x*2+1];
779            y1dst[x*2+0] = bytestream2_get_byte(gbyte) + ly2;
780            ly2 = y1dst[x*2+0];
781            y1dst[x*2+1] = bytestream2_get_byte(gbyte) + ly3;
782            ly3 = y1dst[x*2+1];
783            udst[x] = bytestream2_get_byte(gbyte) + lu;
784            lu = udst[x];
785            vdst[x] = bytestream2_get_byte(gbyte) + lv;
786            lv = vdst[x];
787        }
788
789        y0dst -= 2*frame->linesize[0];
790        y1dst -= 2*frame->linesize[0];
791        udst  -= frame->linesize[1];
792        vdst  -= frame->linesize[2];
793    }
794
795    return 0;
796}
797
798static int decode_intra(AVCodecContext *avctx, GetBitContext *gb, AVFrame *frame)
799{
800    AGMContext *s = avctx->priv_data;
801    int ret;
802
803    compute_quant_matrix(s, (2 * s->compression - 100) / 100.0);
804
805    s->blocks_w = avctx->coded_width  >> 3;
806    s->blocks_h = avctx->coded_height >> 3;
807
808    ret = decode_intra_plane(s, gb, s->size[0], s->luma_quant_matrix, frame, 0);
809    if (ret < 0)
810        return ret;
811
812    bytestream2_skip(&s->gbyte, s->size[0]);
813
814    s->blocks_w = avctx->coded_width  >> 4;
815    s->blocks_h = avctx->coded_height >> 4;
816
817    ret = decode_intra_plane(s, gb, s->size[1], s->chroma_quant_matrix, frame, 2);
818    if (ret < 0)
819        return ret;
820
821    bytestream2_skip(&s->gbyte, s->size[1]);
822
823    s->blocks_w = avctx->coded_width  >> 4;
824    s->blocks_h = avctx->coded_height >> 4;
825
826    ret = decode_intra_plane(s, gb, s->size[2], s->chroma_quant_matrix, frame, 1);
827    if (ret < 0)
828        return ret;
829
830    return 0;
831}
832
833static int decode_motion_vectors(AVCodecContext *avctx, GetBitContext *gb)
834{
835    AGMContext *s = avctx->priv_data;
836    int nb_mvs = ((avctx->coded_height + 15) >> 4) * ((avctx->coded_width + 15) >> 4);
837    int ret, skip = 0, value, map;
838
839    av_fast_padded_malloc(&s->mvectors, &s->mvectors_size,
840                          nb_mvs * sizeof(*s->mvectors));
841    if (!s->mvectors)
842        return AVERROR(ENOMEM);
843
844    if ((ret = init_get_bits8(gb, s->gbyte.buffer, bytestream2_get_bytes_left(&s->gbyte) -
845                                                   (s->size[0] + s->size[1] + s->size[2]))) < 0)
846        return ret;
847
848    memset(s->mvectors, 0, sizeof(*s->mvectors) * nb_mvs);
849
850    for (int i = 0; i < nb_mvs; i++) {
851        ret = read_code(gb, &skip, &value, &map, 1);
852        if (ret < 0)
853            return ret;
854        s->mvectors[i].x = value;
855        i += skip;
856    }
857
858    for (int i = 0; i < nb_mvs; i++) {
859        ret = read_code(gb, &skip, &value, &map, 1);
860        if (ret < 0)
861            return ret;
862        s->mvectors[i].y = value;
863        i += skip;
864    }
865
866    if (get_bits_left(gb) <= 0)
867        return AVERROR_INVALIDDATA;
868    skip = (get_bits_count(gb) >> 3) + 1;
869    bytestream2_skip(&s->gbyte, skip);
870
871    return 0;
872}
873
874static int decode_inter(AVCodecContext *avctx, GetBitContext *gb,
875                        AVFrame *frame, AVFrame *prev)
876{
877    AGMContext *s = avctx->priv_data;
878    int ret;
879
880    compute_quant_matrix(s, (2 * s->compression - 100) / 100.0);
881
882    if (s->flags & 2) {
883        ret = decode_motion_vectors(avctx, gb);
884        if (ret < 0)
885            return ret;
886    }
887
888    s->blocks_w = avctx->coded_width  >> 3;
889    s->blocks_h = avctx->coded_height >> 3;
890
891    ret = decode_inter_plane(s, gb, s->size[0], s->luma_quant_matrix, frame, prev, 0);
892    if (ret < 0)
893        return ret;
894
895    bytestream2_skip(&s->gbyte, s->size[0]);
896
897    s->blocks_w = avctx->coded_width  >> 4;
898    s->blocks_h = avctx->coded_height >> 4;
899
900    ret = decode_inter_plane(s, gb, s->size[1], s->chroma_quant_matrix, frame, prev, 2);
901    if (ret < 0)
902        return ret;
903
904    bytestream2_skip(&s->gbyte, s->size[1]);
905
906    s->blocks_w = avctx->coded_width  >> 4;
907    s->blocks_h = avctx->coded_height >> 4;
908
909    ret = decode_inter_plane(s, gb, s->size[2], s->chroma_quant_matrix, frame, prev, 1);
910    if (ret < 0)
911        return ret;
912
913    return 0;
914}
915
916typedef struct Node {
917    int parent;
918    int child[2];
919} Node;
920
921static void get_tree_codes(uint32_t *codes, Node *nodes, int idx, uint32_t pfx, int bitpos)
922{
923    if (idx < 256 && idx >= 0) {
924        codes[idx] = pfx;
925    } else if (idx >= 0) {
926        get_tree_codes(codes, nodes, nodes[idx].child[0], pfx + (0 << bitpos), bitpos + 1);
927        get_tree_codes(codes, nodes, nodes[idx].child[1], pfx + (1U << bitpos), bitpos + 1);
928    }
929}
930
931static int make_new_tree(const uint8_t *bitlens, uint32_t *codes)
932{
933    int zlcount = 0, curlen, idx, nindex, last, llast;
934    int blcounts[32] = { 0 };
935    int syms[8192];
936    Node nodes[512];
937    int node_idx[1024];
938    int old_idx[512];
939
940    for (int i = 0; i < 256; i++) {
941        int bitlen = bitlens[i];
942        int blcount = blcounts[bitlen];
943
944        zlcount += bitlen < 1;
945        syms[(bitlen << 8) + blcount] = i;
946        blcounts[bitlen]++;
947    }
948
949    for (int i = 0; i < 512; i++) {
950        nodes[i].child[0] = -1;
951        nodes[i].child[1] = -1;
952    }
953
954    for (int i = 0; i < 256; i++) {
955        node_idx[i] = 257 + i;
956    }
957
958    curlen = 1;
959    node_idx[512] = 256;
960    last = 255;
961    nindex = 1;
962
963    for (curlen = 1; curlen < 32; curlen++) {
964        if (blcounts[curlen] > 0) {
965            int max_zlcount = zlcount + blcounts[curlen];
966
967            for (int i = 0; zlcount < 256 && zlcount < max_zlcount; zlcount++, i++) {
968                int p = node_idx[nindex - 1 + 512];
969                int ch = syms[256 * curlen + i];
970
971                if (nindex <= 0)
972                    return AVERROR_INVALIDDATA;
973
974                if (nodes[p].child[0] == -1) {
975                    nodes[p].child[0] = ch;
976                } else {
977                    nodes[p].child[1] = ch;
978                    nindex--;
979                }
980                nodes[ch].parent = p;
981            }
982        }
983        llast = last - 1;
984        idx = 0;
985        while (nindex > 0) {
986            int p, ch;
987
988            last = llast - idx;
989            p = node_idx[nindex - 1 + 512];
990            ch = node_idx[last];
991            if (nodes[p].child[0] == -1) {
992                nodes[p].child[0] = ch;
993            } else {
994                nodes[p].child[1] = ch;
995                nindex--;
996            }
997            old_idx[idx] = ch;
998            nodes[ch].parent = p;
999            if (idx == llast)
1000                goto next;
1001            idx++;
1002            if (nindex <= 0) {
1003                for (int i = 0; i < idx; i++)
1004                    node_idx[512 + i] = old_idx[i];
1005            }
1006        }
1007        nindex = idx;
1008    }
1009
1010next:
1011
1012    get_tree_codes(codes, nodes, 256, 0, 0);
1013    return 0;
1014}
1015
1016static int build_huff(const uint8_t *bitlen, VLC *vlc)
1017{
1018    uint32_t new_codes[256];
1019    uint8_t bits[256];
1020    uint8_t symbols[256];
1021    uint32_t codes[256];
1022    int nb_codes = 0;
1023
1024    int ret = make_new_tree(bitlen, new_codes);
1025    if (ret < 0)
1026        return ret;
1027
1028    for (int i = 0; i < 256; i++) {
1029        if (bitlen[i]) {
1030            bits[nb_codes] = bitlen[i];
1031            codes[nb_codes] = new_codes[i];
1032            symbols[nb_codes] = i;
1033            nb_codes++;
1034        }
1035    }
1036
1037    ff_free_vlc(vlc);
1038    return ff_init_vlc_sparse(vlc, 13, nb_codes,
1039                              bits, 1, 1,
1040                              codes, 4, 4,
1041                              symbols, 1, 1,
1042                              INIT_VLC_LE);
1043}
1044
1045static int decode_huffman2(AVCodecContext *avctx, int header, int size)
1046{
1047    AGMContext *s = avctx->priv_data;
1048    GetBitContext *gb = &s->gb;
1049    uint8_t lens[256];
1050    int ret, x, len;
1051
1052    if ((ret = init_get_bits8(gb, s->gbyte.buffer,
1053                              bytestream2_get_bytes_left(&s->gbyte))) < 0)
1054        return ret;
1055
1056    s->output_size = get_bits_long(gb, 32);
1057
1058    if (s->output_size > avctx->width * avctx->height * 9LL + 10000)
1059        return AVERROR_INVALIDDATA;
1060
1061    av_fast_padded_malloc(&s->output, &s->padded_output_size, s->output_size);
1062    if (!s->output)
1063        return AVERROR(ENOMEM);
1064
1065    x = get_bits(gb, 1);
1066    len = 4 + get_bits(gb, 1);
1067    if (x) {
1068        int cb[8] = { 0 };
1069        int count = get_bits(gb, 3) + 1;
1070
1071        for (int i = 0; i < count; i++)
1072            cb[i] = get_bits(gb, len);
1073
1074        for (int i = 0; i < 256; i++) {
1075            int idx = get_bits(gb, 3);
1076            lens[i] = cb[idx];
1077        }
1078    } else {
1079        for (int i = 0; i < 256; i++)
1080            lens[i] = get_bits(gb, len);
1081    }
1082
1083    if ((ret = build_huff(lens, &s->vlc)) < 0)
1084        return ret;
1085
1086    x = 0;
1087    while (get_bits_left(gb) > 0 && x < s->output_size) {
1088        int val = get_vlc2(gb, s->vlc.table, s->vlc.bits, 3);
1089        if (val < 0)
1090            return AVERROR_INVALIDDATA;
1091        s->output[x++] = val;
1092    }
1093
1094    return 0;
1095}
1096
1097static int decode_frame(AVCodecContext *avctx, AVFrame *frame,
1098                        int *got_frame, AVPacket *avpkt)
1099{
1100    AGMContext *s = avctx->priv_data;
1101    GetBitContext *gb = &s->gb;
1102    GetByteContext *gbyte = &s->gbyte;
1103    int w, h, width, height, header;
1104    unsigned compressed_size;
1105    long skip;
1106    int ret;
1107
1108    if (!avpkt->size)
1109        return 0;
1110
1111    bytestream2_init(gbyte, avpkt->data, avpkt->size);
1112
1113    header = bytestream2_get_le32(gbyte);
1114    s->fflags = bytestream2_get_le32(gbyte);
1115    s->bitstream_size = s->fflags & 0x1FFFFFFF;
1116    s->fflags >>= 29;
1117    av_log(avctx, AV_LOG_DEBUG, "fflags: %X\n", s->fflags);
1118    if (avpkt->size < s->bitstream_size + 8)
1119        return AVERROR_INVALIDDATA;
1120
1121    s->key_frame = (avpkt->flags & AV_PKT_FLAG_KEY);
1122    frame->key_frame = s->key_frame;
1123    frame->pict_type = s->key_frame ? AV_PICTURE_TYPE_I : AV_PICTURE_TYPE_P;
1124
1125    if (!s->key_frame) {
1126        if (!s->prev_frame->data[0]) {
1127            av_log(avctx, AV_LOG_ERROR, "Missing reference frame.\n");
1128            return AVERROR_INVALIDDATA;
1129        }
1130    }
1131
1132    if (header) {
1133        if (avctx->codec_tag == MKTAG('A', 'G', 'M', '0') ||
1134            avctx->codec_tag == MKTAG('A', 'G', 'M', '1'))
1135            return AVERROR_PATCHWELCOME;
1136        else
1137            ret = decode_huffman2(avctx, header, (avpkt->size - s->bitstream_size) - 8);
1138        if (ret < 0)
1139            return ret;
1140        bytestream2_init(gbyte, s->output, s->output_size);
1141    } else if (!s->dct) {
1142        bytestream2_skip(gbyte, 4);
1143    }
1144
1145    if (s->dct) {
1146        s->flags = 0;
1147        w = bytestream2_get_le32(gbyte);
1148        h = bytestream2_get_le32(gbyte);
1149        if (w == INT32_MIN || h == INT32_MIN)
1150            return AVERROR_INVALIDDATA;
1151        if (w < 0) {
1152            w = -w;
1153            s->flags |= 2;
1154        }
1155        if (h < 0) {
1156            h = -h;
1157            s->flags |= 1;
1158        }
1159
1160        width  = avctx->width;
1161        height = avctx->height;
1162        if (w < width || h < height || w & 7 || h & 7)
1163            return AVERROR_INVALIDDATA;
1164
1165        ret = ff_set_dimensions(avctx, w, h);
1166        if (ret < 0)
1167            return ret;
1168        avctx->width = width;
1169        avctx->height = height;
1170
1171        s->compression = bytestream2_get_le32(gbyte);
1172        if (s->compression < 0 || s->compression > 100)
1173            return AVERROR_INVALIDDATA;
1174
1175        for (int i = 0; i < 3; i++)
1176            s->size[i] = bytestream2_get_le32(gbyte);
1177        if (header) {
1178            compressed_size = s->output_size;
1179            skip = 8LL;
1180        } else {
1181            compressed_size = avpkt->size;
1182            skip = 32LL;
1183        }
1184        if (s->size[0] < 0 || s->size[1] < 0 || s->size[2] < 0 ||
1185            skip + s->size[0] + s->size[1] + s->size[2] > compressed_size) {
1186            return AVERROR_INVALIDDATA;
1187        }
1188    }
1189
1190    if ((ret = ff_get_buffer(avctx, frame, AV_GET_BUFFER_FLAG_REF)) < 0)
1191        return ret;
1192
1193    if (frame->key_frame) {
1194        if (!s->dct && !s->rgb)
1195            ret = decode_raw_intra(avctx, gbyte, frame);
1196        else if (!s->dct && s->rgb)
1197            ret = decode_raw_intra_rgb(avctx, gbyte, frame);
1198        else
1199            ret = decode_intra(avctx, gb, frame);
1200    } else {
1201        if (s->prev_frame-> width != frame->width ||
1202            s->prev_frame->height != frame->height)
1203            return AVERROR_INVALIDDATA;
1204
1205        if (!(s->flags & 2)) {
1206            ret = av_frame_copy(frame, s->prev_frame);
1207            if (ret < 0)
1208                return ret;
1209        }
1210
1211        if (s->dct) {
1212            ret = decode_inter(avctx, gb, frame, s->prev_frame);
1213        } else if (!s->dct && !s->rgb) {
1214            ret = decode_runlen(avctx, gbyte, frame);
1215        } else {
1216            ret = decode_runlen_rgb(avctx, gbyte, frame);
1217        }
1218    }
1219    if (ret < 0)
1220        return ret;
1221
1222    av_frame_unref(s->prev_frame);
1223    if ((ret = av_frame_ref(s->prev_frame, frame)) < 0)
1224        return ret;
1225
1226    frame->crop_top  = avctx->coded_height - avctx->height;
1227    frame->crop_left = avctx->coded_width  - avctx->width;
1228
1229    *got_frame = 1;
1230
1231    return avpkt->size;
1232}
1233
1234static av_cold int decode_init(AVCodecContext *avctx)
1235{
1236    AGMContext *s = avctx->priv_data;
1237
1238    s->rgb = avctx->codec_tag == MKTAG('A', 'G', 'M', '4');
1239    avctx->pix_fmt = s->rgb ? AV_PIX_FMT_BGR24 : AV_PIX_FMT_YUV420P;
1240    s->avctx = avctx;
1241    s->plus = avctx->codec_tag == MKTAG('A', 'G', 'M', '3') ||
1242              avctx->codec_tag == MKTAG('A', 'G', 'M', '7');
1243
1244    s->dct = avctx->codec_tag != MKTAG('A', 'G', 'M', '4') &&
1245             avctx->codec_tag != MKTAG('A', 'G', 'M', '5');
1246
1247    if (!s->rgb && !s->dct) {
1248        if ((avctx->width & 1) || (avctx->height & 1))
1249            return AVERROR_INVALIDDATA;
1250    }
1251
1252    avctx->idct_algo = FF_IDCT_SIMPLE;
1253    ff_idctdsp_init(&s->idsp, avctx);
1254    ff_init_scantable(s->idsp.idct_permutation, &s->scantable, ff_zigzag_direct);
1255
1256    s->prev_frame = av_frame_alloc();
1257    if (!s->prev_frame)
1258        return AVERROR(ENOMEM);
1259
1260    return 0;
1261}
1262
1263static void decode_flush(AVCodecContext *avctx)
1264{
1265    AGMContext *s = avctx->priv_data;
1266
1267    av_frame_unref(s->prev_frame);
1268}
1269
1270static av_cold int decode_close(AVCodecContext *avctx)
1271{
1272    AGMContext *s = avctx->priv_data;
1273
1274    ff_free_vlc(&s->vlc);
1275    av_frame_free(&s->prev_frame);
1276    av_freep(&s->mvectors);
1277    s->mvectors_size = 0;
1278    av_freep(&s->wblocks);
1279    s->wblocks_size = 0;
1280    av_freep(&s->output);
1281    s->padded_output_size = 0;
1282    av_freep(&s->map);
1283    s->map_size = 0;
1284
1285    return 0;
1286}
1287
1288const FFCodec ff_agm_decoder = {
1289    .p.name           = "agm",
1290    .p.long_name      = NULL_IF_CONFIG_SMALL("Amuse Graphics Movie"),
1291    .p.type           = AVMEDIA_TYPE_VIDEO,
1292    .p.id             = AV_CODEC_ID_AGM,
1293    .p.capabilities   = AV_CODEC_CAP_DR1,
1294    .priv_data_size   = sizeof(AGMContext),
1295    .init             = decode_init,
1296    .close            = decode_close,
1297    FF_CODEC_DECODE_CB(decode_frame),
1298    .flush            = decode_flush,
1299    .caps_internal    = FF_CODEC_CAP_INIT_THREADSAFE |
1300                        FF_CODEC_CAP_INIT_CLEANUP |
1301                        FF_CODEC_CAP_EXPORTS_CROPPING,
1302};
1303