xref: /third_party/ffmpeg/libavcodec/sanm.c (revision cabdff1a)
1/*
2 * LucasArts Smush video decoder
3 * Copyright (c) 2006 Cyril Zorin
4 * Copyright (c) 2011 Konstantin Shishkov
5 *
6 * This file is part of FFmpeg.
7 *
8 * FFmpeg is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License as published by the Free Software Foundation; either
11 * version 2.1 of the License, or (at your option) any later version.
12 *
13 * FFmpeg is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
16 * Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public
19 * License along with FFmpeg; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 */
22
23#include "libavutil/avassert.h"
24#include "libavutil/bswap.h"
25#include "libavutil/imgutils.h"
26
27#include "avcodec.h"
28#include "bytestream.h"
29#include "copy_block.h"
30#include "codec_internal.h"
31#include "internal.h"
32
33#define NGLYPHS 256
34#define GLYPH_COORD_VECT_SIZE 16
35#define PALETTE_SIZE 256
36#define PALETTE_DELTA 768
37
38static const int8_t glyph4_x[GLYPH_COORD_VECT_SIZE] = {
39    0, 1, 2, 3, 3, 3, 3, 2, 1, 0, 0, 0, 1, 2, 2, 1
40};
41
42static const int8_t glyph4_y[GLYPH_COORD_VECT_SIZE] = {
43    0, 0, 0, 0, 1, 2, 3, 3, 3, 3, 2, 1, 1, 1, 2, 2
44};
45
46static const int8_t glyph8_x[GLYPH_COORD_VECT_SIZE] = {
47    0, 2, 5, 7, 7, 7, 7, 7, 7, 5, 2, 0, 0, 0, 0, 0
48};
49
50static const int8_t glyph8_y[GLYPH_COORD_VECT_SIZE] = {
51    0, 0, 0, 0, 1, 3, 4, 6, 7, 7, 7, 7, 6, 4, 3, 1
52};
53
54static const int8_t motion_vectors[256][2] = {
55    {   0,   0 }, {  -1, -43 }, {   6, -43 }, {  -9, -42 }, {  13, -41 },
56    { -16, -40 }, {  19, -39 }, { -23, -36 }, {  26, -34 }, {  -2, -33 },
57    {   4, -33 }, { -29, -32 }, {  -9, -32 }, {  11, -31 }, { -16, -29 },
58    {  32, -29 }, {  18, -28 }, { -34, -26 }, { -22, -25 }, {  -1, -25 },
59    {   3, -25 }, {  -7, -24 }, {   8, -24 }, {  24, -23 }, {  36, -23 },
60    { -12, -22 }, {  13, -21 }, { -38, -20 }, {   0, -20 }, { -27, -19 },
61    {  -4, -19 }, {   4, -19 }, { -17, -18 }, {  -8, -17 }, {   8, -17 },
62    {  18, -17 }, {  28, -17 }, {  39, -17 }, { -12, -15 }, {  12, -15 },
63    { -21, -14 }, {  -1, -14 }, {   1, -14 }, { -41, -13 }, {  -5, -13 },
64    {   5, -13 }, {  21, -13 }, { -31, -12 }, { -15, -11 }, {  -8, -11 },
65    {   8, -11 }, {  15, -11 }, {  -2, -10 }, {   1, -10 }, {  31, -10 },
66    { -23,  -9 }, { -11,  -9 }, {  -5,  -9 }, {   4,  -9 }, {  11,  -9 },
67    {  42,  -9 }, {   6,  -8 }, {  24,  -8 }, { -18,  -7 }, {  -7,  -7 },
68    {  -3,  -7 }, {  -1,  -7 }, {   2,  -7 }, {  18,  -7 }, { -43,  -6 },
69    { -13,  -6 }, {  -4,  -6 }, {   4,  -6 }, {   8,  -6 }, { -33,  -5 },
70    {  -9,  -5 }, {  -2,  -5 }, {   0,  -5 }, {   2,  -5 }, {   5,  -5 },
71    {  13,  -5 }, { -25,  -4 }, {  -6,  -4 }, {  -3,  -4 }, {   3,  -4 },
72    {   9,  -4 }, { -19,  -3 }, {  -7,  -3 }, {  -4,  -3 }, {  -2,  -3 },
73    {  -1,  -3 }, {   0,  -3 }, {   1,  -3 }, {   2,  -3 }, {   4,  -3 },
74    {   6,  -3 }, {  33,  -3 }, { -14,  -2 }, { -10,  -2 }, {  -5,  -2 },
75    {  -3,  -2 }, {  -2,  -2 }, {  -1,  -2 }, {   0,  -2 }, {   1,  -2 },
76    {   2,  -2 }, {   3,  -2 }, {   5,  -2 }, {   7,  -2 }, {  14,  -2 },
77    {  19,  -2 }, {  25,  -2 }, {  43,  -2 }, {  -7,  -1 }, {  -3,  -1 },
78    {  -2,  -1 }, {  -1,  -1 }, {   0,  -1 }, {   1,  -1 }, {   2,  -1 },
79    {   3,  -1 }, {  10,  -1 }, {  -5,   0 }, {  -3,   0 }, {  -2,   0 },
80    {  -1,   0 }, {   1,   0 }, {   2,   0 }, {   3,   0 }, {   5,   0 },
81    {   7,   0 }, { -10,   1 }, {  -7,   1 }, {  -3,   1 }, {  -2,   1 },
82    {  -1,   1 }, {   0,   1 }, {   1,   1 }, {   2,   1 }, {   3,   1 },
83    { -43,   2 }, { -25,   2 }, { -19,   2 }, { -14,   2 }, {  -5,   2 },
84    {  -3,   2 }, {  -2,   2 }, {  -1,   2 }, {   0,   2 }, {   1,   2 },
85    {   2,   2 }, {   3,   2 }, {   5,   2 }, {   7,   2 }, {  10,   2 },
86    {  14,   2 }, { -33,   3 }, {  -6,   3 }, {  -4,   3 }, {  -2,   3 },
87    {  -1,   3 }, {   0,   3 }, {   1,   3 }, {   2,   3 }, {   4,   3 },
88    {  19,   3 }, {  -9,   4 }, {  -3,   4 }, {   3,   4 }, {   7,   4 },
89    {  25,   4 }, { -13,   5 }, {  -5,   5 }, {  -2,   5 }, {   0,   5 },
90    {   2,   5 }, {   5,   5 }, {   9,   5 }, {  33,   5 }, {  -8,   6 },
91    {  -4,   6 }, {   4,   6 }, {  13,   6 }, {  43,   6 }, { -18,   7 },
92    {  -2,   7 }, {   0,   7 }, {   2,   7 }, {   7,   7 }, {  18,   7 },
93    { -24,   8 }, {  -6,   8 }, { -42,   9 }, { -11,   9 }, {  -4,   9 },
94    {   5,   9 }, {  11,   9 }, {  23,   9 }, { -31,  10 }, {  -1,  10 },
95    {   2,  10 }, { -15,  11 }, {  -8,  11 }, {   8,  11 }, {  15,  11 },
96    {  31,  12 }, { -21,  13 }, {  -5,  13 }, {   5,  13 }, {  41,  13 },
97    {  -1,  14 }, {   1,  14 }, {  21,  14 }, { -12,  15 }, {  12,  15 },
98    { -39,  17 }, { -28,  17 }, { -18,  17 }, {  -8,  17 }, {   8,  17 },
99    {  17,  18 }, {  -4,  19 }, {   0,  19 }, {   4,  19 }, {  27,  19 },
100    {  38,  20 }, { -13,  21 }, {  12,  22 }, { -36,  23 }, { -24,  23 },
101    {  -8,  24 }, {   7,  24 }, {  -3,  25 }, {   1,  25 }, {  22,  25 },
102    {  34,  26 }, { -18,  28 }, { -32,  29 }, {  16,  29 }, { -11,  31 },
103    {   9,  32 }, {  29,  32 }, {  -4,  33 }, {   2,  33 }, { -26,  34 },
104    {  23,  36 }, { -19,  39 }, {  16,  40 }, { -13,  41 }, {   9,  42 },
105    {  -6,  43 }, {   1,  43 }, {   0,   0 }, {   0,   0 }, {   0,   0 },
106};
107
108static const int8_t c37_mv[] = {
109    0,   0,   1,   0,   2,   0,   3,   0,   5,   0,
110    8,   0,  13,   0,  21,   0,  -1,   0,  -2,   0,
111   -3,   0,  -5,   0,  -8,   0, -13,   0, -17,   0,
112  -21,   0,   0,   1,   1,   1,   2,   1,   3,   1,
113    5,   1,   8,   1,  13,   1,  21,   1,  -1,   1,
114   -2,   1,  -3,   1,  -5,   1,  -8,   1, -13,   1,
115  -17,   1, -21,   1,   0,   2,   1,   2,   2,   2,
116    3,   2,   5,   2,   8,   2,  13,   2,  21,   2,
117   -1,   2,  -2,   2,  -3,   2,  -5,   2,  -8,   2,
118  -13,   2, -17,   2, -21,   2,   0,   3,   1,   3,
119    2,   3,   3,   3,   5,   3,   8,   3,  13,   3,
120   21,   3,  -1,   3,  -2,   3,  -3,   3,  -5,   3,
121   -8,   3, -13,   3, -17,   3, -21,   3,   0,   5,
122    1,   5,   2,   5,   3,   5,   5,   5,   8,   5,
123   13,   5,  21,   5,  -1,   5,  -2,   5,  -3,   5,
124   -5,   5,  -8,   5, -13,   5, -17,   5, -21,   5,
125    0,   8,   1,   8,   2,   8,   3,   8,   5,   8,
126    8,   8,  13,   8,  21,   8,  -1,   8,  -2,   8,
127   -3,   8,  -5,   8,  -8,   8, -13,   8, -17,   8,
128  -21,   8,   0,  13,   1,  13,   2,  13,   3,  13,
129    5,  13,   8,  13,  13,  13,  21,  13,  -1,  13,
130   -2,  13,  -3,  13,  -5,  13,  -8,  13, -13,  13,
131  -17,  13, -21,  13,   0,  21,   1,  21,   2,  21,
132    3,  21,   5,  21,   8,  21,  13,  21,  21,  21,
133   -1,  21,  -2,  21,  -3,  21,  -5,  21,  -8,  21,
134  -13,  21, -17,  21, -21,  21,   0,  -1,   1,  -1,
135    2,  -1,   3,  -1,   5,  -1,   8,  -1,  13,  -1,
136   21,  -1,  -1,  -1,  -2,  -1,  -3,  -1,  -5,  -1,
137   -8,  -1, -13,  -1, -17,  -1, -21,  -1,   0,  -2,
138    1,  -2,   2,  -2,   3,  -2,   5,  -2,   8,  -2,
139   13,  -2,  21,  -2,  -1,  -2,  -2,  -2,  -3,  -2,
140   -5,  -2,  -8,  -2, -13,  -2, -17,  -2, -21,  -2,
141    0,  -3,   1,  -3,   2,  -3,   3,  -3,   5,  -3,
142    8,  -3,  13,  -3,  21,  -3,  -1,  -3,  -2,  -3,
143   -3,  -3,  -5,  -3,  -8,  -3, -13,  -3, -17,  -3,
144  -21,  -3,   0,  -5,   1,  -5,   2,  -5,   3,  -5,
145    5,  -5,   8,  -5,  13,  -5,  21,  -5,  -1,  -5,
146   -2,  -5,  -3,  -5,  -5,  -5,  -8,  -5, -13,  -5,
147  -17,  -5, -21,  -5,   0,  -8,   1,  -8,   2,  -8,
148    3,  -8,   5,  -8,   8,  -8,  13,  -8,  21,  -8,
149   -1,  -8,  -2,  -8,  -3,  -8,  -5,  -8,  -8,  -8,
150  -13,  -8, -17,  -8, -21,  -8,   0, -13,   1, -13,
151    2, -13,   3, -13,   5, -13,   8, -13,  13, -13,
152   21, -13,  -1, -13,  -2, -13,  -3, -13,  -5, -13,
153   -8, -13, -13, -13, -17, -13, -21, -13,   0, -17,
154    1, -17,   2, -17,   3, -17,   5, -17,   8, -17,
155   13, -17,  21, -17,  -1, -17,  -2, -17,  -3, -17,
156   -5, -17,  -8, -17, -13, -17, -17, -17, -21, -17,
157    0, -21,   1, -21,   2, -21,   3, -21,   5, -21,
158    8, -21,  13, -21,  21, -21,  -1, -21,  -2, -21,
159   -3, -21,  -5, -21,  -8, -21, -13, -21, -17, -21,
160    0,   0,  -8, -29,   8, -29, -18, -25,  17, -25,
161    0, -23,  -6, -22,   6, -22, -13, -19,  12, -19,
162    0, -18,  25, -18, -25, -17,  -5, -17,   5, -17,
163  -10, -15,  10, -15,   0, -14,  -4, -13,   4, -13,
164   19, -13, -19, -12,  -8, -11,  -2, -11,   0, -11,
165    2, -11,   8, -11, -15, -10,  -4, -10,   4, -10,
166   15, -10,  -6,  -9,  -1,  -9,   1,  -9,   6,  -9,
167  -29,  -8, -11,  -8,  -8,  -8,  -3,  -8,   3,  -8,
168    8,  -8,  11,  -8,  29,  -8,  -5,  -7,  -2,  -7,
169    0,  -7,   2,  -7,   5,  -7, -22,  -6,  -9,  -6,
170   -6,  -6,  -3,  -6,  -1,  -6,   1,  -6,   3,  -6,
171    6,  -6,   9,  -6,  22,  -6, -17,  -5,  -7,  -5,
172   -4,  -5,  -2,  -5,   0,  -5,   2,  -5,   4,  -5,
173    7,  -5,  17,  -5, -13,  -4, -10,  -4,  -5,  -4,
174   -3,  -4,  -1,  -4,   0,  -4,   1,  -4,   3,  -4,
175    5,  -4,  10,  -4,  13,  -4,  -8,  -3,  -6,  -3,
176   -4,  -3,  -3,  -3,  -2,  -3,  -1,  -3,   0,  -3,
177    1,  -3,   2,  -3,   4,  -3,   6,  -3,   8,  -3,
178  -11,  -2,  -7,  -2,  -5,  -2,  -3,  -2,  -2,  -2,
179   -1,  -2,   0,  -2,   1,  -2,   2,  -2,   3,  -2,
180    5,  -2,   7,  -2,  11,  -2,  -9,  -1,  -6,  -1,
181   -4,  -1,  -3,  -1,  -2,  -1,  -1,  -1,   0,  -1,
182    1,  -1,   2,  -1,   3,  -1,   4,  -1,   6,  -1,
183    9,  -1, -31,   0, -23,   0, -18,   0, -14,   0,
184  -11,   0,  -7,   0,  -5,   0,  -4,   0,  -3,   0,
185   -2,   0,  -1,   0,   0, -31,   1,   0,   2,   0,
186    3,   0,   4,   0,   5,   0,   7,   0,  11,   0,
187   14,   0,  18,   0,  23,   0,  31,   0,  -9,   1,
188   -6,   1,  -4,   1,  -3,   1,  -2,   1,  -1,   1,
189    0,   1,   1,   1,   2,   1,   3,   1,   4,   1,
190    6,   1,   9,   1, -11,   2,  -7,   2,  -5,   2,
191   -3,   2,  -2,   2,  -1,   2,   0,   2,   1,   2,
192    2,   2,   3,   2,   5,   2,   7,   2,  11,   2,
193   -8,   3,  -6,   3,  -4,   3,  -2,   3,  -1,   3,
194    0,   3,   1,   3,   2,   3,   3,   3,   4,   3,
195    6,   3,   8,   3, -13,   4, -10,   4,  -5,   4,
196   -3,   4,  -1,   4,   0,   4,   1,   4,   3,   4,
197    5,   4,  10,   4,  13,   4, -17,   5,  -7,   5,
198   -4,   5,  -2,   5,   0,   5,   2,   5,   4,   5,
199    7,   5,  17,   5, -22,   6,  -9,   6,  -6,   6,
200   -3,   6,  -1,   6,   1,   6,   3,   6,   6,   6,
201    9,   6,  22,   6,  -5,   7,  -2,   7,   0,   7,
202    2,   7,   5,   7, -29,   8, -11,   8,  -8,   8,
203   -3,   8,   3,   8,   8,   8,  11,   8,  29,   8,
204   -6,   9,  -1,   9,   1,   9,   6,   9, -15,  10,
205   -4,  10,   4,  10,  15,  10,  -8,  11,  -2,  11,
206    0,  11,   2,  11,   8,  11,  19,  12, -19,  13,
207   -4,  13,   4,  13,   0,  14, -10,  15,  10,  15,
208   -5,  17,   5,  17,  25,  17, -25,  18,   0,  18,
209  -12,  19,  13,  19,  -6,  22,   6,  22,   0,  23,
210  -17,  25,  18,  25,  -8,  29,   8,  29,   0,  31,
211    0,   0,  -6, -22,   6, -22, -13, -19,  12, -19,
212    0, -18,  -5, -17,   5, -17, -10, -15,  10, -15,
213    0, -14,  -4, -13,   4, -13,  19, -13, -19, -12,
214   -8, -11,  -2, -11,   0, -11,   2, -11,   8, -11,
215  -15, -10,  -4, -10,   4, -10,  15, -10,  -6,  -9,
216   -1,  -9,   1,  -9,   6,  -9, -11,  -8,  -8,  -8,
217   -3,  -8,   0,  -8,   3,  -8,   8,  -8,  11,  -8,
218   -5,  -7,  -2,  -7,   0,  -7,   2,  -7,   5,  -7,
219  -22,  -6,  -9,  -6,  -6,  -6,  -3,  -6,  -1,  -6,
220    1,  -6,   3,  -6,   6,  -6,   9,  -6,  22,  -6,
221  -17,  -5,  -7,  -5,  -4,  -5,  -2,  -5,  -1,  -5,
222    0,  -5,   1,  -5,   2,  -5,   4,  -5,   7,  -5,
223   17,  -5, -13,  -4, -10,  -4,  -5,  -4,  -3,  -4,
224   -2,  -4,  -1,  -4,   0,  -4,   1,  -4,   2,  -4,
225    3,  -4,   5,  -4,  10,  -4,  13,  -4,  -8,  -3,
226   -6,  -3,  -4,  -3,  -3,  -3,  -2,  -3,  -1,  -3,
227    0,  -3,   1,  -3,   2,  -3,   3,  -3,   4,  -3,
228    6,  -3,   8,  -3, -11,  -2,  -7,  -2,  -5,  -2,
229   -4,  -2,  -3,  -2,  -2,  -2,  -1,  -2,   0,  -2,
230    1,  -2,   2,  -2,   3,  -2,   4,  -2,   5,  -2,
231    7,  -2,  11,  -2,  -9,  -1,  -6,  -1,  -5,  -1,
232   -4,  -1,  -3,  -1,  -2,  -1,  -1,  -1,   0,  -1,
233    1,  -1,   2,  -1,   3,  -1,   4,  -1,   5,  -1,
234    6,  -1,   9,  -1, -23,   0, -18,   0, -14,   0,
235  -11,   0,  -7,   0,  -5,   0,  -4,   0,  -3,   0,
236   -2,   0,  -1,   0,   0, -23,   1,   0,   2,   0,
237    3,   0,   4,   0,   5,   0,   7,   0,  11,   0,
238   14,   0,  18,   0,  23,   0,  -9,   1,  -6,   1,
239   -5,   1,  -4,   1,  -3,   1,  -2,   1,  -1,   1,
240    0,   1,   1,   1,   2,   1,   3,   1,   4,   1,
241    5,   1,   6,   1,   9,   1, -11,   2,  -7,   2,
242   -5,   2,  -4,   2,  -3,   2,  -2,   2,  -1,   2,
243    0,   2,   1,   2,   2,   2,   3,   2,   4,   2,
244    5,   2,   7,   2,  11,   2,  -8,   3,  -6,   3,
245   -4,   3,  -3,   3,  -2,   3,  -1,   3,   0,   3,
246    1,   3,   2,   3,   3,   3,   4,   3,   6,   3,
247    8,   3, -13,   4, -10,   4,  -5,   4,  -3,   4,
248   -2,   4,  -1,   4,   0,   4,   1,   4,   2,   4,
249    3,   4,   5,   4,  10,   4,  13,   4, -17,   5,
250   -7,   5,  -4,   5,  -2,   5,  -1,   5,   0,   5,
251    1,   5,   2,   5,   4,   5,   7,   5,  17,   5,
252  -22,   6,  -9,   6,  -6,   6,  -3,   6,  -1,   6,
253    1,   6,   3,   6,   6,   6,   9,   6,  22,   6,
254   -5,   7,  -2,   7,   0,   7,   2,   7,   5,   7,
255  -11,   8,  -8,   8,  -3,   8,   0,   8,   3,   8,
256    8,   8,  11,   8,  -6,   9,  -1,   9,   1,   9,
257    6,   9, -15,  10,  -4,  10,   4,  10,  15,  10,
258   -8,  11,  -2,  11,   0,  11,   2,  11,   8,  11,
259   19,  12, -19,  13,  -4,  13,   4,  13,   0,  14,
260  -10,  15,  10,  15,  -5,  17,   5,  17,   0,  18,
261  -12,  19,  13,  19,  -6,  22,   6,  22,   0,  23,
262};
263
264typedef struct SANMVideoContext {
265    AVCodecContext *avctx;
266    GetByteContext gb;
267
268    int version, subversion;
269    uint32_t pal[PALETTE_SIZE];
270    int16_t delta_pal[PALETTE_DELTA];
271
272    ptrdiff_t pitch;
273    int width, height;
274    int aligned_width, aligned_height;
275    int prev_seq;
276
277    AVFrame *frame;
278    uint16_t *frm0, *frm1, *frm2;
279    uint8_t *stored_frame;
280    uint32_t frm0_size, frm1_size, frm2_size;
281    uint32_t stored_frame_size;
282
283    uint8_t *rle_buf;
284    unsigned int rle_buf_size;
285
286    int rotate_code;
287
288    long npixels, buf_size;
289
290    uint16_t codebook[256];
291    uint16_t small_codebook[4];
292
293    int8_t p4x4glyphs[NGLYPHS][16];
294    int8_t p8x8glyphs[NGLYPHS][64];
295} SANMVideoContext;
296
297typedef struct SANMFrameHeader {
298    int seq_num, codec, rotate_code, rle_output_size;
299
300    uint16_t bg_color;
301    uint32_t width, height;
302} SANMFrameHeader;
303
304enum GlyphEdge {
305    LEFT_EDGE,
306    TOP_EDGE,
307    RIGHT_EDGE,
308    BOTTOM_EDGE,
309    NO_EDGE
310};
311
312enum GlyphDir {
313    DIR_LEFT,
314    DIR_UP,
315    DIR_RIGHT,
316    DIR_DOWN,
317    NO_DIR
318};
319
320/**
321 * Return enum GlyphEdge of box where point (x, y) lies.
322 *
323 * @param x x point coordinate
324 * @param y y point coordinate
325 * @param edge_size box width/height.
326 */
327static enum GlyphEdge which_edge(int x, int y, int edge_size)
328{
329    const int edge_max = edge_size - 1;
330
331    if (!y)
332        return BOTTOM_EDGE;
333    else if (y == edge_max)
334        return TOP_EDGE;
335    else if (!x)
336        return LEFT_EDGE;
337    else if (x == edge_max)
338        return RIGHT_EDGE;
339    else
340        return NO_EDGE;
341}
342
343static enum GlyphDir which_direction(enum GlyphEdge edge0, enum GlyphEdge edge1)
344{
345    if ((edge0 == LEFT_EDGE && edge1 == RIGHT_EDGE) ||
346        (edge1 == LEFT_EDGE && edge0 == RIGHT_EDGE) ||
347        (edge0 == BOTTOM_EDGE && edge1 != TOP_EDGE) ||
348        (edge1 == BOTTOM_EDGE && edge0 != TOP_EDGE))
349        return DIR_UP;
350    else if ((edge0 == TOP_EDGE && edge1 != BOTTOM_EDGE) ||
351             (edge1 == TOP_EDGE && edge0 != BOTTOM_EDGE))
352        return DIR_DOWN;
353    else if ((edge0 == LEFT_EDGE && edge1 != RIGHT_EDGE) ||
354             (edge1 == LEFT_EDGE && edge0 != RIGHT_EDGE))
355        return DIR_LEFT;
356    else if ((edge0 == TOP_EDGE && edge1 == BOTTOM_EDGE) ||
357             (edge1 == TOP_EDGE && edge0 == BOTTOM_EDGE) ||
358             (edge0 == RIGHT_EDGE && edge1 != LEFT_EDGE) ||
359             (edge1 == RIGHT_EDGE && edge0 != LEFT_EDGE))
360        return DIR_RIGHT;
361
362    return NO_DIR;
363}
364
365/* Interpolate two points. */
366static void interp_point(int8_t *points, int x0, int y0, int x1, int y1,
367                         int pos, int npoints)
368{
369    if (npoints) {
370        points[0] = (x0 * pos + x1 * (npoints - pos) + (npoints >> 1)) / npoints;
371        points[1] = (y0 * pos + y1 * (npoints - pos) + (npoints >> 1)) / npoints;
372    } else {
373        points[0] = x0;
374        points[1] = y0;
375    }
376}
377
378/**
379 * Construct glyphs by iterating through vector coordinates.
380 *
381 * @param pglyphs pointer to table where glyphs are stored
382 * @param xvec pointer to x component of vector coordinates
383 * @param yvec pointer to y component of vector coordinates
384 * @param side_length glyph width/height.
385 */
386static void make_glyphs(int8_t *pglyphs, const int8_t *xvec, const int8_t *yvec,
387                        const int side_length)
388{
389    const int glyph_size = side_length * side_length;
390    int8_t *pglyph = pglyphs;
391
392    int i, j;
393    for (i = 0; i < GLYPH_COORD_VECT_SIZE; i++) {
394        int x0 = xvec[i];
395        int y0 = yvec[i];
396        enum GlyphEdge edge0 = which_edge(x0, y0, side_length);
397
398        for (j = 0; j < GLYPH_COORD_VECT_SIZE; j++, pglyph += glyph_size) {
399            int x1 = xvec[j];
400            int y1 = yvec[j];
401            enum GlyphEdge edge1 = which_edge(x1, y1, side_length);
402            enum GlyphDir dir = which_direction(edge0, edge1);
403            int npoints = FFMAX(FFABS(x1 - x0), FFABS(y1 - y0));
404            int ipoint;
405
406            for (ipoint = 0; ipoint <= npoints; ipoint++) {
407                int8_t point[2];
408                int irow, icol;
409
410                interp_point(point, x0, y0, x1, y1, ipoint, npoints);
411
412                switch (dir) {
413                case DIR_UP:
414                    for (irow = point[1]; irow >= 0; irow--)
415                        pglyph[point[0] + irow * side_length] = 1;
416                    break;
417
418                case DIR_DOWN:
419                    for (irow = point[1]; irow < side_length; irow++)
420                        pglyph[point[0] + irow * side_length] = 1;
421                    break;
422
423                case DIR_LEFT:
424                    for (icol = point[0]; icol >= 0; icol--)
425                        pglyph[icol + point[1] * side_length] = 1;
426                    break;
427
428                case DIR_RIGHT:
429                    for (icol = point[0]; icol < side_length; icol++)
430                        pglyph[icol + point[1] * side_length] = 1;
431                    break;
432                }
433            }
434        }
435    }
436}
437
438static void init_sizes(SANMVideoContext *ctx, int width, int height)
439{
440    ctx->width   = width;
441    ctx->height  = height;
442    ctx->npixels = width * height;
443
444    ctx->aligned_width  = FFALIGN(width, 8);
445    ctx->aligned_height = FFALIGN(height, 8);
446
447    ctx->buf_size = ctx->aligned_width * ctx->aligned_height * sizeof(ctx->frm0[0]);
448    ctx->pitch    = width;
449}
450
451static void destroy_buffers(SANMVideoContext *ctx)
452{
453    av_freep(&ctx->frm0);
454    av_freep(&ctx->frm1);
455    av_freep(&ctx->frm2);
456    av_freep(&ctx->stored_frame);
457    av_freep(&ctx->rle_buf);
458    ctx->frm0_size =
459    ctx->frm1_size =
460    ctx->frm2_size = 0;
461    init_sizes(ctx, 0, 0);
462}
463
464static av_cold int init_buffers(SANMVideoContext *ctx)
465{
466    av_fast_padded_mallocz(&ctx->frm0, &ctx->frm0_size, ctx->buf_size);
467    av_fast_padded_mallocz(&ctx->frm1, &ctx->frm1_size, ctx->buf_size);
468    av_fast_padded_mallocz(&ctx->frm2, &ctx->frm2_size, ctx->buf_size);
469    if (!ctx->version)
470        av_fast_padded_mallocz(&ctx->stored_frame,
471                              &ctx->stored_frame_size, ctx->buf_size);
472
473    if (!ctx->frm0 || !ctx->frm1 || !ctx->frm2 ||
474        (!ctx->stored_frame && !ctx->version)) {
475        destroy_buffers(ctx);
476        return AVERROR(ENOMEM);
477    }
478
479    return 0;
480}
481
482static void rotate_bufs(SANMVideoContext *ctx, int rotate_code)
483{
484    if (rotate_code == 2)
485        FFSWAP(uint16_t*, ctx->frm1, ctx->frm2);
486    FFSWAP(uint16_t*, ctx->frm2, ctx->frm0);
487}
488
489static av_cold int decode_init(AVCodecContext *avctx)
490{
491    SANMVideoContext *ctx = avctx->priv_data;
492
493    ctx->avctx   = avctx;
494    ctx->version = !avctx->extradata_size;
495    // early sanity check before allocations to avoid need for deallocation code.
496    if (!ctx->version && avctx->extradata_size < 1026) {
497        av_log(avctx, AV_LOG_ERROR, "Not enough extradata.\n");
498        return AVERROR_INVALIDDATA;
499    }
500
501    avctx->pix_fmt = ctx->version ? AV_PIX_FMT_RGB565 : AV_PIX_FMT_PAL8;
502
503    init_sizes(ctx, avctx->width, avctx->height);
504    if (init_buffers(ctx)) {
505        av_log(avctx, AV_LOG_ERROR, "Error allocating buffers.\n");
506        return AVERROR(ENOMEM);
507    }
508
509    make_glyphs(ctx->p4x4glyphs[0], glyph4_x, glyph4_y, 4);
510    make_glyphs(ctx->p8x8glyphs[0], glyph8_x, glyph8_y, 8);
511
512    if (!ctx->version) {
513        int i;
514
515        ctx->subversion = AV_RL16(avctx->extradata);
516        for (i = 0; i < PALETTE_SIZE; i++)
517            ctx->pal[i] = 0xFFU << 24 | AV_RL32(avctx->extradata + 2 + i * 4);
518    }
519
520    return 0;
521}
522
523static av_cold int decode_end(AVCodecContext *avctx)
524{
525    SANMVideoContext *ctx = avctx->priv_data;
526
527    destroy_buffers(ctx);
528
529    return 0;
530}
531
532static int rle_decode(SANMVideoContext *ctx, uint8_t *dst, const int out_size)
533{
534    int opcode, color, run_len, left = out_size;
535
536    while (left > 0) {
537        opcode = bytestream2_get_byte(&ctx->gb);
538        run_len = (opcode >> 1) + 1;
539        if (run_len > left || bytestream2_get_bytes_left(&ctx->gb) <= 0)
540            return AVERROR_INVALIDDATA;
541
542        if (opcode & 1) {
543            color = bytestream2_get_byte(&ctx->gb);
544            memset(dst, color, run_len);
545        } else {
546            if (bytestream2_get_bytes_left(&ctx->gb) < run_len)
547                return AVERROR_INVALIDDATA;
548            bytestream2_get_bufferu(&ctx->gb, dst, run_len);
549        }
550
551        dst  += run_len;
552        left -= run_len;
553    }
554
555    return 0;
556}
557
558static int old_codec1(SANMVideoContext *ctx, int top,
559                      int left, int width, int height)
560{
561    uint8_t *dst = ((uint8_t *)ctx->frm0) + left + top * ctx->pitch;
562    int i, j, len, flag, code, val, pos, end;
563
564    for (i = 0; i < height; i++) {
565        pos = 0;
566
567        if (bytestream2_get_bytes_left(&ctx->gb) < 2)
568            return AVERROR_INVALIDDATA;
569
570        len = bytestream2_get_le16u(&ctx->gb);
571        end = bytestream2_tell(&ctx->gb) + len;
572
573        while (bytestream2_tell(&ctx->gb) < end) {
574            if (bytestream2_get_bytes_left(&ctx->gb) < 2)
575                return AVERROR_INVALIDDATA;
576
577            code = bytestream2_get_byteu(&ctx->gb);
578            flag = code & 1;
579            code = (code >> 1) + 1;
580            if (pos + code > width)
581                return AVERROR_INVALIDDATA;
582            if (flag) {
583                val = bytestream2_get_byteu(&ctx->gb);
584                if (val)
585                    memset(dst + pos, val, code);
586                pos += code;
587            } else {
588                if (bytestream2_get_bytes_left(&ctx->gb) < code)
589                    return AVERROR_INVALIDDATA;
590                for (j = 0; j < code; j++) {
591                    val = bytestream2_get_byteu(&ctx->gb);
592                    if (val)
593                        dst[pos] = val;
594                    pos++;
595                }
596            }
597        }
598        dst += ctx->pitch;
599    }
600    ctx->rotate_code = 0;
601
602    return 0;
603}
604
605static inline void codec37_mv(uint8_t *dst, const uint8_t *src,
606                              int height, int stride, int x, int y)
607{
608    int pos, i, j;
609
610    pos = x + y * stride;
611    for (j = 0; j < 4; j++) {
612        for (i = 0; i < 4; i++) {
613            if ((pos + i) < 0 || (pos + i) >= height * stride)
614                dst[i] = 0;
615            else
616                dst[i] = src[i];
617        }
618        dst += stride;
619        src += stride;
620        pos += stride;
621    }
622}
623
624static int old_codec37(SANMVideoContext *ctx, int top,
625                       int left, int width, int height)
626{
627    ptrdiff_t stride = ctx->pitch;
628    int i, j, k, t;
629    uint8_t *dst, *prev;
630    int skip_run = 0;
631    int compr = bytestream2_get_byte(&ctx->gb);
632    int mvoff = bytestream2_get_byte(&ctx->gb);
633    int seq   = bytestream2_get_le16(&ctx->gb);
634    uint32_t decoded_size = bytestream2_get_le32(&ctx->gb);
635    int flags;
636
637    bytestream2_skip(&ctx->gb, 4);
638    flags = bytestream2_get_byte(&ctx->gb);
639    bytestream2_skip(&ctx->gb, 3);
640
641    if (decoded_size > ctx->height * stride - left - top * stride) {
642        decoded_size = ctx->height * stride - left - top * stride;
643        av_log(ctx->avctx, AV_LOG_WARNING, "Decoded size is too large.\n");
644    }
645
646    ctx->rotate_code = 0;
647
648    if (((seq & 1) || !(flags & 1)) && (compr && compr != 2))
649        rotate_bufs(ctx, 1);
650
651    dst  = ((uint8_t*)ctx->frm0) + left + top * stride;
652    prev = ((uint8_t*)ctx->frm2) + left + top * stride;
653
654    if (mvoff > 2) {
655        av_log(ctx->avctx, AV_LOG_ERROR, "Invalid motion base value %d.\n", mvoff);
656        return AVERROR_INVALIDDATA;
657    }
658
659    switch (compr) {
660    case 0:
661        for (i = 0; i < height; i++) {
662            bytestream2_get_buffer(&ctx->gb, dst, width);
663            dst += stride;
664        }
665        memset(ctx->frm1, 0, ctx->height * stride);
666        memset(ctx->frm2, 0, ctx->height * stride);
667        break;
668    case 2:
669        if (rle_decode(ctx, dst, decoded_size))
670            return AVERROR_INVALIDDATA;
671        memset(ctx->frm1, 0, ctx->frm1_size);
672        memset(ctx->frm2, 0, ctx->frm2_size);
673        break;
674    case 3:
675    case 4:
676        if (flags & 4) {
677            for (j = 0; j < height; j += 4) {
678                for (i = 0; i < width; i += 4) {
679                    int code;
680                    if (skip_run) {
681                        skip_run--;
682                        copy_block4(dst + i, prev + i, stride, stride, 4);
683                        continue;
684                    }
685                    if (bytestream2_get_bytes_left(&ctx->gb) < 1)
686                        return AVERROR_INVALIDDATA;
687                    code = bytestream2_get_byteu(&ctx->gb);
688                    switch (code) {
689                    case 0xFF:
690                        if (bytestream2_get_bytes_left(&ctx->gb) < 16)
691                            return AVERROR_INVALIDDATA;
692                        for (k = 0; k < 4; k++)
693                            bytestream2_get_bufferu(&ctx->gb, dst + i + k * stride, 4);
694                        break;
695                    case 0xFE:
696                        if (bytestream2_get_bytes_left(&ctx->gb) < 4)
697                            return AVERROR_INVALIDDATA;
698                        for (k = 0; k < 4; k++)
699                            memset(dst + i + k * stride, bytestream2_get_byteu(&ctx->gb), 4);
700                        break;
701                    case 0xFD:
702                        if (bytestream2_get_bytes_left(&ctx->gb) < 1)
703                            return AVERROR_INVALIDDATA;
704                        t = bytestream2_get_byteu(&ctx->gb);
705                        for (k = 0; k < 4; k++)
706                            memset(dst + i + k * stride, t, 4);
707                        break;
708                    default:
709                        if (compr == 4 && !code) {
710                            if (bytestream2_get_bytes_left(&ctx->gb) < 1)
711                                return AVERROR_INVALIDDATA;
712                            skip_run = bytestream2_get_byteu(&ctx->gb) + 1;
713                            i -= 4;
714                        } else {
715                            int mx, my;
716
717                            mx = c37_mv[(mvoff * 255 + code) * 2];
718                            my = c37_mv[(mvoff * 255 + code) * 2 + 1];
719                            codec37_mv(dst + i, prev + i + mx + my * stride,
720                                       ctx->height, stride, i + mx, j + my);
721                        }
722                    }
723                }
724                dst  += stride * 4;
725                prev += stride * 4;
726            }
727        } else {
728            for (j = 0; j < height; j += 4) {
729                for (i = 0; i < width; i += 4) {
730                    int code;
731                    if (skip_run) {
732                        skip_run--;
733                        copy_block4(dst + i, prev + i, stride, stride, 4);
734                        continue;
735                    }
736                    code = bytestream2_get_byte(&ctx->gb);
737                    if (code == 0xFF) {
738                        if (bytestream2_get_bytes_left(&ctx->gb) < 16)
739                            return AVERROR_INVALIDDATA;
740                        for (k = 0; k < 4; k++)
741                            bytestream2_get_bufferu(&ctx->gb, dst + i + k * stride, 4);
742                    } else if (compr == 4 && !code) {
743                        if (bytestream2_get_bytes_left(&ctx->gb) < 1)
744                            return AVERROR_INVALIDDATA;
745                        skip_run = bytestream2_get_byteu(&ctx->gb) + 1;
746                        i -= 4;
747                    } else {
748                        int mx, my;
749
750                        mx = c37_mv[(mvoff * 255 + code) * 2];
751                        my = c37_mv[(mvoff * 255 + code) * 2 + 1];
752                        codec37_mv(dst + i, prev + i + mx + my * stride,
753                                   ctx->height, stride, i + mx, j + my);
754                    }
755                }
756                dst  += stride * 4;
757                prev += stride * 4;
758            }
759        }
760        break;
761    default:
762        avpriv_report_missing_feature(ctx->avctx,
763                                      "Subcodec 37 compression %d", compr);
764        return AVERROR_PATCHWELCOME;
765    }
766
767    return 0;
768}
769
770static int process_block(SANMVideoContext *ctx, uint8_t *dst, uint8_t *prev1,
771                         uint8_t *prev2, int stride, int tbl, int size)
772{
773    int code, k, t;
774    uint8_t colors[2];
775    int8_t *pglyph;
776
777    if (bytestream2_get_bytes_left(&ctx->gb) < 1)
778        return AVERROR_INVALIDDATA;
779
780    code = bytestream2_get_byteu(&ctx->gb);
781    if (code >= 0xF8) {
782        switch (code) {
783        case 0xFF:
784            if (size == 2) {
785                if (bytestream2_get_bytes_left(&ctx->gb) < 4)
786                    return AVERROR_INVALIDDATA;
787                dst[0]          = bytestream2_get_byteu(&ctx->gb);
788                dst[1]          = bytestream2_get_byteu(&ctx->gb);
789                dst[0 + stride] = bytestream2_get_byteu(&ctx->gb);
790                dst[1 + stride] = bytestream2_get_byteu(&ctx->gb);
791            } else {
792                size >>= 1;
793                if (process_block(ctx, dst, prev1, prev2, stride, tbl, size))
794                    return AVERROR_INVALIDDATA;
795                if (process_block(ctx, dst + size, prev1 + size, prev2 + size,
796                                  stride, tbl, size))
797                    return AVERROR_INVALIDDATA;
798                dst   += size * stride;
799                prev1 += size * stride;
800                prev2 += size * stride;
801                if (process_block(ctx, dst, prev1, prev2, stride, tbl, size))
802                    return AVERROR_INVALIDDATA;
803                if (process_block(ctx, dst + size, prev1 + size, prev2 + size,
804                                  stride, tbl, size))
805                    return AVERROR_INVALIDDATA;
806            }
807            break;
808        case 0xFE:
809            if (bytestream2_get_bytes_left(&ctx->gb) < 1)
810                return AVERROR_INVALIDDATA;
811
812            t = bytestream2_get_byteu(&ctx->gb);
813            for (k = 0; k < size; k++)
814                memset(dst + k * stride, t, size);
815            break;
816        case 0xFD:
817            if (bytestream2_get_bytes_left(&ctx->gb) < 3)
818                return AVERROR_INVALIDDATA;
819
820            code = bytestream2_get_byteu(&ctx->gb);
821            pglyph = (size == 8) ? ctx->p8x8glyphs[code] : ctx->p4x4glyphs[code];
822            bytestream2_get_bufferu(&ctx->gb, colors, 2);
823
824            for (k = 0; k < size; k++)
825                for (t = 0; t < size; t++)
826                    dst[t + k * stride] = colors[!*pglyph++];
827            break;
828        case 0xFC:
829            for (k = 0; k < size; k++)
830                memcpy(dst + k * stride, prev1 + k * stride, size);
831            break;
832        default:
833            k = bytestream2_tell(&ctx->gb);
834            bytestream2_seek(&ctx->gb, tbl + (code & 7), SEEK_SET);
835            t = bytestream2_get_byte(&ctx->gb);
836            bytestream2_seek(&ctx->gb, k, SEEK_SET);
837            for (k = 0; k < size; k++)
838                memset(dst + k * stride, t, size);
839        }
840    } else {
841        int mx = motion_vectors[code][0];
842        int my = motion_vectors[code][1];
843        int index = prev2 - (const uint8_t *)ctx->frm2;
844
845        av_assert2(index >= 0 && index < (ctx->buf_size >> 1));
846
847        if (index < -mx - my * stride ||
848            (ctx->buf_size >> 1) - index < mx + size + (my + size - 1) * stride) {
849            av_log(ctx->avctx, AV_LOG_ERROR, "MV is invalid.\n");
850            return AVERROR_INVALIDDATA;
851        }
852
853        for (k = 0; k < size; k++)
854            memcpy(dst + k * stride, prev2 + mx + (my + k) * stride, size);
855    }
856
857    return 0;
858}
859
860static int old_codec47(SANMVideoContext *ctx, int top,
861                       int left, int width, int height)
862{
863    uint32_t decoded_size;
864    int i, j;
865    ptrdiff_t stride = ctx->pitch;
866    uint8_t *dst   = (uint8_t *)ctx->frm0 + left + top * stride;
867    uint8_t *prev1 = (uint8_t *)ctx->frm1;
868    uint8_t *prev2 = (uint8_t *)ctx->frm2;
869    int tbl_pos = bytestream2_tell(&ctx->gb);
870    int seq     = bytestream2_get_le16(&ctx->gb);
871    int compr   = bytestream2_get_byte(&ctx->gb);
872    int new_rot = bytestream2_get_byte(&ctx->gb);
873    int skip    = bytestream2_get_byte(&ctx->gb);
874
875    bytestream2_skip(&ctx->gb, 9);
876    decoded_size = bytestream2_get_le32(&ctx->gb);
877    bytestream2_skip(&ctx->gb, 8);
878
879    if (decoded_size > ctx->height * stride - left - top * stride) {
880        decoded_size = ctx->height * stride - left - top * stride;
881        av_log(ctx->avctx, AV_LOG_WARNING, "Decoded size is too large.\n");
882    }
883
884    if (skip & 1)
885        bytestream2_skip(&ctx->gb, 0x8080);
886    if (!seq) {
887        ctx->prev_seq = -1;
888        memset(prev1, 0, ctx->height * stride);
889        memset(prev2, 0, ctx->height * stride);
890    }
891
892    switch (compr) {
893    case 0:
894        if (bytestream2_get_bytes_left(&ctx->gb) < width * height)
895            return AVERROR_INVALIDDATA;
896        for (j = 0; j < height; j++) {
897            bytestream2_get_bufferu(&ctx->gb, dst, width);
898            dst += stride;
899        }
900        break;
901    case 1:
902        if (bytestream2_get_bytes_left(&ctx->gb) < ((width + 1) >> 1) * ((height + 1) >> 1))
903            return AVERROR_INVALIDDATA;
904        for (j = 0; j < height; j += 2) {
905            for (i = 0; i < width; i += 2) {
906                dst[i] =
907                dst[i + 1] =
908                dst[stride + i] =
909                dst[stride + i + 1] = bytestream2_get_byteu(&ctx->gb);
910            }
911            dst += stride * 2;
912        }
913        break;
914    case 2:
915        if (seq == ctx->prev_seq + 1) {
916            for (j = 0; j < height; j += 8) {
917                for (i = 0; i < width; i += 8)
918                    if (process_block(ctx, dst + i, prev1 + i, prev2 + i, stride,
919                                      tbl_pos + 8, 8))
920                        return AVERROR_INVALIDDATA;
921                dst   += stride * 8;
922                prev1 += stride * 8;
923                prev2 += stride * 8;
924            }
925        }
926        break;
927    case 3:
928        memcpy(ctx->frm0, ctx->frm2, ctx->pitch * ctx->height);
929        break;
930    case 4:
931        memcpy(ctx->frm0, ctx->frm1, ctx->pitch * ctx->height);
932        break;
933    case 5:
934        if (rle_decode(ctx, dst, decoded_size))
935            return AVERROR_INVALIDDATA;
936        break;
937    default:
938        avpriv_report_missing_feature(ctx->avctx,
939                                      "Subcodec 47 compression %d", compr);
940        return AVERROR_PATCHWELCOME;
941    }
942    if (seq == ctx->prev_seq + 1)
943        ctx->rotate_code = new_rot;
944    else
945        ctx->rotate_code = 0;
946    ctx->prev_seq = seq;
947
948    return 0;
949}
950
951static int process_frame_obj(SANMVideoContext *ctx)
952{
953    uint16_t codec = bytestream2_get_le16u(&ctx->gb);
954    uint16_t left  = bytestream2_get_le16u(&ctx->gb);
955    uint16_t top   = bytestream2_get_le16u(&ctx->gb);
956    uint16_t w     = bytestream2_get_le16u(&ctx->gb);
957    uint16_t h     = bytestream2_get_le16u(&ctx->gb);
958
959    if (!w || !h) {
960        av_log(ctx->avctx, AV_LOG_ERROR, "Dimensions are invalid.\n");
961        return AVERROR_INVALIDDATA;
962    }
963
964    if (ctx->width < left + w || ctx->height < top + h) {
965        int ret = ff_set_dimensions(ctx->avctx, FFMAX(left + w, ctx->width),
966                                    FFMAX(top + h, ctx->height));
967        if (ret < 0)
968            return ret;
969        init_sizes(ctx, FFMAX(left + w, ctx->width),
970                   FFMAX(top + h, ctx->height));
971        if (init_buffers(ctx)) {
972            av_log(ctx->avctx, AV_LOG_ERROR, "Error resizing buffers.\n");
973            return AVERROR(ENOMEM);
974        }
975    }
976    bytestream2_skip(&ctx->gb, 4);
977
978    switch (codec) {
979    case 1:
980    case 3:
981        return old_codec1(ctx, top, left, w, h);
982    case 37:
983        return old_codec37(ctx, top, left, w, h);
984    case 47:
985        return old_codec47(ctx, top, left, w, h);
986    default:
987        avpriv_request_sample(ctx->avctx, "Subcodec %d", codec);
988        return AVERROR_PATCHWELCOME;
989    }
990}
991
992static int decode_0(SANMVideoContext *ctx)
993{
994    uint16_t *frm = ctx->frm0;
995    int x, y;
996
997    if (bytestream2_get_bytes_left(&ctx->gb) < ctx->width * ctx->height * 2) {
998        av_log(ctx->avctx, AV_LOG_ERROR, "Insufficient data for raw frame.\n");
999        return AVERROR_INVALIDDATA;
1000    }
1001    for (y = 0; y < ctx->height; y++) {
1002        for (x = 0; x < ctx->width; x++)
1003            frm[x] = bytestream2_get_le16u(&ctx->gb);
1004        frm += ctx->pitch;
1005    }
1006    return 0;
1007}
1008
1009static int decode_nop(SANMVideoContext *ctx)
1010{
1011    avpriv_request_sample(ctx->avctx, "Unknown/unsupported compression type");
1012    return AVERROR_PATCHWELCOME;
1013}
1014
1015static void copy_block(uint16_t *pdest, uint16_t *psrc, int block_size, ptrdiff_t pitch)
1016{
1017    uint8_t *dst = (uint8_t *)pdest;
1018    uint8_t *src = (uint8_t *)psrc;
1019    ptrdiff_t stride = pitch * 2;
1020
1021    switch (block_size) {
1022    case 2:
1023        copy_block4(dst, src, stride, stride, 2);
1024        break;
1025    case 4:
1026        copy_block8(dst, src, stride, stride, 4);
1027        break;
1028    case 8:
1029        copy_block16(dst, src, stride, stride, 8);
1030        break;
1031    }
1032}
1033
1034static void fill_block(uint16_t *pdest, uint16_t color, int block_size, ptrdiff_t pitch)
1035{
1036    int x, y;
1037
1038    pitch -= block_size;
1039    for (y = 0; y < block_size; y++, pdest += pitch)
1040        for (x = 0; x < block_size; x++)
1041            *pdest++ = color;
1042}
1043
1044static int draw_glyph(SANMVideoContext *ctx, uint16_t *dst, int index,
1045                      uint16_t fg_color, uint16_t bg_color, int block_size,
1046                      ptrdiff_t pitch)
1047{
1048    int8_t *pglyph;
1049    uint16_t colors[2] = { fg_color, bg_color };
1050    int x, y;
1051
1052    if (index >= NGLYPHS) {
1053        av_log(ctx->avctx, AV_LOG_ERROR, "Ignoring nonexistent glyph #%u.\n", index);
1054        return AVERROR_INVALIDDATA;
1055    }
1056
1057    pglyph = block_size == 8 ? ctx->p8x8glyphs[index] : ctx->p4x4glyphs[index];
1058    pitch -= block_size;
1059
1060    for (y = 0; y < block_size; y++, dst += pitch)
1061        for (x = 0; x < block_size; x++)
1062            *dst++ = colors[*pglyph++];
1063    return 0;
1064}
1065
1066static int opcode_0xf7(SANMVideoContext *ctx, int cx, int cy, int block_size, ptrdiff_t pitch)
1067{
1068    uint16_t *dst = ctx->frm0 + cx + cy * ctx->pitch;
1069
1070    if (block_size == 2) {
1071        uint32_t indices;
1072
1073        if (bytestream2_get_bytes_left(&ctx->gb) < 4)
1074            return AVERROR_INVALIDDATA;
1075
1076        indices        = bytestream2_get_le32u(&ctx->gb);
1077        dst[0]         = ctx->codebook[indices & 0xFF];
1078        indices      >>= 8;
1079        dst[1]         = ctx->codebook[indices & 0xFF];
1080        indices      >>= 8;
1081        dst[pitch]     = ctx->codebook[indices & 0xFF];
1082        indices      >>= 8;
1083        dst[pitch + 1] = ctx->codebook[indices & 0xFF];
1084    } else {
1085        uint16_t fgcolor, bgcolor;
1086        int glyph;
1087
1088        if (bytestream2_get_bytes_left(&ctx->gb) < 3)
1089            return AVERROR_INVALIDDATA;
1090
1091        glyph   = bytestream2_get_byteu(&ctx->gb);
1092        bgcolor = ctx->codebook[bytestream2_get_byteu(&ctx->gb)];
1093        fgcolor = ctx->codebook[bytestream2_get_byteu(&ctx->gb)];
1094
1095        draw_glyph(ctx, dst, glyph, fgcolor, bgcolor, block_size, pitch);
1096    }
1097    return 0;
1098}
1099
1100static int opcode_0xf8(SANMVideoContext *ctx, int cx, int cy, int block_size, ptrdiff_t pitch)
1101{
1102    uint16_t *dst = ctx->frm0 + cx + cy * ctx->pitch;
1103
1104    if (block_size == 2) {
1105        if (bytestream2_get_bytes_left(&ctx->gb) < 8)
1106            return AVERROR_INVALIDDATA;
1107
1108        dst[0]         = bytestream2_get_le16u(&ctx->gb);
1109        dst[1]         = bytestream2_get_le16u(&ctx->gb);
1110        dst[pitch]     = bytestream2_get_le16u(&ctx->gb);
1111        dst[pitch + 1] = bytestream2_get_le16u(&ctx->gb);
1112    } else {
1113        uint16_t fgcolor, bgcolor;
1114        int glyph;
1115
1116        if (bytestream2_get_bytes_left(&ctx->gb) < 5)
1117            return AVERROR_INVALIDDATA;
1118
1119        glyph   = bytestream2_get_byteu(&ctx->gb);
1120        bgcolor = bytestream2_get_le16u(&ctx->gb);
1121        fgcolor = bytestream2_get_le16u(&ctx->gb);
1122
1123        draw_glyph(ctx, dst, glyph, fgcolor, bgcolor, block_size, pitch);
1124    }
1125    return 0;
1126}
1127
1128static int good_mvec(SANMVideoContext *ctx, int cx, int cy, int mx, int my,
1129                     int block_size)
1130{
1131    int start_pos = cx + mx + (cy + my) * ctx->pitch;
1132    int end_pos = start_pos + (block_size - 1) * (ctx->pitch + 1);
1133
1134    int good = start_pos >= 0 && end_pos < (ctx->buf_size >> 1);
1135
1136    if (!good)
1137        av_log(ctx->avctx, AV_LOG_ERROR,
1138               "Ignoring invalid motion vector (%i, %i)->(%u, %u), block size = %u\n",
1139               cx + mx, cy + my, cx, cy, block_size);
1140
1141    return good;
1142}
1143
1144static int codec2subblock(SANMVideoContext *ctx, int cx, int cy, int blk_size)
1145{
1146    int16_t mx, my, index;
1147    int opcode;
1148
1149    if (bytestream2_get_bytes_left(&ctx->gb) < 1)
1150        return AVERROR_INVALIDDATA;
1151
1152    opcode = bytestream2_get_byteu(&ctx->gb);
1153
1154    switch (opcode) {
1155    default:
1156        mx = motion_vectors[opcode][0];
1157        my = motion_vectors[opcode][1];
1158
1159        if (good_mvec(ctx, cx, cy, mx, my, blk_size)) {
1160            copy_block(ctx->frm0 + cx      + ctx->pitch *  cy,
1161                       ctx->frm2 + cx + mx + ctx->pitch * (cy + my),
1162                       blk_size, ctx->pitch);
1163        }
1164        break;
1165    case 0xF5:
1166        if (bytestream2_get_bytes_left(&ctx->gb) < 2)
1167            return AVERROR_INVALIDDATA;
1168        index = bytestream2_get_le16u(&ctx->gb);
1169
1170        mx = index % ctx->width;
1171        my = index / ctx->width;
1172
1173        if (good_mvec(ctx, cx, cy, mx, my, blk_size)) {
1174            copy_block(ctx->frm0 + cx      + ctx->pitch *  cy,
1175                       ctx->frm2 + cx + mx + ctx->pitch * (cy + my),
1176                       blk_size, ctx->pitch);
1177        }
1178        break;
1179    case 0xF6:
1180        copy_block(ctx->frm0 + cx + ctx->pitch * cy,
1181                   ctx->frm1 + cx + ctx->pitch * cy,
1182                   blk_size, ctx->pitch);
1183        break;
1184    case 0xF7:
1185        opcode_0xf7(ctx, cx, cy, blk_size, ctx->pitch);
1186        break;
1187
1188    case 0xF8:
1189        opcode_0xf8(ctx, cx, cy, blk_size, ctx->pitch);
1190        break;
1191    case 0xF9:
1192    case 0xFA:
1193    case 0xFB:
1194    case 0xFC:
1195        fill_block(ctx->frm0 + cx + cy * ctx->pitch,
1196                   ctx->small_codebook[opcode - 0xf9], blk_size, ctx->pitch);
1197        break;
1198    case 0xFD:
1199        if (bytestream2_get_bytes_left(&ctx->gb) < 1)
1200            return AVERROR_INVALIDDATA;
1201        fill_block(ctx->frm0 + cx + cy * ctx->pitch,
1202                   ctx->codebook[bytestream2_get_byteu(&ctx->gb)], blk_size, ctx->pitch);
1203        break;
1204    case 0xFE:
1205        if (bytestream2_get_bytes_left(&ctx->gb) < 2)
1206            return AVERROR_INVALIDDATA;
1207        fill_block(ctx->frm0 + cx + cy * ctx->pitch,
1208                   bytestream2_get_le16u(&ctx->gb), blk_size, ctx->pitch);
1209        break;
1210    case 0xFF:
1211        if (blk_size == 2) {
1212            opcode_0xf8(ctx, cx, cy, blk_size, ctx->pitch);
1213        } else {
1214            blk_size >>= 1;
1215            if (codec2subblock(ctx, cx, cy, blk_size))
1216                return AVERROR_INVALIDDATA;
1217            if (codec2subblock(ctx, cx + blk_size, cy, blk_size))
1218                return AVERROR_INVALIDDATA;
1219            if (codec2subblock(ctx, cx, cy + blk_size, blk_size))
1220                return AVERROR_INVALIDDATA;
1221            if (codec2subblock(ctx, cx + blk_size, cy + blk_size, blk_size))
1222                return AVERROR_INVALIDDATA;
1223        }
1224        break;
1225    }
1226    return 0;
1227}
1228
1229static int decode_2(SANMVideoContext *ctx)
1230{
1231    int cx, cy, ret;
1232
1233    for (cy = 0; cy < ctx->aligned_height; cy += 8)
1234        for (cx = 0; cx < ctx->aligned_width; cx += 8)
1235            if (ret = codec2subblock(ctx, cx, cy, 8))
1236                return ret;
1237
1238    return 0;
1239}
1240
1241static int decode_3(SANMVideoContext *ctx)
1242{
1243    memcpy(ctx->frm0, ctx->frm2, ctx->frm2_size);
1244    return 0;
1245}
1246
1247static int decode_4(SANMVideoContext *ctx)
1248{
1249    memcpy(ctx->frm0, ctx->frm1, ctx->frm1_size);
1250    return 0;
1251}
1252
1253static int decode_5(SANMVideoContext *ctx)
1254{
1255#if HAVE_BIGENDIAN
1256    uint16_t *frm;
1257    int npixels;
1258#endif
1259    uint8_t *dst = (uint8_t*)ctx->frm0;
1260
1261    if (rle_decode(ctx, dst, ctx->buf_size))
1262        return AVERROR_INVALIDDATA;
1263
1264#if HAVE_BIGENDIAN
1265    npixels = ctx->npixels;
1266    frm = ctx->frm0;
1267    while (npixels--) {
1268        *frm = av_bswap16(*frm);
1269        frm++;
1270    }
1271#endif
1272
1273    return 0;
1274}
1275
1276static int decode_6(SANMVideoContext *ctx)
1277{
1278    int npixels = ctx->npixels;
1279    uint16_t *frm = ctx->frm0;
1280
1281    if (bytestream2_get_bytes_left(&ctx->gb) < npixels) {
1282        av_log(ctx->avctx, AV_LOG_ERROR, "Insufficient data for frame.\n");
1283        return AVERROR_INVALIDDATA;
1284    }
1285    while (npixels--)
1286        *frm++ = ctx->codebook[bytestream2_get_byteu(&ctx->gb)];
1287
1288    return 0;
1289}
1290
1291static int decode_8(SANMVideoContext *ctx)
1292{
1293    uint16_t *pdest = ctx->frm0;
1294    uint8_t *rsrc;
1295    long npixels = ctx->npixels;
1296
1297    av_fast_malloc(&ctx->rle_buf, &ctx->rle_buf_size, npixels);
1298    if (!ctx->rle_buf) {
1299        av_log(ctx->avctx, AV_LOG_ERROR, "RLE buffer allocation failed.\n");
1300        return AVERROR(ENOMEM);
1301    }
1302    rsrc = ctx->rle_buf;
1303
1304    if (rle_decode(ctx, rsrc, npixels))
1305        return AVERROR_INVALIDDATA;
1306
1307    while (npixels--)
1308        *pdest++ = ctx->codebook[*rsrc++];
1309
1310    return 0;
1311}
1312
1313typedef int (*frm_decoder)(SANMVideoContext *ctx);
1314
1315static const frm_decoder v1_decoders[] = {
1316    decode_0, decode_nop, decode_2, decode_3, decode_4, decode_5,
1317    decode_6, decode_nop, decode_8
1318};
1319
1320static int read_frame_header(SANMVideoContext *ctx, SANMFrameHeader *hdr)
1321{
1322    int i, ret;
1323
1324    if ((ret = bytestream2_get_bytes_left(&ctx->gb)) < 560) {
1325        av_log(ctx->avctx, AV_LOG_ERROR, "Input frame too short (%d bytes).\n",
1326               ret);
1327        return AVERROR_INVALIDDATA;
1328    }
1329    bytestream2_skip(&ctx->gb, 8); // skip pad
1330
1331    hdr->width  = bytestream2_get_le32u(&ctx->gb);
1332    hdr->height = bytestream2_get_le32u(&ctx->gb);
1333
1334    if (hdr->width != ctx->width || hdr->height != ctx->height) {
1335        avpriv_report_missing_feature(ctx->avctx, "Variable size frames");
1336        return AVERROR_PATCHWELCOME;
1337    }
1338
1339    hdr->seq_num     = bytestream2_get_le16u(&ctx->gb);
1340    hdr->codec       = bytestream2_get_byteu(&ctx->gb);
1341    hdr->rotate_code = bytestream2_get_byteu(&ctx->gb);
1342
1343    bytestream2_skip(&ctx->gb, 4); // skip pad
1344
1345    for (i = 0; i < 4; i++)
1346        ctx->small_codebook[i] = bytestream2_get_le16u(&ctx->gb);
1347    hdr->bg_color = bytestream2_get_le16u(&ctx->gb);
1348
1349    bytestream2_skip(&ctx->gb, 2); // skip pad
1350
1351    hdr->rle_output_size = bytestream2_get_le32u(&ctx->gb);
1352    for (i = 0; i < 256; i++)
1353        ctx->codebook[i] = bytestream2_get_le16u(&ctx->gb);
1354
1355    bytestream2_skip(&ctx->gb, 8); // skip pad
1356
1357    return 0;
1358}
1359
1360static void fill_frame(uint16_t *pbuf, int buf_size, uint16_t color)
1361{
1362    if (buf_size--) {
1363        *pbuf++ = color;
1364        av_memcpy_backptr((uint8_t*)pbuf, 2, 2*buf_size);
1365    }
1366}
1367
1368static int copy_output(SANMVideoContext *ctx, SANMFrameHeader *hdr)
1369{
1370    uint8_t *dst;
1371    const uint8_t *src = (uint8_t*) ctx->frm0;
1372    int ret, height = ctx->height;
1373    ptrdiff_t dstpitch, srcpitch = ctx->pitch * (hdr ? sizeof(ctx->frm0[0]) : 1);
1374
1375    if ((ret = ff_get_buffer(ctx->avctx, ctx->frame, 0)) < 0)
1376        return ret;
1377
1378    dst      = ctx->frame->data[0];
1379    dstpitch = ctx->frame->linesize[0];
1380
1381    while (height--) {
1382        memcpy(dst, src, srcpitch);
1383        src += srcpitch;
1384        dst += dstpitch;
1385    }
1386
1387    return 0;
1388}
1389
1390static int decode_frame(AVCodecContext *avctx, AVFrame *frame,
1391                        int *got_frame_ptr, AVPacket *pkt)
1392{
1393    SANMVideoContext *ctx = avctx->priv_data;
1394    int i, ret;
1395
1396    ctx->frame = frame;
1397    bytestream2_init(&ctx->gb, pkt->data, pkt->size);
1398
1399    if (!ctx->version) {
1400        int to_store = 0;
1401
1402        while (bytestream2_get_bytes_left(&ctx->gb) >= 8) {
1403            uint32_t sig, size;
1404            int pos;
1405
1406            sig  = bytestream2_get_be32u(&ctx->gb);
1407            size = bytestream2_get_be32u(&ctx->gb);
1408            pos  = bytestream2_tell(&ctx->gb);
1409
1410            if (bytestream2_get_bytes_left(&ctx->gb) < size) {
1411                av_log(avctx, AV_LOG_ERROR, "Incorrect chunk size %"PRIu32".\n", size);
1412                break;
1413            }
1414            switch (sig) {
1415            case MKBETAG('N', 'P', 'A', 'L'):
1416                if (size != PALETTE_SIZE * 3) {
1417                    av_log(avctx, AV_LOG_ERROR,
1418                           "Incorrect palette block size %"PRIu32".\n", size);
1419                    return AVERROR_INVALIDDATA;
1420                }
1421                for (i = 0; i < PALETTE_SIZE; i++)
1422                    ctx->pal[i] = 0xFFU << 24 | bytestream2_get_be24u(&ctx->gb);
1423                break;
1424            case MKBETAG('F', 'O', 'B', 'J'):
1425                if (size < 16)
1426                    return AVERROR_INVALIDDATA;
1427                if (ret = process_frame_obj(ctx))
1428                    return ret;
1429                break;
1430            case MKBETAG('X', 'P', 'A', 'L'):
1431                if (size == 6 || size == 4) {
1432                    uint8_t tmp[3];
1433                    int j;
1434
1435                    for (i = 0; i < PALETTE_SIZE; i++) {
1436                        for (j = 0; j < 3; j++) {
1437                            int t = (ctx->pal[i] >> (16 - j * 8)) & 0xFF;
1438                            tmp[j] = av_clip_uint8((t * 129 + ctx->delta_pal[i * 3 + j]) >> 7);
1439                        }
1440                        ctx->pal[i] = 0xFFU << 24 | AV_RB24(tmp);
1441                    }
1442                } else {
1443                    if (size < PALETTE_DELTA * 2 + 4) {
1444                        av_log(avctx, AV_LOG_ERROR,
1445                               "Incorrect palette change block size %"PRIu32".\n",
1446                               size);
1447                        return AVERROR_INVALIDDATA;
1448                    }
1449                    bytestream2_skipu(&ctx->gb, 4);
1450                    for (i = 0; i < PALETTE_DELTA; i++)
1451                        ctx->delta_pal[i] = bytestream2_get_le16u(&ctx->gb);
1452                    if (size >= PALETTE_DELTA * 5 + 4) {
1453                        for (i = 0; i < PALETTE_SIZE; i++)
1454                            ctx->pal[i] = 0xFFU << 24 | bytestream2_get_be24u(&ctx->gb);
1455                    } else {
1456                        memset(ctx->pal, 0, sizeof(ctx->pal));
1457                    }
1458                }
1459                break;
1460            case MKBETAG('S', 'T', 'O', 'R'):
1461                to_store = 1;
1462                break;
1463            case MKBETAG('F', 'T', 'C', 'H'):
1464                memcpy(ctx->frm0, ctx->stored_frame, ctx->buf_size);
1465                break;
1466            default:
1467                bytestream2_skip(&ctx->gb, size);
1468                av_log(avctx, AV_LOG_DEBUG,
1469                       "Unknown/unsupported chunk %"PRIx32".\n", sig);
1470                break;
1471            }
1472
1473            bytestream2_seek(&ctx->gb, pos + size, SEEK_SET);
1474            if (size & 1)
1475                bytestream2_skip(&ctx->gb, 1);
1476        }
1477        if (to_store)
1478            memcpy(ctx->stored_frame, ctx->frm0, ctx->buf_size);
1479        if ((ret = copy_output(ctx, NULL)))
1480            return ret;
1481        memcpy(ctx->frame->data[1], ctx->pal, 1024);
1482    } else {
1483        SANMFrameHeader header;
1484
1485        if ((ret = read_frame_header(ctx, &header)))
1486            return ret;
1487
1488        ctx->rotate_code = header.rotate_code;
1489        if ((ctx->frame->key_frame = !header.seq_num)) {
1490            ctx->frame->pict_type = AV_PICTURE_TYPE_I;
1491            fill_frame(ctx->frm1, ctx->npixels, header.bg_color);
1492            fill_frame(ctx->frm2, ctx->npixels, header.bg_color);
1493        } else {
1494            ctx->frame->pict_type = AV_PICTURE_TYPE_P;
1495        }
1496
1497        if (header.codec < FF_ARRAY_ELEMS(v1_decoders)) {
1498            if ((ret = v1_decoders[header.codec](ctx))) {
1499                av_log(avctx, AV_LOG_ERROR,
1500                       "Subcodec %d: error decoding frame.\n", header.codec);
1501                return ret;
1502            }
1503        } else {
1504            avpriv_request_sample(avctx, "Subcodec %d", header.codec);
1505            return AVERROR_PATCHWELCOME;
1506        }
1507
1508        if ((ret = copy_output(ctx, &header)))
1509            return ret;
1510    }
1511    if (ctx->rotate_code)
1512        rotate_bufs(ctx, ctx->rotate_code);
1513
1514    *got_frame_ptr = 1;
1515
1516    return pkt->size;
1517}
1518
1519const FFCodec ff_sanm_decoder = {
1520    .p.name         = "sanm",
1521    .p.long_name    = NULL_IF_CONFIG_SMALL("LucasArts SANM/Smush video"),
1522    .p.type         = AVMEDIA_TYPE_VIDEO,
1523    .p.id           = AV_CODEC_ID_SANM,
1524    .priv_data_size = sizeof(SANMVideoContext),
1525    .init           = decode_init,
1526    .close          = decode_end,
1527    FF_CODEC_DECODE_CB(decode_frame),
1528    .p.capabilities = AV_CODEC_CAP_DR1,
1529    .caps_internal  = FF_CODEC_CAP_INIT_THREADSAFE,
1530};
1531