1/* 2 * VMware Screen Codec (VMnc) decoder 3 * Copyright (c) 2006 Konstantin Shishkov 4 * 5 * This file is part of FFmpeg. 6 * 7 * FFmpeg is free software; you can redistribute it and/or 8 * modify it under the terms of the GNU Lesser General Public 9 * License as published by the Free Software Foundation; either 10 * version 2.1 of the License, or (at your option) any later version. 11 * 12 * FFmpeg is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 * Lesser General Public License for more details. 16 * 17 * You should have received a copy of the GNU Lesser General Public 18 * License along with FFmpeg; if not, write to the Free Software 19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 20 */ 21 22/** 23 * @file 24 * VMware Screen Codec (VMnc) decoder 25 * As Alex Beregszaszi discovered, this is effectively RFB data dump 26 */ 27 28#include <stdio.h> 29#include <stdlib.h> 30 31#include "libavutil/common.h" 32#include "libavutil/intreadwrite.h" 33#include "avcodec.h" 34#include "codec_internal.h" 35#include "internal.h" 36#include "bytestream.h" 37 38enum EncTypes { 39 MAGIC_WMVd = 0x574D5664, 40 MAGIC_WMVe, 41 MAGIC_WMVf, 42 MAGIC_WMVg, 43 MAGIC_WMVh, 44 MAGIC_WMVi, 45 MAGIC_WMVj 46}; 47 48enum HexTile_Flags { 49 HT_RAW = 1, // tile is raw 50 HT_BKG = 2, // background color is present 51 HT_FG = 4, // foreground color is present 52 HT_SUB = 8, // subrects are present 53 HT_CLR = 16 // each subrect has own color 54}; 55 56/* 57 * Decoder context 58 */ 59typedef struct VmncContext { 60 AVCodecContext *avctx; 61 AVFrame *pic; 62 63 int bpp; 64 int bpp2; 65 int bigendian; 66 uint8_t pal[768]; 67 int width, height; 68 GetByteContext gb; 69 70 /* cursor data */ 71 int cur_w, cur_h; 72 int cur_x, cur_y; 73 int cur_hx, cur_hy; 74 uint8_t *curbits, *curmask; 75 uint8_t *screendta; 76} VmncContext; 77 78/* read pixel value from stream */ 79static av_always_inline int vmnc_get_pixel(GetByteContext *gb, int bpp, int be) 80{ 81 switch (bpp * 2 + be) { 82 case 2: 83 case 3: 84 return bytestream2_get_byte(gb); 85 case 4: 86 return bytestream2_get_le16(gb); 87 case 5: 88 return bytestream2_get_be16(gb); 89 case 8: 90 return bytestream2_get_le32(gb); 91 case 9: 92 return bytestream2_get_be32(gb); 93 default: return 0; 94 } 95} 96 97static void load_cursor(VmncContext *c) 98{ 99 int i, j, p; 100 const int bpp = c->bpp2; 101 uint8_t *dst8 = c->curbits; 102 uint16_t *dst16 = (uint16_t *)c->curbits; 103 uint32_t *dst32 = (uint32_t *)c->curbits; 104 105 for (j = 0; j < c->cur_h; j++) { 106 for (i = 0; i < c->cur_w; i++) { 107 p = vmnc_get_pixel(&c->gb, bpp, c->bigendian); 108 if (bpp == 1) 109 *dst8++ = p; 110 if (bpp == 2) 111 *dst16++ = p; 112 if (bpp == 4) 113 *dst32++ = p; 114 } 115 } 116 dst8 = c->curmask; 117 dst16 = (uint16_t*)c->curmask; 118 dst32 = (uint32_t*)c->curmask; 119 for (j = 0; j < c->cur_h; j++) { 120 for (i = 0; i < c->cur_w; i++) { 121 p = vmnc_get_pixel(&c->gb, bpp, c->bigendian); 122 if (bpp == 1) 123 *dst8++ = p; 124 if (bpp == 2) 125 *dst16++ = p; 126 if (bpp == 4) 127 *dst32++ = p; 128 } 129 } 130} 131 132static void put_cursor(uint8_t *dst, int stride, VmncContext *c, int dx, int dy) 133{ 134 int i, j; 135 int w, h, x, y; 136 w = c->cur_w; 137 if (c->width < c->cur_x + c->cur_w) 138 w = c->width - c->cur_x; 139 h = c->cur_h; 140 if (c->height < c->cur_y + c->cur_h) 141 h = c->height - c->cur_y; 142 x = c->cur_x; 143 y = c->cur_y; 144 if (x < 0) { 145 w += x; 146 x = 0; 147 } 148 if (y < 0) { 149 h += y; 150 y = 0; 151 } 152 153 if ((w < 1) || (h < 1)) 154 return; 155 dst += x * c->bpp2 + y * stride; 156 157 if (c->bpp2 == 1) { 158 uint8_t *cd = c->curbits, *msk = c->curmask; 159 for (j = 0; j < h; j++) { 160 for (i = 0; i < w; i++) 161 dst[i] = (dst[i] & cd[i]) ^ msk[i]; 162 msk += c->cur_w; 163 cd += c->cur_w; 164 dst += stride; 165 } 166 } else if (c->bpp2 == 2) { 167 uint16_t *cd = (uint16_t*)c->curbits, *msk = (uint16_t*)c->curmask; 168 uint16_t *dst2; 169 for (j = 0; j < h; j++) { 170 dst2 = (uint16_t*)dst; 171 for (i = 0; i < w; i++) 172 dst2[i] = (dst2[i] & cd[i]) ^ msk[i]; 173 msk += c->cur_w; 174 cd += c->cur_w; 175 dst += stride; 176 } 177 } else if (c->bpp2 == 4) { 178 uint32_t *cd = (uint32_t*)c->curbits, *msk = (uint32_t*)c->curmask; 179 uint32_t *dst2; 180 for (j = 0; j < h; j++) { 181 dst2 = (uint32_t*)dst; 182 for (i = 0; i < w; i++) 183 dst2[i] = (dst2[i] & cd[i]) ^ msk[i]; 184 msk += c->cur_w; 185 cd += c->cur_w; 186 dst += stride; 187 } 188 } 189} 190 191/* fill rectangle with given color */ 192static av_always_inline void paint_rect(uint8_t *dst, int dx, int dy, 193 int w, int h, int color, 194 int bpp, int stride) 195{ 196 int i, j; 197 dst += dx * bpp + dy * stride; 198 if (bpp == 1) { 199 for (j = 0; j < h; j++) { 200 memset(dst, color, w); 201 dst += stride; 202 } 203 } else if (bpp == 2) { 204 uint16_t *dst2; 205 for (j = 0; j < h; j++) { 206 dst2 = (uint16_t*)dst; 207 for (i = 0; i < w; i++) 208 *dst2++ = color; 209 dst += stride; 210 } 211 } else if (bpp == 4) { 212 uint32_t *dst2; 213 for (j = 0; j < h; j++) { 214 dst2 = (uint32_t*)dst; 215 for (i = 0; i < w; i++) 216 dst2[i] = color; 217 dst += stride; 218 } 219 } 220} 221 222static av_always_inline void paint_raw(uint8_t *dst, int w, int h, 223 GetByteContext *gb, int bpp, 224 int be, int stride) 225{ 226 int i, j, p; 227 for (j = 0; j < h; j++) { 228 for (i = 0; i < w; i++) { 229 p = vmnc_get_pixel(gb, bpp, be); 230 switch (bpp) { 231 case 1: 232 dst[i] = p; 233 break; 234 case 2: 235 ((uint16_t*)dst)[i] = p; 236 break; 237 case 4: 238 ((uint32_t*)dst)[i] = p; 239 break; 240 } 241 } 242 dst += stride; 243 } 244} 245 246static int decode_hextile(VmncContext *c, uint8_t* dst, GetByteContext *gb, 247 int w, int h, int stride) 248{ 249 int i, j, k; 250 int bg = 0, fg = 0, rects, color, flags, xy, wh; 251 const int bpp = c->bpp2; 252 uint8_t *dst2; 253 int bw = 16, bh = 16; 254 255 for (j = 0; j < h; j += 16) { 256 dst2 = dst; 257 bw = 16; 258 if (j + 16 > h) 259 bh = h - j; 260 for (i = 0; i < w; i += 16, dst2 += 16 * bpp) { 261 if (bytestream2_get_bytes_left(gb) <= 0) { 262 av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n"); 263 return AVERROR_INVALIDDATA; 264 } 265 if (i + 16 > w) 266 bw = w - i; 267 flags = bytestream2_get_byte(gb); 268 if (flags & HT_RAW) { 269 if (bytestream2_get_bytes_left(gb) < bw * bh * bpp) { 270 av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n"); 271 return AVERROR_INVALIDDATA; 272 } 273 paint_raw(dst2, bw, bh, gb, bpp, c->bigendian, stride); 274 } else { 275 if (flags & HT_BKG) 276 bg = vmnc_get_pixel(gb, bpp, c->bigendian); 277 if (flags & HT_FG) 278 fg = vmnc_get_pixel(gb, bpp, c->bigendian); 279 rects = 0; 280 if (flags & HT_SUB) 281 rects = bytestream2_get_byte(gb); 282 color = !!(flags & HT_CLR); 283 284 paint_rect(dst2, 0, 0, bw, bh, bg, bpp, stride); 285 286 if (bytestream2_get_bytes_left(gb) < rects * (color * bpp + 2)) { 287 av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n"); 288 return AVERROR_INVALIDDATA; 289 } 290 for (k = 0; k < rects; k++) { 291 int rect_x, rect_y, rect_w, rect_h; 292 if (color) 293 fg = vmnc_get_pixel(gb, bpp, c->bigendian); 294 xy = bytestream2_get_byte(gb); 295 wh = bytestream2_get_byte(gb); 296 297 rect_x = xy >> 4; 298 rect_y = xy & 0xF; 299 rect_w = (wh >> 4) + 1; 300 rect_h = (wh & 0xF) + 1; 301 302 if (rect_x + rect_w > w - i || rect_y + rect_h > h - j) { 303 av_log(c->avctx, AV_LOG_ERROR, "Rectangle outside picture\n"); 304 return AVERROR_INVALIDDATA; 305 } 306 307 paint_rect(dst2, rect_x, rect_y, 308 rect_w, rect_h, fg, bpp, stride); 309 } 310 } 311 } 312 dst += stride * 16; 313 } 314 return 0; 315} 316 317static void reset_buffers(VmncContext *c) 318{ 319 av_freep(&c->curbits); 320 av_freep(&c->curmask); 321 av_freep(&c->screendta); 322 c->cur_w = c->cur_h = 0; 323 c->cur_hx = c->cur_hy = 0; 324 325} 326 327static int decode_frame(AVCodecContext *avctx, AVFrame *rframe, 328 int *got_frame, AVPacket *avpkt) 329{ 330 const uint8_t *buf = avpkt->data; 331 int buf_size = avpkt->size; 332 VmncContext * const c = avctx->priv_data; 333 GetByteContext *gb = &c->gb; 334 uint8_t *outptr; 335 int dx, dy, w, h, depth, enc, chunks, res, size_left, ret; 336 337 bytestream2_init(gb, buf, buf_size); 338 bytestream2_skip(gb, 2); 339 chunks = bytestream2_get_be16(gb); 340 if (12LL * chunks > bytestream2_get_bytes_left(gb)) 341 return AVERROR_INVALIDDATA; 342 343 if ((ret = ff_reget_buffer(avctx, c->pic, 0)) < 0) 344 return ret; 345 346 c->pic->key_frame = 0; 347 c->pic->pict_type = AV_PICTURE_TYPE_P; 348 349 // restore screen after cursor 350 if (c->screendta) { 351 int i; 352 w = c->cur_w; 353 if (c->width < c->cur_x + w) 354 w = c->width - c->cur_x; 355 h = c->cur_h; 356 if (c->height < c->cur_y + h) 357 h = c->height - c->cur_y; 358 dx = c->cur_x; 359 if (dx < 0) { 360 w += dx; 361 dx = 0; 362 } 363 dy = c->cur_y; 364 if (dy < 0) { 365 h += dy; 366 dy = 0; 367 } 368 if ((w > 0) && (h > 0)) { 369 outptr = c->pic->data[0] + dx * c->bpp2 + dy * c->pic->linesize[0]; 370 for (i = 0; i < h; i++) { 371 memcpy(outptr, c->screendta + i * c->cur_w * c->bpp2, 372 w * c->bpp2); 373 outptr += c->pic->linesize[0]; 374 } 375 } 376 } 377 378 while (chunks--) { 379 if (bytestream2_get_bytes_left(gb) < 12) { 380 av_log(avctx, AV_LOG_ERROR, "Premature end of data!\n"); 381 return -1; 382 } 383 dx = bytestream2_get_be16(gb); 384 dy = bytestream2_get_be16(gb); 385 w = bytestream2_get_be16(gb); 386 h = bytestream2_get_be16(gb); 387 enc = bytestream2_get_be32(gb); 388 if ((dx + w > c->width) || (dy + h > c->height)) { 389 av_log(avctx, AV_LOG_ERROR, 390 "Incorrect frame size: %ix%i+%ix%i of %ix%i\n", 391 w, h, dx, dy, c->width, c->height); 392 return AVERROR_INVALIDDATA; 393 } 394 outptr = c->pic->data[0] + dx * c->bpp2 + dy * c->pic->linesize[0]; 395 size_left = bytestream2_get_bytes_left(gb); 396 switch (enc) { 397 case MAGIC_WMVd: // cursor 398 if (w*(int64_t)h*c->bpp2 > INT_MAX/2 - 2) { 399 av_log(avctx, AV_LOG_ERROR, "dimensions too large\n"); 400 return AVERROR_INVALIDDATA; 401 } 402 if (size_left < 2 + w * h * c->bpp2 * 2) { 403 av_log(avctx, AV_LOG_ERROR, 404 "Premature end of data! (need %i got %i)\n", 405 2 + w * h * c->bpp2 * 2, size_left); 406 return AVERROR_INVALIDDATA; 407 } 408 bytestream2_skip(gb, 2); 409 c->cur_w = w; 410 c->cur_h = h; 411 c->cur_hx = dx; 412 c->cur_hy = dy; 413 if ((c->cur_hx > c->cur_w) || (c->cur_hy > c->cur_h)) { 414 av_log(avctx, AV_LOG_ERROR, 415 "Cursor hot spot is not in image: " 416 "%ix%i of %ix%i cursor size\n", 417 c->cur_hx, c->cur_hy, c->cur_w, c->cur_h); 418 c->cur_hx = c->cur_hy = 0; 419 } 420 if (c->cur_w * c->cur_h >= INT_MAX / c->bpp2) { 421 reset_buffers(c); 422 return AVERROR(EINVAL); 423 } else { 424 int screen_size = c->cur_w * c->cur_h * c->bpp2; 425 if ((ret = av_reallocp(&c->curbits, screen_size)) < 0 || 426 (ret = av_reallocp(&c->curmask, screen_size)) < 0 || 427 (ret = av_reallocp(&c->screendta, screen_size)) < 0) { 428 reset_buffers(c); 429 return ret; 430 } 431 } 432 load_cursor(c); 433 break; 434 case MAGIC_WMVe: // unknown 435 bytestream2_skip(gb, 2); 436 break; 437 case MAGIC_WMVf: // update cursor position 438 c->cur_x = dx - c->cur_hx; 439 c->cur_y = dy - c->cur_hy; 440 break; 441 case MAGIC_WMVg: // unknown 442 bytestream2_skip(gb, 10); 443 break; 444 case MAGIC_WMVh: // unknown 445 bytestream2_skip(gb, 4); 446 break; 447 case MAGIC_WMVi: // ServerInitialization struct 448 c->pic->key_frame = 1; 449 c->pic->pict_type = AV_PICTURE_TYPE_I; 450 depth = bytestream2_get_byte(gb); 451 if (depth != c->bpp) { 452 av_log(avctx, AV_LOG_INFO, 453 "Depth mismatch. Container %i bpp, " 454 "Frame data: %i bpp\n", 455 c->bpp, depth); 456 } 457 bytestream2_skip(gb, 1); 458 c->bigendian = bytestream2_get_byte(gb); 459 if (c->bigendian & (~1)) { 460 av_log(avctx, AV_LOG_INFO, 461 "Invalid header: bigendian flag = %i\n", c->bigendian); 462 return AVERROR_INVALIDDATA; 463 } 464 //skip the rest of pixel format data 465 bytestream2_skip(gb, 13); 466 break; 467 case MAGIC_WMVj: // unknown 468 bytestream2_skip(gb, 2); 469 break; 470 case 0x00000000: // raw rectangle data 471 if (size_left < w * h * c->bpp2) { 472 av_log(avctx, AV_LOG_ERROR, 473 "Premature end of data! (need %i got %i)\n", 474 w * h * c->bpp2, size_left); 475 return AVERROR_INVALIDDATA; 476 } 477 paint_raw(outptr, w, h, gb, c->bpp2, c->bigendian, 478 c->pic->linesize[0]); 479 break; 480 case 0x00000005: // HexTile encoded rectangle 481 res = decode_hextile(c, outptr, gb, w, h, c->pic->linesize[0]); 482 if (res < 0) 483 return res; 484 break; 485 default: 486 av_log(avctx, AV_LOG_ERROR, "Unsupported block type 0x%08X\n", enc); 487 chunks = 0; // leave chunks decoding loop 488 } 489 } 490 if (c->screendta) { 491 int i; 492 // save screen data before painting cursor 493 w = c->cur_w; 494 if (c->width < c->cur_x + w) 495 w = c->width - c->cur_x; 496 h = c->cur_h; 497 if (c->height < c->cur_y + h) 498 h = c->height - c->cur_y; 499 dx = c->cur_x; 500 if (dx < 0) { 501 w += dx; 502 dx = 0; 503 } 504 dy = c->cur_y; 505 if (dy < 0) { 506 h += dy; 507 dy = 0; 508 } 509 if ((w > 0) && (h > 0)) { 510 outptr = c->pic->data[0] + dx * c->bpp2 + dy * c->pic->linesize[0]; 511 for (i = 0; i < h; i++) { 512 memcpy(c->screendta + i * c->cur_w * c->bpp2, outptr, 513 w * c->bpp2); 514 outptr += c->pic->linesize[0]; 515 } 516 outptr = c->pic->data[0]; 517 put_cursor(outptr, c->pic->linesize[0], c, c->cur_x, c->cur_y); 518 } 519 } 520 *got_frame = 1; 521 if ((ret = av_frame_ref(rframe, c->pic)) < 0) 522 return ret; 523 524 /* always report that the buffer was completely consumed */ 525 return buf_size; 526} 527 528static av_cold int decode_init(AVCodecContext *avctx) 529{ 530 VmncContext * const c = avctx->priv_data; 531 532 c->avctx = avctx; 533 c->width = avctx->width; 534 c->height = avctx->height; 535 c->bpp = avctx->bits_per_coded_sample; 536 537 switch (c->bpp) { 538 case 8: 539 avctx->pix_fmt = AV_PIX_FMT_PAL8; 540 break; 541 case 16: 542 avctx->pix_fmt = AV_PIX_FMT_RGB555; 543 break; 544 case 24: 545 /* 24 bits is not technically supported, but some clients might 546 * mistakenly set it, so let's assume they actually meant 32 bits */ 547 c->bpp = 32; 548 case 32: 549 avctx->pix_fmt = AV_PIX_FMT_0RGB32; 550 break; 551 default: 552 av_log(avctx, AV_LOG_ERROR, "Unsupported bitdepth %i\n", c->bpp); 553 return AVERROR_INVALIDDATA; 554 } 555 c->bpp2 = c->bpp / 8; 556 557 c->pic = av_frame_alloc(); 558 if (!c->pic) 559 return AVERROR(ENOMEM); 560 561 return 0; 562} 563 564static av_cold int decode_end(AVCodecContext *avctx) 565{ 566 VmncContext * const c = avctx->priv_data; 567 568 av_frame_free(&c->pic); 569 570 av_freep(&c->curbits); 571 av_freep(&c->curmask); 572 av_freep(&c->screendta); 573 return 0; 574} 575 576const FFCodec ff_vmnc_decoder = { 577 .p.name = "vmnc", 578 .p.long_name = NULL_IF_CONFIG_SMALL("VMware Screen Codec / VMware Video"), 579 .p.type = AVMEDIA_TYPE_VIDEO, 580 .p.id = AV_CODEC_ID_VMNC, 581 .priv_data_size = sizeof(VmncContext), 582 .init = decode_init, 583 .close = decode_end, 584 FF_CODEC_DECODE_CB(decode_frame), 585 .p.capabilities = AV_CODEC_CAP_DR1, 586 .caps_internal = FF_CODEC_CAP_INIT_THREADSAFE, 587}; 588