GCC Code Coverage Report
Directory: ../../../ffmpeg/ Exec Total Coverage
File: src/libavcodec/vmnc.c Lines: 254 356 71.3 %
Date: 2020-08-14 10:39:37 Branches: 125 197 63.5 %

Line Branch Exec Source
1
/*
2
 * VMware Screen Codec (VMnc) decoder
3
 * Copyright (c) 2006 Konstantin Shishkov
4
 *
5
 * This file is part of FFmpeg.
6
 *
7
 * FFmpeg is free software; you can redistribute it and/or
8
 * modify it under the terms of the GNU Lesser General Public
9
 * License as published by the Free Software Foundation; either
10
 * version 2.1 of the License, or (at your option) any later version.
11
 *
12
 * FFmpeg is distributed in the hope that it will be useful,
13
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15
 * Lesser General Public License for more details.
16
 *
17
 * You should have received a copy of the GNU Lesser General Public
18
 * License along with FFmpeg; if not, write to the Free Software
19
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20
 */
21
22
/**
23
 * @file
24
 * VMware Screen Codec (VMnc) decoder
25
 * As Alex Beregszaszi discovered, this is effectively RFB data dump
26
 */
27
28
#include <stdio.h>
29
#include <stdlib.h>
30
31
#include "libavutil/common.h"
32
#include "libavutil/intreadwrite.h"
33
#include "avcodec.h"
34
#include "internal.h"
35
#include "bytestream.h"
36
37
enum EncTypes {
38
    MAGIC_WMVd = 0x574D5664,
39
    MAGIC_WMVe,
40
    MAGIC_WMVf,
41
    MAGIC_WMVg,
42
    MAGIC_WMVh,
43
    MAGIC_WMVi,
44
    MAGIC_WMVj
45
};
46
47
enum HexTile_Flags {
48
    HT_RAW =  1, // tile is raw
49
    HT_BKG =  2, // background color is present
50
    HT_FG  =  4, // foreground color is present
51
    HT_SUB =  8, // subrects are present
52
    HT_CLR = 16  // each subrect has own color
53
};
54
55
/*
56
 * Decoder context
57
 */
58
typedef struct VmncContext {
59
    AVCodecContext *avctx;
60
    AVFrame *pic;
61
62
    int bpp;
63
    int bpp2;
64
    int bigendian;
65
    uint8_t pal[768];
66
    int width, height;
67
    GetByteContext gb;
68
69
    /* cursor data */
70
    int cur_w, cur_h;
71
    int cur_x, cur_y;
72
    int cur_hx, cur_hy;
73
    uint8_t *curbits, *curmask;
74
    uint8_t *screendta;
75
} VmncContext;
76
77
/* read pixel value from stream */
78
401741
static av_always_inline int vmnc_get_pixel(GetByteContext *gb, int bpp, int be)
79
{
80

401741
    switch (bpp * 2 + be) {
81
    case 2:
82
    case 3:
83
        return bytestream2_get_byte(gb);
84
58268
    case 4:
85
58268
        return bytestream2_get_le16(gb);
86
    case 5:
87
        return bytestream2_get_be16(gb);
88
343473
    case 8:
89
343473
        return bytestream2_get_le32(gb);
90
    case 9:
91
        return bytestream2_get_be32(gb);
92
    default: return 0;
93
    }
94
}
95
96
16
static void load_cursor(VmncContext *c)
97
{
98
    int i, j, p;
99
16
    const int bpp   = c->bpp2;
100
16
    uint8_t *dst8   =             c->curbits;
101
16
    uint16_t *dst16 = (uint16_t *)c->curbits;
102
16
    uint32_t *dst32 = (uint32_t *)c->curbits;
103
104
432
    for (j = 0; j < c->cur_h; j++) {
105
13728
        for (i = 0; i < c->cur_w; i++) {
106
13312
            p = vmnc_get_pixel(&c->gb, bpp, c->bigendian);
107
13312
            if (bpp == 1)
108
                *dst8++ = p;
109
13312
            if (bpp == 2)
110
                *dst16++ = p;
111
13312
            if (bpp == 4)
112
13312
                *dst32++ = p;
113
        }
114
    }
115
16
    dst8  =            c->curmask;
116
16
    dst16 = (uint16_t*)c->curmask;
117
16
    dst32 = (uint32_t*)c->curmask;
118
432
    for (j = 0; j < c->cur_h; j++) {
119
13728
        for (i = 0; i < c->cur_w; i++) {
120
13312
            p = vmnc_get_pixel(&c->gb, bpp, c->bigendian);
121
13312
            if (bpp == 1)
122
                *dst8++ = p;
123
13312
            if (bpp == 2)
124
                *dst16++ = p;
125
13312
            if (bpp == 4)
126
13312
                *dst32++ = p;
127
        }
128
    }
129
16
}
130
131
49
static void put_cursor(uint8_t *dst, int stride, VmncContext *c, int dx, int dy)
132
{
133
    int i, j;
134
    int w, h, x, y;
135
49
    w = c->cur_w;
136
49
    if (c->width < c->cur_x + c->cur_w)
137
        w = c->width - c->cur_x;
138
49
    h = c->cur_h;
139
49
    if (c->height < c->cur_y + c->cur_h)
140
        h = c->height - c->cur_y;
141
49
    x = c->cur_x;
142
49
    y = c->cur_y;
143
49
    if (x < 0) {
144
        w += x;
145
        x  = 0;
146
    }
147
49
    if (y < 0) {
148
        h += y;
149
        y  = 0;
150
    }
151
152

49
    if ((w < 1) || (h < 1))
153
        return;
154
49
    dst += x * c->bpp2 + y * stride;
155
156
49
    if (c->bpp2 == 1) {
157
        uint8_t *cd = c->curbits, *msk = c->curmask;
158
        for (j = 0; j < h; j++) {
159
            for (i = 0; i < w; i++)
160
                dst[i] = (dst[i] & cd[i]) ^ msk[i];
161
            msk += c->cur_w;
162
            cd  += c->cur_w;
163
            dst += stride;
164
        }
165
49
    } else if (c->bpp2 == 2) {
166
        uint16_t *cd = (uint16_t*)c->curbits, *msk = (uint16_t*)c->curmask;
167
        uint16_t *dst2;
168
        for (j = 0; j < h; j++) {
169
            dst2 = (uint16_t*)dst;
170
            for (i = 0; i < w; i++)
171
                dst2[i] = (dst2[i] & cd[i]) ^ msk[i];
172
            msk += c->cur_w;
173
            cd  += c->cur_w;
174
            dst += stride;
175
        }
176
49
    } else if (c->bpp2 == 4) {
177
49
        uint32_t *cd = (uint32_t*)c->curbits, *msk = (uint32_t*)c->curmask;
178
        uint32_t *dst2;
179
1617
        for (j = 0; j < h; j++) {
180
1568
            dst2 = (uint32_t*)dst;
181
51744
            for (i = 0; i < w; i++)
182
50176
                dst2[i] = (dst2[i] & cd[i]) ^ msk[i];
183
1568
            msk += c->cur_w;
184
1568
            cd  += c->cur_w;
185
1568
            dst += stride;
186
        }
187
    }
188
}
189
190
/* fill rectangle with given color */
191
443805
static av_always_inline void paint_rect(uint8_t *dst, int dx, int dy,
192
                                        int w, int h, int color,
193
                                        int bpp, int stride)
194
{
195
    int i, j;
196
443805
    dst += dx * bpp + dy * stride;
197
443805
    if (bpp == 1) {
198
        for (j = 0; j < h; j++) {
199
            memset(dst, color, w);
200
            dst += stride;
201
        }
202
443805
    } else if (bpp == 2) {
203
        uint16_t *dst2;
204
779284
        for (j = 0; j < h; j++) {
205
687588
            dst2 = (uint16_t*)dst;
206
9596125
            for (i = 0; i < w; i++)
207
8908537
                *dst2++ = color;
208
687588
            dst += stride;
209
        }
210
352109
    } else if (bpp == 4) {
211
        uint32_t *dst2;
212
1672564
        for (j = 0; j < h; j++) {
213
1320455
            dst2 = (uint32_t*)dst;
214
14028614
            for (i = 0; i < w; i++)
215
12708159
                dst2[i] = color;
216
1320455
            dst += stride;
217
        }
218
    }
219
443805
}
220
221
50
static av_always_inline void paint_raw(uint8_t *dst, int w, int h,
222
                                       GetByteContext *gb, int bpp,
223
                                       int be, int stride)
224
{
225
    int i, j, p;
226
467
    for (j = 0; j < h; j++) {
227
6621
        for (i = 0; i < w; i++) {
228
6204
            p = vmnc_get_pixel(gb, bpp, be);
229

6204
            switch (bpp) {
230
            case 1:
231
                dst[i] = p;
232
                break;
233
            case 2:
234
                ((uint16_t*)dst)[i] = p;
235
                break;
236
6204
            case 4:
237
6204
                ((uint32_t*)dst)[i] = p;
238
6204
                break;
239
            }
240
6204
        }
241
417
        dst += stride;
242
    }
243
50
}
244
245
463
static int decode_hextile(VmncContext *c, uint8_t* dst, GetByteContext *gb,
246
                          int w, int h, int stride)
247
{
248
    int i, j, k;
249
463
    int bg = 0, fg = 0, rects, color, flags, xy, wh;
250
463
    const int bpp = c->bpp2;
251
    uint8_t *dst2;
252
463
    int bw = 16, bh = 16;
253
254
2709
    for (j = 0; j < h; j += 16) {
255
2247
        dst2 = dst;
256
2247
        bw   = 16;
257
2247
        if (j + 16 > h)
258
420
            bh = h - j;
259
78438
        for (i = 0; i < w; i += 16, dst2 += 16 * bpp) {
260
76192
            if (bytestream2_get_bytes_left(gb) <= 0) {
261
                av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n");
262
                return AVERROR_INVALIDDATA;
263
            }
264
76192
            if (i + 16 > w)
265
2103
                bw = w - i;
266
76192
            flags = bytestream2_get_byte(gb);
267
76192
            if (flags & HT_RAW) {
268
42
                if (bytestream2_get_bytes_left(gb) < bw * bh * bpp) {
269
                    av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n");
270
                    return AVERROR_INVALIDDATA;
271
                }
272
42
                paint_raw(dst2, bw, bh, gb, bpp, c->bigendian, stride);
273
            } else {
274
76150
                if (flags & HT_BKG)
275
1377
                    bg = vmnc_get_pixel(gb, bpp, c->bigendian);
276
76150
                if (flags & HT_FG)
277
1494
                    fg = vmnc_get_pixel(gb, bpp, c->bigendian);
278
76150
                rects = 0;
279
76150
                if (flags & HT_SUB)
280
30064
                    rects = bytestream2_get_byte(gb);
281
76150
                color = !!(flags & HT_CLR);
282
283
76150
                paint_rect(dst2, 0, 0, bw, bh, bg, bpp, stride);
284
285
76150
                if (bytestream2_get_bytes_left(gb) < rects * (color * bpp + 2)) {
286
1
                    av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n");
287
1
                    return AVERROR_INVALIDDATA;
288
                }
289
443804
                for (k = 0; k < rects; k++) {
290
                    int rect_x, rect_y, rect_w, rect_h;
291
367655
                    if (color)
292
366042
                        fg = vmnc_get_pixel(gb, bpp, c->bigendian);
293
367655
                    xy = bytestream2_get_byte(gb);
294
367655
                    wh = bytestream2_get_byte(gb);
295
296
367655
                    rect_x = xy >> 4;
297
367655
                    rect_y = xy & 0xF;
298
367655
                    rect_w = (wh >> 4) + 1;
299
367655
                    rect_h = (wh & 0xF) + 1;
300
301

367655
                    if (rect_x + rect_w > w - i || rect_y + rect_h > h - j) {
302
                        av_log(c->avctx, AV_LOG_ERROR, "Rectangle outside picture\n");
303
                        return AVERROR_INVALIDDATA;
304
                    }
305
306
367655
                    paint_rect(dst2, rect_x, rect_y,
307
                               rect_w, rect_h, fg, bpp, stride);
308
                }
309
            }
310
        }
311
2246
        dst += stride * 16;
312
    }
313
462
    return 0;
314
}
315
316
static void reset_buffers(VmncContext *c)
317
{
318
    av_freep(&c->curbits);
319
    av_freep(&c->curmask);
320
    av_freep(&c->screendta);
321
    c->cur_w = c->cur_h = 0;
322
    c->cur_hx = c->cur_hy = 0;
323
324
}
325
326
242
static int decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
327
                        AVPacket *avpkt)
328
{
329
242
    const uint8_t *buf = avpkt->data;
330
242
    int buf_size       = avpkt->size;
331
242
    VmncContext * const c = avctx->priv_data;
332
242
    GetByteContext *gb = &c->gb;
333
    uint8_t *outptr;
334
    int dx, dy, w, h, depth, enc, chunks, res, size_left, ret;
335
336
242
    bytestream2_init(gb, buf, buf_size);
337
242
    bytestream2_skip(gb, 2);
338
242
    chunks = bytestream2_get_be16(gb);
339
242
    if (12LL * chunks > bytestream2_get_bytes_left(gb))
340
        return AVERROR_INVALIDDATA;
341
342
242
    if ((ret = ff_reget_buffer(avctx, c->pic, 0)) < 0)
343
        return ret;
344
345
242
    c->pic->key_frame = 0;
346
242
    c->pic->pict_type = AV_PICTURE_TYPE_P;
347
348
    // restore screen after cursor
349
242
    if (c->screendta) {
350
        int i;
351
49
        w = c->cur_w;
352
49
        if (c->width < c->cur_x + w)
353
            w = c->width - c->cur_x;
354
49
        h = c->cur_h;
355
49
        if (c->height < c->cur_y + h)
356
            h = c->height - c->cur_y;
357
49
        dx = c->cur_x;
358
49
        if (dx < 0) {
359
            w += dx;
360
            dx = 0;
361
        }
362
49
        dy = c->cur_y;
363
49
        if (dy < 0) {
364
            h += dy;
365
            dy = 0;
366
        }
367

49
        if ((w > 0) && (h > 0)) {
368
49
            outptr = c->pic->data[0] + dx * c->bpp2 + dy * c->pic->linesize[0];
369
1617
            for (i = 0; i < h; i++) {
370
1568
                memcpy(outptr, c->screendta + i * c->cur_w * c->bpp2,
371
1568
                       w * c->bpp2);
372
1568
                outptr += c->pic->linesize[0];
373
            }
374
        }
375
    }
376
377
808
    while (chunks--) {
378
567
        if (bytestream2_get_bytes_left(gb) < 12) {
379
            av_log(avctx, AV_LOG_ERROR, "Premature end of data!\n");
380
            return -1;
381
        }
382
567
        dx  = bytestream2_get_be16(gb);
383
567
        dy  = bytestream2_get_be16(gb);
384
567
        w   = bytestream2_get_be16(gb);
385
567
        h   = bytestream2_get_be16(gb);
386
567
        enc = bytestream2_get_be32(gb);
387

567
        if ((dx + w > c->width) || (dy + h > c->height)) {
388
            av_log(avctx, AV_LOG_ERROR,
389
                    "Incorrect frame size: %ix%i+%ix%i of %ix%i\n",
390
                    w, h, dx, dy, c->width, c->height);
391
            return AVERROR_INVALIDDATA;
392
        }
393
567
        outptr = c->pic->data[0] + dx * c->bpp2 + dy * c->pic->linesize[0];
394
567
        size_left = bytestream2_get_bytes_left(gb);
395


567
        switch (enc) {
396
16
        case MAGIC_WMVd: // cursor
397
16
            if (w*(int64_t)h*c->bpp2 > INT_MAX/2 - 2) {
398
                av_log(avctx, AV_LOG_ERROR, "dimensions too large\n");
399
                return AVERROR_INVALIDDATA;
400
            }
401
16
            if (size_left < 2 + w * h * c->bpp2 * 2) {
402
                av_log(avctx, AV_LOG_ERROR,
403
                       "Premature end of data! (need %i got %i)\n",
404
                       2 + w * h * c->bpp2 * 2, size_left);
405
                return AVERROR_INVALIDDATA;
406
            }
407
16
            bytestream2_skip(gb, 2);
408
16
            c->cur_w  = w;
409
16
            c->cur_h  = h;
410
16
            c->cur_hx = dx;
411
16
            c->cur_hy = dy;
412

16
            if ((c->cur_hx > c->cur_w) || (c->cur_hy > c->cur_h)) {
413
                av_log(avctx, AV_LOG_ERROR,
414
                       "Cursor hot spot is not in image: "
415
                       "%ix%i of %ix%i cursor size\n",
416
                       c->cur_hx, c->cur_hy, c->cur_w, c->cur_h);
417
                c->cur_hx = c->cur_hy = 0;
418
            }
419
16
            if (c->cur_w * c->cur_h >= INT_MAX / c->bpp2) {
420
                reset_buffers(c);
421
                return AVERROR(EINVAL);
422
            } else {
423
16
                int screen_size = c->cur_w * c->cur_h * c->bpp2;
424

32
                if ((ret = av_reallocp(&c->curbits, screen_size)) < 0 ||
425
32
                    (ret = av_reallocp(&c->curmask, screen_size)) < 0 ||
426
16
                    (ret = av_reallocp(&c->screendta, screen_size)) < 0) {
427
                    reset_buffers(c);
428
                    return ret;
429
                }
430
            }
431
16
            load_cursor(c);
432
16
            break;
433
35
        case MAGIC_WMVe: // unknown
434
35
            bytestream2_skip(gb, 2);
435
35
            break;
436
29
        case MAGIC_WMVf: // update cursor position
437
29
            c->cur_x = dx - c->cur_hx;
438
29
            c->cur_y = dy - c->cur_hy;
439
29
            break;
440
4
        case MAGIC_WMVg: // unknown
441
4
            bytestream2_skip(gb, 10);
442
4
            break;
443
4
        case MAGIC_WMVh: // unknown
444
4
            bytestream2_skip(gb, 4);
445
4
            break;
446
4
        case MAGIC_WMVi: // ServerInitialization struct
447
4
            c->pic->key_frame = 1;
448
4
            c->pic->pict_type = AV_PICTURE_TYPE_I;
449
4
            depth = bytestream2_get_byte(gb);
450
4
            if (depth != c->bpp) {
451
                av_log(avctx, AV_LOG_INFO,
452
                       "Depth mismatch. Container %i bpp, "
453
                       "Frame data: %i bpp\n",
454
                       c->bpp, depth);
455
            }
456
4
            bytestream2_skip(gb, 1);
457
4
            c->bigendian = bytestream2_get_byte(gb);
458
4
            if (c->bigendian & (~1)) {
459
                av_log(avctx, AV_LOG_INFO,
460
                       "Invalid header: bigendian flag = %i\n", c->bigendian);
461
                return AVERROR_INVALIDDATA;
462
            }
463
            //skip the rest of pixel format data
464
4
            bytestream2_skip(gb, 13);
465
4
            break;
466
4
        case MAGIC_WMVj: // unknown
467
4
            bytestream2_skip(gb, 2);
468
4
            break;
469
8
        case 0x00000000: // raw rectangle data
470
8
            if (size_left < w * h * c->bpp2) {
471
                av_log(avctx, AV_LOG_ERROR,
472
                       "Premature end of data! (need %i got %i)\n",
473
                       w * h * c->bpp2, size_left);
474
                return AVERROR_INVALIDDATA;
475
            }
476
8
            paint_raw(outptr, w, h, gb, c->bpp2, c->bigendian,
477
8
                      c->pic->linesize[0]);
478
8
            break;
479
463
        case 0x00000005: // HexTile encoded rectangle
480
463
            res = decode_hextile(c, outptr, gb, w, h, c->pic->linesize[0]);
481
463
            if (res < 0)
482
1
                return res;
483
462
            break;
484
        default:
485
            av_log(avctx, AV_LOG_ERROR, "Unsupported block type 0x%08X\n", enc);
486
            chunks = 0; // leave chunks decoding loop
487
        }
488
    }
489
241
    if (c->screendta) {
490
        int i;
491
        // save screen data before painting cursor
492
49
        w = c->cur_w;
493
49
        if (c->width < c->cur_x + w)
494
            w = c->width - c->cur_x;
495
49
        h = c->cur_h;
496
49
        if (c->height < c->cur_y + h)
497
            h = c->height - c->cur_y;
498
49
        dx = c->cur_x;
499
49
        if (dx < 0) {
500
            w += dx;
501
            dx = 0;
502
        }
503
49
        dy = c->cur_y;
504
49
        if (dy < 0) {
505
            h += dy;
506
            dy = 0;
507
        }
508

49
        if ((w > 0) && (h > 0)) {
509
49
            outptr = c->pic->data[0] + dx * c->bpp2 + dy * c->pic->linesize[0];
510
1617
            for (i = 0; i < h; i++) {
511
1568
                memcpy(c->screendta + i * c->cur_w * c->bpp2, outptr,
512
1568
                       w * c->bpp2);
513
1568
                outptr += c->pic->linesize[0];
514
            }
515
49
            outptr = c->pic->data[0];
516
49
            put_cursor(outptr, c->pic->linesize[0], c, c->cur_x, c->cur_y);
517
        }
518
    }
519
241
    *got_frame = 1;
520
241
    if ((ret = av_frame_ref(data, c->pic)) < 0)
521
        return ret;
522
523
    /* always report that the buffer was completely consumed */
524
241
    return buf_size;
525
}
526
527
4
static av_cold int decode_init(AVCodecContext *avctx)
528
{
529
4
    VmncContext * const c = avctx->priv_data;
530
531
4
    c->avctx  = avctx;
532
4
    c->width  = avctx->width;
533
4
    c->height = avctx->height;
534
4
    c->bpp    = avctx->bits_per_coded_sample;
535
536

4
    switch (c->bpp) {
537
    case 8:
538
        avctx->pix_fmt = AV_PIX_FMT_PAL8;
539
        break;
540
2
    case 16:
541
2
        avctx->pix_fmt = AV_PIX_FMT_RGB555;
542
2
        break;
543
    case 24:
544
        /* 24 bits is not technically supported, but some clients might
545
         * mistakenly set it, so let's assume they actually meant 32 bits */
546
        c->bpp = 32;
547
2
    case 32:
548
2
        avctx->pix_fmt = AV_PIX_FMT_0RGB32;
549
2
        break;
550
    default:
551
        av_log(avctx, AV_LOG_ERROR, "Unsupported bitdepth %i\n", c->bpp);
552
        return AVERROR_INVALIDDATA;
553
    }
554
4
    c->bpp2 = c->bpp / 8;
555
556
4
    c->pic = av_frame_alloc();
557
4
    if (!c->pic)
558
        return AVERROR(ENOMEM);
559
560
4
    return 0;
561
}
562
563
4
static av_cold int decode_end(AVCodecContext *avctx)
564
{
565
4
    VmncContext * const c = avctx->priv_data;
566
567
4
    av_frame_free(&c->pic);
568
569
4
    av_freep(&c->curbits);
570
4
    av_freep(&c->curmask);
571
4
    av_freep(&c->screendta);
572
4
    return 0;
573
}
574
575
AVCodec ff_vmnc_decoder = {
576
    .name           = "vmnc",
577
    .long_name      = NULL_IF_CONFIG_SMALL("VMware Screen Codec / VMware Video"),
578
    .type           = AVMEDIA_TYPE_VIDEO,
579
    .id             = AV_CODEC_ID_VMNC,
580
    .priv_data_size = sizeof(VmncContext),
581
    .init           = decode_init,
582
    .close          = decode_end,
583
    .decode         = decode_frame,
584
    .capabilities   = AV_CODEC_CAP_DR1,
585
};