Line | Branch | Exec | Source |
---|---|---|---|
1 | /* | ||
2 | * VMware Screen Codec (VMnc) decoder | ||
3 | * Copyright (c) 2006 Konstantin Shishkov | ||
4 | * | ||
5 | * This file is part of FFmpeg. | ||
6 | * | ||
7 | * FFmpeg is free software; you can redistribute it and/or | ||
8 | * modify it under the terms of the GNU Lesser General Public | ||
9 | * License as published by the Free Software Foundation; either | ||
10 | * version 2.1 of the License, or (at your option) any later version. | ||
11 | * | ||
12 | * FFmpeg is distributed in the hope that it will be useful, | ||
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | ||
15 | * Lesser General Public License for more details. | ||
16 | * | ||
17 | * You should have received a copy of the GNU Lesser General Public | ||
18 | * License along with FFmpeg; if not, write to the Free Software | ||
19 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA | ||
20 | */ | ||
21 | |||
22 | /** | ||
23 | * @file | ||
24 | * VMware Screen Codec (VMnc) decoder | ||
25 | * As Alex Beregszaszi discovered, this is effectively RFB data dump | ||
26 | */ | ||
27 | |||
28 | #include "libavutil/common.h" | ||
29 | #include "avcodec.h" | ||
30 | #include "codec_internal.h" | ||
31 | #include "decode.h" | ||
32 | #include "bytestream.h" | ||
33 | |||
34 | enum EncTypes { | ||
35 | MAGIC_WMVd = 0x574D5664, | ||
36 | MAGIC_WMVe, | ||
37 | MAGIC_WMVf, | ||
38 | MAGIC_WMVg, | ||
39 | MAGIC_WMVh, | ||
40 | MAGIC_WMVi, | ||
41 | MAGIC_WMVj | ||
42 | }; | ||
43 | |||
44 | enum HexTile_Flags { | ||
45 | HT_RAW = 1, // tile is raw | ||
46 | HT_BKG = 2, // background color is present | ||
47 | HT_FG = 4, // foreground color is present | ||
48 | HT_SUB = 8, // subrects are present | ||
49 | HT_CLR = 16 // each subrect has own color | ||
50 | }; | ||
51 | |||
52 | /* | ||
53 | * Decoder context | ||
54 | */ | ||
55 | typedef struct VmncContext { | ||
56 | AVCodecContext *avctx; | ||
57 | AVFrame *pic; | ||
58 | |||
59 | int bpp; | ||
60 | int bpp2; | ||
61 | int bigendian; | ||
62 | uint8_t pal[768]; | ||
63 | int width, height; | ||
64 | GetByteContext gb; | ||
65 | |||
66 | /* cursor data */ | ||
67 | int cur_w, cur_h; | ||
68 | int cur_x, cur_y; | ||
69 | int cur_hx, cur_hy; | ||
70 | uint8_t *curbits, *curmask; | ||
71 | uint8_t *screendta; | ||
72 | } VmncContext; | ||
73 | |||
74 | /* read pixel value from stream */ | ||
75 | 401741 | static av_always_inline int vmnc_get_pixel(GetByteContext *gb, int bpp, int be) | |
76 | { | ||
77 |
2/6✗ Branch 0 not taken.
✓ Branch 1 taken 58268 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 343473 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
|
401741 | switch (bpp * 2 + be) { |
78 | ✗ | case 2: | |
79 | case 3: | ||
80 | ✗ | return bytestream2_get_byte(gb); | |
81 | 58268 | case 4: | |
82 | 58268 | return bytestream2_get_le16(gb); | |
83 | ✗ | case 5: | |
84 | ✗ | return bytestream2_get_be16(gb); | |
85 | 343473 | case 8: | |
86 | 343473 | return bytestream2_get_le32(gb); | |
87 | ✗ | case 9: | |
88 | ✗ | return bytestream2_get_be32(gb); | |
89 | ✗ | default: return 0; | |
90 | } | ||
91 | } | ||
92 | |||
93 | 16 | static void load_cursor(VmncContext *c) | |
94 | { | ||
95 | int i, j, p; | ||
96 | 16 | const int bpp = c->bpp2; | |
97 | 16 | uint8_t *dst8 = c->curbits; | |
98 | 16 | uint16_t *dst16 = (uint16_t *)c->curbits; | |
99 | 16 | uint32_t *dst32 = (uint32_t *)c->curbits; | |
100 | |||
101 |
2/2✓ Branch 0 taken 416 times.
✓ Branch 1 taken 16 times.
|
432 | for (j = 0; j < c->cur_h; j++) { |
102 |
2/2✓ Branch 0 taken 13312 times.
✓ Branch 1 taken 416 times.
|
13728 | for (i = 0; i < c->cur_w; i++) { |
103 | 13312 | p = vmnc_get_pixel(&c->gb, bpp, c->bigendian); | |
104 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 13312 times.
|
13312 | if (bpp == 1) |
105 | ✗ | *dst8++ = p; | |
106 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 13312 times.
|
13312 | if (bpp == 2) |
107 | ✗ | *dst16++ = p; | |
108 |
1/2✓ Branch 0 taken 13312 times.
✗ Branch 1 not taken.
|
13312 | if (bpp == 4) |
109 | 13312 | *dst32++ = p; | |
110 | } | ||
111 | } | ||
112 | 16 | dst8 = c->curmask; | |
113 | 16 | dst16 = (uint16_t*)c->curmask; | |
114 | 16 | dst32 = (uint32_t*)c->curmask; | |
115 |
2/2✓ Branch 0 taken 416 times.
✓ Branch 1 taken 16 times.
|
432 | for (j = 0; j < c->cur_h; j++) { |
116 |
2/2✓ Branch 0 taken 13312 times.
✓ Branch 1 taken 416 times.
|
13728 | for (i = 0; i < c->cur_w; i++) { |
117 | 13312 | p = vmnc_get_pixel(&c->gb, bpp, c->bigendian); | |
118 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 13312 times.
|
13312 | if (bpp == 1) |
119 | ✗ | *dst8++ = p; | |
120 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 13312 times.
|
13312 | if (bpp == 2) |
121 | ✗ | *dst16++ = p; | |
122 |
1/2✓ Branch 0 taken 13312 times.
✗ Branch 1 not taken.
|
13312 | if (bpp == 4) |
123 | 13312 | *dst32++ = p; | |
124 | } | ||
125 | } | ||
126 | 16 | } | |
127 | |||
128 | 49 | static void put_cursor(uint8_t *dst, int stride, VmncContext *c, int dx, int dy) | |
129 | { | ||
130 | int i, j; | ||
131 | int w, h, x, y; | ||
132 | 49 | w = c->cur_w; | |
133 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 49 times.
|
49 | if (c->width < c->cur_x + c->cur_w) |
134 | ✗ | w = c->width - c->cur_x; | |
135 | 49 | h = c->cur_h; | |
136 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 49 times.
|
49 | if (c->height < c->cur_y + c->cur_h) |
137 | ✗ | h = c->height - c->cur_y; | |
138 | 49 | x = c->cur_x; | |
139 | 49 | y = c->cur_y; | |
140 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 49 times.
|
49 | if (x < 0) { |
141 | ✗ | w += x; | |
142 | ✗ | x = 0; | |
143 | } | ||
144 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 49 times.
|
49 | if (y < 0) { |
145 | ✗ | h += y; | |
146 | ✗ | y = 0; | |
147 | } | ||
148 | |||
149 |
2/4✓ Branch 0 taken 49 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 49 times.
|
49 | if ((w < 1) || (h < 1)) |
150 | ✗ | return; | |
151 | 49 | dst += x * c->bpp2 + y * stride; | |
152 | |||
153 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 49 times.
|
49 | if (c->bpp2 == 1) { |
154 | ✗ | uint8_t *cd = c->curbits, *msk = c->curmask; | |
155 | ✗ | for (j = 0; j < h; j++) { | |
156 | ✗ | for (i = 0; i < w; i++) | |
157 | ✗ | dst[i] = (dst[i] & cd[i]) ^ msk[i]; | |
158 | ✗ | msk += c->cur_w; | |
159 | ✗ | cd += c->cur_w; | |
160 | ✗ | dst += stride; | |
161 | } | ||
162 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 49 times.
|
49 | } else if (c->bpp2 == 2) { |
163 | ✗ | uint16_t *cd = (uint16_t*)c->curbits, *msk = (uint16_t*)c->curmask; | |
164 | uint16_t *dst2; | ||
165 | ✗ | for (j = 0; j < h; j++) { | |
166 | ✗ | dst2 = (uint16_t*)dst; | |
167 | ✗ | for (i = 0; i < w; i++) | |
168 | ✗ | dst2[i] = (dst2[i] & cd[i]) ^ msk[i]; | |
169 | ✗ | msk += c->cur_w; | |
170 | ✗ | cd += c->cur_w; | |
171 | ✗ | dst += stride; | |
172 | } | ||
173 |
1/2✓ Branch 0 taken 49 times.
✗ Branch 1 not taken.
|
49 | } else if (c->bpp2 == 4) { |
174 | 49 | uint32_t *cd = (uint32_t*)c->curbits, *msk = (uint32_t*)c->curmask; | |
175 | uint32_t *dst2; | ||
176 |
2/2✓ Branch 0 taken 1568 times.
✓ Branch 1 taken 49 times.
|
1617 | for (j = 0; j < h; j++) { |
177 | 1568 | dst2 = (uint32_t*)dst; | |
178 |
2/2✓ Branch 0 taken 50176 times.
✓ Branch 1 taken 1568 times.
|
51744 | for (i = 0; i < w; i++) |
179 | 50176 | dst2[i] = (dst2[i] & cd[i]) ^ msk[i]; | |
180 | 1568 | msk += c->cur_w; | |
181 | 1568 | cd += c->cur_w; | |
182 | 1568 | dst += stride; | |
183 | } | ||
184 | } | ||
185 | } | ||
186 | |||
187 | /* fill rectangle with given color */ | ||
188 | 443805 | static av_always_inline void paint_rect(uint8_t *dst, int dx, int dy, | |
189 | int w, int h, int color, | ||
190 | int bpp, int stride) | ||
191 | { | ||
192 | int i, j; | ||
193 | 443805 | dst += dx * bpp + dy * stride; | |
194 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 443805 times.
|
443805 | if (bpp == 1) { |
195 | ✗ | for (j = 0; j < h; j++) { | |
196 | ✗ | memset(dst, color, w); | |
197 | ✗ | dst += stride; | |
198 | } | ||
199 |
2/2✓ Branch 0 taken 91696 times.
✓ Branch 1 taken 352109 times.
|
443805 | } else if (bpp == 2) { |
200 | uint16_t *dst2; | ||
201 |
2/2✓ Branch 0 taken 687588 times.
✓ Branch 1 taken 91696 times.
|
779284 | for (j = 0; j < h; j++) { |
202 | 687588 | dst2 = (uint16_t*)dst; | |
203 |
2/2✓ Branch 0 taken 8908537 times.
✓ Branch 1 taken 687588 times.
|
9596125 | for (i = 0; i < w; i++) |
204 | 8908537 | *dst2++ = color; | |
205 | 687588 | dst += stride; | |
206 | } | ||
207 |
1/2✓ Branch 0 taken 352109 times.
✗ Branch 1 not taken.
|
352109 | } else if (bpp == 4) { |
208 | uint32_t *dst2; | ||
209 |
2/2✓ Branch 0 taken 1320455 times.
✓ Branch 1 taken 352109 times.
|
1672564 | for (j = 0; j < h; j++) { |
210 | 1320455 | dst2 = (uint32_t*)dst; | |
211 |
2/2✓ Branch 0 taken 12708159 times.
✓ Branch 1 taken 1320455 times.
|
14028614 | for (i = 0; i < w; i++) |
212 | 12708159 | dst2[i] = color; | |
213 | 1320455 | dst += stride; | |
214 | } | ||
215 | } | ||
216 | 443805 | } | |
217 | |||
218 | 50 | static av_always_inline void paint_raw(uint8_t *dst, int w, int h, | |
219 | GetByteContext *gb, int bpp, | ||
220 | int be, int stride) | ||
221 | { | ||
222 | int i, j, p; | ||
223 |
2/2✓ Branch 0 taken 417 times.
✓ Branch 1 taken 50 times.
|
467 | for (j = 0; j < h; j++) { |
224 |
2/2✓ Branch 0 taken 6204 times.
✓ Branch 1 taken 417 times.
|
6621 | for (i = 0; i < w; i++) { |
225 | 6204 | p = vmnc_get_pixel(gb, bpp, be); | |
226 |
1/4✗ Branch 0 not taken.
✗ Branch 1 not taken.
✓ Branch 2 taken 6204 times.
✗ Branch 3 not taken.
|
6204 | switch (bpp) { |
227 | ✗ | case 1: | |
228 | ✗ | dst[i] = p; | |
229 | ✗ | break; | |
230 | ✗ | case 2: | |
231 | ✗ | ((uint16_t*)dst)[i] = p; | |
232 | ✗ | break; | |
233 | 6204 | case 4: | |
234 | 6204 | ((uint32_t*)dst)[i] = p; | |
235 | 6204 | break; | |
236 | } | ||
237 | } | ||
238 | 417 | dst += stride; | |
239 | } | ||
240 | 50 | } | |
241 | |||
242 | 463 | static int decode_hextile(VmncContext *c, uint8_t* dst, GetByteContext *gb, | |
243 | int w, int h, int stride) | ||
244 | { | ||
245 | int i, j, k; | ||
246 | 463 | int bg = 0, fg = 0, rects, color, flags, xy, wh; | |
247 | 463 | const int bpp = c->bpp2; | |
248 | uint8_t *dst2; | ||
249 | 463 | int bw = 16, bh = 16; | |
250 | |||
251 |
2/2✓ Branch 0 taken 2247 times.
✓ Branch 1 taken 462 times.
|
2709 | for (j = 0; j < h; j += 16) { |
252 | 2247 | dst2 = dst; | |
253 | 2247 | bw = 16; | |
254 |
2/2✓ Branch 0 taken 420 times.
✓ Branch 1 taken 1827 times.
|
2247 | if (j + 16 > h) |
255 | 420 | bh = h - j; | |
256 |
2/2✓ Branch 0 taken 76192 times.
✓ Branch 1 taken 2246 times.
|
78438 | for (i = 0; i < w; i += 16, dst2 += 16 * bpp) { |
257 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 76192 times.
|
76192 | if (bytestream2_get_bytes_left(gb) <= 0) { |
258 | ✗ | av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n"); | |
259 | ✗ | return AVERROR_INVALIDDATA; | |
260 | } | ||
261 |
2/2✓ Branch 0 taken 2103 times.
✓ Branch 1 taken 74089 times.
|
76192 | if (i + 16 > w) |
262 | 2103 | bw = w - i; | |
263 | 76192 | flags = bytestream2_get_byte(gb); | |
264 |
2/2✓ Branch 0 taken 42 times.
✓ Branch 1 taken 76150 times.
|
76192 | if (flags & HT_RAW) { |
265 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 42 times.
|
42 | if (bytestream2_get_bytes_left(gb) < bw * bh * bpp) { |
266 | ✗ | av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n"); | |
267 | ✗ | return AVERROR_INVALIDDATA; | |
268 | } | ||
269 | 42 | paint_raw(dst2, bw, bh, gb, bpp, c->bigendian, stride); | |
270 | } else { | ||
271 |
2/2✓ Branch 0 taken 1377 times.
✓ Branch 1 taken 74773 times.
|
76150 | if (flags & HT_BKG) |
272 | 1377 | bg = vmnc_get_pixel(gb, bpp, c->bigendian); | |
273 |
2/2✓ Branch 0 taken 1494 times.
✓ Branch 1 taken 74656 times.
|
76150 | if (flags & HT_FG) |
274 | 1494 | fg = vmnc_get_pixel(gb, bpp, c->bigendian); | |
275 | 76150 | rects = 0; | |
276 |
2/2✓ Branch 0 taken 30064 times.
✓ Branch 1 taken 46086 times.
|
76150 | if (flags & HT_SUB) |
277 | 30064 | rects = bytestream2_get_byte(gb); | |
278 | 76150 | color = !!(flags & HT_CLR); | |
279 | |||
280 | 76150 | paint_rect(dst2, 0, 0, bw, bh, bg, bpp, stride); | |
281 | |||
282 |
2/2✓ Branch 1 taken 1 times.
✓ Branch 2 taken 76149 times.
|
76150 | if (bytestream2_get_bytes_left(gb) < rects * (color * bpp + 2)) { |
283 | 1 | av_log(c->avctx, AV_LOG_ERROR, "Premature end of data!\n"); | |
284 | 1 | return AVERROR_INVALIDDATA; | |
285 | } | ||
286 |
2/2✓ Branch 0 taken 367655 times.
✓ Branch 1 taken 76149 times.
|
443804 | for (k = 0; k < rects; k++) { |
287 | int rect_x, rect_y, rect_w, rect_h; | ||
288 |
2/2✓ Branch 0 taken 366042 times.
✓ Branch 1 taken 1613 times.
|
367655 | if (color) |
289 | 366042 | fg = vmnc_get_pixel(gb, bpp, c->bigendian); | |
290 | 367655 | xy = bytestream2_get_byte(gb); | |
291 | 367655 | wh = bytestream2_get_byte(gb); | |
292 | |||
293 | 367655 | rect_x = xy >> 4; | |
294 | 367655 | rect_y = xy & 0xF; | |
295 | 367655 | rect_w = (wh >> 4) + 1; | |
296 | 367655 | rect_h = (wh & 0xF) + 1; | |
297 | |||
298 |
2/4✓ Branch 0 taken 367655 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 367655 times.
|
367655 | if (rect_x + rect_w > w - i || rect_y + rect_h > h - j) { |
299 | ✗ | av_log(c->avctx, AV_LOG_ERROR, "Rectangle outside picture\n"); | |
300 | ✗ | return AVERROR_INVALIDDATA; | |
301 | } | ||
302 | |||
303 | 367655 | paint_rect(dst2, rect_x, rect_y, | |
304 | rect_w, rect_h, fg, bpp, stride); | ||
305 | } | ||
306 | } | ||
307 | } | ||
308 | 2246 | dst += stride * 16; | |
309 | } | ||
310 | 462 | return 0; | |
311 | } | ||
312 | |||
313 | ✗ | static void reset_buffers(VmncContext *c) | |
314 | { | ||
315 | ✗ | av_freep(&c->curbits); | |
316 | ✗ | av_freep(&c->curmask); | |
317 | ✗ | av_freep(&c->screendta); | |
318 | ✗ | c->cur_w = c->cur_h = 0; | |
319 | ✗ | c->cur_hx = c->cur_hy = 0; | |
320 | |||
321 | ✗ | } | |
322 | |||
323 | 242 | static int decode_frame(AVCodecContext *avctx, AVFrame *rframe, | |
324 | int *got_frame, AVPacket *avpkt) | ||
325 | { | ||
326 | 242 | const uint8_t *buf = avpkt->data; | |
327 | 242 | int buf_size = avpkt->size; | |
328 | 242 | VmncContext * const c = avctx->priv_data; | |
329 | 242 | GetByteContext *gb = &c->gb; | |
330 | uint8_t *outptr; | ||
331 | int dx, dy, w, h, depth, enc, chunks, res, size_left, ret; | ||
332 | |||
333 | 242 | bytestream2_init(gb, buf, buf_size); | |
334 | 242 | bytestream2_skip(gb, 2); | |
335 | 242 | chunks = bytestream2_get_be16(gb); | |
336 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 242 times.
|
242 | if (12LL * chunks > bytestream2_get_bytes_left(gb)) |
337 | ✗ | return AVERROR_INVALIDDATA; | |
338 | |||
339 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 242 times.
|
242 | if ((ret = ff_reget_buffer(avctx, c->pic, 0)) < 0) |
340 | ✗ | return ret; | |
341 | |||
342 | 242 | c->pic->flags &= ~AV_FRAME_FLAG_KEY; | |
343 | 242 | c->pic->pict_type = AV_PICTURE_TYPE_P; | |
344 | |||
345 | // restore screen after cursor | ||
346 |
2/2✓ Branch 0 taken 49 times.
✓ Branch 1 taken 193 times.
|
242 | if (c->screendta) { |
347 | int i; | ||
348 | 49 | w = c->cur_w; | |
349 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 49 times.
|
49 | if (c->width < c->cur_x + w) |
350 | ✗ | w = c->width - c->cur_x; | |
351 | 49 | h = c->cur_h; | |
352 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 49 times.
|
49 | if (c->height < c->cur_y + h) |
353 | ✗ | h = c->height - c->cur_y; | |
354 | 49 | dx = c->cur_x; | |
355 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 49 times.
|
49 | if (dx < 0) { |
356 | ✗ | w += dx; | |
357 | ✗ | dx = 0; | |
358 | } | ||
359 | 49 | dy = c->cur_y; | |
360 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 49 times.
|
49 | if (dy < 0) { |
361 | ✗ | h += dy; | |
362 | ✗ | dy = 0; | |
363 | } | ||
364 |
2/4✓ Branch 0 taken 49 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 49 times.
✗ Branch 3 not taken.
|
49 | if ((w > 0) && (h > 0)) { |
365 | 49 | outptr = c->pic->data[0] + dx * c->bpp2 + dy * c->pic->linesize[0]; | |
366 |
2/2✓ Branch 0 taken 1568 times.
✓ Branch 1 taken 49 times.
|
1617 | for (i = 0; i < h; i++) { |
367 | 1568 | memcpy(outptr, c->screendta + i * c->cur_w * c->bpp2, | |
368 | 1568 | w * c->bpp2); | |
369 | 1568 | outptr += c->pic->linesize[0]; | |
370 | } | ||
371 | } | ||
372 | } | ||
373 | |||
374 |
2/2✓ Branch 0 taken 567 times.
✓ Branch 1 taken 241 times.
|
808 | while (chunks--) { |
375 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 567 times.
|
567 | if (bytestream2_get_bytes_left(gb) < 12) { |
376 | ✗ | av_log(avctx, AV_LOG_ERROR, "Premature end of data!\n"); | |
377 | ✗ | return -1; | |
378 | } | ||
379 | 567 | dx = bytestream2_get_be16(gb); | |
380 | 567 | dy = bytestream2_get_be16(gb); | |
381 | 567 | w = bytestream2_get_be16(gb); | |
382 | 567 | h = bytestream2_get_be16(gb); | |
383 | 567 | enc = bytestream2_get_be32(gb); | |
384 |
2/4✓ Branch 0 taken 567 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 567 times.
|
567 | if ((dx + w > c->width) || (dy + h > c->height)) { |
385 | ✗ | av_log(avctx, AV_LOG_ERROR, | |
386 | "Incorrect frame size: %ix%i+%ix%i of %ix%i\n", | ||
387 | w, h, dx, dy, c->width, c->height); | ||
388 | ✗ | return AVERROR_INVALIDDATA; | |
389 | } | ||
390 | 567 | outptr = c->pic->data[0] + dx * c->bpp2 + dy * c->pic->linesize[0]; | |
391 | 567 | size_left = bytestream2_get_bytes_left(gb); | |
392 |
9/10✓ Branch 0 taken 16 times.
✓ Branch 1 taken 35 times.
✓ Branch 2 taken 29 times.
✓ Branch 3 taken 4 times.
✓ Branch 4 taken 4 times.
✓ Branch 5 taken 4 times.
✓ Branch 6 taken 4 times.
✓ Branch 7 taken 8 times.
✓ Branch 8 taken 463 times.
✗ Branch 9 not taken.
|
567 | switch (enc) { |
393 | 16 | case MAGIC_WMVd: // cursor | |
394 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 16 times.
|
16 | if (w*(int64_t)h*c->bpp2 > INT_MAX/2 - 2) { |
395 | ✗ | av_log(avctx, AV_LOG_ERROR, "dimensions too large\n"); | |
396 | ✗ | return AVERROR_INVALIDDATA; | |
397 | } | ||
398 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 16 times.
|
16 | if (size_left < 2 + w * h * c->bpp2 * 2) { |
399 | ✗ | av_log(avctx, AV_LOG_ERROR, | |
400 | "Premature end of data! (need %i got %i)\n", | ||
401 | ✗ | 2 + w * h * c->bpp2 * 2, size_left); | |
402 | ✗ | return AVERROR_INVALIDDATA; | |
403 | } | ||
404 | 16 | bytestream2_skip(gb, 2); | |
405 | 16 | c->cur_w = w; | |
406 | 16 | c->cur_h = h; | |
407 | 16 | c->cur_hx = dx; | |
408 | 16 | c->cur_hy = dy; | |
409 |
2/4✓ Branch 0 taken 16 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 16 times.
|
16 | if ((c->cur_hx > c->cur_w) || (c->cur_hy > c->cur_h)) { |
410 | ✗ | av_log(avctx, AV_LOG_ERROR, | |
411 | "Cursor hot spot is not in image: " | ||
412 | "%ix%i of %ix%i cursor size\n", | ||
413 | c->cur_hx, c->cur_hy, c->cur_w, c->cur_h); | ||
414 | ✗ | c->cur_hx = c->cur_hy = 0; | |
415 | } | ||
416 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 16 times.
|
16 | if (c->cur_w * c->cur_h >= INT_MAX / c->bpp2) { |
417 | ✗ | reset_buffers(c); | |
418 | ✗ | return AVERROR(EINVAL); | |
419 | } else { | ||
420 | 16 | int screen_size = c->cur_w * c->cur_h * c->bpp2; | |
421 |
2/4✓ Branch 1 taken 16 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 16 times.
✗ Branch 4 not taken.
|
32 | if ((ret = av_reallocp(&c->curbits, screen_size)) < 0 || |
422 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 16 times.
|
32 | (ret = av_reallocp(&c->curmask, screen_size)) < 0 || |
423 | 16 | (ret = av_reallocp(&c->screendta, screen_size)) < 0) { | |
424 | ✗ | reset_buffers(c); | |
425 | ✗ | return ret; | |
426 | } | ||
427 | } | ||
428 | 16 | load_cursor(c); | |
429 | 16 | break; | |
430 | 35 | case MAGIC_WMVe: // unknown | |
431 | 35 | bytestream2_skip(gb, 2); | |
432 | 35 | break; | |
433 | 29 | case MAGIC_WMVf: // update cursor position | |
434 | 29 | c->cur_x = dx - c->cur_hx; | |
435 | 29 | c->cur_y = dy - c->cur_hy; | |
436 | 29 | break; | |
437 | 4 | case MAGIC_WMVg: // unknown | |
438 | 4 | bytestream2_skip(gb, 10); | |
439 | 4 | break; | |
440 | 4 | case MAGIC_WMVh: // unknown | |
441 | 4 | bytestream2_skip(gb, 4); | |
442 | 4 | break; | |
443 | 4 | case MAGIC_WMVi: // ServerInitialization struct | |
444 | 4 | c->pic->flags |= AV_FRAME_FLAG_KEY; | |
445 | 4 | c->pic->pict_type = AV_PICTURE_TYPE_I; | |
446 | 4 | depth = bytestream2_get_byte(gb); | |
447 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 4 times.
|
4 | if (depth != c->bpp) { |
448 | ✗ | av_log(avctx, AV_LOG_INFO, | |
449 | "Depth mismatch. Container %i bpp, " | ||
450 | "Frame data: %i bpp\n", | ||
451 | c->bpp, depth); | ||
452 | } | ||
453 | 4 | bytestream2_skip(gb, 1); | |
454 | 4 | c->bigendian = bytestream2_get_byte(gb); | |
455 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 4 times.
|
4 | if (c->bigendian & (~1)) { |
456 | ✗ | av_log(avctx, AV_LOG_INFO, | |
457 | "Invalid header: bigendian flag = %i\n", c->bigendian); | ||
458 | ✗ | return AVERROR_INVALIDDATA; | |
459 | } | ||
460 | //skip the rest of pixel format data | ||
461 | 4 | bytestream2_skip(gb, 13); | |
462 | 4 | break; | |
463 | 4 | case MAGIC_WMVj: // unknown | |
464 | 4 | bytestream2_skip(gb, 2); | |
465 | 4 | break; | |
466 | 8 | case 0x00000000: // raw rectangle data | |
467 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 8 times.
|
8 | if (size_left < w * h * c->bpp2) { |
468 | ✗ | av_log(avctx, AV_LOG_ERROR, | |
469 | "Premature end of data! (need %i got %i)\n", | ||
470 | ✗ | w * h * c->bpp2, size_left); | |
471 | ✗ | return AVERROR_INVALIDDATA; | |
472 | } | ||
473 | 8 | paint_raw(outptr, w, h, gb, c->bpp2, c->bigendian, | |
474 | 8 | c->pic->linesize[0]); | |
475 | 8 | break; | |
476 | 463 | case 0x00000005: // HexTile encoded rectangle | |
477 | 463 | res = decode_hextile(c, outptr, gb, w, h, c->pic->linesize[0]); | |
478 |
2/2✓ Branch 0 taken 1 times.
✓ Branch 1 taken 462 times.
|
463 | if (res < 0) |
479 | 1 | return res; | |
480 | 462 | break; | |
481 | ✗ | default: | |
482 | ✗ | av_log(avctx, AV_LOG_ERROR, "Unsupported block type 0x%08X\n", enc); | |
483 | ✗ | chunks = 0; // leave chunks decoding loop | |
484 | } | ||
485 | } | ||
486 |
2/2✓ Branch 0 taken 49 times.
✓ Branch 1 taken 192 times.
|
241 | if (c->screendta) { |
487 | int i; | ||
488 | // save screen data before painting cursor | ||
489 | 49 | w = c->cur_w; | |
490 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 49 times.
|
49 | if (c->width < c->cur_x + w) |
491 | ✗ | w = c->width - c->cur_x; | |
492 | 49 | h = c->cur_h; | |
493 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 49 times.
|
49 | if (c->height < c->cur_y + h) |
494 | ✗ | h = c->height - c->cur_y; | |
495 | 49 | dx = c->cur_x; | |
496 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 49 times.
|
49 | if (dx < 0) { |
497 | ✗ | w += dx; | |
498 | ✗ | dx = 0; | |
499 | } | ||
500 | 49 | dy = c->cur_y; | |
501 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 49 times.
|
49 | if (dy < 0) { |
502 | ✗ | h += dy; | |
503 | ✗ | dy = 0; | |
504 | } | ||
505 |
2/4✓ Branch 0 taken 49 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 49 times.
✗ Branch 3 not taken.
|
49 | if ((w > 0) && (h > 0)) { |
506 | 49 | outptr = c->pic->data[0] + dx * c->bpp2 + dy * c->pic->linesize[0]; | |
507 |
2/2✓ Branch 0 taken 1568 times.
✓ Branch 1 taken 49 times.
|
1617 | for (i = 0; i < h; i++) { |
508 | 1568 | memcpy(c->screendta + i * c->cur_w * c->bpp2, outptr, | |
509 | 1568 | w * c->bpp2); | |
510 | 1568 | outptr += c->pic->linesize[0]; | |
511 | } | ||
512 | 49 | outptr = c->pic->data[0]; | |
513 | 49 | put_cursor(outptr, c->pic->linesize[0], c, c->cur_x, c->cur_y); | |
514 | } | ||
515 | } | ||
516 | 241 | *got_frame = 1; | |
517 |
1/2✗ Branch 1 not taken.
✓ Branch 2 taken 241 times.
|
241 | if ((ret = av_frame_ref(rframe, c->pic)) < 0) |
518 | ✗ | return ret; | |
519 | |||
520 | /* always report that the buffer was completely consumed */ | ||
521 | 241 | return buf_size; | |
522 | } | ||
523 | |||
524 | 4 | static av_cold int decode_init(AVCodecContext *avctx) | |
525 | { | ||
526 | 4 | VmncContext * const c = avctx->priv_data; | |
527 | |||
528 | 4 | c->avctx = avctx; | |
529 | 4 | c->width = avctx->width; | |
530 | 4 | c->height = avctx->height; | |
531 | 4 | c->bpp = avctx->bits_per_coded_sample; | |
532 | |||
533 |
2/5✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 2 times.
✗ Branch 4 not taken.
|
4 | switch (c->bpp) { |
534 | ✗ | case 8: | |
535 | ✗ | avctx->pix_fmt = AV_PIX_FMT_PAL8; | |
536 | ✗ | break; | |
537 | 2 | case 16: | |
538 | 2 | avctx->pix_fmt = AV_PIX_FMT_RGB555; | |
539 | 2 | break; | |
540 | ✗ | case 24: | |
541 | /* 24 bits is not technically supported, but some clients might | ||
542 | * mistakenly set it, so let's assume they actually meant 32 bits */ | ||
543 | ✗ | c->bpp = 32; | |
544 | 2 | case 32: | |
545 | 2 | avctx->pix_fmt = AV_PIX_FMT_0RGB32; | |
546 | 2 | break; | |
547 | ✗ | default: | |
548 | ✗ | av_log(avctx, AV_LOG_ERROR, "Unsupported bitdepth %i\n", c->bpp); | |
549 | ✗ | return AVERROR_INVALIDDATA; | |
550 | } | ||
551 | 4 | c->bpp2 = c->bpp / 8; | |
552 | |||
553 | 4 | c->pic = av_frame_alloc(); | |
554 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 4 times.
|
4 | if (!c->pic) |
555 | ✗ | return AVERROR(ENOMEM); | |
556 | |||
557 | 4 | return 0; | |
558 | } | ||
559 | |||
560 | 4 | static av_cold int decode_end(AVCodecContext *avctx) | |
561 | { | ||
562 | 4 | VmncContext * const c = avctx->priv_data; | |
563 | |||
564 | 4 | av_frame_free(&c->pic); | |
565 | |||
566 | 4 | av_freep(&c->curbits); | |
567 | 4 | av_freep(&c->curmask); | |
568 | 4 | av_freep(&c->screendta); | |
569 | 4 | return 0; | |
570 | } | ||
571 | |||
572 | const FFCodec ff_vmnc_decoder = { | ||
573 | .p.name = "vmnc", | ||
574 | CODEC_LONG_NAME("VMware Screen Codec / VMware Video"), | ||
575 | .p.type = AVMEDIA_TYPE_VIDEO, | ||
576 | .p.id = AV_CODEC_ID_VMNC, | ||
577 | .priv_data_size = sizeof(VmncContext), | ||
578 | .init = decode_init, | ||
579 | .close = decode_end, | ||
580 | FF_CODEC_DECODE_CB(decode_frame), | ||
581 | .p.capabilities = AV_CODEC_CAP_DR1, | ||
582 | }; | ||
583 |