[SCM] libav/wheezy-security: mpeg12: check scantable indices in all decode_block functions
siretart at users.alioth.debian.org
siretart at users.alioth.debian.org
Sun Jun 1 21:36:13 UTC 2014
The following commit has been merged in the wheezy-security branch:
commit 9938e450c8575e0a4640dcbefb87d26d6bab9e4d
Author: Janne Grunau <janne-libav at jannau.net>
Date: Fri Jan 24 16:22:44 2014 +0100
mpeg12: check scantable indices in all decode_block functions
Add checks to the fast functions used with CODEC_FLAGS2_FAST and move
the check for all other functions to before the invalid memory is
accessed. Fixes https://trac.videolan.org/vlc/ticket/9713 with
CODEC_FLAGS2_FAST.
CC: libav-stable at libav.org
diff --git a/libavcodec/mpeg12.c b/libavcodec/mpeg12.c
index 436b4cf..4db2753 100644
--- a/libavcodec/mpeg12.c
+++ b/libavcodec/mpeg12.c
@@ -80,6 +80,15 @@ static int mpeg_decode_motion(MpegEncContext *s, int fcode, int pred)
return sign_extend(val, 5 + shift);
}
+#define check_scantable_index(ctx, x) \
+ do { \
+ if ((x) > 63) { \
+ av_log(ctx->avctx, AV_LOG_ERROR, "ac-tex damaged at %d %d\n", \
+ ctx->mb_x, ctx->mb_y); \
+ return AVERROR_INVALIDDATA; \
+ } \
+ } while (0) \
+
static inline int mpeg1_decode_block_intra(MpegEncContext *s, DCTELEM *block, int n)
{
int level, dc, diff, i, j, run;
@@ -111,6 +120,7 @@ static inline int mpeg1_decode_block_intra(MpegEncContext *s, DCTELEM *block, in
break;
} else if (level != 0) {
i += run;
+ check_scantable_index(s, i);
j = scantable[i];
level = (level * qscale * quant_matrix[j]) >> 4;
level = (level - 1) | 1;
@@ -127,6 +137,7 @@ static inline int mpeg1_decode_block_intra(MpegEncContext *s, DCTELEM *block, in
level = SHOW_UBITS(re, &s->gb, 8) ; LAST_SKIP_BITS(re, &s->gb, 8);
}
i += run;
+ check_scantable_index(s, i);
j = scantable[i];
if (level < 0) {
level = -level;
@@ -138,10 +149,6 @@ static inline int mpeg1_decode_block_intra(MpegEncContext *s, DCTELEM *block, in
level = (level - 1) | 1;
}
}
- if (i > 63) {
- av_log(s->avctx, AV_LOG_ERROR, "ac-tex damaged at %d %d\n", s->mb_x, s->mb_y);
- return -1;
- }
block[j] = level;
}
@@ -261,6 +268,7 @@ static inline int mpeg1_fast_decode_block_inter(MpegEncContext *s, DCTELEM *bloc
if (level != 0) {
i += run;
+ check_scantable_index(s, i);
j = scantable[i];
level = ((level * 2 + 1) * qscale) >> 1;
level = (level - 1) | 1;
@@ -277,6 +285,7 @@ static inline int mpeg1_fast_decode_block_inter(MpegEncContext *s, DCTELEM *bloc
level = SHOW_UBITS(re, &s->gb, 8) ; SKIP_BITS(re, &s->gb, 8);
}
i += run;
+ check_scantable_index(s, i);
j = scantable[i];
if (level < 0) {
level = -level;
@@ -342,6 +351,7 @@ static inline int mpeg2_decode_block_non_intra(MpegEncContext *s, DCTELEM *block
if (level != 0) {
i += run;
+ check_scantable_index(s, i);
j = scantable[i];
level = ((level * 2 + 1) * qscale * quant_matrix[j]) >> 5;
level = (level ^ SHOW_SBITS(re, &s->gb, 1)) - SHOW_SBITS(re, &s->gb, 1);
@@ -353,6 +363,7 @@ static inline int mpeg2_decode_block_non_intra(MpegEncContext *s, DCTELEM *block
level = SHOW_SBITS(re, &s->gb, 12); SKIP_BITS(re, &s->gb, 12);
i += run;
+ check_scantable_index(s, i);
j = scantable[i];
if (level < 0) {
level = ((-level * 2 + 1) * qscale * quant_matrix[j]) >> 5;
@@ -361,10 +372,6 @@ static inline int mpeg2_decode_block_non_intra(MpegEncContext *s, DCTELEM *block
level = ((level * 2 + 1) * qscale * quant_matrix[j]) >> 5;
}
}
- if (i > 63) {
- av_log(s->avctx, AV_LOG_ERROR, "ac-tex damaged at %d %d\n", s->mb_x, s->mb_y);
- return -1;
- }
mismatch ^= level;
block[j] = level;
@@ -411,6 +418,7 @@ static inline int mpeg2_fast_decode_block_non_intra(MpegEncContext *s,
if (level != 0) {
i += run;
+ check_scantable_index(s, i);
j = scantable[i];
level = ((level * 2 + 1) * qscale) >> 1;
level = (level ^ SHOW_SBITS(re, &s->gb, 1)) - SHOW_SBITS(re, &s->gb, 1);
@@ -422,6 +430,7 @@ static inline int mpeg2_fast_decode_block_non_intra(MpegEncContext *s,
level = SHOW_SBITS(re, &s->gb, 12); SKIP_BITS(re, &s->gb, 12);
i += run;
+ check_scantable_index(s, i);
j = scantable[i];
if (level < 0) {
level = ((-level * 2 + 1) * qscale) >> 1;
@@ -488,6 +497,7 @@ static inline int mpeg2_decode_block_intra(MpegEncContext *s, DCTELEM *block, in
break;
} else if (level != 0) {
i += run;
+ check_scantable_index(s, i);
j = scantable[i];
level = (level * qscale * quant_matrix[j]) >> 4;
level = (level ^ SHOW_SBITS(re, &s->gb, 1)) - SHOW_SBITS(re, &s->gb, 1);
@@ -498,6 +508,7 @@ static inline int mpeg2_decode_block_intra(MpegEncContext *s, DCTELEM *block, in
UPDATE_CACHE(re, &s->gb);
level = SHOW_SBITS(re, &s->gb, 12); SKIP_BITS(re, &s->gb, 12);
i += run;
+ check_scantable_index(s, i);
j = scantable[i];
if (level < 0) {
level = (-level * qscale * quant_matrix[j]) >> 4;
@@ -506,10 +517,6 @@ static inline int mpeg2_decode_block_intra(MpegEncContext *s, DCTELEM *block, in
level = (level * qscale * quant_matrix[j]) >> 4;
}
}
- if (i > 63) {
- av_log(s->avctx, AV_LOG_ERROR, "ac-tex damaged at %d %d\n", s->mb_x, s->mb_y);
- return -1;
- }
mismatch ^= level;
block[j] = level;
@@ -524,10 +531,10 @@ static inline int mpeg2_decode_block_intra(MpegEncContext *s, DCTELEM *block, in
static inline int mpeg2_fast_decode_block_intra(MpegEncContext *s, DCTELEM *block, int n)
{
- int level, dc, diff, j, run;
+ int level, dc, diff, i, j, run;
int component;
RLTable *rl;
- uint8_t * scantable = s->intra_scantable.permutated;
+ uint8_t * const scantable = s->intra_scantable.permutated;
const uint16_t *quant_matrix;
const int qscale = s->qscale;
@@ -546,6 +553,7 @@ static inline int mpeg2_fast_decode_block_intra(MpegEncContext *s, DCTELEM *bloc
dc += diff;
s->last_dc[component] = dc;
block[0] = dc << (3 - s->intra_dc_precision);
+ i = 0;
if (s->intra_vlc_format)
rl = &ff_rl_mpeg2;
else
@@ -561,8 +569,9 @@ static inline int mpeg2_fast_decode_block_intra(MpegEncContext *s, DCTELEM *bloc
if (level == 127) {
break;
} else if (level != 0) {
- scantable += run;
- j = *scantable;
+ i += run;
+ check_scantable_index(s, i);
+ j = scantable[i];
level = (level * qscale * quant_matrix[j]) >> 4;
level = (level ^ SHOW_SBITS(re, &s->gb, 1)) - SHOW_SBITS(re, &s->gb, 1);
LAST_SKIP_BITS(re, &s->gb, 1);
@@ -571,8 +580,9 @@ static inline int mpeg2_fast_decode_block_intra(MpegEncContext *s, DCTELEM *bloc
run = SHOW_UBITS(re, &s->gb, 6) + 1; LAST_SKIP_BITS(re, &s->gb, 6);
UPDATE_CACHE(re, &s->gb);
level = SHOW_SBITS(re, &s->gb, 12); SKIP_BITS(re, &s->gb, 12);
- scantable += run;
- j = *scantable;
+ i += run;
+ check_scantable_index(s, i);
+ j = scantable[i];
if (level < 0) {
level = (-level * qscale * quant_matrix[j]) >> 4;
level = -level;
@@ -586,7 +596,7 @@ static inline int mpeg2_fast_decode_block_intra(MpegEncContext *s, DCTELEM *bloc
CLOSE_READER(re, &s->gb);
}
- s->block_last_index[n] = scantable - s->intra_scantable.permutated;
+ s->block_last_index[n] = i;
return 0;
}
--
Libav/FFmpeg packaging
More information about the pkg-multimedia-commits
mailing list