basic type mechanism working

1 files changed, 18 insertions, 20 deletions

diff --git a/ffmpeg/libavcodec/8bps.c b/ffmpeg/libavcodec/8bps.c
index a910551..1709368 100644
--- a/ffmpeg/libavcodec/8bps.c
+++ b/ffmpeg/libavcodec/8bps.c
@@ -64,7 +64,7 @@ static int decode_frame(AVCodecContext *avctx, void *data,
     unsigned char *pixptr, *pixptr_end;
     unsigned int height = avctx->height; // Real image height
     unsigned int dlen, p, row;
-    const unsigned char *lp, *dp;
+    const unsigned char *lp, *dp, *ep;
     unsigned char count;
     unsigned int planes     = c->planes;
     unsigned char *planemap = c->planemap;
@@ -73,6 +73,8 @@ static int decode_frame(AVCodecContext *avctx, void *data,
     if ((ret = ff_get_buffer(avctx, frame, 0)) < 0)
         return ret;
 
+    ep = encoded + buf_size;
+
     /* Set data pointer after line lengths */
     dp = encoded + planes * (height << 1);
 
@@ -84,19 +86,19 @@ static int decode_frame(AVCodecContext *avctx, void *data,
         for (row = 0; row < height; row++) {
             pixptr = frame->data[0] + row * frame->linesize[0] + planemap[p];
             pixptr_end = pixptr + frame->linesize[0];
-            if(lp - encoded + row*2 + 1 >= buf_size)
-                return -1;
+            if (ep - lp < row * 2 + 2)
+                return AVERROR_INVALIDDATA;
             dlen = av_be2ne16(*(const unsigned short *)(lp + row * 2));
             /* Decode a row of this plane */
             while (dlen > 0) {
-                if (dp + 1 >= buf + buf_size)
+                if (ep - dp <= 1)
                     return AVERROR_INVALIDDATA;
                 if ((count = *dp++) <= 127) {
                     count++;
                     dlen -= count + 1;
-                    if (pixptr + count * planes > pixptr_end)
+                    if (pixptr_end - pixptr < count * planes)
                         break;
-                    if (dp + count > buf + buf_size)
+                    if (ep - dp < count)
                         return AVERROR_INVALIDDATA;
                     while (count--) {
                         *pixptr = *dp++;
@@ -104,7 +106,7 @@ static int decode_frame(AVCodecContext *avctx, void *data,
                     }
                 } else {
                     count = 257 - count;
-                    if (pixptr + count * planes > pixptr_end)
+                    if (pixptr_end - pixptr < count * planes)
                         break;
                     while (count--) {
                         *pixptr = *dp;
@@ -117,7 +119,7 @@ static int decode_frame(AVCodecContext *avctx, void *data,
         }
     }
 
-    if (avctx->bits_per_coded_sample <= 8) {
+    if ((avctx->bits_per_coded_sample & 0x1f) <= 8) {
         const uint8_t *pal = av_packet_get_side_data(avpkt,
                                                      AV_PKT_DATA_PALETTE,
                                                      NULL);
@@ -157,17 +159,7 @@ static av_cold int decode_init(AVCodecContext *avctx)
     case 32:
         avctx->pix_fmt = AV_PIX_FMT_RGB32;
         c->planes      = 4;
-#if HAVE_BIGENDIAN
-        c->planemap[0] = 1; // 1st plane is red
-        c->planemap[1] = 2; // 2nd plane is green
-        c->planemap[2] = 3; // 3rd plane is blue
-        c->planemap[3] = 0; // 4th plane is alpha
-#else
-        c->planemap[0] = 2; // 1st plane is red
-        c->planemap[1] = 1; // 2nd plane is green
-        c->planemap[2] = 0; // 3rd plane is blue
-        c->planemap[3] = 3; // 4th plane is alpha
-#endif
+        /* handle planemap setup later for decoding rgb24 data as rbg32 */
         break;
     default:
         av_log(avctx, AV_LOG_ERROR, "Error: Unsupported color depth: %u.\n",
@@ -175,16 +167,22 @@ static av_cold int decode_init(AVCodecContext *avctx)
         return AVERROR_INVALIDDATA;
     }
 
+    if (avctx->pix_fmt == AV_PIX_FMT_RGB32) {
+        c->planemap[0] = HAVE_BIGENDIAN ? 1 : 2; // 1st plane is red
+        c->planemap[1] = HAVE_BIGENDIAN ? 2 : 1; // 2nd plane is green
+        c->planemap[2] = HAVE_BIGENDIAN ? 3 : 0; // 3rd plane is blue
+        c->planemap[3] = HAVE_BIGENDIAN ? 0 : 3; // 4th plane is alpha
+    }
     return 0;
 }
 
 AVCodec ff_eightbps_decoder = {
     .name           = "8bps",
+    .long_name      = NULL_IF_CONFIG_SMALL("QuickTime 8BPS video"),
     .type           = AVMEDIA_TYPE_VIDEO,
     .id             = AV_CODEC_ID_8BPS,
     .priv_data_size = sizeof(EightBpsContext),
     .init           = decode_init,
     .decode         = decode_frame,
     .capabilities   = CODEC_CAP_DR1,
-    .long_name      = NULL_IF_CONFIG_SMALL("QuickTime 8BPS video"),
 };