[PATCH 12/14] media: vicodec: set pixelformat V4L2_PIX_FMT_FWHT_STATELESS for stateless encoder

[Dafna Hirschfeld]

for stateless encoder, set the capture pixelformat
to V4L2_PIX_FMT_FWHT_STATELESS and the pix info to
pixfmt_stateless_fwht

Signed-off-by: Dafna Hirschfeld <daf...@gmail.com>
---
 drivers/media/platform/vicodec/vicodec-core.c | 31 ++++++++++++-------
 1 file changed, 20 insertions(+), 11 deletions(-)

diff --git a/drivers/media/platform/vicodec/vicodec-core.c 
b/drivers/media/platform/vicodec/vicodec-core.c
index 6bfcd266ffc4..f042c4b618ba 100644
--- a/drivers/media/platform/vicodec/vicodec-core.c
+++ b/drivers/media/platform/vicodec/vicodec-core.c
@@ -895,8 +895,12 @@ static int vidioc_try_fmt_vid_cap(struct file *file, void 
*priv,
                if (multiplanar)
                        return -EINVAL;
                pix = &f->fmt.pix;
-               pix->pixelformat = ctx->is_enc ? V4L2_PIX_FMT_FWHT :
-                                  find_fmt(f->fmt.pix.pixelformat)->id;
+               if (!ctx->is_enc)
+                       pix->pixelformat = find_fmt(f->fmt.pix.pixelformat)->id;
+               else if (ctx->is_stateless)
+                       pix->pixelformat = V4L2_PIX_FMT_FWHT_STATELESS;
+               else
+                       pix->pixelformat = V4L2_PIX_FMT_FWHT;
                pix->colorspace = ctx->state.colorspace;
                pix->xfer_func = ctx->state.xfer_func;
                pix->ycbcr_enc = ctx->state.ycbcr_enc;
@@ -906,8 +910,12 @@ static int vidioc_try_fmt_vid_cap(struct file *file, void 
*priv,
                if (!multiplanar)
                        return -EINVAL;
                pix_mp = &f->fmt.pix_mp;
-               pix_mp->pixelformat = ctx->is_enc ? V4L2_PIX_FMT_FWHT :
-                                     find_fmt(pix_mp->pixelformat)->id;
+               if (!ctx->is_enc)
+                       pix_mp->pixelformat = 
find_fmt(f->fmt.pix_mp.pixelformat)->id;
+               else if (ctx->is_stateless)
+                       pix_mp->pixelformat = V4L2_PIX_FMT_FWHT_STATELESS;
+               else
+                       pix_mp->pixelformat = V4L2_PIX_FMT_FWHT;
                pix_mp->colorspace = ctx->state.colorspace;
                pix_mp->xfer_func = ctx->state.xfer_func;
                pix_mp->ycbcr_enc = ctx->state.ycbcr_enc;
@@ -1631,22 +1639,22 @@ static int vicodec_start_streaming(struct vb2_queue *q,
        state->stride = q_data->coded_width *
                                info->bytesperline_mult;
 
-       if (ctx->is_stateless) {
+       if (!ctx->is_enc && ctx->is_stateless) {
                state->ref_stride = state->stride;
                return 0;
        }
        state->ref_stride = q_data->coded_width * info->luma_alpha_step;
 
        state->ref_frame.buf = kvmalloc(total_planes_size, GFP_KERNEL);
-       state->ref_frame.luma = state->ref_frame.buf;
        new_comp_frame = kvmalloc(ctx->comp_max_size, GFP_KERNEL);
 
-       if (!state->ref_frame.luma || !new_comp_frame) {
-               kvfree(state->ref_frame.luma);
+       if (!state->ref_frame.buf || !new_comp_frame) {
+               kvfree(state->ref_frame.buf);
                kvfree(new_comp_frame);
                vicodec_return_bufs(q, VB2_BUF_STATE_QUEUED);
                return -ENOMEM;
        }
+       state->ref_frame.luma = state->ref_frame.buf;
        /*
         * if state->compressed_frame was already allocated then
         * it contain data of the first frame of the new resolution
@@ -1948,9 +1956,10 @@ static int vicodec_open(struct file *file)
                        comp_size + sizeof(struct fwht_cframe_hdr);
        ctx->q_data[V4L2_M2M_DST] = ctx->q_data[V4L2_M2M_SRC];
        if (ctx->is_enc) {
-               ctx->q_data[V4L2_M2M_DST].info = &pixfmt_fwht;
-               ctx->q_data[V4L2_M2M_DST].sizeimage =
-                       comp_size + sizeof(struct fwht_cframe_hdr);
+               ctx->q_data[V4L2_M2M_DST].info = ctx->is_stateless ?
+                       &pixfmt_stateless_fwht : &pixfmt_fwht;
+               ctx->q_data[V4L2_M2M_DST].sizeimage = comp_size +
+                       sizeof(struct fwht_cframe_hdr);
        } else {
                ctx->q_data[V4L2_M2M_DST].info = info;
                ctx->q_data[V4L2_M2M_DST].sizeimage = raw_size;
-- 
2.17.1