Re: [PATCH 1/6] media: v4l2: Add audio capture and output support

Hi Sakari

On Fri, Jun 30, 2023 at 6:05 PM Sakari Ailus [off-list ref] wrote:

Hi Shengjiu,

On Thu, Jun 29, 2023 at 09:37:48AM +0800, Shengjiu Wang wrote:

quoted

Audio signal processing has the requirement for memory to
memory similar as Video.

This patch is to add this support in v4l2 framework, defined
new buffer type V4L2_BUF_TYPE_AUDIO_CAPTURE and
V4L2_BUF_TYPE_AUDIO_OUTPUT, defined new format v4l2_audio_format
for audio case usage.

Why are you proposing to add this to V4L2 framework instead of doing this
within ALSA?

Also cc Hans and Jacopo.

There is no such memory to memory interface defined in ALSA.  Seems
ALSA is not designed for M2M cases.

V4L2 is designed for video, radio, image, sdr, meta...,   so I think audio
can be
naturally added to the support scope.

Thanks.

Best regards
Shengjiu Wang

quoted

The created audio device is named "/dev/audioX".

Signed-off-by: Shengjiu Wang <redacted>
---
 .../media/common/videobuf2/videobuf2-v4l2.c   |  4 ++
 drivers/media/v4l2-core/v4l2-dev.c            | 17 ++++++
 drivers/media/v4l2-core/v4l2-ioctl.c          | 52 +++++++++++++++++++
 include/media/v4l2-dev.h                      |  2 +
 include/media/v4l2-ioctl.h                    | 34 ++++++++++++
 include/uapi/linux/videodev2.h                | 19 +++++++
 6 files changed, 128 insertions(+)

diff --git a/drivers/media/common/videobuf2/videobuf2-v4l2.c

b/drivers/media/common/videobuf2/videobuf2-v4l2.c

quoted

index c7a54d82a55e..12f2be2773a2 100644

--- a/drivers/media/common/videobuf2/videobuf2-v4l2.c
+++ b/drivers/media/common/videobuf2/videobuf2-v4l2.c

@@ -785,6 +785,10 @@ int vb2_create_bufs(struct vb2_queue *q, struct

v4l2_create_buffers *create)

quoted

      case V4L2_BUF_TYPE_META_OUTPUT:
              requested_sizes[0] = f->fmt.meta.buffersize;
              break;
+     case V4L2_BUF_TYPE_AUDIO_CAPTURE:
+     case V4L2_BUF_TYPE_AUDIO_OUTPUT:
+             requested_sizes[0] = f->fmt.audio.buffersize;
+             break;
      default:
              return -EINVAL;
      }

diff --git a/drivers/media/v4l2-core/v4l2-dev.c

b/drivers/media/v4l2-core/v4l2-dev.c

quoted

index f81279492682..67484f4c6eaf 100644

--- a/drivers/media/v4l2-core/v4l2-dev.c
+++ b/drivers/media/v4l2-core/v4l2-dev.c

@@ -553,6 +553,7 @@ static void determine_valid_ioctls(struct

video_device *vdev)

quoted

      bool is_tch = vdev->vfl_type == VFL_TYPE_TOUCH;
      bool is_meta = vdev->vfl_type == VFL_TYPE_VIDEO &&
                     (vdev->device_caps & meta_caps);
+     bool is_audio = vdev->vfl_type == VFL_TYPE_AUDIO;
      bool is_rx = vdev->vfl_dir != VFL_DIR_TX;
      bool is_tx = vdev->vfl_dir != VFL_DIR_RX;
      bool is_io_mc = vdev->device_caps & V4L2_CAP_IO_MC;

@@ -664,6 +665,19 @@ static void determine_valid_ioctls(struct

video_device *vdev)

quoted

              SET_VALID_IOCTL(ops, VIDIOC_S_FMT, vidioc_s_fmt_meta_out);
              SET_VALID_IOCTL(ops, VIDIOC_TRY_FMT,

vidioc_try_fmt_meta_out);

quoted

      }
+     if (is_audio && is_rx) {
+             /* audio capture specific ioctls */
+             SET_VALID_IOCTL(ops, VIDIOC_ENUM_FMT,

vidioc_enum_fmt_audio_cap);

quoted

+             SET_VALID_IOCTL(ops, VIDIOC_G_FMT, vidioc_g_fmt_audio_cap);
+             SET_VALID_IOCTL(ops, VIDIOC_S_FMT, vidioc_s_fmt_audio_cap);
+             SET_VALID_IOCTL(ops, VIDIOC_TRY_FMT,

vidioc_try_fmt_audio_cap);

quoted

+     } else if (is_audio && is_tx) {
+             /* audio output specific ioctls */
+             SET_VALID_IOCTL(ops, VIDIOC_ENUM_FMT,

vidioc_enum_fmt_audio_out);

quoted

+             SET_VALID_IOCTL(ops, VIDIOC_G_FMT, vidioc_g_fmt_audio_out);
+             SET_VALID_IOCTL(ops, VIDIOC_S_FMT, vidioc_s_fmt_audio_out);
+             SET_VALID_IOCTL(ops, VIDIOC_TRY_FMT,

vidioc_try_fmt_audio_out);

quoted

+     }
      if (is_vbi) {
              /* vbi specific ioctls */
              if ((is_rx && (ops->vidioc_g_fmt_vbi_cap ||

@@ -927,6 +941,9 @@ int __video_register_device(struct video_device

*vdev,

quoted

      case VFL_TYPE_TOUCH:
              name_base = "v4l-touch";
              break;
+     case VFL_TYPE_AUDIO:
+             name_base = "audio";
+             break;
      default:
              pr_err("%s called with unknown type: %d\n",
                     __func__, type);

diff --git a/drivers/media/v4l2-core/v4l2-ioctl.c

b/drivers/media/v4l2-core/v4l2-ioctl.c

quoted

index a858acea6547..26bc4b0d8ef0 100644

--- a/drivers/media/v4l2-core/v4l2-ioctl.c
+++ b/drivers/media/v4l2-core/v4l2-ioctl.c

@@ -188,6 +188,8 @@ const char *v4l2_type_names[] = {
      [V4L2_BUF_TYPE_SDR_OUTPUT]         = "sdr-out",
      [V4L2_BUF_TYPE_META_CAPTURE]       = "meta-cap",
      [V4L2_BUF_TYPE_META_OUTPUT]        = "meta-out",
+     [V4L2_BUF_TYPE_AUDIO_CAPTURE]      = "audio-cap",
+     [V4L2_BUF_TYPE_AUDIO_OUTPUT]       = "audio-out",
 };
 EXPORT_SYMBOL(v4l2_type_names);

@@ -276,6 +278,7 @@ static void v4l_print_format(const void *arg, bool

write_only)

quoted

      const struct v4l2_sliced_vbi_format *sliced;
      const struct v4l2_window *win;
      const struct v4l2_meta_format *meta;
+     const struct v4l2_audio_format *audio;
      u32 pixelformat;
      u32 planes;
      unsigned i;

@@ -346,6 +349,12 @@ static void v4l_print_format(const void *arg, bool

write_only)

quoted

              pr_cont(", dataformat=%p4cc, buffersize=%u\n",
                      &pixelformat, meta->buffersize);
              break;
+     case V4L2_BUF_TYPE_AUDIO_CAPTURE:
+     case V4L2_BUF_TYPE_AUDIO_OUTPUT:
+             audio = &p->fmt.audio;
+             pr_cont(", rate=%u, format=%u, channels=%u,

buffersize=%u\n",

quoted

+                     audio->rate, audio->format, audio->channels,

audio->buffersize);

quoted

+             break;
      }
 }

@@ -927,6 +936,7 @@ static int check_fmt(struct file *file, enum

v4l2_buf_type type)

quoted

      bool is_tch = vfd->vfl_type == VFL_TYPE_TOUCH;
      bool is_meta = vfd->vfl_type == VFL_TYPE_VIDEO &&
                     (vfd->device_caps & meta_caps);
+     bool is_audio = vfd->vfl_type == VFL_TYPE_AUDIO;
      bool is_rx = vfd->vfl_dir != VFL_DIR_TX;
      bool is_tx = vfd->vfl_dir != VFL_DIR_RX;

@@ -992,6 +1002,14 @@ static int check_fmt(struct file *file, enum

v4l2_buf_type type)

quoted

              if (is_meta && is_tx && ops->vidioc_g_fmt_meta_out)
                      return 0;
              break;
+     case V4L2_BUF_TYPE_AUDIO_CAPTURE:
+             if (is_audio && is_rx && ops->vidioc_g_fmt_audio_cap)
+                     return 0;
+             break;
+     case V4L2_BUF_TYPE_AUDIO_OUTPUT:
+             if (is_audio && is_tx && ops->vidioc_g_fmt_audio_out)
+                     return 0;
+             break;
      default:
              break;
      }

@@ -1592,6 +1610,16 @@ static int v4l_enum_fmt(const struct

v4l2_ioctl_ops *ops,

quoted

                      break;
              ret = ops->vidioc_enum_fmt_meta_out(file, fh, arg);
              break;
+     case V4L2_BUF_TYPE_AUDIO_CAPTURE:
+             if (unlikely(!ops->vidioc_enum_fmt_audio_cap))
+                     break;
+             ret = ops->vidioc_enum_fmt_audio_cap(file, fh, arg);
+             break;
+     case V4L2_BUF_TYPE_AUDIO_OUTPUT:
+             if (unlikely(!ops->vidioc_enum_fmt_audio_out))
+                     break;
+             ret = ops->vidioc_enum_fmt_audio_out(file, fh, arg);
+             break;
      }
      if (ret == 0)
              v4l_fill_fmtdesc(p);

@@ -1668,6 +1696,10 @@ static int v4l_g_fmt(const struct v4l2_ioctl_ops

*ops,

quoted

              return ops->vidioc_g_fmt_meta_cap(file, fh, arg);
      case V4L2_BUF_TYPE_META_OUTPUT:
              return ops->vidioc_g_fmt_meta_out(file, fh, arg);
+     case V4L2_BUF_TYPE_AUDIO_CAPTURE:
+             return ops->vidioc_g_fmt_audio_cap(file, fh, arg);
+     case V4L2_BUF_TYPE_AUDIO_OUTPUT:
+             return ops->vidioc_g_fmt_audio_out(file, fh, arg);
      }
      return -EINVAL;
 }

@@ -1779,6 +1811,16 @@ static int v4l_s_fmt(const struct v4l2_ioctl_ops

*ops,

quoted

                      break;
              memset_after(p, 0, fmt.meta);
              return ops->vidioc_s_fmt_meta_out(file, fh, arg);
+     case V4L2_BUF_TYPE_AUDIO_CAPTURE:
+             if (unlikely(!ops->vidioc_s_fmt_audio_cap))
+                     break;
+             memset_after(p, 0, fmt.audio);
+             return ops->vidioc_s_fmt_audio_cap(file, fh, arg);
+     case V4L2_BUF_TYPE_AUDIO_OUTPUT:
+             if (unlikely(!ops->vidioc_s_fmt_audio_out))
+                     break;
+             memset_after(p, 0, fmt.audio);
+             return ops->vidioc_s_fmt_audio_out(file, fh, arg);
      }
      return -EINVAL;
 }

@@ -1887,6 +1929,16 @@ static int v4l_try_fmt(const struct

v4l2_ioctl_ops *ops,

quoted

                      break;
              memset_after(p, 0, fmt.meta);
              return ops->vidioc_try_fmt_meta_out(file, fh, arg);
+     case V4L2_BUF_TYPE_AUDIO_CAPTURE:
+             if (unlikely(!ops->vidioc_try_fmt_audio_cap))
+                     break;
+             memset_after(p, 0, fmt.audio);
+             return ops->vidioc_try_fmt_audio_cap(file, fh, arg);
+     case V4L2_BUF_TYPE_AUDIO_OUTPUT:
+             if (unlikely(!ops->vidioc_try_fmt_audio_out))
+                     break;
+             memset_after(p, 0, fmt.audio);
+             return ops->vidioc_try_fmt_audio_out(file, fh, arg);
      }
      return -EINVAL;
 }

diff --git a/include/media/v4l2-dev.h b/include/media/v4l2-dev.h
index e0a13505f88d..0924e6d1dab1 100644
--- a/include/media/v4l2-dev.h
+++ b/include/media/v4l2-dev.h

@@ -30,6 +30,7 @@
  * @VFL_TYPE_SUBDEV: for V4L2 subdevices
  * @VFL_TYPE_SDR:    for Software Defined Radio tuners
  * @VFL_TYPE_TOUCH:  for touch sensors
+ * @VFL_TYPE_AUDIO:  for audio input/output devices
  * @VFL_TYPE_MAX:    number of VFL types, must always be last in the

enum

quoted

  */
 enum vfl_devnode_type {

@@ -39,6 +40,7 @@ enum vfl_devnode_type {
      VFL_TYPE_SUBDEV,
      VFL_TYPE_SDR,
      VFL_TYPE_TOUCH,
+     VFL_TYPE_AUDIO,
      VFL_TYPE_MAX /* Shall be the last one */
 };

diff --git a/include/media/v4l2-ioctl.h b/include/media/v4l2-ioctl.h
index edb733f21604..f840cf740ce1 100644
--- a/include/media/v4l2-ioctl.h
+++ b/include/media/v4l2-ioctl.h

@@ -45,6 +45,12 @@ struct v4l2_fh;
  * @vidioc_enum_fmt_meta_out: pointer to the function that implements
  *   :ref:`VIDIOC_ENUM_FMT <vidioc_enum_fmt>` ioctl logic
  *   for metadata output
+ * @vidioc_enum_fmt_audio_cap: pointer to the function that implements
+ *   :ref:`VIDIOC_ENUM_FMT <vidioc_enum_fmt>` ioctl logic
+ *   for audio capture
+ * @vidioc_enum_fmt_audio_out: pointer to the function that implements
+ *   :ref:`VIDIOC_ENUM_FMT <vidioc_enum_fmt>` ioctl logic
+ *   for audio output
  * @vidioc_g_fmt_vid_cap: pointer to the function that implements
  *   :ref:`VIDIOC_G_FMT <vidioc_g_fmt>` ioctl logic for video capture
  *   in single plane mode

@@ -79,6 +85,10 @@ struct v4l2_fh;
  *   :ref:`VIDIOC_G_FMT <vidioc_g_fmt>` ioctl logic for metadata capture
  * @vidioc_g_fmt_meta_out: pointer to the function that implements
  *   :ref:`VIDIOC_G_FMT <vidioc_g_fmt>` ioctl logic for metadata output
+ * @vidioc_g_fmt_audio_cap: pointer to the function that implements
+ *   :ref:`VIDIOC_G_FMT <vidioc_g_fmt>` ioctl logic for audio capture
+ * @vidioc_g_fmt_audio_out: pointer to the function that implements
+ *   :ref:`VIDIOC_G_FMT <vidioc_g_fmt>` ioctl logic for audio output
  * @vidioc_s_fmt_vid_cap: pointer to the function that implements
  *   :ref:`VIDIOC_S_FMT <vidioc_g_fmt>` ioctl logic for video capture
  *   in single plane mode

@@ -113,6 +123,10 @@ struct v4l2_fh;
  *   :ref:`VIDIOC_S_FMT <vidioc_g_fmt>` ioctl logic for metadata capture
  * @vidioc_s_fmt_meta_out: pointer to the function that implements
  *   :ref:`VIDIOC_S_FMT <vidioc_g_fmt>` ioctl logic for metadata output
+ * @vidioc_s_fmt_audio_cap: pointer to the function that implements
+ *   :ref:`VIDIOC_S_FMT <vidioc_g_fmt>` ioctl logic for audio capture
+ * @vidioc_s_fmt_audio_out: pointer to the function that implements
+ *   :ref:`VIDIOC_S_FMT <vidioc_g_fmt>` ioctl logic for audio output
  * @vidioc_try_fmt_vid_cap: pointer to the function that implements
  *   :ref:`VIDIOC_TRY_FMT <vidioc_g_fmt>` ioctl logic for video capture
  *   in single plane mode

@@ -149,6 +163,10 @@ struct v4l2_fh;
  *   :ref:`VIDIOC_TRY_FMT <vidioc_g_fmt>` ioctl logic for metadata

capture

quoted

  * @vidioc_try_fmt_meta_out: pointer to the function that implements
  *   :ref:`VIDIOC_TRY_FMT <vidioc_g_fmt>` ioctl logic for metadata

output

quoted

+ * @vidioc_try_fmt_audio_cap: pointer to the function that implements
+ *   :ref:`VIDIOC_TRY_FMT <vidioc_g_fmt>` ioctl logic for audio capture
+ * @vidioc_try_fmt_audio_out: pointer to the function that implements
+ *   :ref:`VIDIOC_TRY_FMT <vidioc_g_fmt>` ioctl logic for audio output
  * @vidioc_reqbufs: pointer to the function that implements
  *   :ref:`VIDIOC_REQBUFS <vidioc_reqbufs>` ioctl
  * @vidioc_querybuf: pointer to the function that implements

@@ -315,6 +333,10 @@ struct v4l2_ioctl_ops {
                                      struct v4l2_fmtdesc *f);
      int (*vidioc_enum_fmt_meta_out)(struct file *file, void *fh,
                                      struct v4l2_fmtdesc *f);
+     int (*vidioc_enum_fmt_audio_cap)(struct file *file, void *fh,
+                                      struct v4l2_fmtdesc *f);
+     int (*vidioc_enum_fmt_audio_out)(struct file *file, void *fh,
+                                      struct v4l2_fmtdesc *f);

      /* VIDIOC_G_FMT handlers */
      int (*vidioc_g_fmt_vid_cap)(struct file *file, void *fh,

@@ -345,6 +367,10 @@ struct v4l2_ioctl_ops {
                                   struct v4l2_format *f);
      int (*vidioc_g_fmt_meta_out)(struct file *file, void *fh,
                                   struct v4l2_format *f);
+     int (*vidioc_g_fmt_audio_cap)(struct file *file, void *fh,
+                                   struct v4l2_format *f);
+     int (*vidioc_g_fmt_audio_out)(struct file *file, void *fh,
+                                   struct v4l2_format *f);

      /* VIDIOC_S_FMT handlers */
      int (*vidioc_s_fmt_vid_cap)(struct file *file, void *fh,

@@ -375,6 +401,10 @@ struct v4l2_ioctl_ops {
                                   struct v4l2_format *f);
      int (*vidioc_s_fmt_meta_out)(struct file *file, void *fh,
                                   struct v4l2_format *f);
+     int (*vidioc_s_fmt_audio_cap)(struct file *file, void *fh,
+                                   struct v4l2_format *f);
+     int (*vidioc_s_fmt_audio_out)(struct file *file, void *fh,
+                                   struct v4l2_format *f);

      /* VIDIOC_TRY_FMT handlers */
      int (*vidioc_try_fmt_vid_cap)(struct file *file, void *fh,

@@ -405,6 +435,10 @@ struct v4l2_ioctl_ops {
                                     struct v4l2_format *f);
      int (*vidioc_try_fmt_meta_out)(struct file *file, void *fh,
                                     struct v4l2_format *f);
+     int (*vidioc_try_fmt_audio_cap)(struct file *file, void *fh,
+                                     struct v4l2_format *f);
+     int (*vidioc_try_fmt_audio_out)(struct file *file, void *fh,
+                                     struct v4l2_format *f);

      /* Buffer handlers */
      int (*vidioc_reqbufs)(struct file *file, void *fh,

diff --git a/include/uapi/linux/videodev2.h

b/include/uapi/linux/videodev2.h

quoted

index aee75eb9e686..a7af28f4c8c3 100644

--- a/include/uapi/linux/videodev2.h
+++ b/include/uapi/linux/videodev2.h

@@ -153,6 +153,8 @@ enum v4l2_buf_type {
      V4L2_BUF_TYPE_SDR_OUTPUT           = 12,
      V4L2_BUF_TYPE_META_CAPTURE         = 13,
      V4L2_BUF_TYPE_META_OUTPUT          = 14,
+     V4L2_BUF_TYPE_AUDIO_CAPTURE        = 15,
+     V4L2_BUF_TYPE_AUDIO_OUTPUT         = 16,
      /* Deprecated, do not use */
      V4L2_BUF_TYPE_PRIVATE              = 0x80,
 };

@@ -169,6 +171,7 @@ enum v4l2_buf_type {
       || (type) == V4L2_BUF_TYPE_VBI_OUTPUT                  \
       || (type) == V4L2_BUF_TYPE_SLICED_VBI_OUTPUT           \
       || (type) == V4L2_BUF_TYPE_SDR_OUTPUT                  \
+      || (type) == V4L2_BUF_TYPE_AUDIO_OUTPUT                \
       || (type) == V4L2_BUF_TYPE_META_OUTPUT)

 #define V4L2_TYPE_IS_CAPTURE(type) (!V4L2_TYPE_IS_OUTPUT(type))

@@ -2404,6 +2407,20 @@ struct v4l2_meta_format {
      __u32                           buffersize;
 } __attribute__ ((packed));

+/**
+ * struct v4l2_audio_format - audio data format definition
+ * @rate:            sample rate
+ * @format:          sample format
+ * @channels:                channel numbers
+ * @buffersize:              maximum size in bytes required for data
+ */
+struct v4l2_audio_format {
+     __u32                           rate;
+     __u32                           format;
+     __u32                           channels;
+     __u32                           buffersize;
+} __attribute__ ((packed));
+
 /**
  * struct v4l2_format - stream data format
  * @type:    enum v4l2_buf_type; type of the data stream

@@ -2412,6 +2429,7 @@ struct v4l2_meta_format {
  * @win:     definition of an overlaid image
  * @vbi:     raw VBI capture or output parameters
  * @sliced:  sliced VBI capture or output parameters
+ * @audio:   definition of an audio format
  * @raw_data:        placeholder for future extensions and custom

formats

quoted

  * @fmt:     union of @pix, @pix_mp, @win, @vbi, @sliced, @sdr, @meta
  *           and @raw_data

@@ -2426,6 +2444,7 @@ struct v4l2_format {
              struct v4l2_sliced_vbi_format   sliced;  /*

V4L2_BUF_TYPE_SLICED_VBI_CAPTURE */

quoted

              struct v4l2_sdr_format          sdr;     /*

V4L2_BUF_TYPE_SDR_CAPTURE */

quoted

              struct v4l2_meta_format         meta;    /*

V4L2_BUF_TYPE_META_CAPTURE */

quoted

+             struct v4l2_audio_format        audio;   /*

V4L2_BUF_TYPE_AUDIO_CAPTURE */

quoted

              __u8    raw_data[200];                   /* user-defined */
      } fmt;
 };
--
2.34.1

--
Sakari Ailus