[PATCH rdma-next 0/2] Real time/free running timestamp support

From: Leon Romanovsky <leonro@nvidia.com>

Add an extra timestamp format for mlx5_ib device.

Thanks

Aharon Landau (2):
  net/mlx5: Add new timestamp mode bits
  RDMA/mlx5: Fail QP creation if the device can not support the CQE TS

 drivers/infiniband/hw/mlx5/qp.c | 104 +++++++++++++++++++++++++++++---
 include/linux/mlx5/mlx5_ifc.h   |  54 +++++++++++++++--
 2 files changed, 145 insertions(+), 13 deletions(-)

--
2.29.2

From: Aharon Landau <redacted>

These fields declare which timestamp mode is supported by the device
per RQ/SQ/QP.

In addiition add the ts_format field to the select the mode for
RQ/SQ/QP.

Signed-off-by: Aharon Landau <redacted>
Signed-off-by: Maor Gottlieb <redacted>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
---
 include/linux/mlx5/mlx5_ifc.h | 54 +++++++++++++++++++++++++++++++----
 1 file changed, 49 insertions(+), 5 deletions(-)

diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h
index b96f99f1198e..77051bd5c1cf 100644
--- a/include/linux/mlx5/mlx5_ifc.h
+++ b/include/linux/mlx5/mlx5_ifc.h

@@ -932,11 +932,18 @@ struct mlx5_ifc_per_protocol_networking_offload_caps_bits {
 	u8         reserved_at_200[0x600];
 };

+enum {
+	MLX5_QP_TIMESTAMP_FORMAT_CAP_FREE_RUNNING               = 0x0,
+	MLX5_QP_TIMESTAMP_FORMAT_CAP_REAL_TIME                  = 0x1,
+	MLX5_QP_TIMESTAMP_FORMAT_CAP_FREE_RUNNING_AND_REAL_TIME = 0x2,
+};
+
 struct mlx5_ifc_roce_cap_bits {
 	u8         roce_apm[0x1];
 	u8         reserved_at_1[0x3];
 	u8         sw_r_roce_src_udp_port[0x1];
-	u8         reserved_at_5[0x1b];
+	u8         reserved_at_5[0x19];
+	u8	   qp_ts_format[0x2];

 	u8         reserved_at_20[0x60];

@@ -1258,6 +1265,18 @@ enum {
 	MLX5_STEERING_FORMAT_CONNECTX_6DX = 1,
 };

+enum {
+	MLX5_SQ_TIMESTAMP_FORMAT_CAP_FREE_RUNNING               = 0x0,
+	MLX5_SQ_TIMESTAMP_FORMAT_CAP_REAL_TIME                  = 0x1,
+	MLX5_SQ_TIMESTAMP_FORMAT_CAP_FREE_RUNNING_AND_REAL_TIME = 0x2,
+};
+
+enum {
+	MLX5_RQ_TIMESTAMP_FORMAT_CAP_FREE_RUNNING               = 0x0,
+	MLX5_RQ_TIMESTAMP_FORMAT_CAP_REAL_TIME                  = 0x1,
+	MLX5_RQ_TIMESTAMP_FORMAT_CAP_FREE_RUNNING_AND_REAL_TIME = 0x2,
+};
+
 struct mlx5_ifc_cmd_hca_cap_bits {
 	u8         reserved_at_0[0x1f];
 	u8         vhca_resource_manager[0x1];

@@ -1569,7 +1588,8 @@ struct mlx5_ifc_cmd_hca_cap_bits {

 	u8         general_obj_types[0x40];

-	u8         reserved_at_440[0x4];
+	u8         sq_ts_format[0x2];
+	u8         rq_ts_format[0x2];
 	u8         steering_format_version[0x4];
 	u8         create_qp_start_hint[0x18];

@@ -2873,6 +2893,12 @@ enum {
 	MLX5_QPC_CS_RES_UP_TO_64B  = 0x2,
 };

+enum {
+	MLX5_QPC_TIMESTAMP_FORMAT_FREE_RUNNING = 0x0,
+	MLX5_QPC_TIMESTAMP_FORMAT_DEFAULT      = 0x1,
+	MLX5_QPC_TIMESTAMP_FORMAT_REAL_TIME    = 0x2,
+};
+
 struct mlx5_ifc_qpc_bits {
 	u8         state[0x4];
 	u8         lag_tx_port_affinity[0x4];

@@ -2901,7 +2927,9 @@ struct mlx5_ifc_qpc_bits {
 	u8         log_rq_stride[0x3];
 	u8         no_sq[0x1];
 	u8         log_sq_size[0x4];
-	u8         reserved_at_55[0x6];
+	u8         reserved_at_55[0x3];
+	u8	   ts_format[0x2];
+	u8         reserved_at_5a[0x1];
 	u8         rlky[0x1];
 	u8         ulp_stateless_offload_mode[0x4];

@@ -3317,6 +3345,12 @@ enum {
 	MLX5_SQC_STATE_ERR  = 0x3,
 };

+enum {
+	MLX5_SQC_TIMESTAMP_FORMAT_FREE_RUNNING = 0x0,
+	MLX5_SQC_TIMESTAMP_FORMAT_DEFAULT      = 0x1,
+	MLX5_SQC_TIMESTAMP_FORMAT_REAL_TIME    = 0x2,
+};
+
 struct mlx5_ifc_sqc_bits {
 	u8         rlky[0x1];
 	u8         cd_master[0x1];

@@ -3328,7 +3362,9 @@ struct mlx5_ifc_sqc_bits {
 	u8         reg_umr[0x1];
 	u8         allow_swp[0x1];
 	u8         hairpin[0x1];
-	u8         reserved_at_f[0x11];
+	u8         reserved_at_f[0xb];
+	u8	   ts_format[0x2];
+	u8	   reserved_at_1c[0x4];

 	u8         reserved_at_20[0x8];
 	u8         user_index[0x18];

@@ -3419,6 +3455,12 @@ enum {
 	MLX5_RQC_STATE_ERR  = 0x3,
 };

+enum {
+	MLX5_RQC_TIMESTAMP_FORMAT_FREE_RUNNING = 0x0,
+	MLX5_RQC_TIMESTAMP_FORMAT_DEFAULT      = 0x1,
+	MLX5_RQC_TIMESTAMP_FORMAT_REAL_TIME    = 0x2,
+};
+
 struct mlx5_ifc_rqc_bits {
 	u8         rlky[0x1];
 	u8	   delay_drop_en[0x1];

@@ -3429,7 +3471,9 @@ struct mlx5_ifc_rqc_bits {
 	u8         reserved_at_c[0x1];
 	u8         flush_in_error_en[0x1];
 	u8         hairpin[0x1];
-	u8         reserved_at_f[0x11];
+	u8         reserved_at_f[0xb];
+	u8	   ts_format[0x2];
+	u8	   reserved_at_1c[0x4];

 	u8         reserved_at_20[0x8];
 	u8         user_index[0x18];
--

2.29.2

On Tue,  9 Feb 2021 15:11:06 +0200 Leon Romanovsky wrote:

From: Aharon Landau <redacted>

These fields declare which timestamp mode is supported by the device
per RQ/SQ/QP.

In addiition add the ts_format field to the select the mode for
RQ/SQ/QP.

Signed-off-by: Aharon Landau <redacted>
Signed-off-by: Maor Gottlieb <redacted>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>

We only got patch 1 which explains very little.

You also need to CC Richard.

On Tue, Feb 09, 2021 at 10:28:25AM -0800, Jakub Kicinski wrote:

On Tue,  9 Feb 2021 15:11:06 +0200 Leon Romanovsky wrote:

quoted

From: Aharon Landau <redacted>

These fields declare which timestamp mode is supported by the device
per RQ/SQ/QP.

In addiition add the ts_format field to the select the mode for
RQ/SQ/QP.

Signed-off-by: Aharon Landau <redacted>
Signed-off-by: Maor Gottlieb <redacted>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>

We only got patch 1 which explains very little.

I will change my scripts to ensure that all people will be always CCed
on whole series.

https://lore.kernel.org/linux-rdma/20210209131107.698833-3-leon@kernel.org

You also need to CC Richard.

We are not talking about PTP, but about specific to RDMA timestamp mechanism
which is added to the CQE (completion queue entry) per-user request when
he/she creates CQ (completion queue). User has an option to choose the format
of it for every QP/RQ/SQ.

https://github.com/linux-rdma/rdma-core/blob/master/libibverbs/man/ibv_create_cq_ex.3#L44

Thanks

On Tue, 9 Feb 2021 21:14:24 +0200 Leon Romanovsky wrote:

On Tue, Feb 09, 2021 at 10:28:25AM -0800, Jakub Kicinski wrote:

quoted

On Tue,  9 Feb 2021 15:11:06 +0200 Leon Romanovsky wrote:  
You also need to CC Richard.  

We are not talking about PTP, but about specific to RDMA timestamp mechanism
which is added to the CQE (completion queue entry) per-user request when
he/she creates CQ (completion queue). User has an option to choose the format
of it for every QP/RQ/SQ.

I see. Perhaps Richard won't be interested then but best to give him 
a chance.

Not directly related to series at hand but how is the clock synchronized
between system and device for the real time option?

On Tue, Feb 09, 2021 at 11:52:54AM -0800, Jakub Kicinski wrote:

On Tue, 9 Feb 2021 21:14:24 +0200 Leon Romanovsky wrote:

quoted

On Tue, Feb 09, 2021 at 10:28:25AM -0800, Jakub Kicinski wrote:

quoted

On Tue,  9 Feb 2021 15:11:06 +0200 Leon Romanovsky wrote:
You also need to CC Richard.

We are not talking about PTP, but about specific to RDMA timestamp mechanism
which is added to the CQE (completion queue entry) per-user request when
he/she creates CQ (completion queue). User has an option to choose the format
of it for every QP/RQ/SQ.

I see. Perhaps Richard won't be interested then but best to give him
a chance.

Not directly related to series at hand but how is the clock synchronized
between system and device for the real time option?

When device works in real time mode, driver can skip cycles to ns translation
it does today in order to provide recv/sent SKB NS TS to the stack. Real time
mode does not require any changes above driver level and the synchronization
to system clock will remain as is today. The mlx5e soon to support this mode.

This series is needed to keep RDMA compatibility and fail QP creation for wrong mode.

Thanks

On Tue, Feb 09, 2021 at 03:11:05PM +0200, Leon Romanovsky wrote:

From: Leon Romanovsky <leonro@nvidia.com>

Add an extra timestamp format for mlx5_ib device.

Thanks

Aharon Landau (2):
  net/mlx5: Add new timestamp mode bits
  RDMA/mlx5: Fail QP creation if the device can not support the CQE TS

 drivers/infiniband/hw/mlx5/qp.c | 104 +++++++++++++++++++++++++++++---
 include/linux/mlx5/mlx5_ifc.h   |  54 +++++++++++++++--
 2 files changed, 145 insertions(+), 13 deletions(-)

Since this is a rdma series, and we are at the end of the cycle, I
took the IFC file directly to the rdma tree instead of through the
shared branch.

Applied to for-next, thanks

Jason

On Fri, 2021-02-12 at 14:10 -0400, Jason Gunthorpe wrote:

On Tue, Feb 09, 2021 at 03:11:05PM +0200, Leon Romanovsky wrote:

quoted

From: Leon Romanovsky <leonro@nvidia.com>

Add an extra timestamp format for mlx5_ib device.

Thanks

Aharon Landau (2):
  net/mlx5: Add new timestamp mode bits
  RDMA/mlx5: Fail QP creation if the device can not support the CQE
TS

 drivers/infiniband/hw/mlx5/qp.c | 104
+++++++++++++++++++++++++++++---
 include/linux/mlx5/mlx5_ifc.h   |  54 +++++++++++++++--
 2 files changed, 145 insertions(+), 13 deletions(-)

Since this is a rdma series, and we are at the end of the cycle, I
took the IFC file directly to the rdma tree instead of through the
shared branch.

Applied to for-next, thanks

mmm, i was planing to resubmit this patch with the netdev real time
support series, since the uplink representor is getting delayed, I
thought I could submit the real time stuff today. can you wait on the
ifc patch, i will re-send it today if you will, but it must go through
the shared branch

Thanks.

On Fri, Feb 12, 2021 at 01:09:20PM -0800, Saeed Mahameed wrote:

On Fri, 2021-02-12 at 14:10 -0400, Jason Gunthorpe wrote:

quoted

On Tue, Feb 09, 2021 at 03:11:05PM +0200, Leon Romanovsky wrote:

quoted

From: Leon Romanovsky <leonro@nvidia.com>

Add an extra timestamp format for mlx5_ib device.

Thanks

Aharon Landau (2):
  net/mlx5: Add new timestamp mode bits
  RDMA/mlx5: Fail QP creation if the device can not support the CQE
TS

 drivers/infiniband/hw/mlx5/qp.c | 104
+++++++++++++++++++++++++++++---
 include/linux/mlx5/mlx5_ifc.h   |  54 +++++++++++++++--
 2 files changed, 145 insertions(+), 13 deletions(-)

Since this is a rdma series, and we are at the end of the cycle, I
took the IFC file directly to the rdma tree instead of through the
shared branch.

Applied to for-next, thanks

mmm, i was planing to resubmit this patch with the netdev real time
support series, since the uplink representor is getting delayed, I
thought I could submit the real time stuff today. can you wait on the
ifc patch, i will re-send it today if you will, but it must go through
the shared branch

Friday of rc7 is a bit late to be sending new patches for the first
time, isn't it??

But sure, if you update the shared branch right now I'll fix up rdma.git

Jason

On Fri, 2021-02-12 at 17:14 -0400, Jason Gunthorpe wrote:

On Fri, Feb 12, 2021 at 01:09:20PM -0800, Saeed Mahameed wrote:

quoted

On Fri, 2021-02-12 at 14:10 -0400, Jason Gunthorpe wrote:

quoted

On Tue, Feb 09, 2021 at 03:11:05PM +0200, Leon Romanovsky wrote:

quoted

From: Leon Romanovsky <leonro@nvidia.com>

Add an extra timestamp format for mlx5_ib device.

Thanks

Aharon Landau (2):
  net/mlx5: Add new timestamp mode bits
  RDMA/mlx5: Fail QP creation if the device can not support the
CQE
TS

 drivers/infiniband/hw/mlx5/qp.c | 104
+++++++++++++++++++++++++++++---
 include/linux/mlx5/mlx5_ifc.h   |  54 +++++++++++++++--
 2 files changed, 145 insertions(+), 13 deletions(-)

Since this is a rdma series, and we are at the end of the cycle,
I
took the IFC file directly to the rdma tree instead of through
the
shared branch.

Applied to for-next, thanks

mmm, i was planing to resubmit this patch with the netdev real time
support series, since the uplink representor is getting delayed, I
thought I could submit the real time stuff today. can you wait on
the
ifc patch, i will re-send it today if you will, but it must go
through
the shared branch

Friday of rc7 is a bit late to be sending new patches for the first
time, isn't it??

I know, uplink representor last minute mess !

But sure, if you update the shared branch right now I'll fix up
rdma.git

I can't put it in the shared brach without review, i will post it to
the netdev/rdma lists for two days at least for review and feedback.

Not critical though, this can wait for next rc1.. if you want to
proceed with what you already did.

Thanks

On Fri, Feb 12, 2021 at 01:19:09PM -0800, Saeed Mahameed wrote:

On Fri, 2021-02-12 at 17:14 -0400, Jason Gunthorpe wrote:

quoted

On Fri, Feb 12, 2021 at 01:09:20PM -0800, Saeed Mahameed wrote:

quoted

On Fri, 2021-02-12 at 14:10 -0400, Jason Gunthorpe wrote:

quoted

On Tue, Feb 09, 2021 at 03:11:05PM +0200, Leon Romanovsky wrote:

quoted

From: Leon Romanovsky <leonro@nvidia.com>

Add an extra timestamp format for mlx5_ib device.

Thanks

Aharon Landau (2):
  net/mlx5: Add new timestamp mode bits
  RDMA/mlx5: Fail QP creation if the device can not support the
CQE
TS

 drivers/infiniband/hw/mlx5/qp.c | 104
+++++++++++++++++++++++++++++---
 include/linux/mlx5/mlx5_ifc.h   |  54 +++++++++++++++--
 2 files changed, 145 insertions(+), 13 deletions(-)

Since this is a rdma series, and we are at the end of the cycle,
I
took the IFC file directly to the rdma tree instead of through
the
shared branch.

Applied to for-next, thanks

mmm, i was planing to resubmit this patch with the netdev real time
support series, since the uplink representor is getting delayed, I
thought I could submit the real time stuff today. can you wait on
the
ifc patch, i will re-send it today if you will, but it must go
through
the shared branch

Friday of rc7 is a bit late to be sending new patches for the first
time, isn't it??

I know, uplink representor last minute mess !

quoted

But sure, if you update the shared branch right now I'll fix up
rdma.git

I can't put it in the shared brach without review, i will post it to
the netdev/rdma lists for two days at least for review and feedback.

Well, I'm not going to take any different patches beyond right now
unless Linus does a rc8??

Just move this one IFC patch to the shared branch, it is obviously OK

Jason

On Fri, Feb 12, 2021 at 05:21:53PM -0400, Jason Gunthorpe wrote:

On Fri, Feb 12, 2021 at 01:19:09PM -0800, Saeed Mahameed wrote:

quoted

On Fri, 2021-02-12 at 17:14 -0400, Jason Gunthorpe wrote:

quoted

On Fri, Feb 12, 2021 at 01:09:20PM -0800, Saeed Mahameed wrote:

quoted

On Fri, 2021-02-12 at 14:10 -0400, Jason Gunthorpe wrote:

quoted

On Tue, Feb 09, 2021 at 03:11:05PM +0200, Leon Romanovsky wrote:

quoted

From: Leon Romanovsky <leonro@nvidia.com>

Add an extra timestamp format for mlx5_ib device.

Thanks

Aharon Landau (2):
  net/mlx5: Add new timestamp mode bits
  RDMA/mlx5: Fail QP creation if the device can not support the
CQE
TS

 drivers/infiniband/hw/mlx5/qp.c | 104
+++++++++++++++++++++++++++++---
 include/linux/mlx5/mlx5_ifc.h   |  54 +++++++++++++++--
 2 files changed, 145 insertions(+), 13 deletions(-)

Since this is a rdma series, and we are at the end of the cycle,
I
took the IFC file directly to the rdma tree instead of through
the
shared branch.

Applied to for-next, thanks

mmm, i was planing to resubmit this patch with the netdev real time
support series, since the uplink representor is getting delayed, I
thought I could submit the real time stuff today. can you wait on
the
ifc patch, i will re-send it today if you will, but it must go
through
the shared branch

Friday of rc7 is a bit late to be sending new patches for the first
time, isn't it??

I know, uplink representor last minute mess !

quoted

But sure, if you update the shared branch right now I'll fix up
rdma.git

I can't put it in the shared brach without review, i will post it to
the netdev/rdma lists for two days at least for review and feedback.

Well, I'm not going to take any different patches beyond right now
unless Linus does a rc8??

Just move this one IFC patch to the shared branch, it is obviously OK

OK, I'm curious to see the end result of all this last minute adventure.

Thanks

Jason

On Sun, Feb 14, 2021 at 08:41:03AM +0200, Leon Romanovsky wrote:

On Fri, Feb 12, 2021 at 05:21:53PM -0400, Jason Gunthorpe wrote:

quoted

On Fri, Feb 12, 2021 at 01:19:09PM -0800, Saeed Mahameed wrote:

quoted

On Fri, 2021-02-12 at 17:14 -0400, Jason Gunthorpe wrote:

quoted

On Fri, Feb 12, 2021 at 01:09:20PM -0800, Saeed Mahameed wrote:

quoted

On Fri, 2021-02-12 at 14:10 -0400, Jason Gunthorpe wrote:

quoted

On Tue, Feb 09, 2021 at 03:11:05PM +0200, Leon Romanovsky wrote:

quoted

From: Leon Romanovsky <leonro@nvidia.com>

Add an extra timestamp format for mlx5_ib device.

Thanks

Aharon Landau (2):
  net/mlx5: Add new timestamp mode bits
  RDMA/mlx5: Fail QP creation if the device can not support the
CQE
TS

 drivers/infiniband/hw/mlx5/qp.c | 104
+++++++++++++++++++++++++++++---
 include/linux/mlx5/mlx5_ifc.h   |  54 +++++++++++++++--
 2 files changed, 145 insertions(+), 13 deletions(-)

Since this is a rdma series, and we are at the end of the cycle,
I
took the IFC file directly to the rdma tree instead of through
the
shared branch.

Applied to for-next, thanks

mmm, i was planing to resubmit this patch with the netdev real time
support series, since the uplink representor is getting delayed, I
thought I could submit the real time stuff today. can you wait on
the
ifc patch, i will re-send it today if you will, but it must go
through
the shared branch

Friday of rc7 is a bit late to be sending new patches for the first
time, isn't it??

I know, uplink representor last minute mess !

quoted

But sure, if you update the shared branch right now I'll fix up
rdma.git

I can't put it in the shared brach without review, i will post it to
the netdev/rdma lists for two days at least for review and feedback.

Well, I'm not going to take any different patches beyond right now
unless Linus does a rc8??

Just move this one IFC patch to the shared branch, it is obviously OK

OK, I'm curious to see the end result of all this last minute adventure.

Jason,

I took first patch to the shared branch.
a6a217dddcd5 net/mlx5: Add new timestamp mode bits

Thanks

Thanks

quoted

Jason