diff --git a/userspace/dpdk/ena/base/ena_com.c b/userspace/dpdk/ena/base/ena_com.c index 24bad198..ed923f57 100644 --- a/userspace/dpdk/ena/base/ena_com.c +++ b/userspace/dpdk/ena/base/ena_com.c @@ -420,11 +420,16 @@ static int ena_com_init_io_cq(struct ena_com_dev *ena_dev, memset(&io_cq->cdesc_addr, 0x0, sizeof(io_cq->cdesc_addr)); - /* Use the basic completion descriptor for Rx */ - io_cq->cdesc_entry_size_in_bytes = - (io_cq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) ? - sizeof(struct ena_eth_io_tx_cdesc) : - sizeof(struct ena_eth_io_rx_cdesc_base); + if (ctx->use_extended_cdesc) + io_cq->cdesc_entry_size_in_bytes = + (io_cq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) ? + sizeof(struct ena_eth_io_tx_cdesc_ext) : + sizeof(struct ena_eth_io_rx_cdesc_ext); + else + io_cq->cdesc_entry_size_in_bytes = + (io_cq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) ? + sizeof(struct ena_eth_io_tx_cdesc) : + sizeof(struct ena_eth_io_rx_cdesc_base); size = io_cq->cdesc_entry_size_in_bytes * io_cq->q_depth; io_cq->bus = ena_dev->bus; @@ -2007,6 +2012,81 @@ int ena_com_phc_get_error_bound(struct ena_com_dev *ena_dev, u32 *error_bound) return ENA_COM_OK; } +bool ena_com_hw_timestamping_supported(struct ena_com_dev *ena_dev) +{ + return ena_com_check_supported_feature_id(ena_dev, + ENA_ADMIN_HW_TIMESTAMP); +} + +int ena_com_get_hw_timestamping_support(struct ena_com_dev *ena_dev, + uint8_t *tx_support, + uint8_t *rx_support) +{ + struct ena_admin_get_feat_resp get_resp; + int ret; + + *tx_support = ENA_ADMIN_HW_TIMESTAMP_TX_SUPPORT_NONE; + *rx_support = ENA_ADMIN_HW_TIMESTAMP_RX_SUPPORT_NONE; + + if (!ena_com_hw_timestamping_supported(ena_dev)) { + ena_trc_dbg(ena_dev, "HW timestamping is not supported\n"); + return ENA_COM_UNSUPPORTED; + } + + ret = ena_com_get_feature(ena_dev, + &get_resp, + ENA_ADMIN_HW_TIMESTAMP, + ENA_ADMIN_HW_TIMESTAMP_FEATURE_VERSION_1); + if (unlikely(ret)) { + ena_trc_err(ena_dev, + "Failed to get HW timestamp configuration, error: %d\n", + ret); + return ret; + } + + *tx_support = get_resp.u.hw_ts.tx; + *rx_support = get_resp.u.hw_ts.rx; + + return 0; +} + +int ena_com_set_hw_timestamping_configuration(struct ena_com_dev *ena_dev, + uint8_t tx_enable, + uint8_t rx_enable) +{ + struct ena_admin_set_feat_resp resp; + struct ena_admin_set_feat_cmd cmd; + int ret; + + if (!ena_com_hw_timestamping_supported(ena_dev)) { + ena_trc_dbg(ena_dev, "HW timestamping is not supported\n"); + return ENA_COM_UNSUPPORTED; + } + + memset(&cmd, 0x0, sizeof(cmd)); + + cmd.aq_common_descriptor.opcode = ENA_ADMIN_SET_FEATURE; + cmd.feat_common.feature_id = ENA_ADMIN_HW_TIMESTAMP; + cmd.u.hw_ts.tx = tx_enable ? ENA_ADMIN_HW_TIMESTAMP_TX_SUPPORT_ALL : + ENA_ADMIN_HW_TIMESTAMP_TX_SUPPORT_NONE; + cmd.u.hw_ts.rx = rx_enable ? ENA_ADMIN_HW_TIMESTAMP_RX_SUPPORT_ALL : + ENA_ADMIN_HW_TIMESTAMP_RX_SUPPORT_NONE; + + ret = ena_com_execute_admin_command(&ena_dev->admin_queue, + (struct ena_admin_aq_entry *)&cmd, + sizeof(cmd), + (struct ena_admin_acq_entry *)&resp, + sizeof(resp)); + if (unlikely(ret)) { + ena_trc_err(ena_dev, + "Failed to set HW timestamping configuration, error: %d\n", + ret); + return ret; + } + + return 0; +} + int ena_com_mmio_reg_read_request_init(struct ena_com_dev *ena_dev) { struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read; diff --git a/userspace/dpdk/ena/base/ena_com.h b/userspace/dpdk/ena/base/ena_com.h index fbb0ea39..5399661b 100644 --- a/userspace/dpdk/ena/base/ena_com.h +++ b/userspace/dpdk/ena/base/ena_com.h @@ -382,6 +382,9 @@ struct ena_com_dev { struct ena_com_mmio_read mmio_read; struct ena_com_phc_info phc; + bool use_extended_tx_cdesc; + bool use_extended_rx_cdesc; + struct ena_rss rss; u32 supported_features; u32 capabilities; @@ -421,6 +424,7 @@ struct ena_com_create_io_ctx { u32 msix_vector; u16 queue_size; u16 qid; + bool use_extended_cdesc; }; typedef void (*ena_aenq_handler)(void *data, @@ -490,6 +494,32 @@ int ena_com_phc_get_timestamp(struct ena_com_dev *ena_dev, u64 *timestamp); */ int ena_com_phc_get_error_bound(struct ena_com_dev *ena_dev, u32 *error_bound); +/* ena_com_hw_timestamping_supported - Check if HW timestamping is supported + * @ena_dev: ENA communication layer struct + * @return - true if supported + */ +bool ena_com_hw_timestamping_supported(struct ena_com_dev *ena_dev); + +/* ena_com_get_hw_timestamping_support - Query HW timestamp TX/RX support + * @ena_dev: ENA communication layer struct + * @tx_support: returned TX support level + * @rx_support: returned RX support level + * @return - 0 on success, negative value on failure + */ +int ena_com_get_hw_timestamping_support(struct ena_com_dev *ena_dev, + uint8_t *tx_support, + uint8_t *rx_support); + +/* ena_com_set_hw_timestamping_configuration - Enable/disable HW timestamps + * @ena_dev: ENA communication layer struct + * @tx_enable: enable TX timestamps + * @rx_enable: enable RX timestamps + * @return - 0 on success, negative value on failure + */ +int ena_com_set_hw_timestamping_configuration(struct ena_com_dev *ena_dev, + uint8_t tx_enable, + uint8_t rx_enable); + /* ena_com_set_mmio_read_mode - Enable/disable the indirect mmio reg read mechanism * @ena_dev: ENA communication layer struct * @readless_supported: readless mode (enable/disable) diff --git a/userspace/dpdk/ena/base/ena_defs/ena_admin_defs.h b/userspace/dpdk/ena/base/ena_defs/ena_admin_defs.h index 8a1bb0bb..bb88dfd7 100644 --- a/userspace/dpdk/ena/base/ena_defs/ena_admin_defs.h +++ b/userspace/dpdk/ena/base/ena_defs/ena_admin_defs.h @@ -67,6 +67,7 @@ enum ena_admin_aq_feature_id { ENA_ADMIN_LINK_CONFIG = 27, ENA_ADMIN_HOST_ATTR_CONFIG = 28, ENA_ADMIN_PHC_CONFIG = 29, + ENA_ADMIN_HW_TIMESTAMP = 31, ENA_ADMIN_FEATURES_OPCODE_NUM = 32, }; @@ -155,6 +156,20 @@ enum ena_admin_phc_error_flags { ENA_ADMIN_PHC_ERROR_FLAG_ERROR_BOUND = BIT(1), }; +enum ena_admin_hw_timestamp_feature_version { + ENA_ADMIN_HW_TIMESTAMP_FEATURE_VERSION_1 = 1, +}; + +enum ena_admin_hw_timestamp_tx_support { + ENA_ADMIN_HW_TIMESTAMP_TX_SUPPORT_NONE = 0, + ENA_ADMIN_HW_TIMESTAMP_TX_SUPPORT_ALL = 1, +}; + +enum ena_admin_hw_timestamp_rx_support { + ENA_ADMIN_HW_TIMESTAMP_RX_SUPPORT_NONE = 0, + ENA_ADMIN_HW_TIMESTAMP_RX_SUPPORT_ALL = 1, +}; + /* ENA SRD configuration for ENI */ enum ena_admin_ena_srd_flags { /* Feature enabled */ @@ -1106,6 +1121,12 @@ struct ena_admin_feature_phc_desc { uint32_t output_length; }; +struct ena_admin_feature_hw_ts_desc { + uint8_t version; + uint8_t tx; + uint8_t rx; +}; + struct ena_admin_get_feat_resp { struct ena_admin_acq_common_desc acq_common_desc; @@ -1138,6 +1159,8 @@ struct ena_admin_get_feat_resp { struct ena_admin_feature_phc_desc phc; + struct ena_admin_feature_hw_ts_desc hw_ts; + struct ena_admin_get_extra_properties_strings_desc extra_properties_strings; struct ena_admin_get_extra_properties_flags_desc extra_properties_flags; @@ -1177,6 +1200,9 @@ struct ena_admin_set_feat_cmd { /* PHC configuration */ struct ena_admin_feature_phc_desc phc; + + /* HW timestamp configuration */ + struct ena_admin_feature_hw_ts_desc hw_ts; } u; }; diff --git a/userspace/dpdk/ena/base/ena_defs/ena_eth_io_defs.h b/userspace/dpdk/ena/base/ena_defs/ena_eth_io_defs.h index c93cd856..78c87c46 100644 --- a/userspace/dpdk/ena/base/ena_defs/ena_eth_io_defs.h +++ b/userspace/dpdk/ena/base/ena_defs/ena_eth_io_defs.h @@ -163,6 +163,14 @@ struct ena_eth_io_tx_cdesc { uint16_t sq_head_idx; }; +struct ena_eth_io_tx_cdesc_ext { + struct ena_eth_io_tx_cdesc base; + + uint32_t timestamp_low; + + uint32_t timestamp_high; +}; + struct ena_eth_io_rx_desc { /* In bytes. 0 means 64KB */ uint16_t length; @@ -255,9 +263,9 @@ struct ena_eth_io_rx_cdesc_ext { uint16_t reserved16; - uint32_t reserved_w6; + uint32_t timestamp_low; - uint32_t reserved_w7; + uint32_t timestamp_high; }; struct ena_eth_io_intr_reg { diff --git a/userspace/dpdk/ena/base/ena_eth_com.c b/userspace/dpdk/ena/base/ena_eth_com.c index 90dd85c7..5823915d 100644 --- a/userspace/dpdk/ena/base/ena_eth_com.c +++ b/userspace/dpdk/ena/base/ena_eth_com.c @@ -680,6 +680,14 @@ int ena_com_rx_pkt(struct ena_com_io_cq *io_cq, /* Get rx flags from the last pkt */ ena_com_rx_set_flags(ena_rx_ctx, cdesc); + if (ena_com_is_extended_rx_cdesc(io_cq)) { + struct ena_eth_io_rx_cdesc_ext *ext = + (struct ena_eth_io_rx_cdesc_ext *)cdesc; + ena_rx_ctx->timestamp = (u64)ext->timestamp_low | + ((u64)ext->timestamp_high << 32); + ena_rx_ctx->has_timestamp = true; + } + ena_trc_dbg(ena_com_io_cq_to_ena_dev(io_cq), "l3_proto %d l4_proto %d l3_csum_err %d l4_csum_err %d hash %d frag %d cdesc_status %x\n", ena_rx_ctx->l3_proto, diff --git a/userspace/dpdk/ena/base/ena_eth_com.h b/userspace/dpdk/ena/base/ena_eth_com.h index 8a12ed5f..ed83fef3 100644 --- a/userspace/dpdk/ena/base/ena_eth_com.h +++ b/userspace/dpdk/ena/base/ena_eth_com.h @@ -60,8 +60,22 @@ struct ena_com_rx_ctx { u16 descs; u16 max_bufs; u8 pkt_offset; + bool has_timestamp; + u64 timestamp; }; +static inline bool ena_com_is_extended_rx_cdesc(struct ena_com_io_cq *io_cq) +{ + return io_cq->cdesc_entry_size_in_bytes == + sizeof(struct ena_eth_io_rx_cdesc_ext); +} + +static inline bool ena_com_is_extended_tx_cdesc(struct ena_com_io_cq *io_cq) +{ + return io_cq->cdesc_entry_size_in_bytes == + sizeof(struct ena_eth_io_tx_cdesc_ext); +} + int ena_com_prepare_tx(struct ena_com_io_sq *io_sq, struct ena_com_tx_ctx *ena_tx_ctx, int *nb_hw_desc); diff --git a/userspace/dpdk/ena/ena_ethdev.c b/userspace/dpdk/ena/ena_ethdev.c index aea2e5c9..7aefdd81 100644 --- a/userspace/dpdk/ena/ena_ethdev.c +++ b/userspace/dpdk/ena/ena_ethdev.c @@ -9,6 +9,7 @@ #include #include #include +#include #include "ena_ethdev.h" #include "ena_logs.h" @@ -219,6 +220,16 @@ static const struct rte_pci_id pci_id_ena_map[] = { { .device_id = 0 }, }; +static const struct rte_mbuf_dynfield ena_timestamp_dynfield_desc = { + .name = RTE_MBUF_DYNFIELD_TIMESTAMP_NAME, + .size = sizeof(uint64_t), + .align = __alignof__(uint64_t), +}; + +static const struct rte_mbuf_dynflag ena_timestamp_dynflag_desc = { + .name = RTE_MBUF_DYNFLAG_RX_TIMESTAMP_NAME, +}; + static struct ena_aenq_handlers aenq_handlers; static int ena_device_init(struct ena_adapter *adapter, @@ -707,6 +718,13 @@ static inline void ena_rx_mbuf_prepare(struct ena_ring *rx_ring, ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN; } + if (rx_ring->hw_ts_enabled && ena_rx_ctx->has_timestamp) { + *RTE_MBUF_DYNFIELD(mbuf, + rx_ring->timestamp_dynfield_offset, + uint64_t *) = ena_rx_ctx->timestamp; + ol_flags |= rx_ring->timestamp_rx_dynflag; + } + mbuf->ol_flags = ol_flags; mbuf->packet_type = packet_type; } @@ -906,6 +924,19 @@ static int ena_close(struct rte_eth_dev *dev) if (adapter->state == ENA_ADAPTER_STATE_RUNNING) ret = ena_stop(dev); + + if (adapter->hw_ts_enabled) { + size_t i; + + ena_com_set_hw_timestamping_configuration(ena_dev, 0, 0); + ena_dev->use_extended_rx_cdesc = false; + adapter->hw_ts_enabled = false; + adapter->timestamp_dynfield_offset = -1; + adapter->timestamp_rx_dynflag = 0; + for (i = 0; i < adapter->max_num_io_queues; i++) + adapter->rx_ring[i].hw_ts_enabled = false; + } + adapter->state = ENA_ADAPTER_STATE_CLOSED; if (!adapter->control_path_poll_interval) { @@ -1442,6 +1473,9 @@ static int ena_create_io_queue(struct rte_eth_dev *dev, struct ena_ring *ring) ctx.qid = ena_qid; ctx.numa_node = ring->numa_socket_id; + if (ring->type == ENA_RING_TYPE_RX) + ctx.use_extended_cdesc = ena_dev->use_extended_rx_cdesc; + rc = ena_com_create_io_queue(ena_dev, &ctx); if (rc) { PMD_DRV_LOG_LINE(ERR, @@ -2355,6 +2389,25 @@ static int eth_ena_dev_init(struct rte_eth_dev *eth_dev) if (!(adapter->all_aenq_groups & BIT(ENA_ADMIN_LINK_CHANGE))) adapter->edev_data->dev_flags &= ~RTE_ETH_DEV_INTR_LSC; + /* Query HW timestamp support */ + { + uint8_t tx_ts_support, rx_ts_support; + + rc = ena_com_get_hw_timestamping_support(ena_dev, + &tx_ts_support, + &rx_ts_support); + if (rc == 0 && + rx_ts_support == ENA_ADMIN_HW_TIMESTAMP_RX_SUPPORT_ALL) { + adapter->hw_ts_rx_supported = true; + PMD_DRV_LOG_LINE(INFO, "HW RX timestamping is supported"); + } else { + adapter->hw_ts_rx_supported = false; + PMD_DRV_LOG_LINE(DEBUG, "HW RX timestamping is not supported"); + } + adapter->hw_ts_enabled = false; + adapter->timestamp_dynfield_offset = -1; + } + bool use_large_llq_hdr = ena_use_large_llq_hdr(adapter, get_feat_ctx.llq.entry_size_recommended); set_default_llq_configurations(&llq_config, &get_feat_ctx.llq, use_large_llq_hdr); @@ -2507,8 +2560,81 @@ static int ena_dev_configure(struct rte_eth_dev *dev) */ adapter->tx_cleanup_stall_delay = adapter->missing_tx_completion_to / 2; + /* TX HW timestamping is not implemented: reject the offload instead of + * silently ignoring it so applications get a clear failure. + */ + if (dev->data->dev_conf.txmode.offloads & RTE_ETH_TX_OFFLOAD_TIMESTAMP) { + PMD_DRV_LOG_LINE(ERR, + "TX HW timestamp offload is not supported"); + return -ENOTSUP; + } + + if (adapter->hw_ts_rx_supported && + (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_TIMESTAMP)) { + struct ena_com_dev *ena_dev = &adapter->ena_dev; + int dynflag_bitnum; + size_t i; + + rc = ena_com_set_hw_timestamping_configuration(ena_dev, 0, 1); + if (rc) { + PMD_DRV_LOG_LINE(ERR, "Failed to enable HW timestamps, rc: %d", rc); + return rc; + } + ena_dev->use_extended_rx_cdesc = true; + adapter->hw_ts_enabled = true; + + adapter->timestamp_dynfield_offset = + rte_mbuf_dynfield_register(&ena_timestamp_dynfield_desc); + if (adapter->timestamp_dynfield_offset < 0) { + PMD_DRV_LOG_LINE(ERR, "Failed to register timestamp dynfield"); + rc = -rte_errno; + goto err_disable_hw_ts; + } + + dynflag_bitnum = rte_mbuf_dynflag_register(&ena_timestamp_dynflag_desc); + if (dynflag_bitnum < 0) { + PMD_DRV_LOG_LINE(ERR, "Failed to register timestamp dynflag"); + rc = -rte_errno; + goto err_disable_hw_ts; + } + adapter->timestamp_rx_dynflag = RTE_BIT64(dynflag_bitnum); + + /* Cache per-ring copies so the RX fast path avoids an + * adapter pointer-chase and cold-cacheline load. + */ + for (i = 0; i < adapter->max_num_io_queues; i++) { + struct ena_ring *ring = &adapter->rx_ring[i]; + ring->hw_ts_enabled = true; + ring->timestamp_dynfield_offset = + adapter->timestamp_dynfield_offset; + ring->timestamp_rx_dynflag = adapter->timestamp_rx_dynflag; + } + + PMD_DRV_LOG_LINE(INFO, "HW RX timestamping enabled"); + } else { + size_t i; + + /* If HW timestamping was previously enabled, disable on the + * device so descriptor sizes match across reconfigure. + */ + if (adapter->hw_ts_enabled) { + ena_com_set_hw_timestamping_configuration(&adapter->ena_dev, 0, 0); + adapter->ena_dev.use_extended_rx_cdesc = false; + } + adapter->hw_ts_enabled = false; + for (i = 0; i < adapter->max_num_io_queues; i++) + adapter->rx_ring[i].hw_ts_enabled = false; + } + rc = ena_configure_aenq(adapter); + return rc; +err_disable_hw_ts: + ena_com_set_hw_timestamping_configuration(&adapter->ena_dev, 0, 0); + adapter->ena_dev.use_extended_rx_cdesc = false; + adapter->hw_ts_enabled = false; + adapter->timestamp_dynfield_offset = -1; + adapter->timestamp_rx_dynflag = 0; return rc; } @@ -2558,6 +2684,9 @@ static uint64_t ena_get_rx_port_offloads(struct ena_adapter *adapter) port_offloads |= RTE_ETH_RX_OFFLOAD_SCATTER; + if (adapter->hw_ts_rx_supported) + port_offloads |= RTE_ETH_RX_OFFLOAD_TIMESTAMP; + return port_offloads; } @@ -2802,6 +2931,8 @@ static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, ena_rx_ctx.ena_bufs = rx_ring->ena_bufs; ena_rx_ctx.descs = 0; ena_rx_ctx.pkt_offset = 0; + ena_rx_ctx.has_timestamp = false; + ena_rx_ctx.timestamp = 0; /* receive packet context */ rc = ena_com_rx_pkt(rx_ring->ena_com_io_cq, rx_ring->ena_com_io_sq, diff --git a/userspace/dpdk/ena/ena_ethdev.h b/userspace/dpdk/ena/ena_ethdev.h index b8aead8f..eeb2ffd8 100644 --- a/userspace/dpdk/ena/ena_ethdev.h +++ b/userspace/dpdk/ena/ena_ethdev.h @@ -194,6 +194,11 @@ struct __rte_cache_aligned ena_ring { bool disable_meta_caching; + /* HW RX timestamp (cached per-ring to avoid adapter pointer chase) */ + bool hw_ts_enabled; + int timestamp_dynfield_offset; + uint64_t timestamp_rx_dynflag; + union { struct ena_stats_rx rx_stats; struct ena_stats_tx tx_stats; @@ -358,6 +363,11 @@ struct ena_adapter { alignas(RTE_CACHE_LINE_SIZE) uint64_t metrics_stats[ENA_MAX_CUSTOMER_METRICS]; uint16_t metrics_num; alignas(RTE_CACHE_LINE_SIZE) struct ena_stats_srd srd_stats; + + bool hw_ts_rx_supported; + bool hw_ts_enabled; + int timestamp_dynfield_offset; + uint64_t timestamp_rx_dynflag; }; int ena_mp_indirect_table_set(struct ena_adapter *adapter);