/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright (C), 2025, Wuxi Stars Micro System Technologies Co., Ltd.
 */

#include <rte_vect.h>

#include "sxe2_ethdev.h"
#include "sxe2_common_log.h"
#include "sxe2_queue.h"
#include "sxe2_txrx_vec.h"
#include "sxe2_txrx_vec_common.h"
#include "sxe2_vsi.h"

static inline void
sxe2_tx_desc_fill_one_avx2(volatile union sxe2_tx_data_desc *desc, struct rte_mbuf *pkt,
			   uint64_t desc_cmd, bool with_offloads)
{
	__m128i data_desc;
	uint64_t desc_qw1;
	uint32_t desc_offset;

	desc_qw1 = (SXE2_TX_DESC_DTYPE_DATA |
		   ((uint64_t)desc_cmd) << SXE2_TX_DATA_DESC_CMD_SHIFT |
		   ((uint64_t)pkt->data_len) << SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);

	desc_offset = SXE2_TX_DATA_DESC_MACLEN_VAL(pkt->l2_len);
	desc_qw1 |= ((uint64_t)desc_offset) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
	if (with_offloads)
		sxe2_tx_desc_fill_offloads(pkt, &desc_qw1);

	data_desc = _mm_set_epi64x(desc_qw1, rte_pktmbuf_iova(pkt));
	_mm_store_si128(RTE_CAST_PTR(__m128i *, desc), data_desc);
}

static __rte_always_inline void
sxe2_tx_desc_fill_avx2(volatile union sxe2_tx_data_desc *desc, struct rte_mbuf **pkts,
		       uint16_t pkts_num, uint64_t desc_cmd, bool with_offloads)
{
	__m256i desc_group0;
	__m256i desc_group1;
	uint64_t desc0_qw1;
	uint64_t desc1_qw1;
	uint64_t desc2_qw1;
	uint64_t desc3_qw1;

	const uint64_t desc_qw1_com = (SXE2_TX_DESC_DTYPE_DATA |
					((uint64_t)desc_cmd) << SXE2_TX_DATA_DESC_CMD_SHIFT);
	uint32_t desc_offset[4] = {0};

	if (((uint64_t)desc & 0x1F) != 0 && pkts_num != 0) {
		sxe2_tx_desc_fill_one_avx2(desc, *pkts, desc_cmd, with_offloads);
		pkts_num--;
		desc++;
		pkts++;
	}

	while (pkts_num > 3) {
		desc3_qw1 = (desc_qw1_com |
					((uint64_t)pkts[3]->data_len)
					<< SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);

		desc_offset[3] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[3]->l2_len);
		desc3_qw1 |= ((uint64_t)desc_offset[3]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
		if (with_offloads)
			sxe2_tx_desc_fill_offloads(pkts[3], &desc3_qw1);

		desc2_qw1 = (desc_qw1_com |
					((uint64_t)pkts[2]->data_len)
					<< SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
		desc_offset[2] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[2]->l2_len);
		desc2_qw1 |= ((uint64_t)desc_offset[2]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
		if (with_offloads)
			sxe2_tx_desc_fill_offloads(pkts[2], &desc2_qw1);

		desc1_qw1 = (desc_qw1_com |
					((uint64_t)pkts[1]->data_len)
					<< SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
		desc_offset[1] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[1]->l2_len);
		desc1_qw1 |= ((uint64_t)desc_offset[1]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
		if (with_offloads)
			sxe2_tx_desc_fill_offloads(pkts[1], &desc1_qw1);

		desc0_qw1 = (desc_qw1_com |
					((uint64_t)pkts[0]->data_len)
					<< SXE2_TX_DATA_DESC_BUF_SZ_SHIFT);
		desc_offset[0] = SXE2_TX_DATA_DESC_MACLEN_VAL(pkts[0]->l2_len);
		desc0_qw1 |= ((uint64_t)desc_offset[0]) << SXE2_TX_DATA_DESC_OFFSET_SHIFT;
		if (with_offloads)
			sxe2_tx_desc_fill_offloads(pkts[0], &desc0_qw1);

		desc_group1 = _mm256_set_epi64x(desc3_qw1, rte_pktmbuf_iova(pkts[3]),
					desc2_qw1, rte_pktmbuf_iova(pkts[2]));

		desc_group0 = _mm256_set_epi64x(desc1_qw1, rte_pktmbuf_iova(pkts[1]),
					desc0_qw1, rte_pktmbuf_iova(pkts[0]));

		_mm256_store_si256(RTE_CAST_PTR(__m256i *, desc + 2), desc_group1);
		_mm256_store_si256(RTE_CAST_PTR(__m256i *, desc), desc_group0);

		pkts_num -= 4;
		desc     += 4;
		pkts     += 4;
	}

	while (pkts_num) {
		sxe2_tx_desc_fill_one_avx2(desc, *pkts, desc_cmd, with_offloads);
		pkts_num--;
		desc++;
		pkts++;
	}
}

static __rte_always_inline uint16_t
sxe2_tx_pkts_vec_avx2_batch(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pkts,
			    uint16_t nb_pkts, bool with_offloads)
{
	volatile union sxe2_tx_data_desc *desc;
	struct sxe2_tx_buffer *buffer;
	uint16_t next_use;
	uint16_t res_num;
	uint16_t tx_num;

	if (txq->desc_free_num < txq->free_thresh)
		(void)sxe2_tx_bufs_free_vec(txq);

	nb_pkts = RTE_MIN(txq->desc_free_num, nb_pkts);
	if (unlikely(nb_pkts == 0)) {
		PMD_LOG_DEBUG(TX, "Tx pkts avx2 batch: may not enough free desc, "
				"free_desc=%u, need_tx_pkts=%u",
				txq->desc_free_num, nb_pkts);
		goto l_end;
	}
	tx_num = nb_pkts;

	next_use = txq->next_use;
	desc     = &txq->desc_ring[next_use];
	buffer   = &txq->buffer_ring[next_use];

	txq->desc_free_num -= nb_pkts;

	res_num = txq->ring_depth - txq->next_use;

	if (tx_num >= res_num) {
		sxe2_tx_pkts_mbuf_fill(buffer, tx_pkts, res_num);

		sxe2_tx_desc_fill_avx2(desc, tx_pkts, res_num,
				SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);
		tx_pkts += (res_num - 1);
		desc    += (res_num - 1);

		sxe2_tx_desc_fill_one_avx2(desc, *tx_pkts++,
				(SXE2_TX_DATA_DESC_CMD_EOP | SXE2_TX_DATA_DESC_CMD_RS),
				with_offloads);

		tx_num -= res_num;

		next_use     = 0;
		txq->next_rs = txq->rs_thresh - 1;
		desc         = &txq->desc_ring[next_use];
		buffer       = &txq->buffer_ring[next_use];
	}

	sxe2_tx_pkts_mbuf_fill(buffer, tx_pkts, tx_num);

	sxe2_tx_desc_fill_avx2(desc, tx_pkts, tx_num,
			SXE2_TX_DATA_DESC_CMD_EOP, with_offloads);

	next_use += tx_num;
	if (next_use > txq->next_rs) {
		txq->desc_ring[txq->next_rs].read.type_cmd_off_bsz_l2t |=
			rte_cpu_to_le_64(SXE2_TX_DATA_DESC_CMD_RS_MASK);

		txq->next_rs += txq->rs_thresh;
	}
	txq->next_use = next_use;
	SXE2_PCI_REG_WRITE_WC(txq->tdt_reg_addr, next_use);
	PMD_LOG_DEBUG(TX, "port_id=%u queue_id=%u next_use=%u send_pkts=%u",
			txq->port_id, txq->queue_id, next_use, nb_pkts);
l_end:
	return nb_pkts;
}

static __rte_always_inline uint16_t
sxe2_tx_pkts_vec_avx2_common(struct sxe2_tx_queue *txq, struct rte_mbuf **tx_pkts,
			     uint16_t nb_pkts, bool with_offloads)
{
	uint16_t tx_done_num = 0;
	uint16_t tx_once_num;
	uint16_t tx_need_num;

	while (nb_pkts) {
		tx_need_num = RTE_MIN(nb_pkts, txq->rs_thresh);
		tx_once_num = sxe2_tx_pkts_vec_avx2_batch(txq,
					tx_pkts + tx_done_num, tx_need_num, with_offloads);

		nb_pkts     -= tx_once_num;
		tx_done_num += tx_once_num;

		if (tx_once_num < tx_need_num)
			break;
	}
	return tx_done_num;
}

uint16_t sxe2_tx_pkts_vec_avx2_simple(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
{
	return sxe2_tx_pkts_vec_avx2_common(tx_queue, tx_pkts, nb_pkts, false);
}

uint16_t sxe2_tx_pkts_vec_avx2(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
{
	return sxe2_tx_pkts_vec_avx2_common(tx_queue, tx_pkts, nb_pkts, true);
}

static inline void sxe2_rx_queue_rearm_avx2(struct sxe2_rx_queue *rxq)
{
	volatile union sxe2_rx_desc *desc;
	struct rte_mbuf **buffer;
	struct rte_mbuf *mbuf0, *mbuf1;
	__m128i dma_addr0, dma_addr1;
	__m128i virt_addr0, virt_addr1;
	__m128i hdr_room = _mm_set_epi64x(RTE_PKTMBUF_HEADROOM, RTE_PKTMBUF_HEADROOM);
	int32_t ret;
	uint16_t i;
	uint16_t new_tail;

	buffer = &rxq->buffer_ring[rxq->realloc_start];
	desc   = &rxq->desc_ring[rxq->realloc_start];

	ret = rte_mempool_get_bulk(rxq->mb_pool, (void *)buffer, SXE2_RX_REARM_THRESH_VEC);
	if (ret != 0) {
		if ((rxq->realloc_num + SXE2_RX_REARM_THRESH_VEC) >= rxq->ring_depth) {
			dma_addr0 = _mm_setzero_si128();
			for (i = 0; i < SXE2_RX_NUM_PER_LOOP_AVX; ++i) {
				buffer[i] = &rxq->fake_mbuf;
				_mm_store_si128(RTE_CAST_PTR(__m128i *, &desc[i].read), dma_addr0);
			}
		}

		rxq->vsi->adapter->dev_info.dev_data->rx_mbuf_alloc_failed +=
				SXE2_RX_REARM_THRESH_VEC;
		return;
	}

	for (i = 0; i < SXE2_RX_REARM_THRESH_VEC; i += 2, buffer += 2) {
		mbuf0 = buffer[0];
		mbuf1 = buffer[1];
#if RTE_IOVA_IN_MBUF

		RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, buf_iova) !=
				 offsetof(struct rte_mbuf, buf_addr) + 8);
#endif
		virt_addr0 = _mm_loadu_si128((__m128i *)&mbuf0->buf_addr);
		virt_addr1 = _mm_loadu_si128((__m128i *)&mbuf1->buf_addr);

#if RTE_IOVA_IN_MBUF

		dma_addr0 = _mm_unpackhi_epi64(virt_addr0, virt_addr0);
		dma_addr1 = _mm_unpackhi_epi64(virt_addr1, virt_addr1);
#else

		dma_addr0 = _mm_unpacklo_epi64(virt_addr0, virt_addr0);
		dma_addr1 = _mm_unpacklo_epi64(virt_addr1, virt_addr1);
#endif

		dma_addr0 = _mm_add_epi64(dma_addr0, hdr_room);
		dma_addr1 = _mm_add_epi64(dma_addr1, hdr_room);

		_mm_store_si128(RTE_CAST_PTR(__m128i *, &desc++->read), dma_addr0);
		_mm_store_si128(RTE_CAST_PTR(__m128i *, &desc++->read), dma_addr1);
	}

	rxq->realloc_start += SXE2_RX_REARM_THRESH_VEC;
	if (rxq->realloc_start >= rxq->ring_depth)
		rxq->realloc_start = 0;
	rxq->realloc_num -= SXE2_RX_REARM_THRESH_VEC;

	new_tail = (rxq->realloc_start == 0) ?
		(rxq->ring_depth - 1) : (rxq->realloc_start - 1);
	SXE2_PCI_REG_WRITE_WC(rxq->rdt_reg_addr, new_tail);
}

static __rte_always_inline uint16_t
sxe2_rx_pkts_common_vec_avx2(struct sxe2_rx_queue *rxq,
			     struct rte_mbuf **rx_pkts, uint16_t nb_pkts, uint8_t *split_rxe_flags,
			     uint8_t *umbcast_flags, bool do_offload)
{
	const __m256i mbuf_init   = _mm256_set_epi64x(0, 0, 0, rxq->mbuf_init_value);
	struct rte_mbuf **buffer;
	volatile union sxe2_rx_desc *desc;
	__m256i mbufs6_7, mbufs4_5, mbufs2_3, mbufs0_1;
	uint32_t bit_num;
	uint16_t done_num;
	uint16_t i = 0;
	uint16_t j = 0;

	buffer   = &rxq->buffer_ring[rxq->processing_idx];
	desc     = &rxq->desc_ring[rxq->processing_idx];
	done_num = 0;

	rte_prefetch0(desc);

	nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, SXE2_RX_NUM_PER_LOOP_AVX);

	if (rxq->realloc_num > SXE2_RX_REARM_THRESH_VEC)
		sxe2_rx_queue_rearm_avx2(rxq);

	if (0 == (rte_le_to_cpu_64(desc->wb.status_err_ptype_len) &
				SXE2_RX_DESC_STATUS_DD_MASK))
		goto l_end;

	const __m256i crc_adjust =
		_mm256_set_epi16(0, 0, 0, -rxq->crc_len,
				 0, -rxq->crc_len, 0,
				 0, 0, 0, 0,
				 -rxq->crc_len, 0, -rxq->crc_len, 0, 0);

	const __m256i dd_mask = _mm256_set1_epi32(1);
	const __m256i rvp_shuf_mask =
		_mm256_set_epi8(7, 6, 5, 4,
				3, 2, 13, 12,
				0xFF, 0xFF, 13, 12,
				0xFF, 0xFF, 0xFF, 0xFF,
				7, 6, 5, 4,
				3, 2, 13, 12,
				0xFF, 0xFF, 13, 12,
				0xFF, 0xFF, 0xFF, 0xFF);

	const __m128i eop_shuf_mask =
		_mm_set_epi8(0xFF, 0xFF, 0xFF, 0xFF,
			     0xFF, 0xFF, 0xFF, 0xFF,
			     8, 0, 10, 2,
			     12, 4, 14, 6);

	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, pkt_len) !=
			offsetof(struct rte_mbuf, rx_descriptor_fields1) + 4);
	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, data_len) !=
			offsetof(struct rte_mbuf, rx_descriptor_fields1) + 8);
	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, vlan_tci) !=
			offsetof(struct rte_mbuf, rx_descriptor_fields1) + 10);
	RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, hash) !=
			offsetof(struct rte_mbuf, rx_descriptor_fields1) + 12);

	for (i = 0; i < nb_pkts; i += SXE2_RX_NUM_PER_LOOP_AVX,
				desc += SXE2_RX_NUM_PER_LOOP_AVX) {
		_mm256_storeu_si256((void *)&rx_pkts[i],
					_mm256_loadu_si256((void *)&buffer[i]));
#ifdef RTE_ARCH_X86_64
		_mm256_storeu_si256((void *)&rx_pkts[i + 4],
					_mm256_loadu_si256((void *)&buffer[i + 4]));
#endif

		const __m128i desc7 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 7));
		rte_compiler_barrier();
		const __m128i desc6 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 6));
		rte_compiler_barrier();
		const __m128i desc5 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 5));
		rte_compiler_barrier();
		const __m128i desc4 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 4));
		rte_compiler_barrier();
		const __m128i desc3 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 3));
		rte_compiler_barrier();
		const __m128i desc2 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 2));
		rte_compiler_barrier();
		const __m128i desc1 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 1));
		rte_compiler_barrier();
		const __m128i desc0 = _mm_loadu_si128(RTE_CAST_PTR(const __m128i *, desc + 0));

		const __m256i descs6_7 =
			_mm256_inserti128_si256(_mm256_castsi128_si256(desc6), desc7, 1);
		const __m256i descs4_5 =
			_mm256_inserti128_si256(_mm256_castsi128_si256(desc4), desc5, 1);
		const __m256i descs2_3 =
			_mm256_inserti128_si256(_mm256_castsi128_si256(desc2), desc3, 1);
		const __m256i descs0_1 =
			_mm256_inserti128_si256(_mm256_castsi128_si256(desc0), desc1, 1);

		if (split_rxe_flags) {
			for (j = 0; j < SXE2_RX_NUM_PER_LOOP_AVX; j++)
				rte_mbuf_prefetch_part2(rx_pkts[i + j]);
		}

		mbufs6_7 = _mm256_shuffle_epi8(descs6_7, rvp_shuf_mask);
		mbufs4_5 = _mm256_shuffle_epi8(descs4_5, rvp_shuf_mask);

		mbufs6_7 = _mm256_add_epi16(mbufs6_7, crc_adjust);
		mbufs4_5 = _mm256_add_epi16(mbufs4_5, crc_adjust);

		const __m256i ptype_mask = _mm256_set1_epi32(SXE2_RX_DESC_PTYPE_MASK);

		const __m256i staterrs4_7 = _mm256_unpackhi_epi32(descs6_7, descs4_5);

		__m256i ptypes4_7 = _mm256_and_si256(staterrs4_7, ptype_mask);

		const uint16_t ptype7 = _mm256_extract_epi16(ptypes4_7, 9);
		const uint16_t ptype6 = _mm256_extract_epi16(ptypes4_7, 1);
		const uint16_t ptype5 = _mm256_extract_epi16(ptypes4_7, 11);
		const uint16_t ptype4 = _mm256_extract_epi16(ptypes4_7, 3);

		mbufs6_7 = _mm256_insert_epi32(mbufs6_7, sxe2_ptype_tbl[ptype7], 4);
		mbufs6_7 = _mm256_insert_epi32(mbufs6_7, sxe2_ptype_tbl[ptype6], 0);
		mbufs4_5 = _mm256_insert_epi32(mbufs4_5, sxe2_ptype_tbl[ptype5], 4);
		mbufs4_5 = _mm256_insert_epi32(mbufs4_5, sxe2_ptype_tbl[ptype4], 0);

		mbufs2_3 = _mm256_shuffle_epi8(descs2_3, rvp_shuf_mask);
		mbufs0_1 = _mm256_shuffle_epi8(descs0_1, rvp_shuf_mask);

		mbufs2_3 = _mm256_add_epi16(mbufs2_3, crc_adjust);
		mbufs0_1 = _mm256_add_epi16(mbufs0_1, crc_adjust);

		const __m256i staterrs0_3 = _mm256_unpackhi_epi32(descs2_3, descs0_1);

		__m256i ptypes0_3 = _mm256_and_si256(staterrs0_3, ptype_mask);

		const uint16_t ptype3 = _mm256_extract_epi16(ptypes0_3, 9);
		const uint16_t ptype2 = _mm256_extract_epi16(ptypes0_3, 1);
		const uint16_t ptype1 = _mm256_extract_epi16(ptypes0_3, 11);
		const uint16_t ptype0 = _mm256_extract_epi16(ptypes0_3, 3);

		mbufs2_3 = _mm256_insert_epi32(mbufs2_3, sxe2_ptype_tbl[ptype3], 4);
		mbufs2_3 = _mm256_insert_epi32(mbufs2_3, sxe2_ptype_tbl[ptype2], 0);
		mbufs0_1 = _mm256_insert_epi32(mbufs0_1, sxe2_ptype_tbl[ptype1], 4);
		mbufs0_1 = _mm256_insert_epi32(mbufs0_1, sxe2_ptype_tbl[ptype0], 0);

		__m256i staterrs0_7 = _mm256_unpacklo_epi64(staterrs4_7, staterrs0_3);

		__m256i stu_len0_7 = _mm256_unpackhi_epi64(staterrs4_7, staterrs0_3);
		__m256i mbuf_flags = _mm256_setzero_si256();

		if (do_offload) {
			const __m256i desc_flags_mask = _mm256_set1_epi32(0x00001C04);
			const __m256i desc_flags_rss_mask = _mm256_set1_epi32(0x20000000);
			const __m256i vlan_flags =
				_mm256_set_epi8
				(0, 0, 0, 0, 0, 0, 0, 0,
				 0, 0, 0,
				 RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED,
				 0, 0, 0, 0,
				 0, 0, 0, 0, 0, 0, 0, 0,
				 0, 0, 0,
				 RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED,
				 0, 0, 0, 0);

			const __m256i rss_flags =
				_mm256_set_epi8
				(0, 0, 0, 0, 0, 0, 0, 0,
				 0, 0, 0, RTE_MBUF_F_RX_RSS_HASH, 0, 0, 0, 0,
				 0, 0, 0, 0, 0, 0, 0, 0,
				 0, 0, 0, RTE_MBUF_F_RX_RSS_HASH, 0, 0, 0, 0);

			const __m256i cksum_flags =
				_mm256_set_epi8
				(0, 0, 0, 0, 0, 0, 0, 0,
				 ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
					RTE_MBUF_F_RX_L4_CKSUM_BAD |
					RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
				 ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
					RTE_MBUF_F_RX_L4_CKSUM_BAD |
					RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
				 ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
					RTE_MBUF_F_RX_L4_CKSUM_GOOD |
					RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
				 ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
					RTE_MBUF_F_RX_L4_CKSUM_GOOD |
					RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
				 ((RTE_MBUF_F_RX_L4_CKSUM_BAD |
					RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
				 ((RTE_MBUF_F_RX_L4_CKSUM_BAD |
					RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
				 ((RTE_MBUF_F_RX_L4_CKSUM_GOOD |
					RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
				 ((RTE_MBUF_F_RX_L4_CKSUM_GOOD |
					RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),

				 0, 0, 0, 0, 0, 0, 0, 0,
				 ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
					RTE_MBUF_F_RX_L4_CKSUM_BAD |
					RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
				 ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
					RTE_MBUF_F_RX_L4_CKSUM_BAD |
					RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
				 ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
					RTE_MBUF_F_RX_L4_CKSUM_GOOD |
					RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
				 ((RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD |
					RTE_MBUF_F_RX_L4_CKSUM_GOOD |
					RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
				 ((RTE_MBUF_F_RX_L4_CKSUM_BAD |
					RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
				 ((RTE_MBUF_F_RX_L4_CKSUM_BAD |
					RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1),
				 ((RTE_MBUF_F_RX_L4_CKSUM_GOOD |
					RTE_MBUF_F_RX_IP_CKSUM_BAD) >> 1),
				 ((RTE_MBUF_F_RX_L4_CKSUM_GOOD |
					RTE_MBUF_F_RX_IP_CKSUM_GOOD) >> 1));

			const __m256i cksum_mask =
				_mm256_set1_epi32
				(RTE_MBUF_F_RX_IP_CKSUM_MASK |
				RTE_MBUF_F_RX_L4_CKSUM_MASK |
				RTE_MBUF_F_RX_OUTER_L4_CKSUM_MASK |
				RTE_MBUF_F_RX_OUTER_IP_CKSUM_BAD);
			const __m256i vlan_mask =
				_mm256_set1_epi32
				(RTE_MBUF_F_RX_VLAN | RTE_MBUF_F_RX_VLAN_STRIPPED);

			__m256i tmp_flags;
			__m256i descs_flags = _mm256_and_si256(staterrs0_7, desc_flags_mask);
			stu_len0_7 = _mm256_and_si256(stu_len0_7, desc_flags_rss_mask);

			tmp_flags = _mm256_shuffle_epi8(vlan_flags, descs_flags);
			mbuf_flags = _mm256_and_si256(tmp_flags, vlan_mask);

			descs_flags = _mm256_srli_epi32(descs_flags, 10);
			tmp_flags = _mm256_shuffle_epi8(cksum_flags, descs_flags);
			tmp_flags = _mm256_slli_epi32(tmp_flags, 1);
			tmp_flags = _mm256_and_si256(tmp_flags, cksum_mask);
			mbuf_flags = _mm256_or_si256(mbuf_flags, tmp_flags);

			descs_flags = _mm256_srli_epi32(stu_len0_7, 27);
			tmp_flags = _mm256_shuffle_epi8(rss_flags, descs_flags);
			mbuf_flags = _mm256_or_si256(mbuf_flags, tmp_flags);

#ifndef RTE_LIBRTE_SXE2_16BYTE_RX_DESC

			if (rxq->fnav_enable) {
				__m256i fnav_vld0_3, fnav_vld4_7;
				__m256i fnav_vld0_7;
				__m256i v_zeros, v_ffff, v_u32_one;
				const __m256i fdir_flags =
					_mm256_set1_epi32
					(RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID);
				fnav_vld0_3 = _mm256_unpacklo_epi32(descs2_3, descs0_1);
				fnav_vld4_7 = _mm256_unpacklo_epi32(descs6_7, descs4_5);

				fnav_vld0_7 = _mm256_unpacklo_epi64(fnav_vld4_7, fnav_vld0_3);

				fnav_vld0_7 = _mm256_slli_epi32(fnav_vld0_7, 26);
				fnav_vld0_7 = _mm256_srli_epi32(fnav_vld0_7, 31);

				v_zeros = _mm256_setzero_si256();
				v_ffff = _mm256_cmpeq_epi32(v_zeros, v_zeros);
				v_u32_one = _mm256_srli_epi32(v_ffff, 31);

				tmp_flags = _mm256_cmpeq_epi32(fnav_vld0_7, v_u32_one);

				tmp_flags = _mm256_and_si256(tmp_flags, fdir_flags);

				mbuf_flags = _mm256_or_si256(mbuf_flags, tmp_flags);

				rx_pkts[i + 0]->hash.fdir.hi = desc[0].wb.fd_filter_id;
				rx_pkts[i + 1]->hash.fdir.hi = desc[1].wb.fd_filter_id;
				rx_pkts[i + 2]->hash.fdir.hi = desc[2].wb.fd_filter_id;
				rx_pkts[i + 3]->hash.fdir.hi = desc[3].wb.fd_filter_id;
				rx_pkts[i + 4]->hash.fdir.hi = desc[4].wb.fd_filter_id;
				rx_pkts[i + 5]->hash.fdir.hi = desc[5].wb.fd_filter_id;
				rx_pkts[i + 6]->hash.fdir.hi = desc[6].wb.fd_filter_id;
				rx_pkts[i + 7]->hash.fdir.hi = desc[7].wb.fd_filter_id;
			}
#endif
		}

		RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, ol_flags) !=
				 offsetof(struct rte_mbuf, rearm_data) + 8);
		RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rx_descriptor_fields1) !=
				 offsetof(struct rte_mbuf, rearm_data) + 16);
		RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf, rearm_data) !=
				 RTE_ALIGN(offsetof(struct rte_mbuf, rearm_data), 16));

		__m256i rearm_arr[8];

		rearm_arr[6] = _mm256_blend_epi32(mbuf_init, _mm256_slli_si256(mbuf_flags, 8), 4);
		rearm_arr[4] = _mm256_blend_epi32(mbuf_init, _mm256_slli_si256(mbuf_flags, 4), 4);
		rearm_arr[2] = _mm256_blend_epi32(mbuf_init, mbuf_flags, 4);
		rearm_arr[0] = _mm256_blend_epi32(mbuf_init, _mm256_srli_si256(mbuf_flags, 4), 4);

		rearm_arr[6] = _mm256_permute2f128_si256(rearm_arr[6], mbufs6_7, 0x20);
		rearm_arr[4] = _mm256_permute2f128_si256(rearm_arr[4], mbufs4_5, 0x20);
		rearm_arr[2] = _mm256_permute2f128_si256(rearm_arr[2], mbufs2_3, 0x20);
		rearm_arr[0] = _mm256_permute2f128_si256(rearm_arr[0], mbufs0_1, 0x20);

		_mm256_storeu_si256((__m256i *)&rx_pkts[i + 6]->rearm_data, rearm_arr[6]);
		_mm256_storeu_si256((__m256i *)&rx_pkts[i + 4]->rearm_data, rearm_arr[4]);
		_mm256_storeu_si256((__m256i *)&rx_pkts[i + 2]->rearm_data, rearm_arr[2]);
		_mm256_storeu_si256((__m256i *)&rx_pkts[i + 0]->rearm_data, rearm_arr[0]);

		const __m256i tmp_mbuf_flags =
			_mm256_castsi128_si256(_mm256_extracti128_si256(mbuf_flags, 1));

		rearm_arr[7] =
			_mm256_blend_epi32(mbuf_init, _mm256_slli_si256(tmp_mbuf_flags, 8), 4);
		rearm_arr[5] =
			_mm256_blend_epi32(mbuf_init, _mm256_slli_si256(tmp_mbuf_flags, 4), 4);
		rearm_arr[3] =
			_mm256_blend_epi32(mbuf_init, tmp_mbuf_flags, 4);
		rearm_arr[1] =
			_mm256_blend_epi32(mbuf_init, _mm256_srli_si256(tmp_mbuf_flags, 4), 4);

		rearm_arr[7] = _mm256_blend_epi32(rearm_arr[7], mbufs6_7, 0XF0);
		rearm_arr[5] = _mm256_blend_epi32(rearm_arr[5], mbufs4_5, 0XF0);
		rearm_arr[3] = _mm256_blend_epi32(rearm_arr[3], mbufs2_3, 0XF0);
		rearm_arr[1] = _mm256_blend_epi32(rearm_arr[1], mbufs0_1, 0XF0);

		_mm256_storeu_si256((__m256i *)&rx_pkts[i + 7]->rearm_data, rearm_arr[7]);
		_mm256_storeu_si256((__m256i *)&rx_pkts[i + 5]->rearm_data, rearm_arr[5]);
		_mm256_storeu_si256((__m256i *)&rx_pkts[i + 3]->rearm_data, rearm_arr[3]);
		_mm256_storeu_si256((__m256i *)&rx_pkts[i + 1]->rearm_data, rearm_arr[1]);

		if (umbcast_flags != NULL) {
			const __m256i umbcast_mask =
					_mm256_set1_epi32(SXE2_RX_DESC_STATUS_UMBCAST_MASK);
			__m256i umbcast_bits_256 = _mm256_and_si256(staterrs0_7,
								    umbcast_mask);

			umbcast_bits_256 = _mm256_srli_epi32(umbcast_bits_256, 24);

			__m128i umbcast_bits_128 = _mm_packs_epi32
							(_mm256_castsi256_si128(umbcast_bits_256),
							 _mm256_extractf128_si256
								(umbcast_bits_256, 1));

			umbcast_bits_128 = _mm_shuffle_epi8(umbcast_bits_128, eop_shuf_mask);

			*(uint64_t *)umbcast_flags = _mm_cvtsi128_si64(umbcast_bits_128);
			umbcast_flags += SXE2_RX_NUM_PER_LOOP_AVX;
		}

		if (split_rxe_flags != NULL) {
			const __m256i eop_rxe_mask = _mm256_set1_epi32
							(SXE2_RX_DESC_STATUS_EOP_MASK |
							 SXE2_RX_DESC_ERROR_RXE_MASK |
							 SXE2_RX_DESC_ERROR_OVERSIZE_MASK);

			const __m128i eop_mask_128 = _mm_set1_epi16(SXE2_RX_DESC_STATUS_EOP_MASK);
			const __m128i rxe_mask_128 = _mm_set1_epi16(SXE2_RX_DESC_ERROR_RXE_MASK |
					SXE2_RX_DESC_ERROR_OVERSIZE_MASK);

			const __m256i tmp_stats = _mm256_and_si256(staterrs0_7, eop_rxe_mask);

			const __m128i eop_rxe_bits = _mm_packs_epi32
							(_mm256_castsi256_si128(tmp_stats),
							 _mm256_extractf128_si256(tmp_stats, 1));

			__m128i not_eop_bits = _mm_andnot_si128(eop_rxe_bits, eop_mask_128);

			not_eop_bits = _mm_or_si128
					(not_eop_bits,
					 _mm_srli_epi16
					(_mm_and_si128(eop_rxe_bits, rxe_mask_128),
					7));

			not_eop_bits = _mm_shuffle_epi8(not_eop_bits, eop_shuf_mask);

			*(uint64_t *)split_rxe_flags = _mm_cvtsi128_si64(not_eop_bits);
			split_rxe_flags += SXE2_RX_NUM_PER_LOOP_AVX;
		}

		staterrs0_7 = _mm256_and_si256(staterrs0_7, dd_mask);

		staterrs0_7 = _mm256_packs_epi32(staterrs0_7, _mm256_setzero_si256());
		bit_num = rte_popcount64
				(_mm_cvtsi128_si64(_mm256_extracti128_si256(staterrs0_7, 1)));
		bit_num += rte_popcount64
				(_mm_cvtsi128_si64(_mm256_castsi256_si128(staterrs0_7)));

		done_num += bit_num;

		if (bit_num != SXE2_RX_NUM_PER_LOOP_AVX)
			break;
	}

	rxq->processing_idx += done_num;
	rxq->processing_idx &= (rxq->ring_depth - 1);
	if ((1 == (rxq->processing_idx & 1)) && done_num > 1) {
		rxq->processing_idx--;
		done_num--;
	}
	rxq->realloc_num     += done_num;

l_end:
	PMD_LOG_DEBUG(RX, "port_id=%u queue_id=%u last_id=%u recv_pkts=%d",
		rxq->port_id, rxq->queue_id, rxq->processing_idx, done_num);
	return done_num;
}

static __rte_always_inline uint16_t
sxe2_rx_pkts_scattered_batch_vec_avx2(struct sxe2_rx_queue *rxq, struct rte_mbuf **rx_pkts,
		uint16_t nb_pkts, bool do_offload)
{
	uint8_t split_rxe_flags[SXE2_RX_PKTS_BURST_BATCH_NUM_VEC] = {0};
	uint8_t umbcast_flags[SXE2_RX_PKTS_BURST_BATCH_NUM_VEC] = {0};
	uint16_t rx_done_num;
	uint16_t rx_pkt_done_num;

	rx_pkt_done_num = 0;

	rx_done_num = sxe2_rx_pkts_common_vec_avx2(rxq, rx_pkts, nb_pkts,
			split_rxe_flags, umbcast_flags, do_offload);
	if (rx_done_num == 0)
		goto l_end;

	rx_pkt_done_num += sxe2_rx_pkts_refactor(rxq, &rx_pkts[rx_pkt_done_num],
			rx_done_num - rx_pkt_done_num, &split_rxe_flags[rx_pkt_done_num],
			&umbcast_flags[rx_pkt_done_num]);

l_end:
	return rx_pkt_done_num;
}

static __rte_always_inline uint16_t
sxe2_rx_pkts_scattered_common_vec_avx2(struct sxe2_rx_queue *rxq, struct rte_mbuf **rx_pkts,
		uint16_t nb_pkts, bool do_offload)
{
	uint16_t done_num = 0;
	uint16_t once_num;

	while (nb_pkts > SXE2_RX_PKTS_BURST_BATCH_NUM) {
		once_num =
		sxe2_rx_pkts_scattered_batch_vec_avx2(rxq,
						      rx_pkts + done_num,
						      SXE2_RX_PKTS_BURST_BATCH_NUM,
						      do_offload);
		done_num += once_num;
		nb_pkts -= once_num;
		if (once_num < SXE2_RX_PKTS_BURST_BATCH_NUM)
			goto l_end;
	}

	done_num += sxe2_rx_pkts_scattered_batch_vec_avx2(rxq,
			rx_pkts + done_num, nb_pkts, do_offload);
l_end:
	return done_num;
}

uint16_t sxe2_rx_pkts_scattered_vec_avx2(void *rx_queue,
		struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
{
	return sxe2_rx_pkts_scattered_common_vec_avx2(rx_queue,
			rx_pkts, nb_pkts, false);
}

uint16_t sxe2_rx_pkts_scattered_vec_avx2_offload(void *rx_queue,
		struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
{
	return sxe2_rx_pkts_scattered_common_vec_avx2(rx_queue,
			rx_pkts, nb_pkts, true);
}
