Receiving Duplicate packets in receiver side when played through DPDK

Question

I have a scenario where i am trying to send packets from a pcap file using dpdk to other other port connected by loopback cable,

Issue: Getting Duplicate packets instead of different ones on receiver side

C-Program:

#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <netinet/in.h>
#include <time.h>
#include <pcap.h>

#include <inttypes.h>
#include <rte_cycles.h>
#include <rte_lcore.h>
#include <rte_mbuf.h>

#include <rte_eal.h>
#include <rte_ethdev.h>

#define RX_RING_SIZE 1024
#define TX_RING_SIZE 1024
#define NUM_MBUFS 8191
#define MBUF_CACHE_SIZE 250
#define BURST_SIZE 32
#define NB_MBUF (1024 * 8)

#define ETH_PORT_ID 0 // Modify this according to your environment

long long int pcap_time_concat(const struct pcap_pkthdr *header) {
    // Implement your concat function
    // This function is assumed to concatenate timestamp seconds and microseconds
    // into a single long long int value
    return header->ts.tv_sec * 1000000000LL + header->ts.tv_usec;
}

long long int system_time_concat(long sec, long nsec) {
    // Convert seconds to nanoseconds and add the nanoseconds
    long long int result = sec * 1000000000LL + nsec;
    return result;
}

static inline int
port_init(uint16_t port, struct rte_mempool *mbuf_pool)
{
    struct rte_eth_conf port_conf;
    const uint16_t rx_rings = 1, tx_rings = 1;
    uint16_t nb_rxd = RX_RING_SIZE;
    uint16_t nb_txd = TX_RING_SIZE;
    int retval;
    uint16_t q;
    struct rte_eth_dev_info dev_info;
    struct rte_eth_txconf txconf;

    if (!rte_eth_dev_is_valid_port(port))
        return -1;

    memset(&port_conf, 0, sizeof(struct rte_eth_conf));

    retval = rte_eth_dev_info_get(port, &dev_info);
    if (retval != 0)
    {
        printf("Error during getting device (port %u) info: %s\n",
               port, strerror(-retval));
        return retval;
    }

    if (dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)
        port_conf.txmode.offloads |=
            RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE;

    /* Configure the Ethernet device. */
    retval = rte_eth_dev_configure(port, rx_rings, tx_rings, &port_conf);
    if (retval != 0)
        return retval;

    retval = rte_eth_dev_set_mtu(port, 9000);
    if (retval != 0)
        return retval;

    retval = rte_eth_dev_adjust_nb_rx_tx_desc(port, &nb_rxd, &nb_txd);
    if (retval != 0)
        return retval;

    /* Allocate and set up 1 RX queue per Ethernet port. */
    for (q = 0; q < rx_rings; q++)
    {
        retval = rte_eth_rx_queue_setup(port, q, nb_rxd,
                                        rte_eth_dev_socket_id(port), NULL, mbuf_pool);
        if (retval < 0)
            return retval;
    }

    txconf = dev_info.default_txconf;
    txconf.offloads = port_conf.txmode.offloads;
    /* Allocate and set up 1 TX queue per Ethernet port. */
    for (q = 0; q < tx_rings; q++)
    {
        retval = rte_eth_tx_queue_setup(port, q, nb_txd,
                                        rte_eth_dev_socket_id(port), &txconf);
        if (retval < 0)
            return retval;
    }

    /* Starting Ethernet port. 8< */
    retval = rte_eth_dev_start(port);
    /* >8 End of starting of ethernet port. */
    if (retval < 0)
        return retval;

    /* Display the port MAC address. */
    struct rte_ether_addr addr;
    retval = rte_eth_macaddr_get(port, &addr);
    if (retval != 0)
        return retval;

    printf("Port %u MAC: %02" PRIx8 " %02" PRIx8 " %02" PRIx8
           " %02" PRIx8 " %02" PRIx8 " %02" PRIx8 "\n",
           port, RTE_ETHER_ADDR_BYTES(&addr));

    /* Enable RX in promiscuous mode for the Ethernet device. */
    retval = rte_eth_promiscuous_enable(port);
    /* End of setting RX port in promiscuous mode. */
    if (retval != 0)
        return retval;

    return 0;
}

static struct rte_mempool *mbuf_pool1;

int main(int argc, char *argv[])
{
    // Initialize DPDK
    int ret = rte_eal_init(argc, argv);
    uint16_t portid;
    struct rte_mempool *mbuf_pool;
    struct timespec system_time;
    if (ret < 0)
    {
        rte_exit(EXIT_FAILURE, "Error initializing DPDK\n");
    }

    // Open Ethernet device
    if (rte_eth_dev_count_avail() == 0)
    {
        rte_exit(EXIT_FAILURE, "No Ethernet ports found\n");
    }

    // Assuming only one port is available
    mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL", NUM_MBUFS * 1,
                                        MBUF_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
    RTE_ETH_FOREACH_DEV(portid)
    if (port_init(portid, mbuf_pool) != 0)
        rte_exit(EXIT_FAILURE, "Cannot init port %" PRIu16 "\n",
                 portid);

    pcap_t *pcap;
    char errbuf[PCAP_ERRBUF_SIZE];

    // Open pcap file
    // pcap = pcap_open_offline("c2_15scs_4_ant_256_frame_128.pcap", errbuf);
    // pcap = pcap_open_offline("TM3.1a_1CC_CC0_8ant_1Slot.pcap", errbuf);
    pcap_t *handle = pcap_open_offline_with_tstamp_precision(argv[1], PCAP_TSTAMP_PRECISION_NANO, errbuf);
    // pcap_t *handle = pcap_open_offline_with_tstamp_precision("c2_15scs_4_ant_256_frame_128.pcap", PCAP_TSTAMP_PRECISION_NANO, errbuf);
    if (handle == NULL) {
        fprintf(stderr, "pcap_open_offline_with_tstamp_precision failed: %s\n", errbuf);
        return 1;
    }

    // Main loop for reading packets and sending them
    const struct pcap_pkthdr header;
    const u_char *packet;
    uint8_t tx_port = portid;
    struct rte_mbuf *mbufs[1];
    mbuf_pool1 = rte_pktmbuf_pool_create("MBUF_POOL1", NB_MBUF, 32,
                                         0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_eth_dev_socket_id(0));
    int packetCount = 1;
    int readstat = 1;
    long long int packet_epoch_time = 0;
    long long int emul_ref_time = -1;
    long long int emul_ref_time_tod = -1;
    long long int epoch_ref_time = -1;
    long long int delta_time = 0;
    long long int current_emul_time = -1;

    packet = pcap_next(handle, &header);
    if (packet == NULL) {
        printf("End of file reached or error occurred while reading packet.\n");
    }
    packet_epoch_time = pcap_time_concat(&header);

    for (int i = 0; i > -1; i++) 
    {

        if (packet == NULL) {
            printf("End of file reached or error occurred while reading packet.\n");
            break;
        }
        if(i == 0)
        {
            clock_gettime(CLOCK_REALTIME, &system_time);
            emul_ref_time = system_time_concat(system_time.tv_sec, system_time.tv_nsec);
            // printf("\n emul_ref_time: %lld\n", emul_ref_time);
            mbufs[0] = rte_pktmbuf_alloc(mbuf_pool1);
            if (mbufs[0] == NULL)
            {
                rte_exit(EXIT_FAILURE, "Failed to allocate mbuf\n");
            }
            rte_memcpy(rte_pktmbuf_mtod(mbufs[0], void *), packet, header.len);
            // rte_memcpy((char*)mbufs[0]->buf_addr, packet, header.len);
            // printf("\npacket: %p\n", (char *)packet);
            mbufs[0]->data_len = header.caplen;
            mbufs[0]->pkt_len = header.len;
            rte_pktmbuf_dump(stdout, mbufs[0], rte_pktmbuf_pkt_len(mbufs[0]));
            int sent = rte_eth_tx_burst(0, 0, mbufs, 1);
            if(!sent)
            {
                rte_exit(EXIT_FAILURE, "Failed to Send Packet\n");
            }
            epoch_ref_time = pcap_time_concat(&header);
            // printf("\n epoch_ref_time: %lld\n", epoch_ref_time);
            packet = pcap_next(handle, &header);
            packet_epoch_time = pcap_time_concat(&header);
            // printf("\n packet_epoch_time: %lld\n", packet_epoch_time);
            rte_pktmbuf_free(mbufs[0]);
        }
        if ((packet_epoch_time - 5100) <= current_emul_time && current_emul_time <= (packet_epoch_time + 5100)) // Transmit Subsequent packets of the respective port when matches time window.
        {
            mbufs[0] = rte_pktmbuf_alloc(mbuf_pool1);
            if (mbufs[0] == NULL)
            {
                rte_exit(EXIT_FAILURE, "Failed to allocate mbuf\n");
            }
            rte_memcpy(rte_pktmbuf_mtod(mbufs[0], void *), packet, header.len);
            // rte_memcpy((char*)mbufs[0]->buf_addr, packet, header.len);
            // printf("\npacket: %p\n", (char *)packet);
            mbufs[0]->data_len = header.caplen;
            mbufs[0]->pkt_len = header.len;
            rte_pktmbuf_dump(stdout, mbufs[0], rte_pktmbuf_pkt_len(mbufs[0]));
            int sent = rte_eth_tx_burst(0, 0, mbufs, 1);
            if(!sent)
            {
                rte_exit(EXIT_FAILURE, "Failed to Send Packet\n");
            }
            packetCount++;
            printf("%d packet sent\n", packetCount);
            packet = pcap_next(handle, &header);
            packet_epoch_time = pcap_time_concat(&header);
            rte_pktmbuf_free(mbufs[0]);
            break;
        }
        else if (current_emul_time > (packet_epoch_time + 5100))
        {
            mbufs[0] = rte_pktmbuf_alloc(mbuf_pool1);
            if (mbufs[0] == NULL)
            {
                rte_exit(EXIT_FAILURE, "Failed to allocate mbuf\n");
            }
            rte_memcpy(rte_pktmbuf_mtod(mbufs[0], void *), packet, header.len);
            // rte_memcpy((char*)mbufs[0]->buf_addr, packet, header.len);
            // printf("\npacket: %p\n", (char *)packet);
            mbufs[0]->data_len = header.caplen;
            mbufs[0]->pkt_len = header.len;
            rte_pktmbuf_dump(stdout, mbufs[0], rte_pktmbuf_pkt_len(mbufs[0]));
            int sent = rte_eth_tx_burst(0, 0, mbufs, 1);
            if(!sent)
            {
                rte_exit(EXIT_FAILURE, "Failed to Send Packet\n");
            }
            packetCount++;
            printf("%d frag packet sent, diff_time: %lld\n", packetCount, current_emul_time - (packet_epoch_time + 5100));
            packet = pcap_next(handle, &header);
            packet_epoch_time = pcap_time_concat(&header);
            rte_pktmbuf_free(mbufs[0]);
            break;
        }
        if (epoch_ref_time != -1)
        {
            //gettimeofday(&loop_time, NULL);
            clock_gettime(CLOCK_REALTIME, &system_time);
            long long int temp_current_time = system_time_concat(system_time.tv_sec, system_time.tv_nsec);
            //long long int temp_current_time_tod = system_time_concat(loop_time.tv_sec, loop_time.tv_usec);
            // delta_time = (loop_time.tv_usec - emul_ref_time)*1000;
            //printf("\n system_time: %lld.%lld\n", system_time.tv_sec, system_time.tv_nsec);
            // printf("\n temp_current_time: %lld\n", temp_current_time);
            //printf("\n temp_current_time_tod: %lld\n", temp_current_time_tod);
            delta_time = temp_current_time - emul_ref_time;
            // printf("\n delta_time: %lld\n", delta_time);
            current_emul_time = epoch_ref_time + delta_time;
            // printf("\n current_emul_time: %lld\n", current_emul_time);
        }
        
    }
    pcap_close(handle);
    rte_eth_dev_stop(0);
    rte_eth_dev_close(0);
    rte_eal_cleanup();

    return 0;
}

In the above code i just tested the sending two packets and see in the receiver side

Transmitting Side: rte_pktmbuf_dump of the mbufs[0] prints two different packets

Receiver Side: tcpdump of incoming packets Getting the second packet transmitted twice on receiver side and there is no first packet at all

Please help me to figure out the issue, if i add delay in the for loop then i don`t see the duplicate packets