1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25 
  26 #include <sys/mac_provider.h>
  27 #include <sys/nxge/nxge_impl.h>
  28 #include <sys/nxge/nxge_hio.h>
  29 #include <npi_tx_wr64.h>
  30 
  31 /* Software LSO required header files */
  32 #include <netinet/tcp.h>
  33 #include <inet/ip_impl.h>
  34 #include <inet/tcp.h>
  35 
  36 extern uint64_t mac_pkt_hash(uint_t, mblk_t *mp, uint8_t policy,
  37     boolean_t is_outbound);
  38 
  39 static mblk_t *nxge_lso_eliminate(mblk_t *);
  40 static mblk_t *nxge_do_softlso(mblk_t *mp, uint32_t mss);
  41 static void nxge_lso_info_get(mblk_t *, uint32_t *, uint32_t *);
  42 static void nxge_hcksum_retrieve(mblk_t *,
  43     uint32_t *, uint32_t *, uint32_t *,
  44     uint32_t *, uint32_t *);
  45 static uint32_t nxge_csgen(uint16_t *, int);
  46 
  47 extern uint32_t         nxge_reclaim_pending;
  48 extern uint32_t         nxge_bcopy_thresh;
  49 extern uint32_t         nxge_dvma_thresh;
  50 extern uint32_t         nxge_dma_stream_thresh;
  51 extern uint32_t         nxge_tx_minfree;
  52 extern uint32_t         nxge_tx_intr_thres;
  53 extern uint32_t         nxge_tx_max_gathers;
  54 extern uint32_t         nxge_tx_tiny_pack;
  55 extern uint32_t         nxge_tx_use_bcopy;
  56 extern nxge_tx_mode_t   nxge_tx_scheme;
  57 uint32_t                nxge_lso_kick_cnt = 2;
  58 
  59 
  60 void
  61 nxge_tx_ring_task(void *arg)
  62 {
  63         p_tx_ring_t     ring = (p_tx_ring_t)arg;
  64 
  65         ASSERT(ring->tx_ring_handle != NULL);
  66 
  67         MUTEX_ENTER(&ring->lock);
  68         (void) nxge_txdma_reclaim(ring->nxgep, ring, 0);
  69         MUTEX_EXIT(&ring->lock);
  70 
  71         if (!ring->tx_ring_offline) {
  72                 mac_tx_ring_update(ring->nxgep->mach, ring->tx_ring_handle);
  73         }
  74 }
  75 
  76 static void
  77 nxge_tx_ring_dispatch(p_tx_ring_t ring)
  78 {
  79         /*
  80          * Kick the ring task to reclaim some buffers.
  81          */
  82         (void) ddi_taskq_dispatch(ring->taskq,
  83             nxge_tx_ring_task, (void *)ring, DDI_SLEEP);
  84 }
  85 
  86 mblk_t *
  87 nxge_tx_ring_send(void *arg, mblk_t *mp)
  88 {
  89         p_nxge_ring_handle_t    nrhp = (p_nxge_ring_handle_t)arg;
  90         p_nxge_t                nxgep;
  91         p_tx_ring_t             tx_ring_p;
  92         int                     status, channel;
  93 
  94         ASSERT(nrhp != NULL);
  95         nxgep = nrhp->nxgep;
  96         channel = nxgep->pt_config.hw_config.tdc.start + nrhp->index;
  97         tx_ring_p = nxgep->tx_rings->rings[channel];
  98 
  99         /*
 100          * We may be in a transition from offlined DMA to onlined
 101          * DMA.
 102          */
 103         if (tx_ring_p == NULL) {
 104                 ASSERT(tx_ring_p != NULL);
 105                 freemsg(mp);
 106                 return ((mblk_t *)NULL);
 107         }
 108 
 109         /*
 110          * Valid DMA?
 111          */
 112         ASSERT(nxgep == tx_ring_p->nxgep);
 113 
 114         /*
 115          * Make sure DMA is not offlined.
 116          */
 117         if (isLDOMservice(nxgep) && tx_ring_p->tx_ring_offline) {
 118                 ASSERT(!tx_ring_p->tx_ring_offline);
 119                 freemsg(mp);
 120                 return ((mblk_t *)NULL);
 121         }
 122 
 123         /*
 124          * Transmit the packet.
 125          */
 126         status = nxge_start(nxgep, tx_ring_p, mp);
 127         if (status) {
 128                 nxge_tx_ring_dispatch(tx_ring_p);
 129                 return (mp);
 130         }
 131 
 132         return ((mblk_t *)NULL);
 133 }
 134 
 135 int
 136 nxge_start(p_nxge_t nxgep, p_tx_ring_t tx_ring_p, p_mblk_t mp)
 137 {
 138         int                     dma_status, status = 0;
 139         p_tx_desc_t             tx_desc_ring_vp;
 140         npi_handle_t            npi_desc_handle;
 141         nxge_os_dma_handle_t    tx_desc_dma_handle;
 142         p_tx_desc_t             tx_desc_p;
 143         p_tx_msg_t              tx_msg_ring;
 144         p_tx_msg_t              tx_msg_p;
 145         tx_desc_t               tx_desc, *tmp_desc_p;
 146         tx_desc_t               sop_tx_desc, *sop_tx_desc_p;
 147         p_tx_pkt_header_t       hdrp;
 148         tx_pkt_hdr_all_t        tmp_hdrp;
 149         p_tx_pkt_hdr_all_t      pkthdrp;
 150         uint8_t                 npads = 0;
 151         uint64_t                dma_ioaddr;
 152         uint32_t                dma_flags;
 153         int                     last_bidx;
 154         uint8_t                 *b_rptr;
 155         caddr_t                 kaddr;
 156         uint32_t                nmblks;
 157         uint32_t                ngathers;
 158         uint32_t                clen;
 159         int                     len;
 160         uint32_t                pkt_len, pack_len, min_len;
 161         uint32_t                bcopy_thresh;
 162         int                     i, cur_index, sop_index;
 163         uint16_t                tail_index;
 164         boolean_t               tail_wrap = B_FALSE;
 165         nxge_dma_common_t       desc_area;
 166         nxge_os_dma_handle_t    dma_handle;
 167         ddi_dma_cookie_t        dma_cookie;
 168         npi_handle_t            npi_handle;
 169         p_mblk_t                nmp;
 170         p_mblk_t                t_mp;
 171         uint32_t                ncookies;
 172         boolean_t               good_packet;
 173         boolean_t               mark_mode = B_FALSE;
 174         p_nxge_stats_t          statsp;
 175         p_nxge_tx_ring_stats_t tdc_stats;
 176         t_uscalar_t             start_offset = 0;
 177         t_uscalar_t             stuff_offset = 0;
 178         t_uscalar_t             end_offset = 0;
 179         t_uscalar_t             value = 0;
 180         t_uscalar_t             cksum_flags = 0;
 181         boolean_t               cksum_on = B_FALSE;
 182         uint32_t                boff = 0;
 183         uint64_t                tot_xfer_len = 0;
 184         boolean_t               header_set = B_FALSE;
 185 #ifdef NXGE_DEBUG
 186         p_tx_desc_t             tx_desc_ring_pp;
 187         p_tx_desc_t             tx_desc_pp;
 188         tx_desc_t               *save_desc_p;
 189         int                     dump_len;
 190         int                     sad_len;
 191         uint64_t                sad;
 192         int                     xfer_len;
 193         uint32_t                msgsize;
 194 #endif
 195         p_mblk_t                mp_chain = NULL;
 196         boolean_t               is_lso = B_FALSE;
 197         boolean_t               lso_again;
 198         int                     cur_index_lso;
 199         p_mblk_t                nmp_lso_save;
 200         uint32_t                lso_ngathers;
 201         boolean_t               lso_tail_wrap = B_FALSE;
 202 
 203         NXGE_DEBUG_MSG((nxgep, TX_CTL,
 204             "==> nxge_start: tx dma channel %d", tx_ring_p->tdc));
 205         NXGE_DEBUG_MSG((nxgep, TX_CTL,
 206             "==> nxge_start: Starting tdc %d desc pending %d",
 207             tx_ring_p->tdc, tx_ring_p->descs_pending));
 208 
 209         statsp = nxgep->statsp;
 210 
 211         if (!isLDOMguest(nxgep)) {
 212                 switch (nxgep->mac.portmode) {
 213                 default:
 214                         if (nxgep->statsp->port_stats.lb_mode ==
 215                             nxge_lb_normal) {
 216                                 if (!statsp->mac_stats.link_up) {
 217                                         freemsg(mp);
 218                                         NXGE_DEBUG_MSG((nxgep, TX_CTL,
 219                                             "==> nxge_start: "
 220                                             "link not up"));
 221                                         goto nxge_start_fail1;
 222                                 }
 223                         }
 224                         break;
 225                 case PORT_10G_FIBER:
 226                         /*
 227                          * For the following modes, check the link status
 228                          * before sending the packet out:
 229                          * nxge_lb_normal,
 230                          * nxge_lb_ext10g,
 231                          * nxge_lb_ext1000,
 232                          * nxge_lb_ext100,
 233                          * nxge_lb_ext10.
 234                          */
 235                         if (nxgep->statsp->port_stats.lb_mode <
 236                             nxge_lb_phy10g) {
 237                                 if (!statsp->mac_stats.link_up) {
 238                                         freemsg(mp);
 239                                         NXGE_DEBUG_MSG((nxgep, TX_CTL,
 240                                             "==> nxge_start: "
 241                                             "link not up"));
 242                                         goto nxge_start_fail1;
 243                                 }
 244                         }
 245                         break;
 246                 }
 247         }
 248 
 249         if ((!(nxgep->drv_state & STATE_HW_INITIALIZED)) ||
 250             (nxgep->nxge_mac_state != NXGE_MAC_STARTED)) {
 251                 NXGE_DEBUG_MSG((nxgep, TX_CTL,
 252                     "==> nxge_start: hardware not initialized or stopped"));
 253                 freemsg(mp);
 254                 goto nxge_start_fail1;
 255         }
 256 
 257         if (nxgep->soft_lso_enable) {
 258                 mp_chain = nxge_lso_eliminate(mp);
 259                 NXGE_DEBUG_MSG((nxgep, TX_CTL,
 260                     "==> nxge_start(0): LSO mp $%p mp_chain $%p",
 261                     mp, mp_chain));
 262                 if (mp_chain == NULL) {
 263                         NXGE_ERROR_MSG((nxgep, TX_CTL,
 264                             "==> nxge_send(0): NULL mp_chain $%p != mp $%p",
 265                             mp_chain, mp));
 266                         goto nxge_start_fail1;
 267                 }
 268                 if (mp_chain != mp) {
 269                         NXGE_DEBUG_MSG((nxgep, TX_CTL,
 270                             "==> nxge_send(1): IS LSO mp_chain $%p != mp $%p",
 271                             mp_chain, mp));
 272                         is_lso = B_TRUE;
 273                         mp = mp_chain;
 274                         mp_chain = mp_chain->b_next;
 275                         mp->b_next = NULL;
 276                 }
 277         }
 278 
 279         mac_hcksum_get(mp, &start_offset, &stuff_offset, &end_offset,
 280             &value, &cksum_flags);
 281         if (!NXGE_IS_VLAN_PACKET(mp->b_rptr)) {
 282                 start_offset += sizeof (ether_header_t);
 283                 stuff_offset += sizeof (ether_header_t);
 284         } else {
 285                 start_offset += sizeof (struct ether_vlan_header);
 286                 stuff_offset += sizeof (struct ether_vlan_header);
 287         }
 288 
 289         if (cksum_flags & HCK_PARTIALCKSUM) {
 290                 NXGE_DEBUG_MSG((nxgep, TX_CTL,
 291                     "==> nxge_start: mp $%p len %d "
 292                     "cksum_flags 0x%x (partial checksum) ",
 293                     mp, MBLKL(mp), cksum_flags));
 294                 cksum_on = B_TRUE;
 295         }
 296 
 297         pkthdrp = (p_tx_pkt_hdr_all_t)&tmp_hdrp;
 298         pkthdrp->reserved = 0;
 299         tmp_hdrp.pkthdr.value = 0;
 300         nxge_fill_tx_hdr(mp, B_FALSE, cksum_on,
 301             0, 0, pkthdrp,
 302             start_offset, stuff_offset);
 303 
 304         lso_again = B_FALSE;
 305         lso_ngathers = 0;
 306 
 307         MUTEX_ENTER(&tx_ring_p->lock);
 308 
 309         if (isLDOMservice(nxgep)) {
 310                 tx_ring_p->tx_ring_busy = B_TRUE;
 311                 if (tx_ring_p->tx_ring_offline) {
 312                         freemsg(mp);
 313                         tx_ring_p->tx_ring_busy = B_FALSE;
 314                         (void) atomic_swap_32(&tx_ring_p->tx_ring_offline,
 315                             NXGE_TX_RING_OFFLINED);
 316                         MUTEX_EXIT(&tx_ring_p->lock);
 317                         return (status);
 318                 }
 319         }
 320 
 321         cur_index_lso = tx_ring_p->wr_index;
 322         lso_tail_wrap = tx_ring_p->wr_index_wrap;
 323 start_again:
 324         ngathers = 0;
 325         sop_index = tx_ring_p->wr_index;
 326 #ifdef  NXGE_DEBUG
 327         if (tx_ring_p->descs_pending) {
 328                 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
 329                     "desc pending %d ", tx_ring_p->descs_pending));
 330         }
 331 
 332         dump_len = (int)(MBLKL(mp));
 333         dump_len = (dump_len > 128) ? 128: dump_len;
 334 
 335         NXGE_DEBUG_MSG((nxgep, TX_CTL,
 336             "==> nxge_start: tdc %d: dumping ...: b_rptr $%p "
 337             "(Before header reserve: ORIGINAL LEN %d)",
 338             tx_ring_p->tdc,
 339             mp->b_rptr,
 340             dump_len));
 341 
 342         NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: dump packets "
 343             "(IP ORIGINAL b_rptr $%p): %s", mp->b_rptr,
 344             nxge_dump_packet((char *)mp->b_rptr, dump_len)));
 345 #endif
 346 
 347         tdc_stats = tx_ring_p->tdc_stats;
 348         mark_mode = (tx_ring_p->descs_pending &&
 349             (((int)tx_ring_p->tx_ring_size - (int)tx_ring_p->descs_pending) <
 350             (int)nxge_tx_minfree));
 351 
 352         NXGE_DEBUG_MSG((nxgep, TX_CTL,
 353             "TX Descriptor ring is channel %d mark mode %d",
 354             tx_ring_p->tdc, mark_mode));
 355 
 356         if ((tx_ring_p->descs_pending + lso_ngathers) >= nxge_reclaim_pending) {
 357                 if (!nxge_txdma_reclaim(nxgep, tx_ring_p,
 358                     (nxge_tx_minfree + lso_ngathers))) {
 359                         NXGE_DEBUG_MSG((nxgep, TX_CTL,
 360                             "TX Descriptor ring is full: channel %d",
 361                             tx_ring_p->tdc));
 362                         NXGE_DEBUG_MSG((nxgep, TX_CTL,
 363                             "TX Descriptor ring is full: channel %d",
 364                             tx_ring_p->tdc));
 365                         if (is_lso) {
 366                                 /*
 367                                  * free the current mp and mp_chain if not FULL.
 368                                  */
 369                                 tdc_stats->tx_no_desc++;
 370                                 NXGE_DEBUG_MSG((nxgep, TX_CTL,
 371                                     "LSO packet: TX Descriptor ring is full: "
 372                                     "channel %d",
 373                                     tx_ring_p->tdc));
 374                                 goto nxge_start_fail_lso;
 375                         } else {
 376                                 (void) cas32((uint32_t *)&tx_ring_p->queueing,
 377                                     0, 1);
 378                                 tdc_stats->tx_no_desc++;
 379 
 380                                 if (isLDOMservice(nxgep)) {
 381                                         tx_ring_p->tx_ring_busy = B_FALSE;
 382                                         if (tx_ring_p->tx_ring_offline) {
 383                                                 (void) atomic_swap_32(
 384                                                     &tx_ring_p->tx_ring_offline,
 385                                                     NXGE_TX_RING_OFFLINED);
 386                                         }
 387                                 }
 388 
 389                                 MUTEX_EXIT(&tx_ring_p->lock);
 390                                 status = 1;
 391                                 goto nxge_start_fail1;
 392                         }
 393                 }
 394         }
 395 
 396         nmp = mp;
 397         i = sop_index = tx_ring_p->wr_index;
 398         nmblks = 0;
 399         ngathers = 0;
 400         pkt_len = 0;
 401         pack_len = 0;
 402         clen = 0;
 403         last_bidx = -1;
 404         good_packet = B_TRUE;
 405 
 406         desc_area = tx_ring_p->tdc_desc;
 407         npi_handle = desc_area.npi_handle;
 408         npi_desc_handle.regh = (nxge_os_acc_handle_t)
 409             DMA_COMMON_ACC_HANDLE(desc_area);
 410         tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area);
 411         tx_desc_dma_handle = (nxge_os_dma_handle_t)
 412             DMA_COMMON_HANDLE(desc_area);
 413         tx_msg_ring = tx_ring_p->tx_msg_ring;
 414 
 415         NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: wr_index %d i %d",
 416             sop_index, i));
 417 
 418 #ifdef  NXGE_DEBUG
 419         msgsize = msgdsize(nmp);
 420         NXGE_DEBUG_MSG((nxgep, TX_CTL,
 421             "==> nxge_start(1): wr_index %d i %d msgdsize %d",
 422             sop_index, i, msgsize));
 423 #endif
 424         /*
 425          * The first 16 bytes of the premapped buffer are reserved
 426          * for header. No padding will be used.
 427          */
 428         pkt_len = pack_len = boff = TX_PKT_HEADER_SIZE;
 429         if (nxge_tx_use_bcopy && (nxgep->niu_type != N2_NIU)) {
 430                 bcopy_thresh = (nxge_bcopy_thresh - TX_PKT_HEADER_SIZE);
 431         } else {
 432                 bcopy_thresh = (TX_BCOPY_SIZE - TX_PKT_HEADER_SIZE);
 433         }
 434         while (nmp) {
 435                 good_packet = B_TRUE;
 436                 b_rptr = nmp->b_rptr;
 437                 len = MBLKL(nmp);
 438                 if (len <= 0) {
 439                         nmp = nmp->b_cont;
 440                         continue;
 441                 }
 442                 nmblks++;
 443 
 444                 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(1): nmblks %d "
 445                     "len %d pkt_len %d pack_len %d",
 446                     nmblks, len, pkt_len, pack_len));
 447                 /*
 448                  * Hardware limits the transfer length to 4K for NIU and
 449                  * 4076 (TX_MAX_TRANSFER_LENGTH) for Neptune. But we just
 450                  * use TX_MAX_TRANSFER_LENGTH as the limit for both.
 451                  * If len is longer than the limit, then we break nmp into
 452                  * two chunks: Make the first chunk equal to the limit and
 453                  * the second chunk for the remaining data. If the second
 454                  * chunk is still larger than the limit, then it will be
 455                  * broken into two in the next pass.
 456                  */
 457                 if (len > TX_MAX_TRANSFER_LENGTH - TX_PKT_HEADER_SIZE) {
 458                         if ((t_mp = dupb(nmp)) != NULL) {
 459                                 nmp->b_wptr = nmp->b_rptr +
 460                                     (TX_MAX_TRANSFER_LENGTH
 461                                     - TX_PKT_HEADER_SIZE);
 462                                 t_mp->b_rptr = nmp->b_wptr;
 463                                 t_mp->b_cont = nmp->b_cont;
 464                                 nmp->b_cont = t_mp;
 465                                 len = MBLKL(nmp);
 466                         } else {
 467                                 if (is_lso) {
 468                                         NXGE_DEBUG_MSG((nxgep, TX_CTL,
 469                                             "LSO packet: dupb failed: "
 470                                             "channel %d",
 471                                             tx_ring_p->tdc));
 472                                         mp = nmp;
 473                                         goto nxge_start_fail_lso;
 474                                 } else {
 475                                         good_packet = B_FALSE;
 476                                         goto nxge_start_fail2;
 477                                 }
 478                         }
 479                 }
 480                 tx_desc.value = 0;
 481                 tx_desc_p = &tx_desc_ring_vp[i];
 482 #ifdef  NXGE_DEBUG
 483                 tx_desc_pp = &tx_desc_ring_pp[i];
 484 #endif
 485                 tx_msg_p = &tx_msg_ring[i];
 486 #if defined(__i386)
 487                 npi_desc_handle.regp = (uint32_t)tx_desc_p;
 488 #else
 489                 npi_desc_handle.regp = (uint64_t)tx_desc_p;
 490 #endif
 491                 if (!header_set &&
 492                     ((!nxge_tx_use_bcopy && (len > TX_BCOPY_SIZE)) ||
 493                     (len >= bcopy_thresh))) {
 494                         header_set = B_TRUE;
 495                         bcopy_thresh += TX_PKT_HEADER_SIZE;
 496                         boff = 0;
 497                         pack_len = 0;
 498                         kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
 499                         hdrp = (p_tx_pkt_header_t)kaddr;
 500                         clen = pkt_len;
 501                         dma_handle = tx_msg_p->buf_dma_handle;
 502                         dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma);
 503                         (void) ddi_dma_sync(dma_handle,
 504                             i * nxge_bcopy_thresh, nxge_bcopy_thresh,
 505                             DDI_DMA_SYNC_FORDEV);
 506 
 507                         tx_msg_p->flags.dma_type = USE_BCOPY;
 508                         goto nxge_start_control_header_only;
 509                 }
 510 
 511                 pkt_len += len;
 512                 pack_len += len;
 513 
 514                 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(3): "
 515                     "desc entry %d "
 516                     "DESC IOADDR $%p "
 517                     "desc_vp $%p tx_desc_p $%p "
 518                     "desc_pp $%p tx_desc_pp $%p "
 519                     "len %d pkt_len %d pack_len %d",
 520                     i,
 521                     DMA_COMMON_IOADDR(desc_area),
 522                     tx_desc_ring_vp, tx_desc_p,
 523                     tx_desc_ring_pp, tx_desc_pp,
 524                     len, pkt_len, pack_len));
 525 
 526                 if (len < bcopy_thresh) {
 527                         NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(4): "
 528                             "USE BCOPY: "));
 529                         if (nxge_tx_tiny_pack) {
 530                                 uint32_t blst =
 531                                     TXDMA_DESC_NEXT_INDEX(i, -1,
 532                                     tx_ring_p->tx_wrap_mask);
 533                                 NXGE_DEBUG_MSG((nxgep, TX_CTL,
 534                                     "==> nxge_start(5): pack"));
 535                                 if ((pack_len <= bcopy_thresh) &&
 536                                     (last_bidx == blst)) {
 537                                         NXGE_DEBUG_MSG((nxgep, TX_CTL,
 538                                             "==> nxge_start: pack(6) "
 539                                             "(pkt_len %d pack_len %d)",
 540                                             pkt_len, pack_len));
 541                                         i = blst;
 542                                         tx_desc_p = &tx_desc_ring_vp[i];
 543 #ifdef  NXGE_DEBUG
 544                                         tx_desc_pp = &tx_desc_ring_pp[i];
 545 #endif
 546                                         tx_msg_p = &tx_msg_ring[i];
 547                                         boff = pack_len - len;
 548                                         ngathers--;
 549                                 } else if (pack_len > bcopy_thresh &&
 550                                     header_set) {
 551                                         pack_len = len;
 552                                         boff = 0;
 553                                         bcopy_thresh = nxge_bcopy_thresh;
 554                                         NXGE_DEBUG_MSG((nxgep, TX_CTL,
 555                                             "==> nxge_start(7): > max NEW "
 556                                             "bcopy thresh %d "
 557                                             "pkt_len %d pack_len %d(next)",
 558                                             bcopy_thresh,
 559                                             pkt_len, pack_len));
 560                                 }
 561                                 last_bidx = i;
 562                         }
 563                         kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
 564                         if ((boff == TX_PKT_HEADER_SIZE) && (nmblks == 1)) {
 565                                 hdrp = (p_tx_pkt_header_t)kaddr;
 566                                 header_set = B_TRUE;
 567                                 NXGE_DEBUG_MSG((nxgep, TX_CTL,
 568                                     "==> nxge_start(7_x2): "
 569                                     "pkt_len %d pack_len %d (new hdrp $%p)",
 570                                     pkt_len, pack_len, hdrp));
 571                         }
 572                         tx_msg_p->flags.dma_type = USE_BCOPY;
 573                         kaddr += boff;
 574                         NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(8): "
 575                             "USE BCOPY: before bcopy "
 576                             "DESC IOADDR $%p entry %d "
 577                             "bcopy packets %d "
 578                             "bcopy kaddr $%p "
 579                             "bcopy ioaddr (SAD) $%p "
 580                             "bcopy clen %d "
 581                             "bcopy boff %d",
 582                             DMA_COMMON_IOADDR(desc_area), i,
 583                             tdc_stats->tx_hdr_pkts,
 584                             kaddr,
 585                             dma_ioaddr,
 586                             clen,
 587                             boff));
 588                         NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
 589                             "1USE BCOPY: "));
 590                         NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
 591                             "2USE BCOPY: "));
 592                         NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: "
 593                             "last USE BCOPY: copy from b_rptr $%p "
 594                             "to KADDR $%p (len %d offset %d",
 595                             b_rptr, kaddr, len, boff));
 596 
 597                         bcopy(b_rptr, kaddr, len);
 598 
 599 #ifdef  NXGE_DEBUG
 600                         dump_len = (len > 128) ? 128: len;
 601                         NXGE_DEBUG_MSG((nxgep, TX_CTL,
 602                             "==> nxge_start: dump packets "
 603                             "(After BCOPY len %d)"
 604                             "(b_rptr $%p): %s", len, nmp->b_rptr,
 605                             nxge_dump_packet((char *)nmp->b_rptr,
 606                             dump_len)));
 607 #endif
 608 
 609                         dma_handle = tx_msg_p->buf_dma_handle;
 610                         dma_ioaddr = DMA_COMMON_IOADDR(tx_msg_p->buf_dma);
 611                         (void) ddi_dma_sync(dma_handle,
 612                             i * nxge_bcopy_thresh, nxge_bcopy_thresh,
 613                             DDI_DMA_SYNC_FORDEV);
 614                         clen = len + boff;
 615                         tdc_stats->tx_hdr_pkts++;
 616                         NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(9): "
 617                             "USE BCOPY: "
 618                             "DESC IOADDR $%p entry %d "
 619                             "bcopy packets %d "
 620                             "bcopy kaddr $%p "
 621                             "bcopy ioaddr (SAD) $%p "
 622                             "bcopy clen %d "
 623                             "bcopy boff %d",
 624                             DMA_COMMON_IOADDR(desc_area),
 625                             i,
 626                             tdc_stats->tx_hdr_pkts,
 627                             kaddr,
 628                             dma_ioaddr,
 629                             clen,
 630                             boff));
 631                 } else {
 632                         NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(12): "
 633                             "USE DVMA: len %d", len));
 634                         tx_msg_p->flags.dma_type = USE_DMA;
 635                         dma_flags = DDI_DMA_WRITE;
 636                         if (len < nxge_dma_stream_thresh) {
 637                                 dma_flags |= DDI_DMA_CONSISTENT;
 638                         } else {
 639                                 dma_flags |= DDI_DMA_STREAMING;
 640                         }
 641 
 642                         dma_handle = tx_msg_p->dma_handle;
 643                         dma_status = ddi_dma_addr_bind_handle(dma_handle, NULL,
 644                             (caddr_t)b_rptr, len, dma_flags,
 645                             DDI_DMA_DONTWAIT, NULL,
 646                             &dma_cookie, &ncookies);
 647                         if (dma_status == DDI_DMA_MAPPED) {
 648                                 dma_ioaddr = dma_cookie.dmac_laddress;
 649                                 len = (int)dma_cookie.dmac_size;
 650                                 clen = (uint32_t)dma_cookie.dmac_size;
 651                                 NXGE_DEBUG_MSG((nxgep, TX_CTL,
 652                                     "==> nxge_start(12_1): "
 653                                     "USE DVMA: len %d clen %d "
 654                                     "ngathers %d",
 655                                     len, clen,
 656                                     ngathers));
 657 #if defined(__i386)
 658                                 npi_desc_handle.regp = (uint32_t)tx_desc_p;
 659 #else
 660                                 npi_desc_handle.regp = (uint64_t)tx_desc_p;
 661 #endif
 662                                 while (ncookies > 1) {
 663                                         ngathers++;
 664                                         /*
 665                                          * this is the fix for multiple
 666                                          * cookies, which are basically
 667                                          * a descriptor entry, we don't set
 668                                          * SOP bit as well as related fields
 669                                          */
 670 
 671                                         (void) npi_txdma_desc_gather_set(
 672                                             npi_desc_handle,
 673                                             &tx_desc,
 674                                             (ngathers -1),
 675                                             mark_mode,
 676                                             ngathers,
 677                                             dma_ioaddr,
 678                                             clen);
 679 
 680                                         tx_msg_p->tx_msg_size = clen;
 681                                         NXGE_DEBUG_MSG((nxgep, TX_CTL,
 682                                             "==> nxge_start:  DMA "
 683                                             "ncookie %d "
 684                                             "ngathers %d "
 685                                             "dma_ioaddr $%p len %d"
 686                                             "desc $%p descp $%p (%d)",
 687                                             ncookies,
 688                                             ngathers,
 689                                             dma_ioaddr, clen,
 690                                             *tx_desc_p, tx_desc_p, i));
 691 
 692                                         ddi_dma_nextcookie(dma_handle,
 693                                             &dma_cookie);
 694                                         dma_ioaddr =
 695                                             dma_cookie.dmac_laddress;
 696 
 697                                         len = (int)dma_cookie.dmac_size;
 698                                         clen = (uint32_t)dma_cookie.dmac_size;
 699                                         NXGE_DEBUG_MSG((nxgep, TX_CTL,
 700                                             "==> nxge_start(12_2): "
 701                                             "USE DVMA: len %d clen %d ",
 702                                             len, clen));
 703 
 704                                         i = TXDMA_DESC_NEXT_INDEX(i, 1,
 705                                             tx_ring_p->tx_wrap_mask);
 706                                         tx_desc_p = &tx_desc_ring_vp[i];
 707 
 708 #if defined(__i386)
 709                                         npi_desc_handle.regp =
 710                                             (uint32_t)tx_desc_p;
 711 #else
 712                                         npi_desc_handle.regp =
 713                                             (uint64_t)tx_desc_p;
 714 #endif
 715                                         tx_msg_p = &tx_msg_ring[i];
 716                                         tx_msg_p->flags.dma_type = USE_NONE;
 717                                         tx_desc.value = 0;
 718 
 719                                         ncookies--;
 720                                 }
 721                                 tdc_stats->tx_ddi_pkts++;
 722                                 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start:"
 723                                     "DMA: ddi packets %d",
 724                                     tdc_stats->tx_ddi_pkts));
 725                         } else {
 726                                 NXGE_ERROR_MSG((nxgep, NXGE_ERR_CTL,
 727                                     "dma mapping failed for %d "
 728                                     "bytes addr $%p flags %x (%d)",
 729                                     len, b_rptr, status, status));
 730                                 good_packet = B_FALSE;
 731                                 tdc_stats->tx_dma_bind_fail++;
 732                                 tx_msg_p->flags.dma_type = USE_NONE;
 733                                 if (is_lso) {
 734                                         mp = nmp;
 735                                         goto nxge_start_fail_lso;
 736                                 } else {
 737                                         status = 1;
 738                                         goto nxge_start_fail2;
 739                                 }
 740                         }
 741                 } /* ddi dvma */
 742 
 743                 if (is_lso) {
 744                         nmp_lso_save = nmp;
 745                 }
 746                 nmp = nmp->b_cont;
 747 nxge_start_control_header_only:
 748 #if defined(__i386)
 749                 npi_desc_handle.regp = (uint32_t)tx_desc_p;
 750 #else
 751                 npi_desc_handle.regp = (uint64_t)tx_desc_p;
 752 #endif
 753                 ngathers++;
 754 
 755                 if (ngathers == 1) {
 756 #ifdef  NXGE_DEBUG
 757                         save_desc_p = &sop_tx_desc;
 758 #endif
 759                         sop_tx_desc_p = &sop_tx_desc;
 760                         sop_tx_desc_p->value = 0;
 761                         sop_tx_desc_p->bits.hdw.tr_len = clen;
 762                         sop_tx_desc_p->bits.hdw.sad = dma_ioaddr >> 32;
 763                         sop_tx_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff;
 764                 } else {
 765 #ifdef  NXGE_DEBUG
 766                         save_desc_p = &tx_desc;
 767 #endif
 768                         tmp_desc_p = &tx_desc;
 769                         tmp_desc_p->value = 0;
 770                         tmp_desc_p->bits.hdw.tr_len = clen;
 771                         tmp_desc_p->bits.hdw.sad = dma_ioaddr >> 32;
 772                         tmp_desc_p->bits.ldw.sad = dma_ioaddr & 0xffffffff;
 773 
 774                         tx_desc_p->value = tmp_desc_p->value;
 775                 }
 776 
 777                 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(13): "
 778                     "Desc_entry %d ngathers %d "
 779                     "desc_vp $%p tx_desc_p $%p "
 780                     "len %d clen %d pkt_len %d pack_len %d nmblks %d "
 781                     "dma_ioaddr (SAD) $%p mark %d",
 782                     i, ngathers,
 783                     tx_desc_ring_vp, tx_desc_p,
 784                     len, clen, pkt_len, pack_len, nmblks,
 785                     dma_ioaddr, mark_mode));
 786 
 787 #ifdef NXGE_DEBUG
 788                 npi_desc_handle.nxgep = nxgep;
 789                 npi_desc_handle.function.function = nxgep->function_num;
 790                 npi_desc_handle.function.instance = nxgep->instance;
 791                 sad = (save_desc_p->value & TX_PKT_DESC_SAD_MASK);
 792                 xfer_len = ((save_desc_p->value & TX_PKT_DESC_TR_LEN_MASK) >>
 793                     TX_PKT_DESC_TR_LEN_SHIFT);
 794 
 795 
 796                 NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n"
 797                     "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\t"
 798                     "mark %d sop %d\n",
 799                     save_desc_p->value,
 800                     sad,
 801                     save_desc_p->bits.hdw.tr_len,
 802                     xfer_len,
 803                     save_desc_p->bits.hdw.num_ptr,
 804                     save_desc_p->bits.hdw.mark,
 805                     save_desc_p->bits.hdw.sop));
 806 
 807                 npi_txdma_dump_desc_one(npi_desc_handle, NULL, i);
 808 #endif
 809 
 810                 tx_msg_p->tx_msg_size = clen;
 811                 i = TXDMA_DESC_NEXT_INDEX(i, 1, tx_ring_p->tx_wrap_mask);
 812                 if (ngathers > nxge_tx_max_gathers) {
 813                         good_packet = B_FALSE;
 814                         mac_hcksum_get(mp, &start_offset,
 815                             &stuff_offset, &end_offset, &value,
 816                             &cksum_flags);
 817 
 818                         NXGE_DEBUG_MSG((NULL, TX_CTL,
 819                             "==> nxge_start(14): pull msg - "
 820                             "len %d pkt_len %d ngathers %d",
 821                             len, pkt_len, ngathers));
 822 
 823                         /*
 824                          * Just give up on this packet.
 825                          */
 826                         if (is_lso) {
 827                                 mp = nmp_lso_save;
 828                                 goto nxge_start_fail_lso;
 829                         }
 830                         status = 0;
 831                         goto nxge_start_fail2;
 832                 }
 833         } /* while (nmp) */
 834 
 835         tx_msg_p->tx_message = mp;
 836         tx_desc_p = &tx_desc_ring_vp[sop_index];
 837 #if defined(__i386)
 838         npi_desc_handle.regp = (uint32_t)tx_desc_p;
 839 #else
 840         npi_desc_handle.regp = (uint64_t)tx_desc_p;
 841 #endif
 842 
 843         pkthdrp = (p_tx_pkt_hdr_all_t)hdrp;
 844         pkthdrp->reserved = 0;
 845         hdrp->value = 0;
 846         bcopy(&tmp_hdrp, hdrp, sizeof (tx_pkt_header_t));
 847 
 848         if (pkt_len > NXGE_MTU_DEFAULT_MAX) {
 849                 tdc_stats->tx_jumbo_pkts++;
 850         }
 851 
 852         min_len = (ETHERMIN + TX_PKT_HEADER_SIZE + (npads * 2));
 853         if (pkt_len < min_len) {
 854                 /* Assume we use bcopy to premapped buffers */
 855                 kaddr = (caddr_t)DMA_COMMON_VPTR(tx_msg_p->buf_dma);
 856                 NXGE_DEBUG_MSG((NULL, TX_CTL,
 857                     "==> nxge_start(14-1): < (msg_min + 16)"
 858                     "len %d pkt_len %d min_len %d bzero %d ngathers %d",
 859                     len, pkt_len, min_len, (min_len - pkt_len), ngathers));
 860                 bzero((kaddr + pkt_len), (min_len - pkt_len));
 861                 pkt_len = tx_msg_p->tx_msg_size = min_len;
 862 
 863                 sop_tx_desc_p->bits.hdw.tr_len = min_len;
 864 
 865                 NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value);
 866                 tx_desc_p->value = sop_tx_desc_p->value;
 867 
 868                 NXGE_DEBUG_MSG((NULL, TX_CTL,
 869                     "==> nxge_start(14-2): < msg_min - "
 870                     "len %d pkt_len %d min_len %d ngathers %d",
 871                     len, pkt_len, min_len, ngathers));
 872         }
 873 
 874         NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: cksum_flags 0x%x ",
 875             cksum_flags));
 876         {
 877                 uint64_t        tmp_len;
 878 
 879                 /* pkt_len already includes 16 + paddings!! */
 880                 /* Update the control header length */
 881                 tot_xfer_len = (pkt_len - TX_PKT_HEADER_SIZE);
 882                 tmp_len = hdrp->value |
 883                     (tot_xfer_len << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT);
 884 
 885                 NXGE_DEBUG_MSG((nxgep, TX_CTL,
 886                     "==> nxge_start(15_x1): setting SOP "
 887                     "tot_xfer_len 0x%llx (%d) pkt_len %d tmp_len "
 888                     "0x%llx hdrp->value 0x%llx",
 889                     tot_xfer_len, tot_xfer_len, pkt_len,
 890                     tmp_len, hdrp->value));
 891 #if defined(_BIG_ENDIAN)
 892                 hdrp->value = ddi_swap64(tmp_len);
 893 #else
 894                 hdrp->value = tmp_len;
 895 #endif
 896                 NXGE_DEBUG_MSG((nxgep,
 897                     TX_CTL, "==> nxge_start(15_x2): setting SOP "
 898                     "after SWAP: tot_xfer_len 0x%llx pkt_len %d "
 899                     "tmp_len 0x%llx hdrp->value 0x%llx",
 900                     tot_xfer_len, pkt_len,
 901                     tmp_len, hdrp->value));
 902         }
 903 
 904         NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(15): setting SOP "
 905             "wr_index %d "
 906             "tot_xfer_len (%d) pkt_len %d npads %d",
 907             sop_index,
 908             tot_xfer_len, pkt_len,
 909             npads));
 910 
 911         sop_tx_desc_p->bits.hdw.sop = 1;
 912         sop_tx_desc_p->bits.hdw.mark = mark_mode;
 913         sop_tx_desc_p->bits.hdw.num_ptr = ngathers;
 914 
 915         NXGE_MEM_PIO_WRITE64(npi_desc_handle, sop_tx_desc_p->value);
 916 
 917         NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start(16): set SOP done"));
 918 
 919 #ifdef NXGE_DEBUG
 920         npi_desc_handle.nxgep = nxgep;
 921         npi_desc_handle.function.function = nxgep->function_num;
 922         npi_desc_handle.function.instance = nxgep->instance;
 923 
 924         NXGE_DEBUG_MSG((nxgep, TX_CTL, "\n\t: value 0x%llx\n"
 925             "\t\tsad $%p\ttr_len %d len %d\tnptrs %d\tmark %d sop %d\n",
 926             save_desc_p->value,
 927             sad,
 928             save_desc_p->bits.hdw.tr_len,
 929             xfer_len,
 930             save_desc_p->bits.hdw.num_ptr,
 931             save_desc_p->bits.hdw.mark,
 932             save_desc_p->bits.hdw.sop));
 933         (void) npi_txdma_dump_desc_one(npi_desc_handle, NULL, sop_index);
 934 
 935         dump_len = (pkt_len > 128) ? 128: pkt_len;
 936         NXGE_DEBUG_MSG((nxgep, TX_CTL,
 937             "==> nxge_start: dump packets(17) (after sop set, len "
 938             " (len/dump_len/pkt_len/tot_xfer_len) %d/%d/%d/%d):\n"
 939             "ptr $%p: %s", len, dump_len, pkt_len, tot_xfer_len,
 940             (char *)hdrp,
 941             nxge_dump_packet((char *)hdrp, dump_len)));
 942         NXGE_DEBUG_MSG((nxgep, TX_CTL,
 943             "==> nxge_start(18): TX desc sync: sop_index %d",
 944             sop_index));
 945 #endif
 946 
 947         if ((ngathers == 1) || tx_ring_p->wr_index < i) {
 948                 (void) ddi_dma_sync(tx_desc_dma_handle,
 949                     sop_index * sizeof (tx_desc_t),
 950                     ngathers * sizeof (tx_desc_t),
 951                     DDI_DMA_SYNC_FORDEV);
 952 
 953                 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(19): sync 1 "
 954                     "cs_off = 0x%02X cs_s_off = 0x%02X "
 955                     "pkt_len %d ngathers %d sop_index %d\n",
 956                     stuff_offset, start_offset,
 957                     pkt_len, ngathers, sop_index));
 958         } else { /* more than one descriptor and wrap around */
 959                 uint32_t nsdescs = tx_ring_p->tx_ring_size - sop_index;
 960                 (void) ddi_dma_sync(tx_desc_dma_handle,
 961                     sop_index * sizeof (tx_desc_t),
 962                     nsdescs * sizeof (tx_desc_t),
 963                     DDI_DMA_SYNC_FORDEV);
 964                 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(20): sync 1 "
 965                     "cs_off = 0x%02X cs_s_off = 0x%02X "
 966                     "pkt_len %d ngathers %d sop_index %d\n",
 967                     stuff_offset, start_offset,
 968                     pkt_len, ngathers, sop_index));
 969 
 970                 (void) ddi_dma_sync(tx_desc_dma_handle,
 971                     0,
 972                     (ngathers - nsdescs) * sizeof (tx_desc_t),
 973                     DDI_DMA_SYNC_FORDEV);
 974                 NXGE_DEBUG_MSG((nxgep, TX_CTL, "nxge_start(21): sync 2 "
 975                     "cs_off = 0x%02X cs_s_off = 0x%02X "
 976                     "pkt_len %d ngathers %d sop_index %d\n",
 977                     stuff_offset, start_offset,
 978                     pkt_len, ngathers, sop_index));
 979         }
 980 
 981         tail_index = tx_ring_p->wr_index;
 982         tail_wrap = tx_ring_p->wr_index_wrap;
 983 
 984         tx_ring_p->wr_index = i;
 985         if (tx_ring_p->wr_index <= tail_index) {
 986                 tx_ring_p->wr_index_wrap = ((tail_wrap == B_TRUE) ?
 987                     B_FALSE : B_TRUE);
 988         }
 989 
 990         NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX kick: "
 991             "channel %d wr_index %d wrap %d ngathers %d desc_pend %d",
 992             tx_ring_p->tdc,
 993             tx_ring_p->wr_index,
 994             tx_ring_p->wr_index_wrap,
 995             ngathers,
 996             tx_ring_p->descs_pending));
 997 
 998         if (is_lso) {
 999                 lso_ngathers += ngathers;
1000                 if (mp_chain != NULL) {
1001                         mp = mp_chain;
1002                         mp_chain = mp_chain->b_next;
1003                         mp->b_next = NULL;
1004                         if (nxge_lso_kick_cnt == lso_ngathers) {
1005                                 tx_ring_p->descs_pending += lso_ngathers;
1006                                 {
1007                                         tx_ring_kick_t          kick;
1008 
1009                                         kick.value = 0;
1010                                         kick.bits.ldw.wrap =
1011                                             tx_ring_p->wr_index_wrap;
1012                                         kick.bits.ldw.tail =
1013                                             (uint16_t)tx_ring_p->wr_index;
1014 
1015                                         /* Kick the Transmit kick register */
1016                                         TXDMA_REG_WRITE64(
1017                                             NXGE_DEV_NPI_HANDLE(nxgep),
1018                                             TX_RING_KICK_REG,
1019                                             (uint8_t)tx_ring_p->tdc,
1020                                             kick.value);
1021                                         tdc_stats->tx_starts++;
1022 
1023                                         NXGE_DEBUG_MSG((nxgep, TX_CTL,
1024                                             "==> nxge_start: more LSO: "
1025                                             "LSO_CNT %d",
1026                                             lso_ngathers));
1027                                 }
1028                                 lso_ngathers = 0;
1029                                 ngathers = 0;
1030                                 cur_index_lso = sop_index = tx_ring_p->wr_index;
1031                                 lso_tail_wrap = tx_ring_p->wr_index_wrap;
1032                         }
1033                         NXGE_DEBUG_MSG((nxgep, TX_CTL,
1034                             "==> nxge_start: lso again: "
1035                             "lso_gathers %d ngathers %d cur_index_lso %d "
1036                             "wr_index %d sop_index %d",
1037                             lso_ngathers, ngathers, cur_index_lso,
1038                             tx_ring_p->wr_index, sop_index));
1039 
1040                         NXGE_DEBUG_MSG((nxgep, TX_CTL,
1041                             "==> nxge_start: next : count %d",
1042                             lso_ngathers));
1043                         lso_again = B_TRUE;
1044                         goto start_again;
1045                 }
1046                 ngathers = lso_ngathers;
1047         }
1048 
1049         NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: TX KICKING: "));
1050 
1051         {
1052                 tx_ring_kick_t          kick;
1053 
1054                 kick.value = 0;
1055                 kick.bits.ldw.wrap = tx_ring_p->wr_index_wrap;
1056                 kick.bits.ldw.tail = (uint16_t)tx_ring_p->wr_index;
1057 
1058                 /* Kick start the Transmit kick register */
1059                 TXDMA_REG_WRITE64(NXGE_DEV_NPI_HANDLE(nxgep),
1060                     TX_RING_KICK_REG,
1061                     (uint8_t)tx_ring_p->tdc,
1062                     kick.value);
1063         }
1064 
1065         tx_ring_p->descs_pending += ngathers;
1066         tdc_stats->tx_starts++;
1067 
1068         if (isLDOMservice(nxgep)) {
1069                 tx_ring_p->tx_ring_busy = B_FALSE;
1070                 if (tx_ring_p->tx_ring_offline) {
1071                         (void) atomic_swap_32(&tx_ring_p->tx_ring_offline,
1072                             NXGE_TX_RING_OFFLINED);
1073                 }
1074         }
1075 
1076         MUTEX_EXIT(&tx_ring_p->lock);
1077 
1078         NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start"));
1079         return (status);
1080 
1081 nxge_start_fail_lso:
1082         status = 0;
1083         good_packet = B_FALSE;
1084         if (mp != NULL)
1085                 freemsg(mp);
1086         if (mp_chain != NULL)
1087                 freemsgchain(mp_chain);
1088 
1089         if (!lso_again && !ngathers) {
1090                 if (isLDOMservice(nxgep)) {
1091                         tx_ring_p->tx_ring_busy = B_FALSE;
1092                         if (tx_ring_p->tx_ring_offline) {
1093                                 (void) atomic_swap_32(
1094                                     &tx_ring_p->tx_ring_offline,
1095                                     NXGE_TX_RING_OFFLINED);
1096                         }
1097                 }
1098 
1099                 MUTEX_EXIT(&tx_ring_p->lock);
1100                 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1101                     "==> nxge_start: lso exit (nothing changed)"));
1102                 goto nxge_start_fail1;
1103         }
1104 
1105         NXGE_DEBUG_MSG((nxgep, TX_CTL,
1106             "==> nxge_start (channel %d): before lso "
1107             "lso_gathers %d ngathers %d cur_index_lso %d "
1108             "wr_index %d sop_index %d lso_again %d",
1109             tx_ring_p->tdc,
1110             lso_ngathers, ngathers, cur_index_lso,
1111             tx_ring_p->wr_index, sop_index, lso_again));
1112 
1113         if (lso_again) {
1114                 lso_ngathers += ngathers;
1115                 ngathers = lso_ngathers;
1116                 sop_index = cur_index_lso;
1117                 tx_ring_p->wr_index = sop_index;
1118                 tx_ring_p->wr_index_wrap = lso_tail_wrap;
1119         }
1120 
1121         NXGE_DEBUG_MSG((nxgep, TX_CTL,
1122             "==> nxge_start (channel %d): after lso "
1123             "lso_gathers %d ngathers %d cur_index_lso %d "
1124             "wr_index %d sop_index %d lso_again %d",
1125             tx_ring_p->tdc,
1126             lso_ngathers, ngathers, cur_index_lso,
1127             tx_ring_p->wr_index, sop_index, lso_again));
1128 
1129 nxge_start_fail2:
1130         if (good_packet == B_FALSE) {
1131                 cur_index = sop_index;
1132                 NXGE_DEBUG_MSG((nxgep, TX_CTL, "==> nxge_start: clean up"));
1133                 for (i = 0; i < ngathers; i++) {
1134                         tx_desc_p = &tx_desc_ring_vp[cur_index];
1135 #if defined(__i386)
1136                         npi_handle.regp = (uint32_t)tx_desc_p;
1137 #else
1138                         npi_handle.regp = (uint64_t)tx_desc_p;
1139 #endif
1140                         tx_msg_p = &tx_msg_ring[cur_index];
1141                         (void) npi_txdma_desc_set_zero(npi_handle, 1);
1142                         if (tx_msg_p->flags.dma_type == USE_DVMA) {
1143                                 NXGE_DEBUG_MSG((nxgep, TX_CTL,
1144                                     "tx_desc_p = %X index = %d",
1145                                     tx_desc_p, tx_ring_p->rd_index));
1146                                 (void) dvma_unload(tx_msg_p->dvma_handle,
1147                                     0, -1);
1148                                 tx_msg_p->dvma_handle = NULL;
1149                                 if (tx_ring_p->dvma_wr_index ==
1150                                     tx_ring_p->dvma_wrap_mask)
1151                                         tx_ring_p->dvma_wr_index = 0;
1152                                 else
1153                                         tx_ring_p->dvma_wr_index++;
1154                                 tx_ring_p->dvma_pending--;
1155                         } else if (tx_msg_p->flags.dma_type == USE_DMA) {
1156                                 if (ddi_dma_unbind_handle(
1157                                     tx_msg_p->dma_handle)) {
1158                                         cmn_err(CE_WARN, "!nxge_start: "
1159                                             "ddi_dma_unbind_handle failed");
1160                                 }
1161                         }
1162                         tx_msg_p->flags.dma_type = USE_NONE;
1163                         cur_index = TXDMA_DESC_NEXT_INDEX(cur_index, 1,
1164                             tx_ring_p->tx_wrap_mask);
1165 
1166                 }
1167         }
1168 
1169         if (isLDOMservice(nxgep)) {
1170                 tx_ring_p->tx_ring_busy = B_FALSE;
1171                 if (tx_ring_p->tx_ring_offline) {
1172                         (void) atomic_swap_32(&tx_ring_p->tx_ring_offline,
1173                             NXGE_TX_RING_OFFLINED);
1174                 }
1175         }
1176 
1177         MUTEX_EXIT(&tx_ring_p->lock);
1178 
1179 nxge_start_fail1:
1180         /* Add FMA to check the access handle nxge_hregh */
1181 
1182         NXGE_DEBUG_MSG((nxgep, TX_CTL, "<== nxge_start"));
1183         return (status);
1184 }
1185 
1186 /* Software LSO starts here */
1187 static void
1188 nxge_hcksum_retrieve(mblk_t *mp,
1189     uint32_t *start, uint32_t *stuff, uint32_t *end,
1190     uint32_t *value, uint32_t *flags)
1191 {
1192         if (mp->b_datap->db_type == M_DATA) {
1193                 if (flags != NULL) {
1194                         *flags = DB_CKSUMFLAGS(mp) & (HCK_IPV4_HDRCKSUM |
1195                             HCK_PARTIALCKSUM | HCK_FULLCKSUM |
1196                             HCK_FULLCKSUM_OK);
1197                         if ((*flags & (HCK_PARTIALCKSUM |
1198                             HCK_FULLCKSUM)) != 0) {
1199                                 if (value != NULL)
1200                                         *value = (uint32_t)DB_CKSUM16(mp);
1201                                 if ((*flags & HCK_PARTIALCKSUM) != 0) {
1202                                         if (start != NULL)
1203                                                 *start =
1204                                                     (uint32_t)DB_CKSUMSTART(mp);
1205                                         if (stuff != NULL)
1206                                                 *stuff =
1207                                                     (uint32_t)DB_CKSUMSTUFF(mp);
1208                                         if (end != NULL)
1209                                                 *end =
1210                                                     (uint32_t)DB_CKSUMEND(mp);
1211                                 }
1212                         }
1213                 }
1214         }
1215 }
1216 
1217 static void
1218 nxge_lso_info_get(mblk_t *mp, uint32_t *mss, uint32_t *flags)
1219 {
1220         ASSERT(DB_TYPE(mp) == M_DATA);
1221 
1222         *mss = 0;
1223         if (flags != NULL) {
1224                 *flags = DB_CKSUMFLAGS(mp) & HW_LSO;
1225                 if ((*flags != 0) && (mss != NULL)) {
1226                         *mss = (uint32_t)DB_LSOMSS(mp);
1227                 }
1228                 NXGE_DEBUG_MSG((NULL, TX_CTL,
1229                     "==> nxge_lso_info_get(flag !=NULL): mss %d *flags 0x%x",
1230                     *mss, *flags));
1231         }
1232 
1233         NXGE_DEBUG_MSG((NULL, TX_CTL,
1234             "<== nxge_lso_info_get: mss %d", *mss));
1235 }
1236 
1237 /*
1238  * Do Soft LSO on the oversized packet.
1239  *
1240  * 1. Create a chain of message for headers.
1241  * 2. Fill up header messages with proper information.
1242  * 3. Copy Eithernet, IP, and TCP headers from the original message to
1243  *    each new message with necessary adjustments.
1244  *    * Unchange the ethernet header for DIX frames. (by default)
1245  *    * IP Total Length field is updated to MSS or less(only for the last one).
1246  *    * IP Identification value is incremented by one for each packet.
1247  *    * TCP sequence Number is recalculated according to the payload length.
1248  *    * Set FIN and/or PSH flags for the *last* packet if applied.
1249  *    * TCP partial Checksum
1250  * 4. Update LSO information in the first message header.
1251  * 5. Release the original message header.
1252  */
1253 static mblk_t *
1254 nxge_do_softlso(mblk_t *mp, uint32_t mss)
1255 {
1256         uint32_t        hckflags;
1257         int             pktlen;
1258         int             hdrlen;
1259         int             segnum;
1260         int             i;
1261         struct ether_vlan_header *evh;
1262         int             ehlen, iphlen, tcphlen;
1263         struct ip       *oiph, *niph;
1264         struct tcphdr *otcph, *ntcph;
1265         int             available, len, left;
1266         uint16_t        ip_id;
1267         uint32_t        tcp_seq;
1268 #ifdef __sparc
1269         uint32_t        tcp_seq_tmp;
1270 #endif
1271         mblk_t          *datamp;
1272         uchar_t         *rptr;
1273         mblk_t          *nmp;
1274         mblk_t          *cmp;
1275         mblk_t          *mp_chain;
1276         boolean_t do_cleanup = B_FALSE;
1277         t_uscalar_t start_offset = 0;
1278         t_uscalar_t stuff_offset = 0;
1279         t_uscalar_t value = 0;
1280         uint16_t        l4_len;
1281         ipaddr_t        src, dst;
1282         uint32_t        cksum, sum, l4cksum;
1283 
1284         NXGE_DEBUG_MSG((NULL, TX_CTL,
1285             "==> nxge_do_softlso"));
1286         /*
1287          * check the length of LSO packet payload and calculate the number of
1288          * segments to be generated.
1289          */
1290         pktlen = msgsize(mp);
1291         evh = (struct ether_vlan_header *)mp->b_rptr;
1292 
1293         /* VLAN? */
1294         if (evh->ether_tpid == htons(ETHERTYPE_VLAN))
1295                 ehlen = sizeof (struct ether_vlan_header);
1296         else
1297                 ehlen = sizeof (struct ether_header);
1298         oiph = (struct ip *)(mp->b_rptr + ehlen);
1299         iphlen = oiph->ip_hl * 4;
1300         otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen);
1301         tcphlen = otcph->th_off * 4;
1302 
1303         l4_len = pktlen - ehlen - iphlen;
1304 
1305         NXGE_DEBUG_MSG((NULL, TX_CTL,
1306             "==> nxge_do_softlso: mss %d oiph $%p "
1307             "original ip_sum oiph->ip_sum 0x%x "
1308             "original tcp_sum otcph->th_sum 0x%x "
1309             "oiph->ip_len %d pktlen %d ehlen %d "
1310             "l4_len %d (0x%x) ip_len - iphlen %d ",
1311             mss,
1312             oiph,
1313             oiph->ip_sum,
1314             otcph->th_sum,
1315             ntohs(oiph->ip_len), pktlen,
1316             ehlen,
1317             l4_len,
1318             l4_len,
1319             ntohs(oiph->ip_len) - iphlen));
1320 
1321         /* IPv4 + TCP */
1322         if (!(oiph->ip_v == IPV4_VERSION)) {
1323                 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
1324                     "<== nxge_do_softlso: not IPV4 "
1325                     "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d",
1326                     ntohs(oiph->ip_len), pktlen, ehlen,
1327                     tcphlen));
1328                 freemsg(mp);
1329                 return (NULL);
1330         }
1331 
1332         if (!(oiph->ip_p == IPPROTO_TCP)) {
1333                 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
1334                     "<== nxge_do_softlso: not TCP "
1335                     "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d",
1336                     ntohs(oiph->ip_len), pktlen, ehlen,
1337                     tcphlen));
1338                 freemsg(mp);
1339                 return (NULL);
1340         }
1341 
1342         if (!(ntohs(oiph->ip_len) == pktlen - ehlen)) {
1343                 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
1344                     "<== nxge_do_softlso: len not matched  "
1345                     "oiph->ip_len %d pktlen %d ehlen %d tcphlen %d",
1346                     ntohs(oiph->ip_len), pktlen, ehlen,
1347                     tcphlen));
1348                 freemsg(mp);
1349                 return (NULL);
1350         }
1351 
1352         otcph = (struct tcphdr *)(mp->b_rptr + ehlen + iphlen);
1353         tcphlen = otcph->th_off * 4;
1354 
1355         /* TCP flags can not include URG, RST, or SYN */
1356         VERIFY((otcph->th_flags & (TH_SYN | TH_RST | TH_URG)) == 0);
1357 
1358         hdrlen = ehlen + iphlen + tcphlen;
1359 
1360         VERIFY(MBLKL(mp) >= hdrlen);
1361 
1362         if (MBLKL(mp) > hdrlen) {
1363                 datamp = mp;
1364                 rptr = mp->b_rptr + hdrlen;
1365         } else { /* = */
1366                 datamp = mp->b_cont;
1367                 rptr = datamp->b_rptr;
1368         }
1369 
1370         NXGE_DEBUG_MSG((NULL, TX_CTL,
1371             "nxge_do_softlso: otcph $%p pktlen: %d, "
1372             "hdrlen %d ehlen %d iphlen %d tcphlen %d "
1373             "mblkl(mp): %d, mblkl(datamp): %d",
1374             otcph,
1375             pktlen, hdrlen, ehlen, iphlen, tcphlen,
1376             (int)MBLKL(mp), (int)MBLKL(datamp)));
1377 
1378         hckflags = 0;
1379         nxge_hcksum_retrieve(mp,
1380             &start_offset, &stuff_offset, &value, NULL, &hckflags);
1381 
1382         dst = oiph->ip_dst.s_addr;
1383         src = oiph->ip_src.s_addr;
1384 
1385         cksum = (dst >> 16) + (dst & 0xFFFF) +
1386             (src >> 16) + (src & 0xFFFF);
1387         l4cksum = cksum + IP_TCP_CSUM_COMP;
1388 
1389         sum = l4_len + l4cksum;
1390         sum = (sum & 0xFFFF) + (sum >> 16);
1391 
1392         NXGE_DEBUG_MSG((NULL, TX_CTL,
1393             "==> nxge_do_softlso: dst 0x%x src 0x%x sum 0x%x ~new 0x%x "
1394             "hckflags 0x%x start_offset %d stuff_offset %d "
1395             "value (original) 0x%x th_sum 0x%x "
1396             "pktlen %d l4_len %d (0x%x) "
1397             "MBLKL(mp): %d, MBLKL(datamp): %d dump header %s",
1398             dst, src,
1399             (sum & 0xffff), (~sum & 0xffff),
1400             hckflags, start_offset, stuff_offset,
1401             value, otcph->th_sum,
1402             pktlen,
1403             l4_len,
1404             l4_len,
1405             ntohs(oiph->ip_len) - (int)MBLKL(mp),
1406             (int)MBLKL(datamp),
1407             nxge_dump_packet((char *)evh, 12)));
1408 
1409         /*
1410          * Start to process.
1411          */
1412         available = pktlen - hdrlen;
1413         segnum = (available - 1) / mss + 1;
1414 
1415         NXGE_DEBUG_MSG((NULL, TX_CTL,
1416             "==> nxge_do_softlso: pktlen %d "
1417             "MBLKL(mp): %d, MBLKL(datamp): %d "
1418             "available %d mss %d segnum %d",
1419             pktlen, (int)MBLKL(mp), (int)MBLKL(datamp),
1420             available,
1421             mss,
1422             segnum));
1423 
1424         VERIFY(segnum >= 2);
1425 
1426         /*
1427          * Try to pre-allocate all header messages
1428          */
1429         mp_chain = NULL;
1430         for (i = 0; i < segnum; i++) {
1431                 if ((nmp = allocb(hdrlen, 0)) == NULL) {
1432                         /* Clean up the mp_chain */
1433                         while (mp_chain != NULL) {
1434                                 nmp = mp_chain;
1435                                 mp_chain = mp_chain->b_next;
1436                                 freemsg(nmp);
1437                         }
1438                         NXGE_DEBUG_MSG((NULL, TX_CTL,
1439                             "<== nxge_do_softlso: "
1440                             "Could not allocate enough messages for headers!"));
1441                         freemsg(mp);
1442                         return (NULL);
1443                 }
1444                 nmp->b_next = mp_chain;
1445                 mp_chain = nmp;
1446 
1447                 NXGE_DEBUG_MSG((NULL, TX_CTL,
1448                     "==> nxge_do_softlso: "
1449                     "mp $%p nmp $%p mp_chain $%p mp_chain->b_next $%p",
1450                     mp, nmp, mp_chain, mp_chain->b_next));
1451         }
1452 
1453         NXGE_DEBUG_MSG((NULL, TX_CTL,
1454             "==> nxge_do_softlso: mp $%p nmp $%p mp_chain $%p",
1455             mp, nmp, mp_chain));
1456 
1457         /*
1458          * Associate payload with new packets
1459          */
1460         cmp = mp_chain;
1461         left = available;
1462         while (cmp != NULL) {
1463                 nmp = dupb(datamp);
1464                 if (nmp == NULL) {
1465                         do_cleanup = B_TRUE;
1466                         NXGE_DEBUG_MSG((NULL, TX_CTL,
1467                             "==>nxge_do_softlso: "
1468                             "Can not dupb(datamp), have to do clean up"));
1469                         goto cleanup_allocated_msgs;
1470                 }
1471 
1472                 NXGE_DEBUG_MSG((NULL, TX_CTL,
1473                     "==> nxge_do_softlso: (loop) before mp $%p cmp $%p "
1474                     "dupb nmp $%p len %d left %d msd %d ",
1475                     mp, cmp, nmp, len, left, mss));
1476 
1477                 cmp->b_cont = nmp;
1478                 nmp->b_rptr = rptr;
1479                 len = (left < mss) ? left : mss;
1480                 left -= len;
1481 
1482                 NXGE_DEBUG_MSG((NULL, TX_CTL,
1483                     "==> nxge_do_softlso: (loop) after mp $%p cmp $%p "
1484                     "dupb nmp $%p len %d left %d mss %d ",
1485                     mp, cmp, nmp, len, left, mss));
1486                 NXGE_DEBUG_MSG((NULL, TX_CTL,
1487                     "nxge_do_softlso: before available: %d, "
1488                     "left: %d, len: %d, segnum: %d MBLK(nmp): %d",
1489                     available, left, len, segnum, (int)MBLKL(nmp)));
1490 
1491                 len -= MBLKL(nmp);
1492                 NXGE_DEBUG_MSG((NULL, TX_CTL,
1493                     "nxge_do_softlso: after available: %d, "
1494                     "left: %d, len: %d, segnum: %d MBLK(nmp): %d",
1495                     available, left, len, segnum, (int)MBLKL(nmp)));
1496 
1497                 while (len > 0) {
1498                         mblk_t *mmp = NULL;
1499 
1500                         NXGE_DEBUG_MSG((NULL, TX_CTL,
1501                             "nxge_do_softlso: (4) len > 0 available: %d, "
1502                             "left: %d, len: %d, segnum: %d MBLK(nmp): %d",
1503                             available, left, len, segnum, (int)MBLKL(nmp)));
1504 
1505                         if (datamp->b_cont != NULL) {
1506                                 datamp = datamp->b_cont;
1507                                 rptr = datamp->b_rptr;
1508                                 mmp = dupb(datamp);
1509                                 if (mmp == NULL) {
1510                                         do_cleanup = B_TRUE;
1511                                         NXGE_DEBUG_MSG((NULL, TX_CTL,
1512                                             "==> nxge_do_softlso: "
1513                                             "Can not dupb(datamp) (1), :"
1514                                             "have to do clean up"));
1515                                         NXGE_DEBUG_MSG((NULL, TX_CTL,
1516                                             "==> nxge_do_softlso: "
1517                                             "available: %d, left: %d, "
1518                                             "len: %d, MBLKL(nmp): %d",
1519                                             available, left, len,
1520                                             (int)MBLKL(nmp)));
1521                                         goto cleanup_allocated_msgs;
1522                                 }
1523                         } else {
1524                                 NXGE_ERROR_MSG((NULL, NXGE_ERR_CTL,
1525                                     "==> nxge_do_softlso: "
1526                                     "(1)available: %d, left: %d, "
1527                                     "len: %d, MBLKL(nmp): %d",
1528                                     available, left, len,
1529                                     (int)MBLKL(nmp)));
1530                                 cmn_err(CE_PANIC,
1531                                     "==> nxge_do_softlso: "
1532                                     "Pointers must have been corrupted!\n"
1533                                     "datamp: $%p, nmp: $%p, rptr: $%p",
1534                                     (void *)datamp,
1535                                     (void *)nmp,
1536                                     (void *)rptr);
1537                         }
1538                         nmp->b_cont = mmp;
1539                         nmp = mmp;
1540                         len -= MBLKL(nmp);
1541                 }
1542                 if (len < 0) {
1543                         nmp->b_wptr += len;
1544                         rptr = nmp->b_wptr;
1545                         NXGE_DEBUG_MSG((NULL, TX_CTL,
1546                             "(5) len < 0 (less than 0)"
1547                             "available: %d, left: %d, len: %d, MBLKL(nmp): %d",
1548                             available, left, len, (int)MBLKL(nmp)));
1549 
1550                 } else if (len == 0) {
1551                         if (datamp->b_cont != NULL) {
1552                                 NXGE_DEBUG_MSG((NULL, TX_CTL,
1553                                     "(5) len == 0"
1554                                     "available: %d, left: %d, len: %d, "
1555                                     "MBLKL(nmp): %d",
1556                                     available, left, len, (int)MBLKL(nmp)));
1557                                 datamp = datamp->b_cont;
1558                                 rptr = datamp->b_rptr;
1559                         } else {
1560                                 NXGE_DEBUG_MSG((NULL, TX_CTL,
1561                                     "(6)available b_cont == NULL : %d, "
1562                                     "left: %d, len: %d, MBLKL(nmp): %d",
1563                                     available, left, len, (int)MBLKL(nmp)));
1564 
1565                                 VERIFY(cmp->b_next == NULL);
1566                                 VERIFY(left == 0);
1567                                 break; /* Done! */
1568                         }
1569                 }
1570                 cmp = cmp->b_next;
1571 
1572                 NXGE_DEBUG_MSG((NULL, TX_CTL,
1573                     "(7) do_softlso: "
1574                     "next mp in mp_chain available len != 0 : %d, "
1575                     "left: %d, len: %d, MBLKL(nmp): %d",
1576                     available, left, len, (int)MBLKL(nmp)));
1577         }
1578 
1579         /*
1580          * From now, start to fill up all headers for the first message
1581          * Hardware checksum flags need to be updated separately for FULLCKSUM
1582          * and PARTIALCKSUM cases. For full checksum, copy the original flags
1583          * into every new packet is enough. But for HCK_PARTIALCKSUM, all
1584          * required fields need to be updated properly.
1585          */
1586         nmp = mp_chain;
1587         bcopy(mp->b_rptr, nmp->b_rptr, hdrlen);
1588         nmp->b_wptr = nmp->b_rptr + hdrlen;
1589         niph = (struct ip *)(nmp->b_rptr + ehlen);
1590         niph->ip_len = htons(mss + iphlen + tcphlen);
1591         ip_id = ntohs(niph->ip_id);
1592         ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen);
1593 #ifdef __sparc
1594         bcopy((char *)&ntcph->th_seq, &tcp_seq_tmp, 4);
1595         tcp_seq = ntohl(tcp_seq_tmp);
1596 #else
1597         tcp_seq = ntohl(ntcph->th_seq);
1598 #endif
1599 
1600         ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST);
1601 
1602         DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags;
1603         DB_CKSUMSTART(nmp) = start_offset;
1604         DB_CKSUMSTUFF(nmp) = stuff_offset;
1605 
1606         /* calculate IP checksum and TCP pseudo header checksum */
1607         niph->ip_sum = 0;
1608         niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen);
1609 
1610         l4_len = mss + tcphlen;
1611         sum = htons(l4_len) + l4cksum;
1612         sum = (sum & 0xFFFF) + (sum >> 16);
1613         ntcph->th_sum = (sum & 0xffff);
1614 
1615         NXGE_DEBUG_MSG((NULL, TX_CTL,
1616             "==> nxge_do_softlso: first mp $%p (mp_chain $%p) "
1617             "mss %d pktlen %d l4_len %d (0x%x) "
1618             "MBLKL(mp): %d, MBLKL(datamp): %d "
1619             "ip_sum 0x%x "
1620             "th_sum 0x%x sum 0x%x ) "
1621             "dump first ip->tcp %s",
1622             nmp, mp_chain,
1623             mss,
1624             pktlen,
1625             l4_len,
1626             l4_len,
1627             (int)MBLKL(mp), (int)MBLKL(datamp),
1628             niph->ip_sum,
1629             ntcph->th_sum,
1630             sum,
1631             nxge_dump_packet((char *)niph, 52)));
1632 
1633         cmp = nmp;
1634         while ((nmp = nmp->b_next)->b_next != NULL) {
1635                 NXGE_DEBUG_MSG((NULL, TX_CTL,
1636                     "==>nxge_do_softlso: middle l4_len %d ", l4_len));
1637                 bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen);
1638                 nmp->b_wptr = nmp->b_rptr + hdrlen;
1639                 niph = (struct ip *)(nmp->b_rptr + ehlen);
1640                 niph->ip_id = htons(++ip_id);
1641                 niph->ip_len = htons(mss + iphlen + tcphlen);
1642                 ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen);
1643                 tcp_seq += mss;
1644 
1645                 ntcph->th_flags &= ~(TH_FIN | TH_PUSH | TH_RST | TH_URG);
1646 
1647 #ifdef __sparc
1648                 tcp_seq_tmp = htonl(tcp_seq);
1649                 bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4);
1650 #else
1651                 ntcph->th_seq = htonl(tcp_seq);
1652 #endif
1653                 DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags;
1654                 DB_CKSUMSTART(nmp) = start_offset;
1655                 DB_CKSUMSTUFF(nmp) = stuff_offset;
1656 
1657                 /* calculate IP checksum and TCP pseudo header checksum */
1658                 niph->ip_sum = 0;
1659                 niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen);
1660                 ntcph->th_sum = (sum & 0xffff);
1661 
1662                 NXGE_DEBUG_MSG((NULL, TX_CTL,
1663                     "==> nxge_do_softlso: middle ip_sum 0x%x "
1664                     "th_sum 0x%x "
1665                     " mp $%p (mp_chain $%p) pktlen %d "
1666                     "MBLKL(mp): %d, MBLKL(datamp): %d ",
1667                     niph->ip_sum,
1668                     ntcph->th_sum,
1669                     nmp, mp_chain,
1670                     pktlen, (int)MBLKL(mp), (int)MBLKL(datamp)));
1671         }
1672 
1673         /* Last segment */
1674         /*
1675          * Set FIN and/or PSH flags if present only in the last packet.
1676          * The ip_len could be different from prior packets.
1677          */
1678         bcopy(cmp->b_rptr, nmp->b_rptr, hdrlen);
1679         nmp->b_wptr = nmp->b_rptr + hdrlen;
1680         niph = (struct ip *)(nmp->b_rptr + ehlen);
1681         niph->ip_id = htons(++ip_id);
1682         niph->ip_len = htons(msgsize(nmp->b_cont) + iphlen + tcphlen);
1683         ntcph = (struct tcphdr *)(nmp->b_rptr + ehlen + iphlen);
1684         tcp_seq += mss;
1685 #ifdef __sparc
1686         tcp_seq_tmp = htonl(tcp_seq);
1687         bcopy(&tcp_seq_tmp, (char *)&ntcph->th_seq, 4);
1688 #else
1689         ntcph->th_seq = htonl(tcp_seq);
1690 #endif
1691         ntcph->th_flags = (otcph->th_flags & ~TH_URG);
1692 
1693         DB_CKSUMFLAGS(nmp) = (uint16_t)hckflags;
1694         DB_CKSUMSTART(nmp) = start_offset;
1695         DB_CKSUMSTUFF(nmp) = stuff_offset;
1696 
1697         /* calculate IP checksum and TCP pseudo header checksum */
1698         niph->ip_sum = 0;
1699         niph->ip_sum = (uint16_t)nxge_csgen((uint16_t *)niph, iphlen);
1700 
1701         l4_len = ntohs(niph->ip_len) - iphlen;
1702         sum = htons(l4_len) + l4cksum;
1703         sum = (sum & 0xFFFF) + (sum >> 16);
1704         ntcph->th_sum = (sum & 0xffff);
1705 
1706         NXGE_DEBUG_MSG((NULL, TX_CTL,
1707             "==> nxge_do_softlso: last next "
1708             "niph->ip_sum 0x%x "
1709             "ntcph->th_sum 0x%x sum 0x%x "
1710             "dump last ip->tcp %s "
1711             "cmp $%p mp $%p (mp_chain $%p) pktlen %d (0x%x) "
1712             "l4_len %d (0x%x) "
1713             "MBLKL(mp): %d, MBLKL(datamp): %d ",
1714             niph->ip_sum,
1715             ntcph->th_sum, sum,
1716             nxge_dump_packet((char *)niph, 52),
1717             cmp, nmp, mp_chain,
1718             pktlen, pktlen,
1719             l4_len,
1720             l4_len,
1721             (int)MBLKL(mp), (int)MBLKL(datamp)));
1722 
1723 cleanup_allocated_msgs:
1724         if (do_cleanup) {
1725                 NXGE_DEBUG_MSG((NULL, TX_CTL,
1726                     "==> nxge_do_softlso: "
1727                     "Failed allocating messages, "
1728                     "have to clean up and fail!"));
1729                 while (mp_chain != NULL) {
1730                         nmp = mp_chain;
1731                         mp_chain = mp_chain->b_next;
1732                         freemsg(nmp);
1733                 }
1734         }
1735         /*
1736          * We're done here, so just free the original message and return the
1737          * new message chain, that could be NULL if failed, back to the caller.
1738          */
1739         freemsg(mp);
1740 
1741         NXGE_DEBUG_MSG((NULL, TX_CTL,
1742             "<== nxge_do_softlso:mp_chain $%p", mp_chain));
1743         return (mp_chain);
1744 }
1745 
1746 /*
1747  * Will be called before NIC driver do further operation on the message.
1748  * The input message may include LSO information, if so, go to softlso logic
1749  * to eliminate the oversized LSO packet for the incapable underlying h/w.
1750  * The return could be the same non-LSO message or a message chain for LSO case.
1751  *
1752  * The driver needs to call this function per packet and process the whole chain
1753  * if applied.
1754  */
1755 static mblk_t *
1756 nxge_lso_eliminate(mblk_t *mp)
1757 {
1758         uint32_t lsoflags;
1759         uint32_t mss;
1760 
1761         NXGE_DEBUG_MSG((NULL, TX_CTL,
1762             "==>nxge_lso_eliminate:"));
1763         nxge_lso_info_get(mp, &mss, &lsoflags);
1764 
1765         if (lsoflags & HW_LSO) {
1766                 mblk_t *nmp;
1767 
1768                 NXGE_DEBUG_MSG((NULL, TX_CTL,
1769                     "==>nxge_lso_eliminate:"
1770                     "HW_LSO:mss %d mp $%p",
1771                     mss, mp));
1772                 if ((nmp = nxge_do_softlso(mp, mss)) != NULL) {
1773                         NXGE_DEBUG_MSG((NULL, TX_CTL,
1774                             "<== nxge_lso_eliminate: "
1775                             "LSO: nmp not NULL nmp $%p mss %d mp $%p",
1776                             nmp, mss, mp));
1777                         return (nmp);
1778                 } else {
1779                         NXGE_DEBUG_MSG((NULL, TX_CTL,
1780                             "<== nxge_lso_eliminate_ "
1781                             "LSO: failed nmp NULL nmp $%p mss %d mp $%p",
1782                             nmp, mss, mp));
1783                         return (NULL);
1784                 }
1785         }
1786 
1787         NXGE_DEBUG_MSG((NULL, TX_CTL,
1788             "<== nxge_lso_eliminate"));
1789         return (mp);
1790 }
1791 
1792 static uint32_t
1793 nxge_csgen(uint16_t *adr, int len)
1794 {
1795         int             i, odd;
1796         uint32_t        sum = 0;
1797         uint32_t        c = 0;
1798 
1799         odd = len % 2;
1800         for (i = 0; i < (len / 2); i++) {
1801                 sum += (adr[i] & 0xffff);
1802         }
1803         if (odd) {
1804                 sum += adr[len / 2] & 0xff00;
1805         }
1806         while ((c = ((sum & 0xffff0000) >> 16)) != 0) {
1807                 sum &= 0xffff;
1808                 sum += c;
1809         }
1810         return (~sum & 0xffff);
1811 }