1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 #include <hxge_impl.h>
27 #include <hxge_txdma.h>
28 #include <sys/llc1.h>
29
30 uint32_t hxge_reclaim_pending = TXDMA_RECLAIM_PENDING_DEFAULT;
31 uint32_t hxge_tx_minfree = 64;
32 uint32_t hxge_tx_intr_thres = 0;
33 uint32_t hxge_tx_max_gathers = TX_MAX_GATHER_POINTERS;
34 uint32_t hxge_tx_tiny_pack = 1;
35 uint32_t hxge_tx_use_bcopy = 1;
36
37 extern uint32_t hxge_tx_ring_size;
38 extern uint32_t hxge_bcopy_thresh;
39 extern uint32_t hxge_dvma_thresh;
40 extern uint32_t hxge_dma_stream_thresh;
41 extern dma_method_t hxge_force_dma;
42
43 /* Device register access attributes for PIO. */
44 extern ddi_device_acc_attr_t hxge_dev_reg_acc_attr;
45
46 /* Device descriptor access attributes for DMA. */
47 extern ddi_device_acc_attr_t hxge_dev_desc_dma_acc_attr;
48
49 /* Device buffer access attributes for DMA. */
50 extern ddi_device_acc_attr_t hxge_dev_buf_dma_acc_attr;
51 extern ddi_dma_attr_t hxge_desc_dma_attr;
52 extern ddi_dma_attr_t hxge_tx_dma_attr;
53
54 static hxge_status_t hxge_map_txdma(p_hxge_t hxgep);
55 static void hxge_unmap_txdma(p_hxge_t hxgep);
56 static hxge_status_t hxge_txdma_hw_start(p_hxge_t hxgep);
57 static void hxge_txdma_hw_stop(p_hxge_t hxgep);
58
59 static hxge_status_t hxge_map_txdma_channel(p_hxge_t hxgep, uint16_t channel,
60 p_hxge_dma_common_t *dma_buf_p, p_tx_ring_t *tx_desc_p,
61 uint32_t num_chunks, p_hxge_dma_common_t *dma_cntl_p,
62 p_tx_mbox_t *tx_mbox_p);
63 static void hxge_unmap_txdma_channel(p_hxge_t hxgep, uint16_t channel,
64 p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p);
65 static hxge_status_t hxge_map_txdma_channel_buf_ring(p_hxge_t hxgep, uint16_t,
66 p_hxge_dma_common_t *, p_tx_ring_t *, uint32_t);
67 static void hxge_unmap_txdma_channel_buf_ring(p_hxge_t hxgep,
68 p_tx_ring_t tx_ring_p);
69 static void hxge_map_txdma_channel_cfg_ring(p_hxge_t, uint16_t,
70 p_hxge_dma_common_t *, p_tx_ring_t, p_tx_mbox_t *);
71 static void hxge_unmap_txdma_channel_cfg_ring(p_hxge_t hxgep,
72 p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p);
73 static hxge_status_t hxge_txdma_start_channel(p_hxge_t hxgep, uint16_t channel,
74 p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p);
75 static hxge_status_t hxge_txdma_stop_channel(p_hxge_t hxgep, uint16_t channel,
76 p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p);
77 static p_tx_ring_t hxge_txdma_get_ring(p_hxge_t hxgep, uint16_t channel);
78 static hxge_status_t hxge_tx_err_evnts(p_hxge_t hxgep, uint_t index,
79 p_hxge_ldv_t ldvp, tdc_stat_t cs);
80 static p_tx_mbox_t hxge_txdma_get_mbox(p_hxge_t hxgep, uint16_t channel);
81 static hxge_status_t hxge_txdma_fatal_err_recover(p_hxge_t hxgep,
82 uint16_t channel, p_tx_ring_t tx_ring_p);
83 static hxge_status_t hxge_tx_port_fatal_err_recover(p_hxge_t hxgep);
84
85 hxge_status_t
86 hxge_init_txdma_channels(p_hxge_t hxgep)
87 {
88 hxge_status_t status = HXGE_OK;
89 block_reset_t reset_reg;
90
91 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_init_txdma_channels"));
92
93 /*
94 * Reset TDC block from PEU to cleanup any unknown configuration.
95 * This may be resulted from previous reboot.
96 */
97 reset_reg.value = 0;
98 reset_reg.bits.tdc_rst = 1;
99 HXGE_REG_WR32(hxgep->hpi_handle, BLOCK_RESET, reset_reg.value);
100
101 HXGE_DELAY(1000);
102
103 status = hxge_map_txdma(hxgep);
104 if (status != HXGE_OK) {
105 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
106 "<== hxge_init_txdma_channels: status 0x%x", status));
107 return (status);
108 }
109
110 status = hxge_txdma_hw_start(hxgep);
111 if (status != HXGE_OK) {
112 hxge_unmap_txdma(hxgep);
113 return (status);
114 }
115
116 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
117 "<== hxge_init_txdma_channels: status 0x%x", status));
118
119 return (HXGE_OK);
120 }
121
122 void
123 hxge_uninit_txdma_channels(p_hxge_t hxgep)
124 {
125 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_uninit_txdma_channels"));
126
127 hxge_txdma_hw_stop(hxgep);
128 hxge_unmap_txdma(hxgep);
129
130 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "<== hxge_uinit_txdma_channels"));
131 }
132
133 void
134 hxge_setup_dma_common(p_hxge_dma_common_t dest_p, p_hxge_dma_common_t src_p,
135 uint32_t entries, uint32_t size)
136 {
137 size_t tsize;
138 *dest_p = *src_p;
139 tsize = size * entries;
140 dest_p->alength = tsize;
141 dest_p->nblocks = entries;
142 dest_p->block_size = size;
143 dest_p->offset += tsize;
144
145 src_p->kaddrp = (caddr_t)dest_p->kaddrp + tsize;
146 src_p->alength -= tsize;
147 src_p->dma_cookie.dmac_laddress += tsize;
148 src_p->dma_cookie.dmac_size -= tsize;
149 }
150
151 hxge_status_t
152 hxge_reset_txdma_channel(p_hxge_t hxgep, uint16_t channel, uint64_t reg_data)
153 {
154 hpi_status_t rs = HPI_SUCCESS;
155 hxge_status_t status = HXGE_OK;
156 hpi_handle_t handle;
157
158 HXGE_DEBUG_MSG((hxgep, TX_CTL, " ==> hxge_reset_txdma_channel"));
159
160 handle = HXGE_DEV_HPI_HANDLE(hxgep);
161 if ((reg_data & TDC_TDR_RST_MASK) == TDC_TDR_RST_MASK) {
162 rs = hpi_txdma_channel_reset(handle, channel);
163 } else {
164 rs = hpi_txdma_channel_control(handle, TXDMA_RESET, channel);
165 }
166
167 if (rs != HPI_SUCCESS) {
168 status = HXGE_ERROR | rs;
169 }
170
171 /*
172 * Reset the tail (kick) register to 0. (Hardware will not reset it. Tx
173 * overflow fatal error if tail is not set to 0 after reset!
174 */
175 TXDMA_REG_WRITE64(handle, TDC_TDR_KICK, channel, 0);
176
177 HXGE_DEBUG_MSG((hxgep, TX_CTL, " <== hxge_reset_txdma_channel"));
178
179 return (status);
180 }
181
182 hxge_status_t
183 hxge_init_txdma_channel_event_mask(p_hxge_t hxgep, uint16_t channel,
184 tdc_int_mask_t *mask_p)
185 {
186 hpi_handle_t handle;
187 hpi_status_t rs = HPI_SUCCESS;
188 hxge_status_t status = HXGE_OK;
189
190 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
191 "<== hxge_init_txdma_channel_event_mask"));
192
193 handle = HXGE_DEV_HPI_HANDLE(hxgep);
194
195 /*
196 * Mask off tx_rng_oflow since it is a false alarm. The driver
197 * ensures not over flowing the hardware and check the hardware
198 * status.
199 */
200 mask_p->bits.tx_rng_oflow = 1;
201 rs = hpi_txdma_event_mask(handle, OP_SET, channel, mask_p);
202 if (rs != HPI_SUCCESS) {
203 status = HXGE_ERROR | rs;
204 }
205
206 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
207 "==> hxge_init_txdma_channel_event_mask"));
208 return (status);
209 }
210
211 hxge_status_t
212 hxge_enable_txdma_channel(p_hxge_t hxgep,
213 uint16_t channel, p_tx_ring_t tx_desc_p, p_tx_mbox_t mbox_p)
214 {
215 hpi_handle_t handle;
216 hpi_status_t rs = HPI_SUCCESS;
217 hxge_status_t status = HXGE_OK;
218
219 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_enable_txdma_channel"));
220
221 handle = HXGE_DEV_HPI_HANDLE(hxgep);
222 /*
223 * Use configuration data composed at init time. Write to hardware the
224 * transmit ring configurations.
225 */
226 rs = hpi_txdma_ring_config(handle, OP_SET, channel,
227 (uint64_t *)&(tx_desc_p->tx_ring_cfig.value));
228
229 if (rs != HPI_SUCCESS) {
230 return (HXGE_ERROR | rs);
231 }
232
233 /* Write to hardware the mailbox */
234 rs = hpi_txdma_mbox_config(handle, OP_SET, channel,
235 (uint64_t *)&mbox_p->tx_mbox.dma_cookie.dmac_laddress);
236
237 if (rs != HPI_SUCCESS) {
238 return (HXGE_ERROR | rs);
239 }
240
241 /* Start the DMA engine. */
242 rs = hpi_txdma_channel_init_enable(handle, channel);
243 if (rs != HPI_SUCCESS) {
244 return (HXGE_ERROR | rs);
245 }
246 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "<== hxge_enable_txdma_channel"));
247 return (status);
248 }
249
250 void
251 hxge_fill_tx_hdr(p_mblk_t mp, boolean_t fill_len, boolean_t l4_cksum,
252 int pkt_len, uint8_t npads, p_tx_pkt_hdr_all_t pkthdrp)
253 {
254 p_tx_pkt_header_t hdrp;
255 p_mblk_t nmp;
256 uint64_t tmp;
257 size_t mblk_len;
258 size_t iph_len;
259 size_t hdrs_size;
260 uint8_t *ip_buf;
261 uint16_t eth_type;
262 uint8_t ipproto;
263 boolean_t is_vlan = B_FALSE;
264 size_t eth_hdr_size;
265 uint8_t hdrs_buf[sizeof (struct ether_header) + 64 + sizeof (uint32_t)];
266
267 HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_fill_tx_hdr: mp $%p", mp));
268
269 /*
270 * Caller should zero out the headers first.
271 */
272 hdrp = (p_tx_pkt_header_t)&pkthdrp->pkthdr;
273
274 if (fill_len) {
275 HXGE_DEBUG_MSG((NULL, TX_CTL,
276 "==> hxge_fill_tx_hdr: pkt_len %d npads %d",
277 pkt_len, npads));
278 tmp = (uint64_t)pkt_len;
279 hdrp->value |= (tmp << TX_PKT_HEADER_TOT_XFER_LEN_SHIFT);
280
281 goto fill_tx_header_done;
282 }
283 tmp = (uint64_t)npads;
284 hdrp->value |= (tmp << TX_PKT_HEADER_PAD_SHIFT);
285
286 /*
287 * mp is the original data packet (does not include the Neptune
288 * transmit header).
289 */
290 nmp = mp;
291 mblk_len = (size_t)nmp->b_wptr - (size_t)nmp->b_rptr;
292 HXGE_DEBUG_MSG((NULL, TX_CTL,
293 "==> hxge_fill_tx_hdr: mp $%p b_rptr $%p len %d",
294 mp, nmp->b_rptr, mblk_len));
295 ip_buf = NULL;
296 bcopy(nmp->b_rptr, &hdrs_buf[0], sizeof (struct ether_vlan_header));
297 eth_type = ntohs(((p_ether_header_t)hdrs_buf)->ether_type);
298 HXGE_DEBUG_MSG((NULL, TX_CTL,
299 "==> : hxge_fill_tx_hdr: (value 0x%llx) ether type 0x%x",
300 eth_type, hdrp->value));
301
302 if (eth_type < ETHERMTU) {
303 tmp = 1ull;
304 hdrp->value |= (tmp << TX_PKT_HEADER_LLC_SHIFT);
305 HXGE_DEBUG_MSG((NULL, TX_CTL,
306 "==> hxge_tx_pkt_hdr_init: LLC value 0x%llx", hdrp->value));
307 if (*(hdrs_buf + sizeof (struct ether_header)) ==
308 LLC_SNAP_SAP) {
309 eth_type = ntohs(*((uint16_t *)(hdrs_buf +
310 sizeof (struct ether_header) + 6)));
311 HXGE_DEBUG_MSG((NULL, TX_CTL,
312 "==> hxge_tx_pkt_hdr_init: LLC ether type 0x%x",
313 eth_type));
314 } else {
315 goto fill_tx_header_done;
316 }
317 } else if (eth_type == VLAN_ETHERTYPE) {
318 tmp = 1ull;
319 hdrp->value |= (tmp << TX_PKT_HEADER_VLAN__SHIFT);
320
321 eth_type = ntohs(((struct ether_vlan_header *)
322 hdrs_buf)->ether_type);
323 is_vlan = B_TRUE;
324 HXGE_DEBUG_MSG((NULL, TX_CTL,
325 "==> hxge_tx_pkt_hdr_init: VLAN value 0x%llx",
326 hdrp->value));
327 }
328 if (!is_vlan) {
329 eth_hdr_size = sizeof (struct ether_header);
330 } else {
331 eth_hdr_size = sizeof (struct ether_vlan_header);
332 }
333
334 switch (eth_type) {
335 case ETHERTYPE_IP:
336 if (mblk_len > eth_hdr_size + sizeof (uint8_t)) {
337 ip_buf = nmp->b_rptr + eth_hdr_size;
338 mblk_len -= eth_hdr_size;
339 iph_len = ((*ip_buf) & 0x0f);
340 if (mblk_len > (iph_len + sizeof (uint32_t))) {
341 ip_buf = nmp->b_rptr;
342 ip_buf += eth_hdr_size;
343 } else {
344 ip_buf = NULL;
345 }
346 }
347 if (ip_buf == NULL) {
348 hdrs_size = 0;
349 ((p_ether_header_t)hdrs_buf)->ether_type = 0;
350 while ((nmp) && (hdrs_size < sizeof (hdrs_buf))) {
351 mblk_len = (size_t)nmp->b_wptr -
352 (size_t)nmp->b_rptr;
353 if (mblk_len >=
354 (sizeof (hdrs_buf) - hdrs_size))
355 mblk_len = sizeof (hdrs_buf) -
356 hdrs_size;
357 bcopy(nmp->b_rptr,
358 &hdrs_buf[hdrs_size], mblk_len);
359 hdrs_size += mblk_len;
360 nmp = nmp->b_cont;
361 }
362 ip_buf = hdrs_buf;
363 ip_buf += eth_hdr_size;
364 iph_len = ((*ip_buf) & 0x0f);
365 }
366 ipproto = ip_buf[9];
367
368 tmp = (uint64_t)iph_len;
369 hdrp->value |= (tmp << TX_PKT_HEADER_IHL_SHIFT);
370 tmp = (uint64_t)(eth_hdr_size >> 1);
371 hdrp->value |= (tmp << TX_PKT_HEADER_L3START_SHIFT);
372
373 HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_fill_tx_hdr: IPv4 "
374 " iph_len %d l3start %d eth_hdr_size %d proto 0x%x"
375 "tmp 0x%x", iph_len, hdrp->bits.l3start, eth_hdr_size,
376 ipproto, tmp));
377 HXGE_DEBUG_MSG((NULL, TX_CTL,
378 "==> hxge_tx_pkt_hdr_init: IP value 0x%llx", hdrp->value));
379 break;
380
381 case ETHERTYPE_IPV6:
382 hdrs_size = 0;
383 ((p_ether_header_t)hdrs_buf)->ether_type = 0;
384 while ((nmp) && (hdrs_size < sizeof (hdrs_buf))) {
385 mblk_len = (size_t)nmp->b_wptr - (size_t)nmp->b_rptr;
386 if (mblk_len >= (sizeof (hdrs_buf) - hdrs_size))
387 mblk_len = sizeof (hdrs_buf) - hdrs_size;
388 bcopy(nmp->b_rptr, &hdrs_buf[hdrs_size], mblk_len);
389 hdrs_size += mblk_len;
390 nmp = nmp->b_cont;
391 }
392 ip_buf = hdrs_buf;
393 ip_buf += eth_hdr_size;
394
395 tmp = 1ull;
396 hdrp->value |= (tmp << TX_PKT_HEADER_IP_VER_SHIFT);
397
398 tmp = (eth_hdr_size >> 1);
399 hdrp->value |= (tmp << TX_PKT_HEADER_L3START_SHIFT);
400
401 /* byte 6 is the next header protocol */
402 ipproto = ip_buf[6];
403
404 HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_fill_tx_hdr: IPv6 "
405 " iph_len %d l3start %d eth_hdr_size %d proto 0x%x",
406 iph_len, hdrp->bits.l3start, eth_hdr_size, ipproto));
407 HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_tx_pkt_hdr_init: IPv6 "
408 "value 0x%llx", hdrp->value));
409 break;
410
411 default:
412 HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_fill_tx_hdr: non-IP"));
413 goto fill_tx_header_done;
414 }
415
416 switch (ipproto) {
417 case IPPROTO_TCP:
418 HXGE_DEBUG_MSG((NULL, TX_CTL,
419 "==> hxge_fill_tx_hdr: TCP (cksum flag %d)", l4_cksum));
420 if (l4_cksum) {
421 tmp = 1ull;
422 hdrp->value |= (tmp << TX_PKT_HEADER_PKT_TYPE_SHIFT);
423 HXGE_DEBUG_MSG((NULL, TX_CTL,
424 "==> hxge_tx_pkt_hdr_init: TCP CKSUM"
425 "value 0x%llx", hdrp->value));
426 }
427 HXGE_DEBUG_MSG((NULL, TX_CTL,
428 "==> hxge_tx_pkt_hdr_init: TCP value 0x%llx", hdrp->value));
429 break;
430
431 case IPPROTO_UDP:
432 HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_fill_tx_hdr: UDP"));
433 if (l4_cksum) {
434 tmp = 0x2ull;
435 hdrp->value |= (tmp << TX_PKT_HEADER_PKT_TYPE_SHIFT);
436 }
437 HXGE_DEBUG_MSG((NULL, TX_CTL,
438 "==> hxge_tx_pkt_hdr_init: UDP value 0x%llx",
439 hdrp->value));
440 break;
441
442 default:
443 goto fill_tx_header_done;
444 }
445
446 fill_tx_header_done:
447 HXGE_DEBUG_MSG((NULL, TX_CTL,
448 "==> hxge_fill_tx_hdr: pkt_len %d npads %d value 0x%llx",
449 pkt_len, npads, hdrp->value));
450 HXGE_DEBUG_MSG((NULL, TX_CTL, "<== hxge_fill_tx_hdr"));
451 }
452
453 /*ARGSUSED*/
454 p_mblk_t
455 hxge_tx_pkt_header_reserve(p_mblk_t mp, uint8_t *npads)
456 {
457 p_mblk_t newmp = NULL;
458
459 if ((newmp = allocb(TX_PKT_HEADER_SIZE, BPRI_MED)) == NULL) {
460 HXGE_DEBUG_MSG((NULL, TX_CTL,
461 "<== hxge_tx_pkt_header_reserve: allocb failed"));
462 return (NULL);
463 }
464 HXGE_DEBUG_MSG((NULL, TX_CTL,
465 "==> hxge_tx_pkt_header_reserve: get new mp"));
466 DB_TYPE(newmp) = M_DATA;
467 newmp->b_rptr = newmp->b_wptr = DB_LIM(newmp);
468 linkb(newmp, mp);
469 newmp->b_rptr -= TX_PKT_HEADER_SIZE;
470
471 HXGE_DEBUG_MSG((NULL, TX_CTL,
472 "==>hxge_tx_pkt_header_reserve: b_rptr $%p b_wptr $%p",
473 newmp->b_rptr, newmp->b_wptr));
474 HXGE_DEBUG_MSG((NULL, TX_CTL,
475 "<== hxge_tx_pkt_header_reserve: use new mp"));
476 return (newmp);
477 }
478
479 int
480 hxge_tx_pkt_nmblocks(p_mblk_t mp, int *tot_xfer_len_p)
481 {
482 uint_t nmblks;
483 ssize_t len;
484 uint_t pkt_len;
485 p_mblk_t nmp, bmp, tmp;
486 uint8_t *b_wptr;
487
488 HXGE_DEBUG_MSG((NULL, TX_CTL,
489 "==> hxge_tx_pkt_nmblocks: mp $%p rptr $%p wptr $%p len %d",
490 mp, mp->b_rptr, mp->b_wptr, MBLKL(mp)));
491
492 nmp = mp;
493 bmp = mp;
494 nmblks = 0;
495 pkt_len = 0;
496 *tot_xfer_len_p = 0;
497
498 while (nmp) {
499 len = MBLKL(nmp);
500 HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_tx_pkt_nmblocks: "
501 "len %d pkt_len %d nmblks %d tot_xfer_len %d",
502 len, pkt_len, nmblks, *tot_xfer_len_p));
503
504 if (len <= 0) {
505 bmp = nmp;
506 nmp = nmp->b_cont;
507 HXGE_DEBUG_MSG((NULL, TX_CTL,
508 "==> hxge_tx_pkt_nmblocks:"
509 " len (0) pkt_len %d nmblks %d", pkt_len, nmblks));
510 continue;
511 }
512 *tot_xfer_len_p += len;
513 HXGE_DEBUG_MSG((NULL, TX_CTL, "==> hxge_tx_pkt_nmblocks: "
514 "len %d pkt_len %d nmblks %d tot_xfer_len %d",
515 len, pkt_len, nmblks, *tot_xfer_len_p));
516
517 if (len < hxge_bcopy_thresh) {
518 HXGE_DEBUG_MSG((NULL, TX_CTL,
519 "==> hxge_tx_pkt_nmblocks: "
520 "len %d (< thresh) pkt_len %d nmblks %d",
521 len, pkt_len, nmblks));
522 if (pkt_len == 0)
523 nmblks++;
524 pkt_len += len;
525 if (pkt_len >= hxge_bcopy_thresh) {
526 pkt_len = 0;
527 len = 0;
528 nmp = bmp;
529 }
530 } else {
531 HXGE_DEBUG_MSG((NULL, TX_CTL,
532 "==> hxge_tx_pkt_nmblocks: "
533 "len %d (> thresh) pkt_len %d nmblks %d",
534 len, pkt_len, nmblks));
535 pkt_len = 0;
536 nmblks++;
537 /*
538 * Hardware limits the transfer length to 4K. If len is
539 * more than 4K, we need to break it up to at most 2
540 * more blocks.
541 */
542 if (len > TX_MAX_TRANSFER_LENGTH) {
543 uint32_t nsegs;
544
545 HXGE_DEBUG_MSG((NULL, TX_CTL,
546 "==> hxge_tx_pkt_nmblocks: "
547 "len %d pkt_len %d nmblks %d nsegs %d",
548 len, pkt_len, nmblks, nsegs));
549 nsegs = 1;
550 if (len % (TX_MAX_TRANSFER_LENGTH * 2)) {
551 ++nsegs;
552 }
553 do {
554 b_wptr = nmp->b_rptr +
555 TX_MAX_TRANSFER_LENGTH;
556 nmp->b_wptr = b_wptr;
557 if ((tmp = dupb(nmp)) == NULL) {
558 return (0);
559 }
560 tmp->b_rptr = b_wptr;
561 tmp->b_wptr = nmp->b_wptr;
562 tmp->b_cont = nmp->b_cont;
563 nmp->b_cont = tmp;
564 nmblks++;
565 if (--nsegs) {
566 nmp = tmp;
567 }
568 } while (nsegs);
569 nmp = tmp;
570 }
571 }
572
573 /*
574 * Hardware limits the transmit gather pointers to 15.
575 */
576 if (nmp->b_cont && (nmblks + TX_GATHER_POINTERS_THRESHOLD) >
577 TX_MAX_GATHER_POINTERS) {
578 HXGE_DEBUG_MSG((NULL, TX_CTL,
579 "==> hxge_tx_pkt_nmblocks: pull msg - "
580 "len %d pkt_len %d nmblks %d",
581 len, pkt_len, nmblks));
582 /* Pull all message blocks from b_cont */
583 if ((tmp = msgpullup(nmp->b_cont, -1)) == NULL) {
584 return (0);
585 }
586 freemsg(nmp->b_cont);
587 nmp->b_cont = tmp;
588 pkt_len = 0;
589 }
590 bmp = nmp;
591 nmp = nmp->b_cont;
592 }
593
594 HXGE_DEBUG_MSG((NULL, TX_CTL,
595 "<== hxge_tx_pkt_nmblocks: rptr $%p wptr $%p "
596 "nmblks %d len %d tot_xfer_len %d",
597 mp->b_rptr, mp->b_wptr, nmblks, MBLKL(mp), *tot_xfer_len_p));
598 return (nmblks);
599 }
600
601 boolean_t
602 hxge_txdma_reclaim(p_hxge_t hxgep, p_tx_ring_t tx_ring_p, int nmblks)
603 {
604 boolean_t status = B_TRUE;
605 p_hxge_dma_common_t tx_desc_dma_p;
606 hxge_dma_common_t desc_area;
607 p_tx_desc_t tx_desc_ring_vp;
608 p_tx_desc_t tx_desc_p;
609 p_tx_desc_t tx_desc_pp;
610 tx_desc_t r_tx_desc;
611 p_tx_msg_t tx_msg_ring;
612 p_tx_msg_t tx_msg_p;
613 hpi_handle_t handle;
614 tdc_tdr_head_t tx_head;
615 uint32_t pkt_len;
616 uint_t tx_rd_index;
617 uint16_t head_index, tail_index;
618 uint8_t tdc;
619 boolean_t head_wrap, tail_wrap;
620 p_hxge_tx_ring_stats_t tdc_stats;
621 tdc_byte_cnt_t byte_cnt;
622 tdc_tdr_qlen_t qlen;
623 int rc;
624
625 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_reclaim"));
626
627 status = ((tx_ring_p->descs_pending < hxge_reclaim_pending) &&
628 (nmblks != 0));
629 HXGE_DEBUG_MSG((hxgep, TX_CTL,
630 "==> hxge_txdma_reclaim: pending %d reclaim %d nmblks %d",
631 tx_ring_p->descs_pending, hxge_reclaim_pending, nmblks));
632
633 if (!status) {
634 tx_desc_dma_p = &tx_ring_p->tdc_desc;
635 desc_area = tx_ring_p->tdc_desc;
636 tx_desc_ring_vp = tx_desc_dma_p->kaddrp;
637 tx_desc_ring_vp = (p_tx_desc_t)DMA_COMMON_VPTR(desc_area);
638 tx_rd_index = tx_ring_p->rd_index;
639 tx_desc_p = &tx_desc_ring_vp[tx_rd_index];
640 tx_msg_ring = tx_ring_p->tx_msg_ring;
641 tx_msg_p = &tx_msg_ring[tx_rd_index];
642 tdc = tx_ring_p->tdc;
643 tdc_stats = tx_ring_p->tdc_stats;
644 if (tx_ring_p->descs_pending > tdc_stats->tx_max_pend) {
645 tdc_stats->tx_max_pend = tx_ring_p->descs_pending;
646 }
647 tail_index = tx_ring_p->wr_index;
648 tail_wrap = tx_ring_p->wr_index_wrap;
649
650 /*
651 * tdc_byte_cnt reg can be used to get bytes transmitted. It
652 * includes padding too in case of runt packets.
653 */
654 handle = HXGE_DEV_HPI_HANDLE(hxgep);
655 TXDMA_REG_READ64(handle, TDC_BYTE_CNT, tdc, &byte_cnt.value);
656 tdc_stats->obytes_with_pad += byte_cnt.bits.byte_count;
657
658 HXGE_DEBUG_MSG((hxgep, TX_CTL,
659 "==> hxge_txdma_reclaim: tdc %d tx_rd_index %d "
660 "tail_index %d tail_wrap %d tx_desc_p $%p ($%p) ",
661 tdc, tx_rd_index, tail_index, tail_wrap,
662 tx_desc_p, (*(uint64_t *)tx_desc_p)));
663
664 /*
665 * Read the hardware maintained transmit head and wrap around
666 * bit.
667 */
668 TXDMA_REG_READ64(handle, TDC_TDR_HEAD, tdc, &tx_head.value);
669 head_index = tx_head.bits.head;
670 head_wrap = tx_head.bits.wrap;
671 HXGE_DEBUG_MSG((hxgep, TX_CTL,
672 "==> hxge_txdma_reclaim: "
673 "tx_rd_index %d tail %d tail_wrap %d head %d wrap %d",
674 tx_rd_index, tail_index, tail_wrap, head_index, head_wrap));
675
676 /*
677 * For debug only. This can be used to verify the qlen and make
678 * sure the hardware is wrapping the Tdr correctly.
679 */
680 TXDMA_REG_READ64(handle, TDC_TDR_QLEN, tdc, &qlen.value);
681 HXGE_DEBUG_MSG((hxgep, TX_CTL,
682 "==> hxge_txdma_reclaim: tdr_qlen %d tdr_pref_qlen %d",
683 qlen.bits.tdr_qlen, qlen.bits.tdr_pref_qlen));
684
685 if (head_index == tail_index) {
686 if (TXDMA_RING_EMPTY(head_index, head_wrap, tail_index,
687 tail_wrap) && (head_index == tx_rd_index)) {
688 HXGE_DEBUG_MSG((hxgep, TX_CTL,
689 "==> hxge_txdma_reclaim: EMPTY"));
690 return (B_TRUE);
691 }
692 HXGE_DEBUG_MSG((hxgep, TX_CTL,
693 "==> hxge_txdma_reclaim: Checking if ring full"));
694 if (TXDMA_RING_FULL(head_index, head_wrap, tail_index,
695 tail_wrap)) {
696 HXGE_DEBUG_MSG((hxgep, TX_CTL,
697 "==> hxge_txdma_reclaim: full"));
698 return (B_FALSE);
699 }
700 }
701 HXGE_DEBUG_MSG((hxgep, TX_CTL,
702 "==> hxge_txdma_reclaim: tx_rd_index and head_index"));
703
704 /* XXXX: limit the # of reclaims */
705 tx_desc_pp = &r_tx_desc;
706 while ((tx_rd_index != head_index) &&
707 (tx_ring_p->descs_pending != 0)) {
708 HXGE_DEBUG_MSG((hxgep, TX_CTL,
709 "==> hxge_txdma_reclaim: Checking if pending"));
710 HXGE_DEBUG_MSG((hxgep, TX_CTL,
711 "==> hxge_txdma_reclaim: descs_pending %d ",
712 tx_ring_p->descs_pending));
713 HXGE_DEBUG_MSG((hxgep, TX_CTL,
714 "==> hxge_txdma_reclaim: "
715 "(tx_rd_index %d head_index %d (tx_desc_p $%p)",
716 tx_rd_index, head_index, tx_desc_p));
717
718 tx_desc_pp->value = tx_desc_p->value;
719 HXGE_DEBUG_MSG((hxgep, TX_CTL,
720 "==> hxge_txdma_reclaim: "
721 "(tx_rd_index %d head_index %d "
722 "tx_desc_p $%p (desc value 0x%llx) ",
723 tx_rd_index, head_index,
724 tx_desc_pp, (*(uint64_t *)tx_desc_pp)));
725 HXGE_DEBUG_MSG((hxgep, TX_CTL,
726 "==> hxge_txdma_reclaim: dump desc:"));
727
728 /*
729 * tdc_byte_cnt reg can be used to get bytes
730 * transmitted
731 */
732 pkt_len = tx_desc_pp->bits.tr_len;
733 tdc_stats->obytes += pkt_len;
734 tdc_stats->opackets += tx_desc_pp->bits.sop;
735 HXGE_DEBUG_MSG((hxgep, TX_CTL,
736 "==> hxge_txdma_reclaim: pkt_len %d "
737 "tdc channel %d opackets %d",
738 pkt_len, tdc, tdc_stats->opackets));
739
740 if (tx_msg_p->flags.dma_type == USE_DVMA) {
741 HXGE_DEBUG_MSG((hxgep, TX_CTL,
742 "tx_desc_p = $%p tx_desc_pp = $%p "
743 "index = %d",
744 tx_desc_p, tx_desc_pp,
745 tx_ring_p->rd_index));
746 (void) dvma_unload(tx_msg_p->dvma_handle,
747 0, -1);
748 tx_msg_p->dvma_handle = NULL;
749 if (tx_ring_p->dvma_wr_index ==
750 tx_ring_p->dvma_wrap_mask) {
751 tx_ring_p->dvma_wr_index = 0;
752 } else {
753 tx_ring_p->dvma_wr_index++;
754 }
755 tx_ring_p->dvma_pending--;
756 } else if (tx_msg_p->flags.dma_type == USE_DMA) {
757 HXGE_DEBUG_MSG((hxgep, TX_CTL,
758 "==> hxge_txdma_reclaim: USE DMA"));
759 if (rc = ddi_dma_unbind_handle
760 (tx_msg_p->dma_handle)) {
761 cmn_err(CE_WARN, "hxge_reclaim: "
762 "ddi_dma_unbind_handle "
763 "failed. status %d", rc);
764 }
765 }
766
767 HXGE_DEBUG_MSG((hxgep, TX_CTL,
768 "==> hxge_txdma_reclaim: count packets"));
769
770 /*
771 * count a chained packet only once.
772 */
773 if (tx_msg_p->tx_message != NULL) {
774 freemsg(tx_msg_p->tx_message);
775 tx_msg_p->tx_message = NULL;
776 }
777 tx_msg_p->flags.dma_type = USE_NONE;
778 tx_rd_index = tx_ring_p->rd_index;
779 tx_rd_index = (tx_rd_index + 1) &
780 tx_ring_p->tx_wrap_mask;
781 tx_ring_p->rd_index = tx_rd_index;
782 tx_ring_p->descs_pending--;
783 tx_desc_p = &tx_desc_ring_vp[tx_rd_index];
784 tx_msg_p = &tx_msg_ring[tx_rd_index];
785 }
786
787 status = (nmblks <= ((int)tx_ring_p->tx_ring_size -
788 (int)tx_ring_p->descs_pending - TX_FULL_MARK));
789 if (status) {
790 (void) cas32((uint32_t *)&tx_ring_p->queueing, 1, 0);
791 }
792 } else {
793 status = (nmblks <= ((int)tx_ring_p->tx_ring_size -
794 (int)tx_ring_p->descs_pending - TX_FULL_MARK));
795 }
796
797 HXGE_DEBUG_MSG((hxgep, TX_CTL,
798 "<== hxge_txdma_reclaim status = 0x%08x", status));
799 return (status);
800 }
801
802 uint_t
803 hxge_tx_intr(caddr_t arg1, caddr_t arg2)
804 {
805 p_hxge_ldv_t ldvp = (p_hxge_ldv_t)arg1;
806 p_hxge_t hxgep = (p_hxge_t)arg2;
807 p_hxge_ldg_t ldgp;
808 uint8_t channel;
809 uint32_t vindex;
810 hpi_handle_t handle;
811 tdc_stat_t cs;
812 p_tx_ring_t *tx_rings;
813 p_tx_ring_t tx_ring_p;
814 hpi_status_t rs = HPI_SUCCESS;
815 uint_t serviced = DDI_INTR_UNCLAIMED;
816 hxge_status_t status = HXGE_OK;
817
818 if (ldvp == NULL) {
819 HXGE_DEBUG_MSG((NULL, INT_CTL,
820 "<== hxge_tx_intr: hxgep $%p ldvp $%p", hxgep, ldvp));
821 return (DDI_INTR_UNCLAIMED);
822 }
823
824 if (arg2 == NULL || (void *) ldvp->hxgep != arg2) {
825 hxgep = ldvp->hxgep;
826 }
827
828 /*
829 * If the interface is not started, just swallow the interrupt
830 * and don't rearm the logical device.
831 */
832 if (hxgep->hxge_mac_state != HXGE_MAC_STARTED)
833 return (DDI_INTR_CLAIMED);
834
835 HXGE_DEBUG_MSG((hxgep, INT_CTL,
836 "==> hxge_tx_intr: hxgep(arg2) $%p ldvp(arg1) $%p", hxgep, ldvp));
837
838 /*
839 * This interrupt handler is for a specific transmit dma channel.
840 */
841 handle = HXGE_DEV_HPI_HANDLE(hxgep);
842
843 /* Get the control and status for this channel. */
844 channel = ldvp->channel;
845 ldgp = ldvp->ldgp;
846 HXGE_DEBUG_MSG((hxgep, INT_CTL,
847 "==> hxge_tx_intr: hxgep $%p ldvp (ldvp) $%p channel %d",
848 hxgep, ldvp, channel));
849
850 rs = hpi_txdma_control_status(handle, OP_GET, channel, &cs);
851 vindex = ldvp->vdma_index;
852 HXGE_DEBUG_MSG((hxgep, INT_CTL,
853 "==> hxge_tx_intr:channel %d ring index %d status 0x%08x",
854 channel, vindex, rs));
855
856 if (!rs && cs.bits.marked) {
857 HXGE_DEBUG_MSG((hxgep, INT_CTL,
858 "==> hxge_tx_intr:channel %d ring index %d "
859 "status 0x%08x (marked bit set)", channel, vindex, rs));
860 tx_rings = hxgep->tx_rings->rings;
861 tx_ring_p = tx_rings[vindex];
862 HXGE_DEBUG_MSG((hxgep, INT_CTL,
863 "==> hxge_tx_intr:channel %d ring index %d "
864 "status 0x%08x (marked bit set, calling reclaim)",
865 channel, vindex, rs));
866
867 MUTEX_ENTER(&tx_ring_p->lock);
868 (void) hxge_txdma_reclaim(hxgep, tx_rings[vindex], 0);
869 MUTEX_EXIT(&tx_ring_p->lock);
870 mac_tx_update(hxgep->mach);
871 }
872
873 /*
874 * Process other transmit control and status. Check the ldv state.
875 */
876 status = hxge_tx_err_evnts(hxgep, ldvp->vdma_index, ldvp, cs);
877
878 /* Clear the error bits */
879 RXDMA_REG_WRITE64(handle, TDC_STAT, channel, cs.value);
880
881 /*
882 * Rearm this logical group if this is a single device group.
883 */
884 if (ldgp->nldvs == 1) {
885 HXGE_DEBUG_MSG((hxgep, INT_CTL, "==> hxge_tx_intr: rearm"));
886 if (status == HXGE_OK) {
887 (void) hpi_intr_ldg_mgmt_set(handle, ldgp->ldg,
888 B_TRUE, ldgp->ldg_timer);
889 }
890 }
891 HXGE_DEBUG_MSG((hxgep, INT_CTL, "<== hxge_tx_intr"));
892 serviced = DDI_INTR_CLAIMED;
893 return (serviced);
894 }
895
896 void
897 hxge_txdma_stop(p_hxge_t hxgep)
898 {
899 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_stop"));
900
901 (void) hxge_tx_vmac_disable(hxgep);
902 (void) hxge_txdma_hw_mode(hxgep, HXGE_DMA_STOP);
903
904 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_stop"));
905 }
906
907 hxge_status_t
908 hxge_txdma_hw_mode(p_hxge_t hxgep, boolean_t enable)
909 {
910 int i, ndmas;
911 uint16_t channel;
912 p_tx_rings_t tx_rings;
913 p_tx_ring_t *tx_desc_rings;
914 hpi_handle_t handle;
915 hpi_status_t rs = HPI_SUCCESS;
916 hxge_status_t status = HXGE_OK;
917
918 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
919 "==> hxge_txdma_hw_mode: enable mode %d", enable));
920
921 if (!(hxgep->drv_state & STATE_HW_INITIALIZED)) {
922 HXGE_DEBUG_MSG((hxgep, TX_CTL,
923 "<== hxge_txdma_mode: not initialized"));
924 return (HXGE_ERROR);
925 }
926 tx_rings = hxgep->tx_rings;
927 if (tx_rings == NULL) {
928 HXGE_DEBUG_MSG((hxgep, TX_CTL,
929 "<== hxge_txdma_hw_mode: NULL global ring pointer"));
930 return (HXGE_ERROR);
931 }
932 tx_desc_rings = tx_rings->rings;
933 if (tx_desc_rings == NULL) {
934 HXGE_DEBUG_MSG((hxgep, TX_CTL,
935 "<== hxge_txdma_hw_mode: NULL rings pointer"));
936 return (HXGE_ERROR);
937 }
938 ndmas = tx_rings->ndmas;
939 if (!ndmas) {
940 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
941 "<== hxge_txdma_hw_mode: no dma channel allocated"));
942 return (HXGE_ERROR);
943 }
944 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_hw_mode: "
945 "tx_rings $%p tx_desc_rings $%p ndmas %d",
946 tx_rings, tx_desc_rings, ndmas));
947
948 handle = HXGE_DEV_HPI_HANDLE(hxgep);
949 for (i = 0; i < ndmas; i++) {
950 if (tx_desc_rings[i] == NULL) {
951 continue;
952 }
953 channel = tx_desc_rings[i]->tdc;
954 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
955 "==> hxge_txdma_hw_mode: channel %d", channel));
956 if (enable) {
957 rs = hpi_txdma_channel_enable(handle, channel);
958 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
959 "==> hxge_txdma_hw_mode: channel %d (enable) "
960 "rs 0x%x", channel, rs));
961 } else {
962 /*
963 * Stop the dma channel and waits for the stop done. If
964 * the stop done bit is not set, then force an error so
965 * TXC will stop. All channels bound to this port need
966 * to be stopped and reset after injecting an interrupt
967 * error.
968 */
969 rs = hpi_txdma_channel_disable(handle, channel);
970 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
971 "==> hxge_txdma_hw_mode: channel %d (disable) "
972 "rs 0x%x", channel, rs));
973 }
974 }
975
976 status = ((rs == HPI_SUCCESS) ? HXGE_OK : HXGE_ERROR | rs);
977
978 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
979 "<== hxge_txdma_hw_mode: status 0x%x", status));
980
981 return (status);
982 }
983
984 void
985 hxge_txdma_enable_channel(p_hxge_t hxgep, uint16_t channel)
986 {
987 hpi_handle_t handle;
988
989 HXGE_DEBUG_MSG((hxgep, DMA_CTL,
990 "==> hxge_txdma_enable_channel: channel %d", channel));
991
992 handle = HXGE_DEV_HPI_HANDLE(hxgep);
993 /* enable the transmit dma channels */
994 (void) hpi_txdma_channel_enable(handle, channel);
995
996 HXGE_DEBUG_MSG((hxgep, DMA_CTL, "<== hxge_txdma_enable_channel"));
997 }
998
999 void
1000 hxge_txdma_disable_channel(p_hxge_t hxgep, uint16_t channel)
1001 {
1002 hpi_handle_t handle;
1003
1004 HXGE_DEBUG_MSG((hxgep, DMA_CTL,
1005 "==> hxge_txdma_disable_channel: channel %d", channel));
1006
1007 handle = HXGE_DEV_HPI_HANDLE(hxgep);
1008 /* stop the transmit dma channels */
1009 (void) hpi_txdma_channel_disable(handle, channel);
1010
1011 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_disable_channel"));
1012 }
1013
1014 int
1015 hxge_txdma_stop_inj_err(p_hxge_t hxgep, int channel)
1016 {
1017 hpi_handle_t handle;
1018 int status;
1019 hpi_status_t rs = HPI_SUCCESS;
1020
1021 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_stop_inj_err"));
1022
1023 /*
1024 * Stop the dma channel waits for the stop done. If the stop done bit
1025 * is not set, then create an error.
1026 */
1027 handle = HXGE_DEV_HPI_HANDLE(hxgep);
1028 rs = hpi_txdma_channel_disable(handle, channel);
1029 status = ((rs == HPI_SUCCESS) ? HXGE_OK : HXGE_ERROR | rs);
1030 if (status == HXGE_OK) {
1031 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1032 "<== hxge_txdma_stop_inj_err (channel %d): "
1033 "stopped OK", channel));
1034 return (status);
1035 }
1036
1037 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
1038 "==> hxge_txdma_stop_inj_err (channel): stop failed (0x%x) "
1039 " (injected error but still not stopped)", channel, rs));
1040
1041 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_stop_inj_err"));
1042
1043 return (status);
1044 }
1045
1046 /*ARGSUSED*/
1047 void
1048 hxge_fixup_txdma_rings(p_hxge_t hxgep)
1049 {
1050 int index, ndmas;
1051 uint16_t channel;
1052 p_tx_rings_t tx_rings;
1053
1054 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_fixup_txdma_rings"));
1055
1056 /*
1057 * For each transmit channel, reclaim each descriptor and free buffers.
1058 */
1059 tx_rings = hxgep->tx_rings;
1060 if (tx_rings == NULL) {
1061 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1062 "<== hxge_fixup_txdma_rings: NULL ring pointer"));
1063 return;
1064 }
1065
1066 ndmas = tx_rings->ndmas;
1067 if (!ndmas) {
1068 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1069 "<== hxge_fixup_txdma_rings: no channel allocated"));
1070 return;
1071 }
1072
1073 if (tx_rings->rings == NULL) {
1074 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1075 "<== hxge_fixup_txdma_rings: NULL rings pointer"));
1076 return;
1077 }
1078
1079 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_fixup_txdma_rings: "
1080 "tx_rings $%p tx_desc_rings $%p ndmas %d",
1081 tx_rings, tx_rings->rings, ndmas));
1082
1083 for (index = 0; index < ndmas; index++) {
1084 channel = tx_rings->rings[index]->tdc;
1085 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1086 "==> hxge_fixup_txdma_rings: channel %d", channel));
1087 hxge_txdma_fixup_channel(hxgep, tx_rings->rings[index],
1088 channel);
1089 }
1090
1091 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_fixup_txdma_rings"));
1092 }
1093
1094 /*ARGSUSED*/
1095 void
1096 hxge_txdma_fix_channel(p_hxge_t hxgep, uint16_t channel)
1097 {
1098 p_tx_ring_t ring_p;
1099
1100 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_fix_channel"));
1101
1102 ring_p = hxge_txdma_get_ring(hxgep, channel);
1103 if (ring_p == NULL) {
1104 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_fix_channel"));
1105 return;
1106 }
1107
1108 if (ring_p->tdc != channel) {
1109 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1110 "<== hxge_txdma_fix_channel: channel not matched "
1111 "ring tdc %d passed channel", ring_p->tdc, channel));
1112 return;
1113 }
1114
1115 hxge_txdma_fixup_channel(hxgep, ring_p, channel);
1116
1117 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_fix_channel"));
1118 }
1119
1120 /*ARGSUSED*/
1121 void
1122 hxge_txdma_fixup_channel(p_hxge_t hxgep, p_tx_ring_t ring_p, uint16_t channel)
1123 {
1124 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_fixup_channel"));
1125
1126 if (ring_p == NULL) {
1127 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1128 "<== hxge_txdma_fixup_channel: NULL ring pointer"));
1129 return;
1130 }
1131 if (ring_p->tdc != channel) {
1132 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1133 "<== hxge_txdma_fixup_channel: channel not matched "
1134 "ring tdc %d passed channel", ring_p->tdc, channel));
1135 return;
1136 }
1137 MUTEX_ENTER(&ring_p->lock);
1138 (void) hxge_txdma_reclaim(hxgep, ring_p, 0);
1139
1140 ring_p->rd_index = 0;
1141 ring_p->wr_index = 0;
1142 ring_p->ring_head.value = 0;
1143 ring_p->ring_kick_tail.value = 0;
1144 ring_p->descs_pending = 0;
1145 MUTEX_EXIT(&ring_p->lock);
1146
1147 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_fixup_channel"));
1148 }
1149
1150 /*ARGSUSED*/
1151 void
1152 hxge_txdma_hw_kick(p_hxge_t hxgep)
1153 {
1154 int index, ndmas;
1155 uint16_t channel;
1156 p_tx_rings_t tx_rings;
1157
1158 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_hw_kick"));
1159
1160 tx_rings = hxgep->tx_rings;
1161 if (tx_rings == NULL) {
1162 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1163 "<== hxge_txdma_hw_kick: NULL ring pointer"));
1164 return;
1165 }
1166 ndmas = tx_rings->ndmas;
1167 if (!ndmas) {
1168 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1169 "<== hxge_txdma_hw_kick: no channel allocated"));
1170 return;
1171 }
1172 if (tx_rings->rings == NULL) {
1173 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1174 "<== hxge_txdma_hw_kick: NULL rings pointer"));
1175 return;
1176 }
1177 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_hw_kick: "
1178 "tx_rings $%p tx_desc_rings $%p ndmas %d",
1179 tx_rings, tx_rings->rings, ndmas));
1180
1181 for (index = 0; index < ndmas; index++) {
1182 channel = tx_rings->rings[index]->tdc;
1183 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1184 "==> hxge_txdma_hw_kick: channel %d", channel));
1185 hxge_txdma_hw_kick_channel(hxgep, tx_rings->rings[index],
1186 channel);
1187 }
1188
1189 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_hw_kick"));
1190 }
1191
1192 /*ARGSUSED*/
1193 void
1194 hxge_txdma_kick_channel(p_hxge_t hxgep, uint16_t channel)
1195 {
1196 p_tx_ring_t ring_p;
1197
1198 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_kick_channel"));
1199
1200 ring_p = hxge_txdma_get_ring(hxgep, channel);
1201 if (ring_p == NULL) {
1202 HXGE_DEBUG_MSG((hxgep, TX_CTL, " hxge_txdma_kick_channel"));
1203 return;
1204 }
1205
1206 if (ring_p->tdc != channel) {
1207 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1208 "<== hxge_txdma_kick_channel: channel not matched "
1209 "ring tdc %d passed channel", ring_p->tdc, channel));
1210 return;
1211 }
1212
1213 hxge_txdma_hw_kick_channel(hxgep, ring_p, channel);
1214
1215 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_kick_channel"));
1216 }
1217
1218 /*ARGSUSED*/
1219 void
1220 hxge_txdma_hw_kick_channel(p_hxge_t hxgep, p_tx_ring_t ring_p, uint16_t channel)
1221 {
1222 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_hw_kick_channel"));
1223
1224 if (ring_p == NULL) {
1225 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1226 "<== hxge_txdma_hw_kick_channel: NULL ring pointer"));
1227 return;
1228 }
1229
1230 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_hw_kick_channel"));
1231 }
1232
1233 /*ARGSUSED*/
1234 void
1235 hxge_check_tx_hang(p_hxge_t hxgep)
1236 {
1237 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_check_tx_hang"));
1238
1239 /*
1240 * Needs inputs from hardware for regs: head index had not moved since
1241 * last timeout. packets not transmitted or stuffed registers.
1242 */
1243 if (hxge_txdma_hung(hxgep)) {
1244 hxge_fixup_hung_txdma_rings(hxgep);
1245 }
1246
1247 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_check_tx_hang"));
1248 }
1249
1250 int
1251 hxge_txdma_hung(p_hxge_t hxgep)
1252 {
1253 int index, ndmas;
1254 uint16_t channel;
1255 p_tx_rings_t tx_rings;
1256 p_tx_ring_t tx_ring_p;
1257
1258 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_hung"));
1259
1260 tx_rings = hxgep->tx_rings;
1261 if (tx_rings == NULL) {
1262 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1263 "<== hxge_txdma_hung: NULL ring pointer"));
1264 return (B_FALSE);
1265 }
1266
1267 ndmas = tx_rings->ndmas;
1268 if (!ndmas) {
1269 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1270 "<== hxge_txdma_hung: no channel allocated"));
1271 return (B_FALSE);
1272 }
1273
1274 if (tx_rings->rings == NULL) {
1275 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1276 "<== hxge_txdma_hung: NULL rings pointer"));
1277 return (B_FALSE);
1278 }
1279
1280 for (index = 0; index < ndmas; index++) {
1281 channel = tx_rings->rings[index]->tdc;
1282 tx_ring_p = tx_rings->rings[index];
1283 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1284 "==> hxge_txdma_hung: channel %d", channel));
1285 if (hxge_txdma_channel_hung(hxgep, tx_ring_p, channel)) {
1286 return (B_TRUE);
1287 }
1288 }
1289
1290 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_hung"));
1291
1292 return (B_FALSE);
1293 }
1294
1295 int
1296 hxge_txdma_channel_hung(p_hxge_t hxgep, p_tx_ring_t tx_ring_p, uint16_t channel)
1297 {
1298 uint16_t head_index, tail_index;
1299 boolean_t head_wrap, tail_wrap;
1300 hpi_handle_t handle;
1301 tdc_tdr_head_t tx_head;
1302 uint_t tx_rd_index;
1303
1304 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_channel_hung"));
1305
1306 handle = HXGE_DEV_HPI_HANDLE(hxgep);
1307 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1308 "==> hxge_txdma_channel_hung: channel %d", channel));
1309 MUTEX_ENTER(&tx_ring_p->lock);
1310 (void) hxge_txdma_reclaim(hxgep, tx_ring_p, 0);
1311
1312 tail_index = tx_ring_p->wr_index;
1313 tail_wrap = tx_ring_p->wr_index_wrap;
1314 tx_rd_index = tx_ring_p->rd_index;
1315 MUTEX_EXIT(&tx_ring_p->lock);
1316
1317 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1318 "==> hxge_txdma_channel_hung: tdc %d tx_rd_index %d "
1319 "tail_index %d tail_wrap %d ",
1320 channel, tx_rd_index, tail_index, tail_wrap));
1321 /*
1322 * Read the hardware maintained transmit head and wrap around bit.
1323 */
1324 (void) hpi_txdma_ring_head_get(handle, channel, &tx_head);
1325 head_index = tx_head.bits.head;
1326 head_wrap = tx_head.bits.wrap;
1327 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_channel_hung: "
1328 "tx_rd_index %d tail %d tail_wrap %d head %d wrap %d",
1329 tx_rd_index, tail_index, tail_wrap, head_index, head_wrap));
1330
1331 if (TXDMA_RING_EMPTY(head_index, head_wrap, tail_index, tail_wrap) &&
1332 (head_index == tx_rd_index)) {
1333 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1334 "==> hxge_txdma_channel_hung: EMPTY"));
1335 return (B_FALSE);
1336 }
1337 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1338 "==> hxge_txdma_channel_hung: Checking if ring full"));
1339 if (TXDMA_RING_FULL(head_index, head_wrap, tail_index, tail_wrap)) {
1340 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1341 "==> hxge_txdma_channel_hung: full"));
1342 return (B_TRUE);
1343 }
1344
1345 /* If not full, check with hardware to see if it is hung */
1346 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_channel_hung"));
1347
1348 return (B_FALSE);
1349 }
1350
1351 /*ARGSUSED*/
1352 void
1353 hxge_fixup_hung_txdma_rings(p_hxge_t hxgep)
1354 {
1355 int index, ndmas;
1356 uint16_t channel;
1357 p_tx_rings_t tx_rings;
1358
1359 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_fixup_hung_txdma_rings"));
1360 tx_rings = hxgep->tx_rings;
1361 if (tx_rings == NULL) {
1362 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1363 "<== hxge_fixup_hung_txdma_rings: NULL ring pointer"));
1364 return;
1365 }
1366 ndmas = tx_rings->ndmas;
1367 if (!ndmas) {
1368 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1369 "<== hxge_fixup_hung_txdma_rings: no channel allocated"));
1370 return;
1371 }
1372 if (tx_rings->rings == NULL) {
1373 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1374 "<== hxge_fixup_hung_txdma_rings: NULL rings pointer"));
1375 return;
1376 }
1377 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_fixup_hung_txdma_rings: "
1378 "tx_rings $%p tx_desc_rings $%p ndmas %d",
1379 tx_rings, tx_rings->rings, ndmas));
1380
1381 for (index = 0; index < ndmas; index++) {
1382 channel = tx_rings->rings[index]->tdc;
1383 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1384 "==> hxge_fixup_hung_txdma_rings: channel %d", channel));
1385 hxge_txdma_fixup_hung_channel(hxgep, tx_rings->rings[index],
1386 channel);
1387 }
1388
1389 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_fixup_hung_txdma_rings"));
1390 }
1391
1392 /*ARGSUSED*/
1393 void
1394 hxge_txdma_fix_hung_channel(p_hxge_t hxgep, uint16_t channel)
1395 {
1396 p_tx_ring_t ring_p;
1397
1398 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_fix_hung_channel"));
1399 ring_p = hxge_txdma_get_ring(hxgep, channel);
1400 if (ring_p == NULL) {
1401 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1402 "<== hxge_txdma_fix_hung_channel"));
1403 return;
1404 }
1405 if (ring_p->tdc != channel) {
1406 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1407 "<== hxge_txdma_fix_hung_channel: channel not matched "
1408 "ring tdc %d passed channel", ring_p->tdc, channel));
1409 return;
1410 }
1411 hxge_txdma_fixup_channel(hxgep, ring_p, channel);
1412
1413 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_fix_hung_channel"));
1414 }
1415
1416 /*ARGSUSED*/
1417 void
1418 hxge_txdma_fixup_hung_channel(p_hxge_t hxgep, p_tx_ring_t ring_p,
1419 uint16_t channel)
1420 {
1421 hpi_handle_t handle;
1422 int status = HXGE_OK;
1423
1424 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_fixup_hung_channel"));
1425
1426 if (ring_p == NULL) {
1427 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1428 "<== hxge_txdma_fixup_hung_channel: NULL ring pointer"));
1429 return;
1430 }
1431 if (ring_p->tdc != channel) {
1432 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1433 "<== hxge_txdma_fixup_hung_channel: channel "
1434 "not matched ring tdc %d passed channel",
1435 ring_p->tdc, channel));
1436 return;
1437 }
1438 /* Reclaim descriptors */
1439 MUTEX_ENTER(&ring_p->lock);
1440 (void) hxge_txdma_reclaim(hxgep, ring_p, 0);
1441 MUTEX_EXIT(&ring_p->lock);
1442
1443 handle = HXGE_DEV_HPI_HANDLE(hxgep);
1444 /*
1445 * Stop the dma channel waits for the stop done. If the stop done bit
1446 * is not set, then force an error.
1447 */
1448 status = hpi_txdma_channel_disable(handle, channel);
1449 if (!(status & HPI_TXDMA_STOP_FAILED)) {
1450 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1451 "<== hxge_txdma_fixup_hung_channel: stopped OK "
1452 "ring tdc %d passed channel %d", ring_p->tdc, channel));
1453 return;
1454 }
1455 /* Stop done bit will be set as a result of error injection */
1456 status = hpi_txdma_channel_disable(handle, channel);
1457 if (!(status & HPI_TXDMA_STOP_FAILED)) {
1458 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1459 "<== hxge_txdma_fixup_hung_channel: stopped again"
1460 "ring tdc %d passed channel", ring_p->tdc, channel));
1461 return;
1462 }
1463
1464 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1465 "<== hxge_txdma_fixup_hung_channel: stop done still not set!! "
1466 "ring tdc %d passed channel", ring_p->tdc, channel));
1467 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_fixup_hung_channel"));
1468 }
1469
1470 /*ARGSUSED*/
1471 void
1472 hxge_reclaim_rings(p_hxge_t hxgep)
1473 {
1474 int index, ndmas;
1475 uint16_t channel;
1476 p_tx_rings_t tx_rings;
1477 p_tx_ring_t tx_ring_p;
1478
1479 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_reclaim_ring"));
1480 tx_rings = hxgep->tx_rings;
1481 if (tx_rings == NULL) {
1482 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1483 "<== hxge_reclain_rimgs: NULL ring pointer"));
1484 return;
1485 }
1486 ndmas = tx_rings->ndmas;
1487 if (!ndmas) {
1488 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1489 "<== hxge_reclain_rimgs: no channel allocated"));
1490 return;
1491 }
1492 if (tx_rings->rings == NULL) {
1493 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1494 "<== hxge_reclain_rimgs: NULL rings pointer"));
1495 return;
1496 }
1497 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_reclain_rimgs: "
1498 "tx_rings $%p tx_desc_rings $%p ndmas %d",
1499 tx_rings, tx_rings->rings, ndmas));
1500
1501 for (index = 0; index < ndmas; index++) {
1502 channel = tx_rings->rings[index]->tdc;
1503 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> reclain_rimgs: channel %d",
1504 channel));
1505 tx_ring_p = tx_rings->rings[index];
1506 MUTEX_ENTER(&tx_ring_p->lock);
1507 (void) hxge_txdma_reclaim(hxgep, tx_ring_p, channel);
1508 MUTEX_EXIT(&tx_ring_p->lock);
1509 }
1510
1511 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_reclaim_rings"));
1512 }
1513
1514 /*
1515 * Static functions start here.
1516 */
1517 static hxge_status_t
1518 hxge_map_txdma(p_hxge_t hxgep)
1519 {
1520 int i, ndmas;
1521 uint16_t channel;
1522 p_tx_rings_t tx_rings;
1523 p_tx_ring_t *tx_desc_rings;
1524 p_tx_mbox_areas_t tx_mbox_areas_p;
1525 p_tx_mbox_t *tx_mbox_p;
1526 p_hxge_dma_pool_t dma_buf_poolp;
1527 p_hxge_dma_pool_t dma_cntl_poolp;
1528 p_hxge_dma_common_t *dma_buf_p;
1529 p_hxge_dma_common_t *dma_cntl_p;
1530 hxge_status_t status = HXGE_OK;
1531
1532 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_map_txdma"));
1533
1534 dma_buf_poolp = hxgep->tx_buf_pool_p;
1535 dma_cntl_poolp = hxgep->tx_cntl_pool_p;
1536
1537 if (!dma_buf_poolp->buf_allocated || !dma_cntl_poolp->buf_allocated) {
1538 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1539 "==> hxge_map_txdma: buf not allocated"));
1540 return (HXGE_ERROR);
1541 }
1542 ndmas = dma_buf_poolp->ndmas;
1543 if (!ndmas) {
1544 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1545 "<== hxge_map_txdma: no dma allocated"));
1546 return (HXGE_ERROR);
1547 }
1548 dma_buf_p = dma_buf_poolp->dma_buf_pool_p;
1549 dma_cntl_p = dma_cntl_poolp->dma_buf_pool_p;
1550
1551 tx_rings = (p_tx_rings_t)KMEM_ZALLOC(sizeof (tx_rings_t), KM_SLEEP);
1552 tx_desc_rings = (p_tx_ring_t *)KMEM_ZALLOC(
1553 sizeof (p_tx_ring_t) * ndmas, KM_SLEEP);
1554
1555 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_map_txdma: "
1556 "tx_rings $%p tx_desc_rings $%p", tx_rings, tx_desc_rings));
1557
1558 tx_mbox_areas_p = (p_tx_mbox_areas_t)
1559 KMEM_ZALLOC(sizeof (tx_mbox_areas_t), KM_SLEEP);
1560 tx_mbox_p = (p_tx_mbox_t *)KMEM_ZALLOC(
1561 sizeof (p_tx_mbox_t) * ndmas, KM_SLEEP);
1562
1563 /*
1564 * Map descriptors from the buffer pools for each dma channel.
1565 */
1566 for (i = 0; i < ndmas; i++) {
1567 /*
1568 * Set up and prepare buffer blocks, descriptors and mailbox.
1569 */
1570 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel;
1571 status = hxge_map_txdma_channel(hxgep, channel,
1572 (p_hxge_dma_common_t *)&dma_buf_p[i],
1573 (p_tx_ring_t *)&tx_desc_rings[i],
1574 dma_buf_poolp->num_chunks[i],
1575 (p_hxge_dma_common_t *)&dma_cntl_p[i],
1576 (p_tx_mbox_t *)&tx_mbox_p[i]);
1577 if (status != HXGE_OK) {
1578 goto hxge_map_txdma_fail1;
1579 }
1580 tx_desc_rings[i]->index = (uint16_t)i;
1581 tx_desc_rings[i]->tdc_stats = &hxgep->statsp->tdc_stats[i];
1582 }
1583
1584 tx_rings->ndmas = ndmas;
1585 tx_rings->rings = tx_desc_rings;
1586 hxgep->tx_rings = tx_rings;
1587 tx_mbox_areas_p->txmbox_areas_p = tx_mbox_p;
1588 hxgep->tx_mbox_areas_p = tx_mbox_areas_p;
1589
1590 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_map_txdma: "
1591 "tx_rings $%p rings $%p", hxgep->tx_rings, hxgep->tx_rings->rings));
1592 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_map_txdma: "
1593 "tx_rings $%p tx_desc_rings $%p",
1594 hxgep->tx_rings, tx_desc_rings));
1595
1596 goto hxge_map_txdma_exit;
1597
1598 hxge_map_txdma_fail1:
1599 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1600 "==> hxge_map_txdma: uninit tx desc "
1601 "(status 0x%x channel %d i %d)", hxgep, status, channel, i));
1602 i--;
1603 for (; i >= 0; i--) {
1604 channel = ((p_hxge_dma_common_t)dma_buf_p[i])->dma_channel;
1605 hxge_unmap_txdma_channel(hxgep, channel, tx_desc_rings[i],
1606 tx_mbox_p[i]);
1607 }
1608
1609 KMEM_FREE(tx_desc_rings, sizeof (p_tx_ring_t) * ndmas);
1610 KMEM_FREE(tx_rings, sizeof (tx_rings_t));
1611 KMEM_FREE(tx_mbox_p, sizeof (p_tx_mbox_t) * ndmas);
1612 KMEM_FREE(tx_mbox_areas_p, sizeof (tx_mbox_areas_t));
1613
1614 hxge_map_txdma_exit:
1615 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1616 "==> hxge_map_txdma: (status 0x%x channel %d)", status, channel));
1617
1618 return (status);
1619 }
1620
1621 static void
1622 hxge_unmap_txdma(p_hxge_t hxgep)
1623 {
1624 int i, ndmas;
1625 uint8_t channel;
1626 p_tx_rings_t tx_rings;
1627 p_tx_ring_t *tx_desc_rings;
1628 p_tx_mbox_areas_t tx_mbox_areas_p;
1629 p_tx_mbox_t *tx_mbox_p;
1630 p_hxge_dma_pool_t dma_buf_poolp;
1631
1632 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_unmap_txdma"));
1633
1634 dma_buf_poolp = hxgep->tx_buf_pool_p;
1635 if (!dma_buf_poolp->buf_allocated) {
1636 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1637 "==> hxge_unmap_txdma: buf not allocated"));
1638 return;
1639 }
1640 ndmas = dma_buf_poolp->ndmas;
1641 if (!ndmas) {
1642 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1643 "<== hxge_unmap_txdma: no dma allocated"));
1644 return;
1645 }
1646 tx_rings = hxgep->tx_rings;
1647 tx_desc_rings = tx_rings->rings;
1648 if (tx_rings == NULL) {
1649 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1650 "<== hxge_unmap_txdma: NULL ring pointer"));
1651 return;
1652 }
1653 tx_desc_rings = tx_rings->rings;
1654 if (tx_desc_rings == NULL) {
1655 HXGE_DEBUG_MSG((hxgep, TX_CTL,
1656 "<== hxge_unmap_txdma: NULL ring pointers"));
1657 return;
1658 }
1659 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_unmap_txdma: "
1660 "tx_rings $%p tx_desc_rings $%p ndmas %d",
1661 tx_rings, tx_desc_rings, ndmas));
1662
1663 tx_mbox_areas_p = hxgep->tx_mbox_areas_p;
1664 tx_mbox_p = tx_mbox_areas_p->txmbox_areas_p;
1665
1666 for (i = 0; i < ndmas; i++) {
1667 channel = tx_desc_rings[i]->tdc;
1668 (void) hxge_unmap_txdma_channel(hxgep, channel,
1669 (p_tx_ring_t)tx_desc_rings[i],
1670 (p_tx_mbox_t)tx_mbox_p[i]);
1671 }
1672
1673 KMEM_FREE(tx_desc_rings, sizeof (p_tx_ring_t) * ndmas);
1674 KMEM_FREE(tx_rings, sizeof (tx_rings_t));
1675 KMEM_FREE(tx_mbox_p, sizeof (p_tx_mbox_t) * ndmas);
1676 KMEM_FREE(tx_mbox_areas_p, sizeof (tx_mbox_areas_t));
1677
1678 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "<== hxge_unmap_txdma"));
1679 }
1680
1681 static hxge_status_t
1682 hxge_map_txdma_channel(p_hxge_t hxgep, uint16_t channel,
1683 p_hxge_dma_common_t *dma_buf_p, p_tx_ring_t *tx_desc_p,
1684 uint32_t num_chunks, p_hxge_dma_common_t *dma_cntl_p,
1685 p_tx_mbox_t *tx_mbox_p)
1686 {
1687 int status = HXGE_OK;
1688
1689 /*
1690 * Set up and prepare buffer blocks, descriptors and mailbox.
1691 */
1692 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1693 "==> hxge_map_txdma_channel (channel %d)", channel));
1694
1695 /*
1696 * Transmit buffer blocks
1697 */
1698 status = hxge_map_txdma_channel_buf_ring(hxgep, channel,
1699 dma_buf_p, tx_desc_p, num_chunks);
1700 if (status != HXGE_OK) {
1701 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
1702 "==> hxge_map_txdma_channel (channel %d): "
1703 "map buffer failed 0x%x", channel, status));
1704 goto hxge_map_txdma_channel_exit;
1705 }
1706 /*
1707 * Transmit block ring, and mailbox.
1708 */
1709 hxge_map_txdma_channel_cfg_ring(hxgep, channel, dma_cntl_p, *tx_desc_p,
1710 tx_mbox_p);
1711
1712 goto hxge_map_txdma_channel_exit;
1713
1714 hxge_map_txdma_channel_fail1:
1715 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1716 "==> hxge_map_txdma_channel: unmap buf"
1717 "(status 0x%x channel %d)", status, channel));
1718 hxge_unmap_txdma_channel_buf_ring(hxgep, *tx_desc_p);
1719
1720 hxge_map_txdma_channel_exit:
1721 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1722 "<== hxge_map_txdma_channel: (status 0x%x channel %d)",
1723 status, channel));
1724
1725 return (status);
1726 }
1727
1728 /*ARGSUSED*/
1729 static void
1730 hxge_unmap_txdma_channel(p_hxge_t hxgep, uint16_t channel,
1731 p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p)
1732 {
1733 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1734 "==> hxge_unmap_txdma_channel (channel %d)", channel));
1735
1736 /* unmap tx block ring, and mailbox. */
1737 (void) hxge_unmap_txdma_channel_cfg_ring(hxgep, tx_ring_p, tx_mbox_p);
1738
1739 /* unmap buffer blocks */
1740 (void) hxge_unmap_txdma_channel_buf_ring(hxgep, tx_ring_p);
1741
1742 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "<== hxge_unmap_txdma_channel"));
1743 }
1744
1745 /*ARGSUSED*/
1746 static void
1747 hxge_map_txdma_channel_cfg_ring(p_hxge_t hxgep, uint16_t dma_channel,
1748 p_hxge_dma_common_t *dma_cntl_p, p_tx_ring_t tx_ring_p,
1749 p_tx_mbox_t *tx_mbox_p)
1750 {
1751 p_tx_mbox_t mboxp;
1752 p_hxge_dma_common_t cntl_dmap;
1753 p_hxge_dma_common_t dmap;
1754 tdc_tdr_cfg_t *tx_ring_cfig_p;
1755 tdc_tdr_kick_t *tx_ring_kick_p;
1756 tdc_tdr_cfg_t *tx_cs_p;
1757 tdc_int_mask_t *tx_evmask_p;
1758 tdc_mbh_t *mboxh_p;
1759 tdc_mbl_t *mboxl_p;
1760 uint64_t tx_desc_len;
1761
1762 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1763 "==> hxge_map_txdma_channel_cfg_ring"));
1764
1765 cntl_dmap = *dma_cntl_p;
1766
1767 dmap = (p_hxge_dma_common_t)&tx_ring_p->tdc_desc;
1768 hxge_setup_dma_common(dmap, cntl_dmap, tx_ring_p->tx_ring_size,
1769 sizeof (tx_desc_t));
1770
1771 /*
1772 * Zero out transmit ring descriptors.
1773 */
1774 bzero((caddr_t)dmap->kaddrp, dmap->alength);
1775 tx_ring_cfig_p = &(tx_ring_p->tx_ring_cfig);
1776 tx_ring_kick_p = &(tx_ring_p->tx_ring_kick);
1777 tx_cs_p = &(tx_ring_p->tx_cs);
1778 tx_evmask_p = &(tx_ring_p->tx_evmask);
1779 tx_ring_cfig_p->value = 0;
1780 tx_ring_kick_p->value = 0;
1781 tx_cs_p->value = 0;
1782 tx_evmask_p->value = 0;
1783
1784 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1785 "==> hxge_map_txdma_channel_cfg_ring: channel %d des $%p",
1786 dma_channel, dmap->dma_cookie.dmac_laddress));
1787
1788 tx_ring_cfig_p->value = 0;
1789
1790 /* Hydra len is 11 bits and the lower 5 bits are 0s */
1791 tx_desc_len = (uint64_t)(tx_ring_p->tx_ring_size >> 5);
1792 tx_ring_cfig_p->value =
1793 (dmap->dma_cookie.dmac_laddress & TDC_TDR_CFG_ADDR_MASK) |
1794 (tx_desc_len << TDC_TDR_CFG_LEN_SHIFT);
1795
1796 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1797 "==> hxge_map_txdma_channel_cfg_ring: channel %d cfg 0x%llx",
1798 dma_channel, tx_ring_cfig_p->value));
1799
1800 tx_cs_p->bits.reset = 1;
1801
1802 /* Map in mailbox */
1803 mboxp = (p_tx_mbox_t)KMEM_ZALLOC(sizeof (tx_mbox_t), KM_SLEEP);
1804 dmap = (p_hxge_dma_common_t)&mboxp->tx_mbox;
1805 hxge_setup_dma_common(dmap, cntl_dmap, 1, sizeof (txdma_mailbox_t));
1806 mboxh_p = (tdc_mbh_t *)&tx_ring_p->tx_mbox_mbh;
1807 mboxl_p = (tdc_mbl_t *)&tx_ring_p->tx_mbox_mbl;
1808 mboxh_p->value = mboxl_p->value = 0;
1809
1810 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1811 "==> hxge_map_txdma_channel_cfg_ring: mbox 0x%lx",
1812 dmap->dma_cookie.dmac_laddress));
1813
1814 mboxh_p->bits.mbaddr = ((dmap->dma_cookie.dmac_laddress >>
1815 TDC_MBH_ADDR_SHIFT) & TDC_MBH_MASK);
1816 mboxl_p->bits.mbaddr = ((dmap->dma_cookie.dmac_laddress &
1817 TDC_MBL_MASK) >> TDC_MBL_SHIFT);
1818
1819 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1820 "==> hxge_map_txdma_channel_cfg_ring: mbox 0x%lx",
1821 dmap->dma_cookie.dmac_laddress));
1822 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1823 "==> hxge_map_txdma_channel_cfg_ring: hmbox $%p mbox $%p",
1824 mboxh_p->bits.mbaddr, mboxl_p->bits.mbaddr));
1825
1826 /*
1827 * Set page valid and no mask
1828 */
1829 tx_ring_p->page_hdl.value = 0;
1830
1831 *tx_mbox_p = mboxp;
1832
1833 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1834 "<== hxge_map_txdma_channel_cfg_ring"));
1835 }
1836
1837 /*ARGSUSED*/
1838 static void
1839 hxge_unmap_txdma_channel_cfg_ring(p_hxge_t hxgep,
1840 p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p)
1841 {
1842 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1843 "==> hxge_unmap_txdma_channel_cfg_ring: channel %d",
1844 tx_ring_p->tdc));
1845
1846 KMEM_FREE(tx_mbox_p, sizeof (tx_mbox_t));
1847
1848 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1849 "<== hxge_unmap_txdma_channel_cfg_ring"));
1850 }
1851
1852 static hxge_status_t
1853 hxge_map_txdma_channel_buf_ring(p_hxge_t hxgep, uint16_t channel,
1854 p_hxge_dma_common_t *dma_buf_p,
1855 p_tx_ring_t *tx_desc_p, uint32_t num_chunks)
1856 {
1857 p_hxge_dma_common_t dma_bufp, tmp_bufp;
1858 p_hxge_dma_common_t dmap;
1859 hxge_os_dma_handle_t tx_buf_dma_handle;
1860 p_tx_ring_t tx_ring_p;
1861 p_tx_msg_t tx_msg_ring;
1862 hxge_status_t status = HXGE_OK;
1863 int ddi_status = DDI_SUCCESS;
1864 int i, j, index;
1865 uint32_t size, bsize;
1866 uint32_t nblocks, nmsgs;
1867 char qname[TASKQ_NAMELEN];
1868
1869 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1870 "==> hxge_map_txdma_channel_buf_ring"));
1871
1872 dma_bufp = tmp_bufp = *dma_buf_p;
1873 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1874 " hxge_map_txdma_channel_buf_ring: channel %d to map %d "
1875 "chunks bufp $%p", channel, num_chunks, dma_bufp));
1876
1877 nmsgs = 0;
1878 for (i = 0; i < num_chunks; i++, tmp_bufp++) {
1879 nmsgs += tmp_bufp->nblocks;
1880 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1881 "==> hxge_map_txdma_channel_buf_ring: channel %d "
1882 "bufp $%p nblocks %d nmsgs %d",
1883 channel, tmp_bufp, tmp_bufp->nblocks, nmsgs));
1884 }
1885 if (!nmsgs) {
1886 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1887 "<== hxge_map_txdma_channel_buf_ring: channel %d "
1888 "no msg blocks", channel));
1889 status = HXGE_ERROR;
1890
1891 goto hxge_map_txdma_channel_buf_ring_exit;
1892 }
1893
1894 tx_ring_p = (p_tx_ring_t)KMEM_ZALLOC(sizeof (tx_ring_t), KM_SLEEP);
1895 tx_ring_p->hxgep = hxgep;
1896 (void) snprintf(qname, TASKQ_NAMELEN, "hxge_%d_%d",
1897 hxgep->instance, channel);
1898 tx_ring_p->taskq = ddi_taskq_create(hxgep->dip, qname, 1,
1899 TASKQ_DEFAULTPRI, 0);
1900 if (tx_ring_p->taskq == NULL) {
1901 goto hxge_map_txdma_channel_buf_ring_fail1;
1902 }
1903
1904 MUTEX_INIT(&tx_ring_p->lock, NULL, MUTEX_DRIVER,
1905 (void *) hxgep->interrupt_cookie);
1906 /*
1907 * Allocate transmit message rings and handles for packets not to be
1908 * copied to premapped buffers.
1909 */
1910 size = nmsgs * sizeof (tx_msg_t);
1911 tx_msg_ring = KMEM_ZALLOC(size, KM_SLEEP);
1912 for (i = 0; i < nmsgs; i++) {
1913 ddi_status = ddi_dma_alloc_handle(hxgep->dip, &hxge_tx_dma_attr,
1914 DDI_DMA_DONTWAIT, 0, &tx_msg_ring[i].dma_handle);
1915 if (ddi_status != DDI_SUCCESS) {
1916 status |= HXGE_DDI_FAILED;
1917 break;
1918 }
1919 }
1920
1921 if (i < nmsgs) {
1922 HXGE_DEBUG_MSG((hxgep, HXGE_ERR_CTL,
1923 "Allocate handles failed."));
1924
1925 goto hxge_map_txdma_channel_buf_ring_fail1;
1926 }
1927 tx_ring_p->tdc = channel;
1928 tx_ring_p->tx_msg_ring = tx_msg_ring;
1929 tx_ring_p->tx_ring_size = nmsgs;
1930 tx_ring_p->num_chunks = num_chunks;
1931 if (!hxge_tx_intr_thres) {
1932 hxge_tx_intr_thres = tx_ring_p->tx_ring_size / 4;
1933 }
1934 tx_ring_p->tx_wrap_mask = tx_ring_p->tx_ring_size - 1;
1935 tx_ring_p->rd_index = 0;
1936 tx_ring_p->wr_index = 0;
1937 tx_ring_p->ring_head.value = 0;
1938 tx_ring_p->ring_kick_tail.value = 0;
1939 tx_ring_p->descs_pending = 0;
1940
1941 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1942 "==> hxge_map_txdma_channel_buf_ring: channel %d "
1943 "actual tx desc max %d nmsgs %d (config hxge_tx_ring_size %d)",
1944 channel, tx_ring_p->tx_ring_size, nmsgs, hxge_tx_ring_size));
1945
1946 /*
1947 * Map in buffers from the buffer pool.
1948 */
1949 index = 0;
1950 bsize = dma_bufp->block_size;
1951
1952 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_map_txdma_channel_buf_ring: "
1953 "dma_bufp $%p tx_rng_p $%p tx_msg_rng_p $%p bsize %d",
1954 dma_bufp, tx_ring_p, tx_msg_ring, bsize));
1955
1956 for (i = 0; i < num_chunks; i++, dma_bufp++) {
1957 bsize = dma_bufp->block_size;
1958 nblocks = dma_bufp->nblocks;
1959 tx_buf_dma_handle = dma_bufp->dma_handle;
1960 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1961 "==> hxge_map_txdma_channel_buf_ring: dma chunk %d "
1962 "size %d dma_bufp $%p",
1963 i, sizeof (hxge_dma_common_t), dma_bufp));
1964
1965 for (j = 0; j < nblocks; j++) {
1966 tx_msg_ring[index].buf_dma_handle = tx_buf_dma_handle;
1967 tx_msg_ring[index].offset_index = j;
1968 dmap = &tx_msg_ring[index++].buf_dma;
1969 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
1970 "==> hxge_map_txdma_channel_buf_ring: j %d"
1971 "dmap $%p", i, dmap));
1972 hxge_setup_dma_common(dmap, dma_bufp, 1, bsize);
1973 }
1974 }
1975
1976 if (i < num_chunks) {
1977 status = HXGE_ERROR;
1978
1979 goto hxge_map_txdma_channel_buf_ring_fail1;
1980 }
1981
1982 *tx_desc_p = tx_ring_p;
1983
1984 goto hxge_map_txdma_channel_buf_ring_exit;
1985
1986 hxge_map_txdma_channel_buf_ring_fail1:
1987 if (tx_ring_p->taskq) {
1988 ddi_taskq_destroy(tx_ring_p->taskq);
1989 tx_ring_p->taskq = NULL;
1990 }
1991
1992 index--;
1993 for (; index >= 0; index--) {
1994 if (tx_msg_ring[index].dma_handle != NULL) {
1995 ddi_dma_free_handle(&tx_msg_ring[index].dma_handle);
1996 }
1997 }
1998 MUTEX_DESTROY(&tx_ring_p->lock);
1999 KMEM_FREE(tx_msg_ring, size);
2000 KMEM_FREE(tx_ring_p, sizeof (tx_ring_t));
2001
2002 status = HXGE_ERROR;
2003
2004 hxge_map_txdma_channel_buf_ring_exit:
2005 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2006 "<== hxge_map_txdma_channel_buf_ring status 0x%x", status));
2007
2008 return (status);
2009 }
2010
2011 /*ARGSUSED*/
2012 static void
2013 hxge_unmap_txdma_channel_buf_ring(p_hxge_t hxgep, p_tx_ring_t tx_ring_p)
2014 {
2015 p_tx_msg_t tx_msg_ring;
2016 p_tx_msg_t tx_msg_p;
2017 int i;
2018
2019 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2020 "==> hxge_unmap_txdma_channel_buf_ring"));
2021 if (tx_ring_p == NULL) {
2022 HXGE_DEBUG_MSG((hxgep, TX_CTL,
2023 "<== hxge_unmap_txdma_channel_buf_ring: NULL ringp"));
2024 return;
2025 }
2026 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2027 "==> hxge_unmap_txdma_channel_buf_ring: channel %d",
2028 tx_ring_p->tdc));
2029
2030 MUTEX_ENTER(&tx_ring_p->lock);
2031 tx_msg_ring = tx_ring_p->tx_msg_ring;
2032 for (i = 0; i < tx_ring_p->tx_ring_size; i++) {
2033 tx_msg_p = &tx_msg_ring[i];
2034 if (tx_msg_p->flags.dma_type == USE_DVMA) {
2035 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "entry = %d", i));
2036 (void) dvma_unload(tx_msg_p->dvma_handle, 0, -1);
2037 tx_msg_p->dvma_handle = NULL;
2038 if (tx_ring_p->dvma_wr_index ==
2039 tx_ring_p->dvma_wrap_mask) {
2040 tx_ring_p->dvma_wr_index = 0;
2041 } else {
2042 tx_ring_p->dvma_wr_index++;
2043 }
2044 tx_ring_p->dvma_pending--;
2045 } else if (tx_msg_p->flags.dma_type == USE_DMA) {
2046 if (ddi_dma_unbind_handle(tx_msg_p->dma_handle)) {
2047 cmn_err(CE_WARN, "hxge_unmap_tx_bug_ring: "
2048 "ddi_dma_unbind_handle failed.");
2049 }
2050 }
2051 if (tx_msg_p->tx_message != NULL) {
2052 freemsg(tx_msg_p->tx_message);
2053 tx_msg_p->tx_message = NULL;
2054 }
2055 }
2056
2057 for (i = 0; i < tx_ring_p->tx_ring_size; i++) {
2058 if (tx_msg_ring[i].dma_handle != NULL) {
2059 ddi_dma_free_handle(&tx_msg_ring[i].dma_handle);
2060 }
2061 }
2062 MUTEX_EXIT(&tx_ring_p->lock);
2063
2064 if (tx_ring_p->taskq) {
2065 ddi_taskq_destroy(tx_ring_p->taskq);
2066 tx_ring_p->taskq = NULL;
2067 }
2068
2069 MUTEX_DESTROY(&tx_ring_p->lock);
2070 KMEM_FREE(tx_msg_ring, sizeof (tx_msg_t) * tx_ring_p->tx_ring_size);
2071 KMEM_FREE(tx_ring_p, sizeof (tx_ring_t));
2072
2073 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2074 "<== hxge_unmap_txdma_channel_buf_ring"));
2075 }
2076
2077 static hxge_status_t
2078 hxge_txdma_hw_start(p_hxge_t hxgep)
2079 {
2080 int i, ndmas;
2081 uint16_t channel;
2082 p_tx_rings_t tx_rings;
2083 p_tx_ring_t *tx_desc_rings;
2084 p_tx_mbox_areas_t tx_mbox_areas_p;
2085 p_tx_mbox_t *tx_mbox_p;
2086 hxge_status_t status = HXGE_OK;
2087 uint64_t tmp;
2088
2089 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_hw_start"));
2090
2091 /*
2092 * Initialize REORD Table 1. Disable VMAC 2. Reset the FIFO Err Stat.
2093 * 3. Scrub memory and check for errors.
2094 */
2095 (void) hxge_tx_vmac_disable(hxgep);
2096
2097 /*
2098 * Clear the error status
2099 */
2100 HXGE_REG_WR64(hxgep->hpi_handle, TDC_FIFO_ERR_STAT, 0x7);
2101
2102 /*
2103 * Scrub the rtab memory for the TDC and reset the TDC.
2104 */
2105 HXGE_REG_WR64(hxgep->hpi_handle, TDC_REORD_TBL_DATA_HI, 0x0ULL);
2106 HXGE_REG_WR64(hxgep->hpi_handle, TDC_REORD_TBL_DATA_LO, 0x0ULL);
2107
2108 for (i = 0; i < 256; i++) {
2109 HXGE_REG_WR64(hxgep->hpi_handle, TDC_REORD_TBL_CMD,
2110 (uint64_t)i);
2111
2112 /*
2113 * Write the command register with an indirect read instruction
2114 */
2115 tmp = (0x1ULL << 30) | i;
2116 HXGE_REG_WR64(hxgep->hpi_handle, TDC_REORD_TBL_CMD, tmp);
2117
2118 /*
2119 * Wait for status done
2120 */
2121 tmp = 0;
2122 do {
2123 HXGE_REG_RD64(hxgep->hpi_handle, TDC_REORD_TBL_CMD,
2124 &tmp);
2125 } while (((tmp >> 31) & 0x1ULL) == 0x0);
2126 }
2127
2128 for (i = 0; i < 256; i++) {
2129 /*
2130 * Write the command register with an indirect read instruction
2131 */
2132 tmp = (0x1ULL << 30) | i;
2133 HXGE_REG_WR64(hxgep->hpi_handle, TDC_REORD_TBL_CMD, tmp);
2134
2135 /*
2136 * Wait for status done
2137 */
2138 tmp = 0;
2139 do {
2140 HXGE_REG_RD64(hxgep->hpi_handle, TDC_REORD_TBL_CMD,
2141 &tmp);
2142 } while (((tmp >> 31) & 0x1ULL) == 0x0);
2143
2144 HXGE_REG_RD64(hxgep->hpi_handle, TDC_REORD_TBL_DATA_HI, &tmp);
2145 if (0x1ff00ULL != (0x1ffffULL & tmp)) {
2146 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "PANIC ReordTbl "
2147 "unexpected data (hi), entry: %x, value: 0x%0llx\n",
2148 i, (unsigned long long)tmp));
2149 status = HXGE_ERROR;
2150 }
2151
2152 HXGE_REG_RD64(hxgep->hpi_handle, TDC_REORD_TBL_DATA_LO, &tmp);
2153 if (tmp != 0) {
2154 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "PANIC ReordTbl "
2155 "unexpected data (lo), entry: %x\n", i));
2156 status = HXGE_ERROR;
2157 }
2158
2159 HXGE_REG_RD64(hxgep->hpi_handle, TDC_FIFO_ERR_STAT, &tmp);
2160 if (tmp != 0) {
2161 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "PANIC ReordTbl "
2162 "parity error, entry: %x, val 0x%llx\n",
2163 i, (unsigned long long)tmp));
2164 status = HXGE_ERROR;
2165 }
2166
2167 HXGE_REG_RD64(hxgep->hpi_handle, TDC_FIFO_ERR_STAT, &tmp);
2168 if (tmp != 0) {
2169 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "PANIC ReordTbl "
2170 "parity error, entry: %x\n", i));
2171 status = HXGE_ERROR;
2172 }
2173 }
2174
2175 if (status != HXGE_OK)
2176 goto hxge_txdma_hw_start_exit;
2177
2178 /*
2179 * Reset FIFO Error Status for the TDC and enable FIFO error events.
2180 */
2181 HXGE_REG_WR64(hxgep->hpi_handle, TDC_FIFO_ERR_STAT, 0x7);
2182 HXGE_REG_WR64(hxgep->hpi_handle, TDC_FIFO_ERR_MASK, 0x0);
2183
2184 /*
2185 * Initialize the Transmit DMAs.
2186 */
2187 tx_rings = hxgep->tx_rings;
2188 if (tx_rings == NULL) {
2189 HXGE_DEBUG_MSG((hxgep, TX_CTL,
2190 "<== hxge_txdma_hw_start: NULL ring pointer"));
2191 return (HXGE_ERROR);
2192 }
2193
2194 tx_desc_rings = tx_rings->rings;
2195 if (tx_desc_rings == NULL) {
2196 HXGE_DEBUG_MSG((hxgep, TX_CTL,
2197 "<== hxge_txdma_hw_start: NULL ring pointers"));
2198 return (HXGE_ERROR);
2199 }
2200 ndmas = tx_rings->ndmas;
2201 if (!ndmas) {
2202 HXGE_DEBUG_MSG((hxgep, TX_CTL,
2203 "<== hxge_txdma_hw_start: no dma channel allocated"));
2204 return (HXGE_ERROR);
2205 }
2206 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_hw_start: "
2207 "tx_rings $%p tx_desc_rings $%p ndmas %d",
2208 tx_rings, tx_desc_rings, ndmas));
2209
2210 tx_mbox_areas_p = hxgep->tx_mbox_areas_p;
2211 tx_mbox_p = tx_mbox_areas_p->txmbox_areas_p;
2212
2213 /*
2214 * Init the DMAs.
2215 */
2216 for (i = 0; i < ndmas; i++) {
2217 channel = tx_desc_rings[i]->tdc;
2218 status = hxge_txdma_start_channel(hxgep, channel,
2219 (p_tx_ring_t)tx_desc_rings[i],
2220 (p_tx_mbox_t)tx_mbox_p[i]);
2221 if (status != HXGE_OK) {
2222 goto hxge_txdma_hw_start_fail1;
2223 }
2224 }
2225
2226 (void) hxge_tx_vmac_enable(hxgep);
2227
2228 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2229 "==> hxge_txdma_hw_start: tx_rings $%p rings $%p",
2230 hxgep->tx_rings, hxgep->tx_rings->rings));
2231 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2232 "==> hxge_txdma_hw_start: tx_rings $%p tx_desc_rings $%p",
2233 hxgep->tx_rings, tx_desc_rings));
2234
2235 goto hxge_txdma_hw_start_exit;
2236
2237 hxge_txdma_hw_start_fail1:
2238 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2239 "==> hxge_txdma_hw_start: disable (status 0x%x channel %d i %d)",
2240 status, channel, i));
2241
2242 for (; i >= 0; i--) {
2243 channel = tx_desc_rings[i]->tdc,
2244 (void) hxge_txdma_stop_channel(hxgep, channel,
2245 (p_tx_ring_t)tx_desc_rings[i],
2246 (p_tx_mbox_t)tx_mbox_p[i]);
2247 }
2248
2249 hxge_txdma_hw_start_exit:
2250 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2251 "==> hxge_txdma_hw_start: (status 0x%x)", status));
2252
2253 return (status);
2254 }
2255
2256 static void
2257 hxge_txdma_hw_stop(p_hxge_t hxgep)
2258 {
2259 int i, ndmas;
2260 uint16_t channel;
2261 p_tx_rings_t tx_rings;
2262 p_tx_ring_t *tx_desc_rings;
2263 p_tx_mbox_areas_t tx_mbox_areas_p;
2264 p_tx_mbox_t *tx_mbox_p;
2265
2266 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_hw_stop"));
2267
2268 tx_rings = hxgep->tx_rings;
2269 if (tx_rings == NULL) {
2270 HXGE_DEBUG_MSG((hxgep, TX_CTL,
2271 "<== hxge_txdma_hw_stop: NULL ring pointer"));
2272 return;
2273 }
2274
2275 tx_desc_rings = tx_rings->rings;
2276 if (tx_desc_rings == NULL) {
2277 HXGE_DEBUG_MSG((hxgep, TX_CTL,
2278 "<== hxge_txdma_hw_stop: NULL ring pointers"));
2279 return;
2280 }
2281
2282 ndmas = tx_rings->ndmas;
2283 if (!ndmas) {
2284 HXGE_DEBUG_MSG((hxgep, TX_CTL,
2285 "<== hxge_txdma_hw_stop: no dma channel allocated"));
2286 return;
2287 }
2288
2289 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_hw_stop: "
2290 "tx_rings $%p tx_desc_rings $%p", tx_rings, tx_desc_rings));
2291
2292 tx_mbox_areas_p = hxgep->tx_mbox_areas_p;
2293 tx_mbox_p = tx_mbox_areas_p->txmbox_areas_p;
2294
2295 for (i = 0; i < ndmas; i++) {
2296 channel = tx_desc_rings[i]->tdc;
2297 (void) hxge_txdma_stop_channel(hxgep, channel,
2298 (p_tx_ring_t)tx_desc_rings[i],
2299 (p_tx_mbox_t)tx_mbox_p[i]);
2300 }
2301
2302 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_hw_stop: "
2303 "tx_rings $%p tx_desc_rings $%p", tx_rings, tx_desc_rings));
2304 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "<== hxge_txdma_hw_stop"));
2305 }
2306
2307 static hxge_status_t
2308 hxge_txdma_start_channel(p_hxge_t hxgep, uint16_t channel,
2309 p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p)
2310 {
2311 hxge_status_t status = HXGE_OK;
2312
2313 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2314 "==> hxge_txdma_start_channel (channel %d)", channel));
2315 /*
2316 * TXDMA/TXC must be in stopped state.
2317 */
2318 (void) hxge_txdma_stop_inj_err(hxgep, channel);
2319
2320 /*
2321 * Reset TXDMA channel
2322 */
2323 tx_ring_p->tx_cs.value = 0;
2324 tx_ring_p->tx_cs.bits.reset = 1;
2325 status = hxge_reset_txdma_channel(hxgep, channel,
2326 tx_ring_p->tx_cs.value);
2327 if (status != HXGE_OK) {
2328 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2329 "==> hxge_txdma_start_channel (channel %d)"
2330 " reset channel failed 0x%x", channel, status));
2331
2332 goto hxge_txdma_start_channel_exit;
2333 }
2334
2335 /*
2336 * Initialize the TXDMA channel specific FZC control configurations.
2337 * These FZC registers are pertaining to each TX channel (i.e. logical
2338 * pages).
2339 */
2340 status = hxge_init_fzc_txdma_channel(hxgep, channel,
2341 tx_ring_p, tx_mbox_p);
2342 if (status != HXGE_OK) {
2343 goto hxge_txdma_start_channel_exit;
2344 }
2345
2346 /*
2347 * Initialize the event masks.
2348 */
2349 tx_ring_p->tx_evmask.value = 0;
2350 status = hxge_init_txdma_channel_event_mask(hxgep,
2351 channel, &tx_ring_p->tx_evmask);
2352 if (status != HXGE_OK) {
2353 goto hxge_txdma_start_channel_exit;
2354 }
2355
2356 /*
2357 * Load TXDMA descriptors, buffers, mailbox, initialise the DMA
2358 * channels and enable each DMA channel.
2359 */
2360 status = hxge_enable_txdma_channel(hxgep, channel,
2361 tx_ring_p, tx_mbox_p);
2362 if (status != HXGE_OK) {
2363 goto hxge_txdma_start_channel_exit;
2364 }
2365
2366 hxge_txdma_start_channel_exit:
2367 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "<== hxge_txdma_start_channel"));
2368
2369 return (status);
2370 }
2371
2372 /*ARGSUSED*/
2373 static hxge_status_t
2374 hxge_txdma_stop_channel(p_hxge_t hxgep, uint16_t channel,
2375 p_tx_ring_t tx_ring_p, p_tx_mbox_t tx_mbox_p)
2376 {
2377 int status = HXGE_OK;
2378
2379 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2380 "==> hxge_txdma_stop_channel: channel %d", channel));
2381
2382 /*
2383 * Stop (disable) TXDMA and TXC (if stop bit is set and STOP_N_GO bit
2384 * not set, the TXDMA reset state will not be set if reset TXDMA.
2385 */
2386 (void) hxge_txdma_stop_inj_err(hxgep, channel);
2387
2388 /*
2389 * Reset TXDMA channel
2390 */
2391 tx_ring_p->tx_cs.value = 0;
2392 tx_ring_p->tx_cs.bits.reset = 1;
2393 status = hxge_reset_txdma_channel(hxgep, channel,
2394 tx_ring_p->tx_cs.value);
2395 if (status != HXGE_OK) {
2396 goto hxge_txdma_stop_channel_exit;
2397 }
2398
2399 hxge_txdma_stop_channel_exit:
2400 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "<== hxge_txdma_stop_channel"));
2401
2402 return (status);
2403 }
2404
2405 static p_tx_ring_t
2406 hxge_txdma_get_ring(p_hxge_t hxgep, uint16_t channel)
2407 {
2408 int index, ndmas;
2409 uint16_t tdc;
2410 p_tx_rings_t tx_rings;
2411
2412 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_get_ring"));
2413
2414 tx_rings = hxgep->tx_rings;
2415 if (tx_rings == NULL) {
2416 HXGE_DEBUG_MSG((hxgep, TX_CTL,
2417 "<== hxge_txdma_get_ring: NULL ring pointer"));
2418 return (NULL);
2419 }
2420 ndmas = tx_rings->ndmas;
2421 if (!ndmas) {
2422 HXGE_DEBUG_MSG((hxgep, TX_CTL,
2423 "<== hxge_txdma_get_ring: no channel allocated"));
2424 return (NULL);
2425 }
2426 if (tx_rings->rings == NULL) {
2427 HXGE_DEBUG_MSG((hxgep, TX_CTL,
2428 "<== hxge_txdma_get_ring: NULL rings pointer"));
2429 return (NULL);
2430 }
2431 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_get_ring: "
2432 "tx_rings $%p tx_desc_rings $%p ndmas %d",
2433 tx_rings, tx_rings, ndmas));
2434
2435 for (index = 0; index < ndmas; index++) {
2436 tdc = tx_rings->rings[index]->tdc;
2437 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2438 "==> hxge_fixup_txdma_rings: channel %d", tdc));
2439 if (channel == tdc) {
2440 HXGE_DEBUG_MSG((hxgep, TX_CTL,
2441 "<== hxge_txdma_get_ring: tdc %d ring $%p",
2442 tdc, tx_rings->rings[index]));
2443 return (p_tx_ring_t)(tx_rings->rings[index]);
2444 }
2445 }
2446
2447 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_get_ring"));
2448
2449 return (NULL);
2450 }
2451
2452 static p_tx_mbox_t
2453 hxge_txdma_get_mbox(p_hxge_t hxgep, uint16_t channel)
2454 {
2455 int index, tdc, ndmas;
2456 p_tx_rings_t tx_rings;
2457 p_tx_mbox_areas_t tx_mbox_areas_p;
2458 p_tx_mbox_t *tx_mbox_p;
2459
2460 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_get_mbox"));
2461
2462 tx_rings = hxgep->tx_rings;
2463 if (tx_rings == NULL) {
2464 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2465 "<== hxge_txdma_get_mbox: NULL ring pointer"));
2466 return (NULL);
2467 }
2468 tx_mbox_areas_p = hxgep->tx_mbox_areas_p;
2469 if (tx_mbox_areas_p == NULL) {
2470 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2471 "<== hxge_txdma_get_mbox: NULL mbox pointer"));
2472 return (NULL);
2473 }
2474 tx_mbox_p = tx_mbox_areas_p->txmbox_areas_p;
2475
2476 ndmas = tx_rings->ndmas;
2477 if (!ndmas) {
2478 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2479 "<== hxge_txdma_get_mbox: no channel allocated"));
2480 return (NULL);
2481 }
2482 if (tx_rings->rings == NULL) {
2483 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2484 "<== hxge_txdma_get_mbox: NULL rings pointer"));
2485 return (NULL);
2486 }
2487 HXGE_DEBUG_MSG((hxgep, MEM3_CTL, "==> hxge_txdma_get_mbox: "
2488 "tx_rings $%p tx_desc_rings $%p ndmas %d",
2489 tx_rings, tx_rings, ndmas));
2490
2491 for (index = 0; index < ndmas; index++) {
2492 tdc = tx_rings->rings[index]->tdc;
2493 HXGE_DEBUG_MSG((hxgep, MEM3_CTL,
2494 "==> hxge_txdma_get_mbox: channel %d", tdc));
2495 if (channel == tdc) {
2496 HXGE_DEBUG_MSG((hxgep, TX_CTL,
2497 "<== hxge_txdma_get_mbox: tdc %d ring $%p",
2498 tdc, tx_rings->rings[index]));
2499 return (p_tx_mbox_t)(tx_mbox_p[index]);
2500 }
2501 }
2502
2503 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_get_mbox"));
2504
2505 return (NULL);
2506 }
2507
2508 /*ARGSUSED*/
2509 static hxge_status_t
2510 hxge_tx_err_evnts(p_hxge_t hxgep, uint_t index, p_hxge_ldv_t ldvp,
2511 tdc_stat_t cs)
2512 {
2513 hpi_handle_t handle;
2514 uint8_t channel;
2515 p_tx_ring_t *tx_rings;
2516 p_tx_ring_t tx_ring_p;
2517 p_hxge_tx_ring_stats_t tdc_stats;
2518 boolean_t txchan_fatal = B_FALSE;
2519 hxge_status_t status = HXGE_OK;
2520 tdc_drop_cnt_t drop_cnt;
2521
2522 HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "==> hxge_tx_err_evnts"));
2523 handle = HXGE_DEV_HPI_HANDLE(hxgep);
2524 channel = ldvp->channel;
2525
2526 tx_rings = hxgep->tx_rings->rings;
2527 tx_ring_p = tx_rings[index];
2528 tdc_stats = tx_ring_p->tdc_stats;
2529
2530 /* Get the error counts if any */
2531 TXDMA_REG_READ64(handle, TDC_DROP_CNT, channel, &drop_cnt.value);
2532 tdc_stats->count_hdr_size_err += drop_cnt.bits.hdr_size_error_count;
2533 tdc_stats->count_runt += drop_cnt.bits.runt_count;
2534 tdc_stats->count_abort += drop_cnt.bits.abort_count;
2535
2536 if (cs.bits.peu_resp_err) {
2537 tdc_stats->peu_resp_err++;
2538 HXGE_FM_REPORT_ERROR(hxgep, channel,
2539 HXGE_FM_EREPORT_TDMC_PEU_RESP_ERR);
2540 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2541 "==> hxge_tx_err_evnts(channel %d): "
2542 "fatal error: peu_resp_err", channel));
2543 txchan_fatal = B_TRUE;
2544 }
2545
2546 if (cs.bits.pkt_size_hdr_err) {
2547 tdc_stats->pkt_size_hdr_err++;
2548 HXGE_FM_REPORT_ERROR(hxgep, channel,
2549 HXGE_FM_EREPORT_TDMC_PKT_SIZE_HDR_ERR);
2550 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2551 "==> hxge_tx_err_evnts(channel %d): "
2552 "fatal error: pkt_size_hdr_err", channel));
2553 txchan_fatal = B_TRUE;
2554 }
2555
2556 if (cs.bits.runt_pkt_drop_err) {
2557 tdc_stats->runt_pkt_drop_err++;
2558 HXGE_FM_REPORT_ERROR(hxgep, channel,
2559 HXGE_FM_EREPORT_TDMC_RUNT_PKT_DROP_ERR);
2560 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2561 "==> hxge_tx_err_evnts(channel %d): "
2562 "fatal error: runt_pkt_drop_err", channel));
2563 txchan_fatal = B_TRUE;
2564 }
2565
2566 if (cs.bits.pkt_size_err) {
2567 tdc_stats->pkt_size_err++;
2568 HXGE_FM_REPORT_ERROR(hxgep, channel,
2569 HXGE_FM_EREPORT_TDMC_PKT_SIZE_ERR);
2570 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2571 "==> hxge_tx_err_evnts(channel %d): "
2572 "fatal error: pkt_size_err", channel));
2573 txchan_fatal = B_TRUE;
2574 }
2575
2576 if (cs.bits.tx_rng_oflow) {
2577 tdc_stats->tx_rng_oflow++;
2578 if (tdc_stats->tx_rng_oflow)
2579 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2580 "==> hxge_tx_err_evnts(channel %d): "
2581 "fatal error: tx_rng_oflow", channel));
2582 }
2583
2584 if (cs.bits.pref_par_err) {
2585 tdc_stats->pref_par_err++;
2586
2587 /* Get the address of parity error read data */
2588 TXDMA_REG_READ64(hxgep->hpi_handle, TDC_PREF_PAR_LOG,
2589 channel, &tdc_stats->errlog.value);
2590
2591 HXGE_FM_REPORT_ERROR(hxgep, channel,
2592 HXGE_FM_EREPORT_TDMC_PREF_PAR_ERR);
2593 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2594 "==> hxge_tx_err_evnts(channel %d): "
2595 "fatal error: pref_par_err", channel));
2596 txchan_fatal = B_TRUE;
2597 }
2598
2599 if (cs.bits.tdr_pref_cpl_to) {
2600 tdc_stats->tdr_pref_cpl_to++;
2601 HXGE_FM_REPORT_ERROR(hxgep, channel,
2602 HXGE_FM_EREPORT_TDMC_TDR_PREF_CPL_TO);
2603 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2604 "==> hxge_tx_err_evnts(channel %d): "
2605 "fatal error: tdr_pref_cpl_to", channel));
2606 txchan_fatal = B_TRUE;
2607 }
2608
2609 if (cs.bits.pkt_cpl_to) {
2610 tdc_stats->pkt_cpl_to++;
2611 HXGE_FM_REPORT_ERROR(hxgep, channel,
2612 HXGE_FM_EREPORT_TDMC_PKT_CPL_TO);
2613 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2614 "==> hxge_tx_err_evnts(channel %d): "
2615 "fatal error: pkt_cpl_to", channel));
2616 txchan_fatal = B_TRUE;
2617 }
2618
2619 if (cs.bits.invalid_sop) {
2620 tdc_stats->invalid_sop++;
2621 HXGE_FM_REPORT_ERROR(hxgep, channel,
2622 HXGE_FM_EREPORT_TDMC_INVALID_SOP);
2623 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2624 "==> hxge_tx_err_evnts(channel %d): "
2625 "fatal error: invalid_sop", channel));
2626 txchan_fatal = B_TRUE;
2627 }
2628
2629 if (cs.bits.unexpected_sop) {
2630 tdc_stats->unexpected_sop++;
2631 HXGE_FM_REPORT_ERROR(hxgep, channel,
2632 HXGE_FM_EREPORT_TDMC_UNEXPECTED_SOP);
2633 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2634 "==> hxge_tx_err_evnts(channel %d): "
2635 "fatal error: unexpected_sop", channel));
2636 txchan_fatal = B_TRUE;
2637 }
2638
2639 /* Clear error injection source in case this is an injected error */
2640 TXDMA_REG_WRITE64(hxgep->hpi_handle, TDC_STAT_INT_DBG, channel, 0);
2641
2642 if (txchan_fatal) {
2643 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2644 " hxge_tx_err_evnts: "
2645 " fatal error on channel %d cs 0x%llx\n",
2646 channel, cs.value));
2647 status = hxge_txdma_fatal_err_recover(hxgep, channel,
2648 tx_ring_p);
2649 if (status == HXGE_OK) {
2650 FM_SERVICE_RESTORED(hxgep);
2651 }
2652 }
2653
2654 HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "<== hxge_tx_err_evnts"));
2655
2656 return (status);
2657 }
2658
2659 hxge_status_t
2660 hxge_txdma_handle_sys_errors(p_hxge_t hxgep)
2661 {
2662 hpi_handle_t handle;
2663 hxge_status_t status = HXGE_OK;
2664 tdc_fifo_err_stat_t fifo_stat;
2665 hxge_tdc_sys_stats_t *tdc_sys_stats;
2666
2667 HXGE_DEBUG_MSG((hxgep, TX_CTL, "==> hxge_txdma_handle_sys_errors"));
2668
2669 handle = HXGE_DEV_HPI_HANDLE(hxgep);
2670
2671 /*
2672 * The FIFO is shared by all channels.
2673 * Get the status of Reorder Buffer and Reorder Table Buffer Errors
2674 */
2675 HXGE_REG_RD64(handle, TDC_FIFO_ERR_STAT, &fifo_stat.value);
2676
2677 /*
2678 * Clear the error bits. Note that writing a 1 clears the bit. Writing
2679 * a 0 does nothing.
2680 */
2681 HXGE_REG_WR64(handle, TDC_FIFO_ERR_STAT, fifo_stat.value);
2682
2683 tdc_sys_stats = &hxgep->statsp->tdc_sys_stats;
2684 if (fifo_stat.bits.reord_tbl_par_err) {
2685 tdc_sys_stats->reord_tbl_par_err++;
2686 HXGE_FM_REPORT_ERROR(hxgep, NULL,
2687 HXGE_FM_EREPORT_TDMC_REORD_TBL_PAR);
2688 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2689 "==> hxge_txdma_handle_sys_errors: fatal error: "
2690 "reord_tbl_par_err"));
2691 }
2692
2693 if (fifo_stat.bits.reord_buf_ded_err) {
2694 tdc_sys_stats->reord_buf_ded_err++;
2695 HXGE_FM_REPORT_ERROR(hxgep, NULL,
2696 HXGE_FM_EREPORT_TDMC_REORD_BUF_DED);
2697 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2698 "==> hxge_txdma_handle_sys_errors: "
2699 "fatal error: reord_buf_ded_err"));
2700 }
2701
2702 if (fifo_stat.bits.reord_buf_sec_err) {
2703 tdc_sys_stats->reord_buf_sec_err++;
2704 if (tdc_sys_stats->reord_buf_sec_err == 1)
2705 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2706 "==> hxge_txdma_handle_sys_errors: "
2707 "reord_buf_sec_err"));
2708 }
2709
2710 if (fifo_stat.bits.reord_tbl_par_err ||
2711 fifo_stat.bits.reord_buf_ded_err) {
2712 status = hxge_tx_port_fatal_err_recover(hxgep);
2713 if (status == HXGE_OK) {
2714 FM_SERVICE_RESTORED(hxgep);
2715 }
2716 }
2717
2718 HXGE_DEBUG_MSG((hxgep, TX_CTL, "<== hxge_txdma_handle_sys_errors"));
2719
2720 return (status);
2721 }
2722
2723 static hxge_status_t
2724 hxge_txdma_fatal_err_recover(p_hxge_t hxgep, uint16_t channel,
2725 p_tx_ring_t tx_ring_p)
2726 {
2727 hpi_handle_t handle;
2728 hpi_status_t rs = HPI_SUCCESS;
2729 p_tx_mbox_t tx_mbox_p;
2730 hxge_status_t status = HXGE_OK;
2731
2732 HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "==> hxge_txdma_fatal_err_recover"));
2733 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2734 "Recovering from TxDMAChannel#%d error...", channel));
2735
2736 /*
2737 * Stop the dma channel waits for the stop done. If the stop done bit
2738 * is not set, then create an error.
2739 */
2740 handle = HXGE_DEV_HPI_HANDLE(hxgep);
2741 HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "stopping txdma channel(%d)",
2742 channel));
2743 MUTEX_ENTER(&tx_ring_p->lock);
2744 rs = hpi_txdma_channel_control(handle, TXDMA_STOP, channel);
2745 if (rs != HPI_SUCCESS) {
2746 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2747 "==> hxge_txdma_fatal_err_recover (channel %d): "
2748 "stop failed ", channel));
2749
2750 goto fail;
2751 }
2752 HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "reclaiming txdma channel(%d)",
2753 channel));
2754 (void) hxge_txdma_reclaim(hxgep, tx_ring_p, 0);
2755
2756 /*
2757 * Reset TXDMA channel
2758 */
2759 HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "resetting txdma channel(%d)",
2760 channel));
2761 if ((rs = hpi_txdma_channel_control(handle, TXDMA_RESET, channel)) !=
2762 HPI_SUCCESS) {
2763 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2764 "==> hxge_txdma_fatal_err_recover (channel %d)"
2765 " reset channel failed 0x%x", channel, rs));
2766
2767 goto fail;
2768 }
2769 /*
2770 * Reset the tail (kick) register to 0. (Hardware will not reset it. Tx
2771 * overflow fatal error if tail is not set to 0 after reset!
2772 */
2773 TXDMA_REG_WRITE64(handle, TDC_TDR_KICK, channel, 0);
2774
2775 /*
2776 * Restart TXDMA channel
2777 *
2778 * Initialize the TXDMA channel specific FZC control configurations.
2779 * These FZC registers are pertaining to each TX channel (i.e. logical
2780 * pages).
2781 */
2782 tx_mbox_p = hxge_txdma_get_mbox(hxgep, channel);
2783 HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "restarting txdma channel(%d)",
2784 channel));
2785 status = hxge_init_fzc_txdma_channel(hxgep, channel,
2786 tx_ring_p, tx_mbox_p);
2787 if (status != HXGE_OK)
2788 goto fail;
2789
2790 /*
2791 * Initialize the event masks.
2792 */
2793 tx_ring_p->tx_evmask.value = 0;
2794 status = hxge_init_txdma_channel_event_mask(hxgep, channel,
2795 &tx_ring_p->tx_evmask);
2796 if (status != HXGE_OK)
2797 goto fail;
2798
2799 tx_ring_p->wr_index_wrap = B_FALSE;
2800 tx_ring_p->wr_index = 0;
2801 tx_ring_p->rd_index = 0;
2802
2803 /*
2804 * Load TXDMA descriptors, buffers, mailbox, initialise the DMA
2805 * channels and enable each DMA channel.
2806 */
2807 HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "enabling txdma channel(%d)",
2808 channel));
2809 status = hxge_enable_txdma_channel(hxgep, channel,
2810 tx_ring_p, tx_mbox_p);
2811 MUTEX_EXIT(&tx_ring_p->lock);
2812 if (status != HXGE_OK)
2813 goto fail;
2814
2815 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2816 "Recovery Successful, TxDMAChannel#%d Restored", channel));
2817 HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "==> hxge_txdma_fatal_err_recover"));
2818
2819 return (HXGE_OK);
2820
2821 fail:
2822 MUTEX_EXIT(&tx_ring_p->lock);
2823 HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL,
2824 "hxge_txdma_fatal_err_recover (channel %d): "
2825 "failed to recover this txdma channel", channel));
2826 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "Recovery failed"));
2827
2828 return (status);
2829 }
2830
2831 static hxge_status_t
2832 hxge_tx_port_fatal_err_recover(p_hxge_t hxgep)
2833 {
2834 hpi_handle_t handle;
2835 hpi_status_t rs = HPI_SUCCESS;
2836 hxge_status_t status = HXGE_OK;
2837 p_tx_ring_t *tx_desc_rings;
2838 p_tx_rings_t tx_rings;
2839 p_tx_ring_t tx_ring_p;
2840 int i, ndmas;
2841 uint16_t channel;
2842 block_reset_t reset_reg;
2843
2844 HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL,
2845 "==> hxge_tx_port_fatal_err_recover"));
2846 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2847 "Recovering from TxPort error..."));
2848
2849 handle = HXGE_DEV_HPI_HANDLE(hxgep);
2850
2851 /* Reset TDC block from PEU for this fatal error */
2852 reset_reg.value = 0;
2853 reset_reg.bits.tdc_rst = 1;
2854 HXGE_REG_WR32(handle, BLOCK_RESET, reset_reg.value);
2855
2856 HXGE_DELAY(1000);
2857
2858 /*
2859 * Stop the dma channel waits for the stop done. If the stop done bit
2860 * is not set, then create an error.
2861 */
2862 HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "stopping all DMA channels..."));
2863
2864 tx_rings = hxgep->tx_rings;
2865 tx_desc_rings = tx_rings->rings;
2866 ndmas = tx_rings->ndmas;
2867
2868 for (i = 0; i < ndmas; i++) {
2869 if (tx_desc_rings[i] == NULL) {
2870 continue;
2871 }
2872 tx_ring_p = tx_rings->rings[i];
2873 MUTEX_ENTER(&tx_ring_p->lock);
2874 }
2875
2876 for (i = 0; i < ndmas; i++) {
2877 if (tx_desc_rings[i] == NULL) {
2878 continue;
2879 }
2880 channel = tx_desc_rings[i]->tdc;
2881 tx_ring_p = tx_rings->rings[i];
2882 rs = hpi_txdma_channel_control(handle, TXDMA_STOP, channel);
2883 if (rs != HPI_SUCCESS) {
2884 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2885 "==> hxge_txdma_fatal_err_recover (channel %d): "
2886 "stop failed ", channel));
2887
2888 goto fail;
2889 }
2890 }
2891
2892 /*
2893 * Do reclaim on all of th DMAs.
2894 */
2895 HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL, "reclaiming all DMA channels..."));
2896 for (i = 0; i < ndmas; i++) {
2897 if (tx_desc_rings[i] == NULL) {
2898 continue;
2899 }
2900 tx_ring_p = tx_rings->rings[i];
2901 (void) hxge_txdma_reclaim(hxgep, tx_ring_p, 0);
2902 }
2903
2904 /* Restart the TDC */
2905 if ((status = hxge_txdma_hw_start(hxgep)) != HXGE_OK)
2906 goto fail;
2907
2908 for (i = 0; i < ndmas; i++) {
2909 if (tx_desc_rings[i] == NULL) {
2910 continue;
2911 }
2912 tx_ring_p = tx_rings->rings[i];
2913 MUTEX_EXIT(&tx_ring_p->lock);
2914 }
2915
2916 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL,
2917 "Recovery Successful, TxPort Restored"));
2918 HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL,
2919 "<== hxge_tx_port_fatal_err_recover"));
2920 return (HXGE_OK);
2921
2922 fail:
2923 for (i = 0; i < ndmas; i++) {
2924 if (tx_desc_rings[i] == NULL) {
2925 continue;
2926 }
2927 tx_ring_p = tx_rings->rings[i];
2928 MUTEX_EXIT(&tx_ring_p->lock);
2929 }
2930
2931 HXGE_ERROR_MSG((hxgep, HXGE_ERR_CTL, "Recovery failed"));
2932 HXGE_DEBUG_MSG((hxgep, TX_ERR_CTL,
2933 "hxge_txdma_fatal_err_recover (channel %d): "
2934 "failed to recover this txdma channel"));
2935
2936 return (status);
2937 }