Print this page
5255 uts shouldn't open-code ISP2
Split |
Close |
Expand all |
Collapse all |
--- old/usr/src/uts/common/io/ib/adapters/tavor/tavor_rsrc.c
+++ new/usr/src/uts/common/io/ib/adapters/tavor/tavor_rsrc.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
24 24 * Use is subject to license terms.
25 25 */
↓ open down ↓ |
25 lines elided |
↑ open up ↑ |
26 26
27 27 /*
28 28 * tavor_rsrc.c
29 29 * Tavor Resource Management Routines
30 30 *
31 31 * Implements all the routines necessary for setup, teardown, and
32 32 * alloc/free of all Tavor resources, including those that are managed
33 33 * by Tavor hardware or which live in Tavor's direct attached DDR memory.
34 34 */
35 35
36 +#include <sys/sysmacros.h>
36 37 #include <sys/types.h>
37 38 #include <sys/conf.h>
38 39 #include <sys/ddi.h>
39 40 #include <sys/sunddi.h>
40 41 #include <sys/modctl.h>
41 42 #include <sys/vmem.h>
42 43 #include <sys/bitmap.h>
43 44
44 45 #include <sys/ib/adapters/tavor/tavor.h>
45 46
46 47 /*
47 48 * The following routines are used for initializing and destroying
48 49 * the resource pools used by the Tavor resource allocation routines.
49 50 * They consist of four classes of object:
50 51 *
51 52 * Mailboxes: The "In" and "Out" mailbox types are used by the Tavor
52 53 * command interface routines. Mailboxes are used to pass information
53 54 * back and forth to the Tavor firmware. Either type of mailbox may
54 55 * be allocated from Tavor's direct attached DDR memory or from system
55 56 * memory (although currently all "In" mailboxes are in DDR and all "out"
56 57 * mailboxes come from system memory.
57 58 *
58 59 * HW entry objects: These objects represent resources required by the Tavor
59 60 * hardware. These objects include things like Queue Pair contexts (QPC),
60 61 * Completion Queue contexts (CQC), Event Queue contexts (EQC), RDB (for
61 62 * supporting RDMA Read/Atomic), Multicast Group entries (MCG), Memory
62 63 * Protection Table entries (MPT), Memory Translation Table entries (MTT).
63 64 *
64 65 * What these objects all have in common is that they are each required
65 66 * to come from DDR memory, they are always allocated from tables, and
66 67 * they are not to be directly accessed (read or written) by driver
67 68 * software.
68 69 * One notable exceptions to this rule are the Extended QP contexts (EQPC),
69 70 * and the UAR scratch area (UAR_SCR), both of which are not directly
70 71 * accessible through the Tavor resource allocation routines, but both
71 72 * of which are also required to reside in DDR memory and are not to be
72 73 * manipulated by driver software (they are privately managed by Tavor
73 74 * hardware).
74 75 * The other notable exceptions are the UAR pages (UAR_PG) which are
75 76 * allocated from the UAR address space rather than DDR, and the UD
76 77 * address vectors (UDAV) which are similar to the common object types
77 78 * with the major difference being that UDAVs _are_ directly read and
78 79 * written by driver software.
79 80 *
80 81 * SW handle objects: These objects represent resources required by Tavor
81 82 * driver software. They are primarily software tracking structures,
82 83 * which are allocated from system memory (using kmem_cache). Several of
83 84 * the objects have both a "constructor" and "destructor" method
84 85 * associated with them (see below).
85 86 *
86 87 * Protection Domain (PD) handle objects: These objects are very much like
87 88 * a SW handle object with the notable difference that all PD handle
88 89 * objects have an actual Protection Domain number (PD) associated with
89 90 * them (and the PD number is allocated/managed through a separate
90 91 * vmem_arena specifically set aside for this purpose.
91 92 */
92 93
93 94 static int tavor_rsrc_mbox_init(tavor_state_t *state,
94 95 tavor_rsrc_mbox_info_t *info);
95 96 static void tavor_rsrc_mbox_fini(tavor_state_t *state,
96 97 tavor_rsrc_mbox_info_t *info);
97 98
98 99 static int tavor_rsrc_hw_entries_init(tavor_state_t *state,
99 100 tavor_rsrc_hw_entry_info_t *info);
100 101 static void tavor_rsrc_hw_entries_fini(tavor_state_t *state,
101 102 tavor_rsrc_hw_entry_info_t *info);
102 103
103 104 static int tavor_rsrc_sw_handles_init(tavor_state_t *state,
104 105 tavor_rsrc_sw_hdl_info_t *info);
105 106 static void tavor_rsrc_sw_handles_fini(tavor_state_t *state,
106 107 tavor_rsrc_sw_hdl_info_t *info);
107 108
108 109 static int tavor_rsrc_pd_handles_init(tavor_state_t *state,
109 110 tavor_rsrc_sw_hdl_info_t *info);
110 111 static void tavor_rsrc_pd_handles_fini(tavor_state_t *state,
111 112 tavor_rsrc_sw_hdl_info_t *info);
112 113
113 114 /*
114 115 * The following routines are used for allocating and freeing the specific
115 116 * types of objects described above from their associated resource pools.
116 117 */
117 118 static int tavor_rsrc_mbox_alloc(tavor_rsrc_pool_info_t *pool_info,
118 119 uint_t num, tavor_rsrc_t *hdl);
119 120 static void tavor_rsrc_mbox_free(tavor_rsrc_pool_info_t *pool_info,
120 121 tavor_rsrc_t *hdl);
121 122
122 123 static int tavor_rsrc_hw_entry_alloc(tavor_rsrc_pool_info_t *pool_info,
123 124 uint_t num, uint_t num_align, ddi_acc_handle_t acc_handle,
124 125 uint_t sleepflag, tavor_rsrc_t *hdl);
125 126 static void tavor_rsrc_hw_entry_free(tavor_rsrc_pool_info_t *pool_info,
126 127 tavor_rsrc_t *hdl);
127 128
128 129 static int tavor_rsrc_swhdl_alloc(tavor_rsrc_pool_info_t *pool_info,
129 130 uint_t sleepflag, tavor_rsrc_t *hdl);
130 131 static void tavor_rsrc_swhdl_free(tavor_rsrc_pool_info_t *pool_info,
131 132 tavor_rsrc_t *hdl);
132 133
133 134 static int tavor_rsrc_pdhdl_alloc(tavor_rsrc_pool_info_t *pool_info,
134 135 uint_t sleepflag, tavor_rsrc_t *hdl);
135 136 static void tavor_rsrc_pdhdl_free(tavor_rsrc_pool_info_t *pool_info,
136 137 tavor_rsrc_t *hdl);
137 138
138 139 /*
139 140 * The following routines are the constructors and destructors for several
140 141 * of the SW handle type objects. For certain types of SW handles objects
141 142 * (all of which are implemented using kmem_cache), we need to do some
142 143 * special field initialization (specifically, mutex_init/destroy). These
143 144 * routines enable that init and teardown.
144 145 */
145 146 static int tavor_rsrc_pdhdl_constructor(void *pd, void *priv, int flags);
146 147 static void tavor_rsrc_pdhdl_destructor(void *pd, void *state);
147 148 static int tavor_rsrc_cqhdl_constructor(void *cq, void *priv, int flags);
148 149 static void tavor_rsrc_cqhdl_destructor(void *cq, void *state);
149 150 static int tavor_rsrc_qphdl_constructor(void *cq, void *priv, int flags);
150 151 static void tavor_rsrc_qphdl_destructor(void *cq, void *state);
151 152 static int tavor_rsrc_srqhdl_constructor(void *srq, void *priv, int flags);
152 153 static void tavor_rsrc_srqhdl_destructor(void *srq, void *state);
153 154 static int tavor_rsrc_refcnt_constructor(void *rc, void *priv, int flags);
154 155 static void tavor_rsrc_refcnt_destructor(void *rc, void *state);
155 156 static int tavor_rsrc_ahhdl_constructor(void *ah, void *priv, int flags);
156 157 static void tavor_rsrc_ahhdl_destructor(void *ah, void *state);
157 158 static int tavor_rsrc_mrhdl_constructor(void *mr, void *priv, int flags);
158 159 static void tavor_rsrc_mrhdl_destructor(void *mr, void *state);
159 160
160 161 /*
161 162 * Special routine to calculate and return the size of a MCG object based
162 163 * on current driver configuration (specifically, the number of QP per MCG
163 164 * that has been configured.
164 165 */
165 166 static int tavor_rsrc_mcg_entry_get_size(tavor_state_t *state,
166 167 uint_t *mcg_size_shift);
167 168
168 169
169 170 /*
170 171 * tavor_rsrc_alloc()
171 172 *
172 173 * Context: Can be called from interrupt or base context.
173 174 * The "sleepflag" parameter is used by all object allocators to
174 175 * determine whether to SLEEP for resources or not.
175 176 */
176 177 int
177 178 tavor_rsrc_alloc(tavor_state_t *state, tavor_rsrc_type_t rsrc, uint_t num,
178 179 uint_t sleepflag, tavor_rsrc_t **hdl)
179 180 {
180 181 tavor_rsrc_pool_info_t *rsrc_pool;
181 182 tavor_rsrc_t *tmp_rsrc_hdl;
182 183 int flag, status = DDI_FAILURE;
183 184
184 185 TAVOR_TNF_ENTER(tavor_rsrc_alloc);
185 186
186 187 ASSERT(state != NULL);
187 188 ASSERT(hdl != NULL);
188 189
189 190 rsrc_pool = &state->ts_rsrc_hdl[rsrc];
190 191 ASSERT(rsrc_pool != NULL);
191 192
192 193 /*
193 194 * Allocate space for the object used to track the resource handle
194 195 */
195 196 flag = (sleepflag == TAVOR_SLEEP) ? KM_SLEEP : KM_NOSLEEP;
196 197 tmp_rsrc_hdl = (tavor_rsrc_t *)kmem_cache_alloc(state->ts_rsrc_cache,
197 198 flag);
198 199 if (tmp_rsrc_hdl == NULL) {
199 200 TNF_PROBE_0(tavor_rsrc_alloc_kmca_fail, TAVOR_TNF_ERROR, "");
200 201 TAVOR_TNF_EXIT(tavor_rsrc_alloc);
201 202 return (DDI_FAILURE);
202 203 }
203 204 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*tmp_rsrc_hdl))
204 205
205 206 /*
206 207 * Set rsrc_hdl type. This is later used by the tavor_rsrc_free call
207 208 * to know what type of resource is being freed.
208 209 */
209 210 tmp_rsrc_hdl->rsrc_type = rsrc;
210 211
211 212 /*
212 213 * Depending on resource type, call the appropriate alloc routine
213 214 */
214 215 switch (rsrc_pool->rsrc_type) {
215 216 case TAVOR_IN_MBOX:
216 217 case TAVOR_OUT_MBOX:
217 218 case TAVOR_INTR_IN_MBOX:
218 219 case TAVOR_INTR_OUT_MBOX:
219 220 status = tavor_rsrc_mbox_alloc(rsrc_pool, num, tmp_rsrc_hdl);
220 221 break;
221 222
222 223 case TAVOR_QPC:
223 224 case TAVOR_CQC:
224 225 case TAVOR_SRQC:
225 226 case TAVOR_EQC:
226 227 case TAVOR_RDB:
227 228 /*
228 229 * Because these objects are NOT accessed by Tavor driver
229 230 * software, we set the acc_handle parameter to zero. But
230 231 * if they are allocated in multiples, we specify here that
231 232 * they must be aligned on a more restrictive boundary.
232 233 */
233 234 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, num, 0,
234 235 sleepflag, tmp_rsrc_hdl);
235 236 break;
236 237
237 238 case TAVOR_MPT:
238 239 /*
239 240 * Because these MPT objects are sometimes accessed by Tavor
240 241 * driver software (FMR), we set the acc_handle parameter. But
241 242 * if they are allocated in multiples, we specify here that
242 243 * they must be aligned on a more restrictive boundary.
243 244 */
244 245 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, num,
245 246 state->ts_reg_ddrhdl, sleepflag, tmp_rsrc_hdl);
246 247 break;
247 248
248 249 case TAVOR_MCG:
249 250 /*
250 251 * Tavor MCG entries are also NOT accessed by Tavor driver
251 252 * software, but because MCG entries do not have the same
252 253 * alignnment restrictions we loosen the constraint here.
253 254 */
254 255 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1, 0,
255 256 sleepflag, tmp_rsrc_hdl);
256 257 break;
257 258
258 259 case TAVOR_MTT:
259 260 case TAVOR_UDAV:
260 261 /*
261 262 * Because MTT segments are among the few HW resources that
262 263 * may be allocated in odd numbers, we specify a less
263 264 * restrictive alignment than for the above resources.
264 265 *
265 266 * Also because both UDAV and MTT segment objects are read
266 267 * and/or written by Tavor driver software, we set the
267 268 * acc_handle parameter to point to the ddi_acc_handle_t for
268 269 * the Tavor DDR memory.
269 270 */
270 271 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1,
271 272 state->ts_reg_ddrhdl, sleepflag, tmp_rsrc_hdl);
272 273 break;
273 274
274 275 case TAVOR_UARPG:
275 276 /*
276 277 * Because UAR pages are written by Tavor driver software (for
277 278 * doorbells), we set the acc_handle parameter to point to
278 279 * the ddi_acc_handle_t for the Tavor UAR memory.
279 280 */
280 281 status = tavor_rsrc_hw_entry_alloc(rsrc_pool, num, 1,
281 282 state->ts_reg_uarhdl, sleepflag, tmp_rsrc_hdl);
282 283 break;
283 284
284 285 case TAVOR_MRHDL:
285 286 case TAVOR_EQHDL:
286 287 case TAVOR_CQHDL:
287 288 case TAVOR_SRQHDL:
288 289 case TAVOR_AHHDL:
289 290 case TAVOR_QPHDL:
290 291 case TAVOR_REFCNT:
291 292 status = tavor_rsrc_swhdl_alloc(rsrc_pool, sleepflag,
292 293 tmp_rsrc_hdl);
293 294 break;
294 295
295 296 case TAVOR_PDHDL:
296 297 status = tavor_rsrc_pdhdl_alloc(rsrc_pool, sleepflag,
297 298 tmp_rsrc_hdl);
298 299 break;
299 300
300 301 default:
301 302 TAVOR_WARNING(state, "unexpected resource type in alloc");
302 303 TNF_PROBE_0(tavor_rsrc_alloc_inv_rsrctype_fail,
303 304 TAVOR_TNF_ERROR, "");
304 305 break;
305 306 }
306 307
307 308 /*
308 309 * If the resource allocation failed, then free the special resource
309 310 * tracking structure and return failure. Otherwise return the
310 311 * handle for the resource tracking structure.
311 312 */
312 313 if (status != DDI_SUCCESS) {
313 314 kmem_cache_free(state->ts_rsrc_cache, tmp_rsrc_hdl);
314 315 tmp_rsrc_hdl = NULL;
315 316 TNF_PROBE_1(tavor_rsrc_alloc_fail, TAVOR_TNF_ERROR, "",
316 317 tnf_uint, rsrc_type, rsrc_pool->rsrc_type);
317 318 TAVOR_TNF_EXIT(tavor_rsrc_alloc);
318 319 return (DDI_FAILURE);
319 320 } else {
320 321 *hdl = tmp_rsrc_hdl;
321 322 TAVOR_TNF_EXIT(tavor_rsrc_alloc);
322 323 return (DDI_SUCCESS);
323 324 }
324 325 }
325 326
326 327
327 328 /*
328 329 * tavor_rsrc_free()
329 330 * Context: Can be called from interrupt or base context.
330 331 */
331 332 void
332 333 tavor_rsrc_free(tavor_state_t *state, tavor_rsrc_t **hdl)
333 334 {
334 335 tavor_rsrc_pool_info_t *rsrc_pool;
335 336
336 337 TAVOR_TNF_ENTER(tavor_rsrc_free);
337 338
338 339 ASSERT(state != NULL);
339 340 ASSERT(hdl != NULL);
340 341
341 342 rsrc_pool = &state->ts_rsrc_hdl[(*hdl)->rsrc_type];
342 343 ASSERT(rsrc_pool != NULL);
343 344
344 345 /*
345 346 * Depending on resource type, call the appropriate free routine
346 347 */
347 348 switch (rsrc_pool->rsrc_type) {
348 349 case TAVOR_IN_MBOX:
349 350 case TAVOR_OUT_MBOX:
350 351 case TAVOR_INTR_IN_MBOX:
351 352 case TAVOR_INTR_OUT_MBOX:
352 353 tavor_rsrc_mbox_free(rsrc_pool, *hdl);
353 354 break;
354 355
355 356 case TAVOR_QPC:
356 357 case TAVOR_CQC:
357 358 case TAVOR_SRQC:
358 359 case TAVOR_EQC:
359 360 case TAVOR_RDB:
360 361 case TAVOR_MCG:
361 362 case TAVOR_MPT:
362 363 case TAVOR_MTT:
363 364 case TAVOR_UDAV:
364 365 case TAVOR_UARPG:
365 366 tavor_rsrc_hw_entry_free(rsrc_pool, *hdl);
366 367 break;
367 368
368 369 case TAVOR_MRHDL:
369 370 case TAVOR_EQHDL:
370 371 case TAVOR_CQHDL:
371 372 case TAVOR_SRQHDL:
372 373 case TAVOR_AHHDL:
373 374 case TAVOR_QPHDL:
374 375 case TAVOR_REFCNT:
375 376 tavor_rsrc_swhdl_free(rsrc_pool, *hdl);
376 377 break;
377 378
378 379 case TAVOR_PDHDL:
379 380 tavor_rsrc_pdhdl_free(rsrc_pool, *hdl);
380 381 break;
381 382
382 383 default:
383 384 TAVOR_WARNING(state, "unexpected resource type in free");
384 385 TNF_PROBE_0(tavor_rsrc_free_inv_rsrctype_fail,
385 386 TAVOR_TNF_ERROR, "");
386 387 break;
387 388 }
388 389
389 390 /*
390 391 * Free the special resource tracking structure, set the handle to
391 392 * NULL, and return.
392 393 */
393 394 kmem_cache_free(state->ts_rsrc_cache, *hdl);
394 395 *hdl = NULL;
395 396
396 397 TAVOR_TNF_EXIT(tavor_rsrc_free);
397 398 }
398 399
399 400
400 401 /*
401 402 * tavor_rsrc_init_phase1()
402 403 *
403 404 * Completes the first phase of Tavor resource/configuration init.
404 405 * This involves creating the kmem_cache for the "tavor_rsrc_t"
405 406 * structs, allocating the space for the resource pool handles,
406 407 * and setting up the "Out" mailboxes.
407 408 *
408 409 * When this function completes, the Tavor driver is ready to
409 410 * post the following commands which return information only in the
410 411 * "Out" mailbox: QUERY_DDR, QUERY_FW, QUERY_DEV_LIM, and QUERY_ADAPTER
411 412 * If any of these commands are to be posted at this time, they must be
412 413 * done so only when "spinning" (as the outstanding command list and
413 414 * EQ setup code has not yet run)
414 415 *
415 416 * Context: Only called from attach() path context
416 417 */
417 418 int
418 419 tavor_rsrc_init_phase1(tavor_state_t *state)
419 420 {
420 421 tavor_rsrc_pool_info_t *rsrc_pool;
421 422 tavor_rsrc_mbox_info_t mbox_info;
422 423 tavor_rsrc_cleanup_level_t cleanup;
423 424 tavor_cfg_profile_t *cfgprof;
424 425 uint64_t num, size;
425 426 int status;
426 427 char *errormsg, *rsrc_name;
427 428
428 429 TAVOR_TNF_ENTER(tavor_rsrc_init_phase1);
429 430
430 431 ASSERT(state != NULL);
431 432
432 433 /* This is where Phase 1 of resource initialization begins */
433 434 cleanup = TAVOR_RSRC_CLEANUP_LEVEL0;
434 435
435 436 /* Build kmem cache name from Tavor instance */
436 437 rsrc_name = (char *)kmem_zalloc(TAVOR_RSRC_NAME_MAXLEN, KM_SLEEP);
437 438 TAVOR_RSRC_NAME(rsrc_name, TAVOR_RSRC_CACHE);
438 439
439 440 /*
440 441 * Create the kmem_cache for "tavor_rsrc_t" structures
441 442 * (kmem_cache_create will SLEEP until successful)
442 443 */
443 444 state->ts_rsrc_cache = kmem_cache_create(rsrc_name,
444 445 sizeof (tavor_rsrc_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
445 446
446 447 /*
447 448 * Allocate an array of tavor_rsrc_pool_info_t's (used in all
448 449 * subsequent resource allocations)
449 450 */
450 451 state->ts_rsrc_hdl = kmem_zalloc(TAVOR_NUM_RESOURCES *
451 452 sizeof (tavor_rsrc_pool_info_t), KM_SLEEP);
452 453
453 454 cfgprof = state->ts_cfg_profile;
454 455
455 456 /*
456 457 * Initialize the resource pool for "Out" mailboxes. Notice that
457 458 * the number of "Out" mailboxes, their size, and their location
458 459 * (DDR or system memory) is configurable. By default, however,
459 460 * all "Out" mailboxes are located in system memory only (because
460 461 * they are primarily read from and never written to)
461 462 */
462 463 num = ((uint64_t)1 << cfgprof->cp_log_num_outmbox);
463 464 size = ((uint64_t)1 << cfgprof->cp_log_outmbox_size);
464 465 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_OUT_MBOX];
465 466 rsrc_pool->rsrc_type = TAVOR_OUT_MBOX;
466 467 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
467 468 rsrc_pool->rsrc_pool_size = (size * num);
468 469 rsrc_pool->rsrc_shift = cfgprof->cp_log_outmbox_size;
469 470 rsrc_pool->rsrc_quantum = size;
470 471 rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN;
471 472 rsrc_pool->rsrc_state = state;
472 473 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_OUTMBOX_VMEM);
473 474 mbox_info.mbi_num = num;
474 475 mbox_info.mbi_size = size;
475 476 mbox_info.mbi_rsrcpool = rsrc_pool;
476 477 mbox_info.mbi_rsrcname = rsrc_name;
477 478 status = tavor_rsrc_mbox_init(state, &mbox_info);
478 479 if (status != DDI_SUCCESS) {
479 480 tavor_rsrc_fini(state, cleanup);
480 481 /* Set "status" and "errormsg" and goto failure */
481 482 TAVOR_TNF_FAIL(DDI_FAILURE, "out mailboxes");
482 483 goto rsrcinitp1_fail;
483 484 }
484 485 cleanup = TAVOR_RSRC_CLEANUP_LEVEL1;
485 486
486 487 /*
487 488 * Initialize the Tavor "Out" mailbox list. This step actually uses
488 489 * the tavor_rsrc_alloc() for TAVOR_OUT_MBOX to preallocate the
489 490 * "Out" mailboxes, bind them for DMA access, and arrange them into
490 491 * an easily accessed fast-allocation mechanism (see tavor_cmd.c
491 492 * for more details)
492 493 */
493 494 status = tavor_outmbox_list_init(state);
494 495 if (status != DDI_SUCCESS) {
495 496 tavor_rsrc_fini(state, cleanup);
496 497 /* Set "status" and "errormsg" and goto failure */
497 498 TAVOR_TNF_FAIL(DDI_FAILURE, "out mailbox list");
498 499 goto rsrcinitp1_fail;
499 500 }
500 501 cleanup = TAVOR_RSRC_CLEANUP_LEVEL2;
501 502
502 503 /*
503 504 * Initialize the resource pool for interrupt "Out" mailboxes. Notice
504 505 * that the number of interrupt "Out" mailboxes, their size, and their
505 506 * location (DDR or system memory) is configurable. By default,
506 507 * however, all interrupt "Out" mailboxes are located in system memory
507 508 * only (because they are primarily read from and never written to)
508 509 */
509 510 num = ((uint64_t)1 << cfgprof->cp_log_num_intr_outmbox);
510 511 size = ((uint64_t)1 << cfgprof->cp_log_outmbox_size);
511 512 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_INTR_OUT_MBOX];
512 513 rsrc_pool->rsrc_type = TAVOR_INTR_OUT_MBOX;
513 514 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
514 515 rsrc_pool->rsrc_pool_size = (size * num);
515 516 rsrc_pool->rsrc_shift = cfgprof->cp_log_outmbox_size;
516 517 rsrc_pool->rsrc_quantum = size;
517 518 rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN;
518 519 rsrc_pool->rsrc_state = state;
519 520 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INTR_OUTMBOX_VMEM);
520 521 mbox_info.mbi_num = num;
521 522 mbox_info.mbi_size = size;
522 523 mbox_info.mbi_rsrcpool = rsrc_pool;
523 524 mbox_info.mbi_rsrcname = rsrc_name;
524 525 status = tavor_rsrc_mbox_init(state, &mbox_info);
525 526 if (status != DDI_SUCCESS) {
526 527 tavor_rsrc_fini(state, cleanup);
527 528 /* Set "status" and "errormsg" and goto failure */
528 529 TAVOR_TNF_FAIL(DDI_FAILURE, "out intr mailboxes");
529 530 goto rsrcinitp1_fail;
530 531 }
531 532 cleanup = TAVOR_RSRC_CLEANUP_LEVEL3;
532 533
533 534 /*
534 535 * Initialize the Tavor "Out" mailbox list. This step actually uses
535 536 * the tavor_rsrc_alloc() for TAVOR_OUT_MBOX to preallocate the
536 537 * "Out" mailboxes, bind them for DMA access, and arrange them into
537 538 * an easily accessed fast-allocation mechanism (see tavor_cmd.c
538 539 * for more details)
539 540 */
540 541 status = tavor_intr_outmbox_list_init(state);
541 542 if (status != DDI_SUCCESS) {
542 543 tavor_rsrc_fini(state, cleanup);
543 544 /* Set "status" and "errormsg" and goto failure */
544 545 TAVOR_TNF_FAIL(DDI_FAILURE, "out intr mailbox list");
545 546 goto rsrcinitp1_fail;
546 547 }
547 548 cleanup = TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE;
548 549
549 550 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN);
550 551 TAVOR_TNF_EXIT(tavor_rsrc_init_phase1);
551 552 return (DDI_SUCCESS);
552 553
553 554 rsrcinitp1_fail:
554 555 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN);
555 556 TNF_PROBE_1(tavor_rsrc_init_phase1_fail, TAVOR_TNF_ERROR, "",
556 557 tnf_string, msg, errormsg);
557 558 TAVOR_TNF_EXIT(tavor_rsrc_init_phase1);
558 559 return (status);
559 560 }
560 561
561 562
562 563 /*
563 564 * tavor_rsrc_init_phase2()
564 565 * Context: Only called from attach() path context
565 566 */
566 567 int
567 568 tavor_rsrc_init_phase2(tavor_state_t *state)
568 569 {
569 570 tavor_rsrc_sw_hdl_info_t hdl_info;
570 571 tavor_rsrc_hw_entry_info_t entry_info;
571 572 tavor_rsrc_mbox_info_t mbox_info;
572 573 tavor_rsrc_pool_info_t *rsrc_pool;
573 574 tavor_rsrc_cleanup_level_t cleanup;
574 575 tavor_cfg_profile_t *cfgprof;
575 576 uint64_t num, max, size, num_prealloc;
576 577 uint64_t ddr_size, fw_size;
577 578 uint_t mcg_size, mcg_size_shift;
578 579 uint_t uarscr_size, mttsegment_sz;
579 580 int status;
580 581 char *errormsg, *rsrc_name;
581 582
582 583 TAVOR_TNF_ENTER(tavor_rsrc_init_phase2);
583 584
584 585 ASSERT(state != NULL);
585 586
586 587 /* Phase 2 initialization begins where Phase 1 left off */
587 588 cleanup = TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE;
588 589
589 590 /*
590 591 * Calculate the extent of the DDR size and portion of which that
591 592 * is already reserved for Tavor firmware. (Note: this information
592 593 * is available because the QUERY_DDR and QUERY_FW commands have
593 594 * been posted to Tavor firmware prior to calling this routine)
594 595 */
595 596 ddr_size = state->ts_ddr.ddr_endaddr - state->ts_ddr.ddr_baseaddr + 1;
596 597 fw_size = state->ts_fw.fw_endaddr - state->ts_fw.fw_baseaddr + 1;
597 598
598 599 /* Build the DDR vmem arena name from Tavor instance */
599 600 rsrc_name = (char *)kmem_zalloc(TAVOR_RSRC_NAME_MAXLEN, KM_SLEEP);
600 601 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_VMEM);
601 602
602 603 /*
603 604 * Do a vmem_create for the entire DDR range (not including the
604 605 * portion consumed by Tavor firmware). This creates the vmem arena
605 606 * from which all other DDR objects (specifically, tables of HW
606 607 * entries) will be allocated.
607 608 */
608 609 state->ts_ddrvmem = vmem_create(rsrc_name,
609 610 (void *)(uintptr_t)state->ts_ddr.ddr_baseaddr, (ddr_size - fw_size),
610 611 sizeof (uint64_t), NULL, NULL, NULL, 0, VM_SLEEP);
611 612 if (state->ts_ddrvmem == NULL) {
612 613 tavor_rsrc_fini(state, cleanup);
613 614 /* Set "status" and "errormsg" and goto failure */
614 615 TAVOR_TNF_FAIL(DDI_FAILURE, "DDR vmem");
615 616 goto rsrcinitp2_fail;
616 617 }
617 618 cleanup = TAVOR_RSRC_CLEANUP_LEVEL5;
618 619
619 620 /*
620 621 * Initialize the resource pools for all objects that exist in
621 622 * Tavor DDR memory. This includes ("In") mailboxes, context tables
622 623 * (QPC, CQC, EQC, etc...), and other miscellaneous HW objects.
623 624 */
624 625 cfgprof = state->ts_cfg_profile;
625 626
626 627 /*
627 628 * Initialize the resource pool for the MPT table entries. Notice
628 629 * that the number of MPTs is configurable. The configured value must
629 630 * be less that the maximum value (obtained from the QUERY_DEV_LIM
630 631 * command) or the initialization will fail. Note also that a certain
631 632 * number of MPTs must be set aside for Tavor firmware use.
632 633 */
633 634 num = ((uint64_t)1 << cfgprof->cp_log_num_mpt);
634 635 max = ((uint64_t)1 << state->ts_devlim.log_max_mpt);
635 636 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_mpt);
636 637 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MPT];
637 638 rsrc_pool->rsrc_type = TAVOR_MPT;
638 639 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
639 640 rsrc_pool->rsrc_pool_size = (TAVOR_MPT_SIZE * num);
640 641 rsrc_pool->rsrc_shift = TAVOR_MPT_SIZE_SHIFT;
641 642 rsrc_pool->rsrc_quantum = TAVOR_MPT_SIZE;
642 643 rsrc_pool->rsrc_align = (TAVOR_MPT_SIZE * num);
643 644 rsrc_pool->rsrc_state = state;
644 645 rsrc_pool->rsrc_start = NULL;
645 646 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MPT_VMEM);
646 647 entry_info.hwi_num = num;
647 648 entry_info.hwi_max = max;
648 649 entry_info.hwi_prealloc = num_prealloc;
649 650 entry_info.hwi_rsrcpool = rsrc_pool;
650 651 entry_info.hwi_rsrcname = rsrc_name;
651 652 status = tavor_rsrc_hw_entries_init(state, &entry_info);
652 653 if (status != DDI_SUCCESS) {
653 654 tavor_rsrc_fini(state, cleanup);
654 655 /* Set "status" and "errormsg" and goto failure */
655 656 TAVOR_TNF_FAIL(DDI_FAILURE, "MPT table");
656 657 goto rsrcinitp2_fail;
657 658 }
658 659 cleanup = TAVOR_RSRC_CLEANUP_LEVEL6;
659 660
660 661 /*
661 662 * Initialize the resource pool for the MTT table entries. Notice
662 663 * that the number of MTTs is configurable. The configured value must
663 664 * be less that the maximum value (obtained from the QUERY_DEV_LIM
664 665 * command) or the initialization will fail. Note also that a certain
665 666 * number of MTT segments must be set aside for Tavor firmware use.
666 667 */
667 668 mttsegment_sz = (TAVOR_MTTSEG_SIZE << TAVOR_MTT_SIZE_SHIFT);
668 669 num = ((uint64_t)1 << cfgprof->cp_log_num_mttseg);
669 670 max = ((uint64_t)1 << state->ts_devlim.log_max_mttseg);
670 671 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_mttseg);
671 672 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MTT];
672 673 rsrc_pool->rsrc_type = TAVOR_MTT;
673 674 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
674 675 rsrc_pool->rsrc_pool_size = (TAVOR_MTT_SIZE * num);
675 676 rsrc_pool->rsrc_shift = TAVOR_MTT_SIZE_SHIFT;
676 677 rsrc_pool->rsrc_quantum = mttsegment_sz;
677 678 rsrc_pool->rsrc_align = (TAVOR_MTT_SIZE * num);
678 679 rsrc_pool->rsrc_state = state;
679 680 rsrc_pool->rsrc_start = NULL;
680 681 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MTT_VMEM);
681 682 entry_info.hwi_num = num;
682 683 entry_info.hwi_max = max;
683 684 entry_info.hwi_prealloc = num_prealloc;
684 685 entry_info.hwi_rsrcpool = rsrc_pool;
685 686 entry_info.hwi_rsrcname = rsrc_name;
686 687 status = tavor_rsrc_hw_entries_init(state, &entry_info);
687 688 if (status != DDI_SUCCESS) {
688 689 tavor_rsrc_fini(state, cleanup);
689 690 /* Set "status" and "errormsg" and goto failure */
690 691 TAVOR_TNF_FAIL(DDI_FAILURE, "MTT table");
691 692 goto rsrcinitp2_fail;
692 693 }
693 694 cleanup = TAVOR_RSRC_CLEANUP_LEVEL7;
694 695
695 696 /*
696 697 * Initialize the resource pool for the QPC table entries. Notice
697 698 * that the number of QPs is configurable. The configured value must
698 699 * be less that the maximum value (obtained from the QUERY_DEV_LIM
699 700 * command) or the initialization will fail. Note also that a certain
700 701 * number of QP contexts must be set aside for Tavor firmware use.
701 702 */
702 703 num = ((uint64_t)1 << cfgprof->cp_log_num_qp);
703 704 max = ((uint64_t)1 << state->ts_devlim.log_max_qp);
704 705 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_qp);
705 706 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_QPC];
706 707 rsrc_pool->rsrc_type = TAVOR_QPC;
707 708 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
708 709 rsrc_pool->rsrc_pool_size = (TAVOR_QPC_SIZE * num);
709 710 rsrc_pool->rsrc_shift = TAVOR_QPC_SIZE_SHIFT;
710 711 rsrc_pool->rsrc_quantum = TAVOR_QPC_SIZE;
711 712 rsrc_pool->rsrc_align = (TAVOR_QPC_SIZE * num);
712 713 rsrc_pool->rsrc_state = state;
713 714 rsrc_pool->rsrc_start = NULL;
714 715 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_QPC_VMEM);
715 716 entry_info.hwi_num = num;
716 717 entry_info.hwi_max = max;
717 718 entry_info.hwi_prealloc = num_prealloc;
718 719 entry_info.hwi_rsrcpool = rsrc_pool;
719 720 entry_info.hwi_rsrcname = rsrc_name;
720 721 status = tavor_rsrc_hw_entries_init(state, &entry_info);
721 722 if (status != DDI_SUCCESS) {
722 723 tavor_rsrc_fini(state, cleanup);
723 724 /* Set "status" and "errormsg" and goto failure */
724 725 TAVOR_TNF_FAIL(DDI_FAILURE, "QPC table");
725 726 goto rsrcinitp2_fail;
726 727 }
727 728 cleanup = TAVOR_RSRC_CLEANUP_LEVEL8;
728 729
729 730 /*
730 731 * Initialize the resource pool for the RDB table entries. Notice
731 732 * that the number of RDBs is configurable. The configured value must
732 733 * be less that the maximum value (obtained from the QUERY_DEV_LIM
733 734 * command) or the initialization will fail.
734 735 */
735 736 num = ((uint64_t)1 << cfgprof->cp_log_num_rdb);
736 737 max = ((uint64_t)1 << state->ts_devlim.log_max_ra_glob);
737 738 num_prealloc = 0;
738 739 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_RDB];
739 740 rsrc_pool->rsrc_type = TAVOR_RDB;
740 741 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
741 742 rsrc_pool->rsrc_pool_size = (TAVOR_RDB_SIZE * num);
742 743 rsrc_pool->rsrc_shift = TAVOR_RDB_SIZE_SHIFT;
743 744 rsrc_pool->rsrc_quantum = TAVOR_RDB_SIZE;
744 745 rsrc_pool->rsrc_align = (TAVOR_RDB_SIZE * num);
745 746 rsrc_pool->rsrc_state = state;
746 747 rsrc_pool->rsrc_start = NULL;
747 748 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_RDB_VMEM);
748 749 entry_info.hwi_num = num;
749 750 entry_info.hwi_max = max;
750 751 entry_info.hwi_prealloc = num_prealloc;
751 752 entry_info.hwi_rsrcpool = rsrc_pool;
752 753 entry_info.hwi_rsrcname = rsrc_name;
753 754 status = tavor_rsrc_hw_entries_init(state, &entry_info);
754 755 if (status != DDI_SUCCESS) {
755 756 tavor_rsrc_fini(state, cleanup);
756 757 /* Set "status" and "errormsg" and goto failure */
757 758 TAVOR_TNF_FAIL(DDI_FAILURE, "RDB table");
758 759 goto rsrcinitp2_fail;
759 760 }
760 761 cleanup = TAVOR_RSRC_CLEANUP_LEVEL9;
761 762
762 763 /*
763 764 * Initialize the resource pool for the CQC table entries. Notice
764 765 * that the number of CQs is configurable. The configured value must
765 766 * be less that the maximum value (obtained from the QUERY_DEV_LIM
766 767 * command) or the initialization will fail. Note also that a certain
767 768 * number of CQ contexts must be set aside for Tavor firmware use.
768 769 */
769 770 num = ((uint64_t)1 << cfgprof->cp_log_num_cq);
770 771 max = ((uint64_t)1 << state->ts_devlim.log_max_cq);
771 772 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_cq);
772 773 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_CQC];
773 774 rsrc_pool->rsrc_type = TAVOR_CQC;
774 775 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
775 776 rsrc_pool->rsrc_pool_size = (TAVOR_CQC_SIZE * num);
776 777 rsrc_pool->rsrc_shift = TAVOR_CQC_SIZE_SHIFT;
777 778 rsrc_pool->rsrc_quantum = TAVOR_CQC_SIZE;
778 779 rsrc_pool->rsrc_align = (TAVOR_CQC_SIZE * num);
779 780 rsrc_pool->rsrc_state = state;
780 781 rsrc_pool->rsrc_start = NULL;
781 782 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_CQC_VMEM);
782 783 entry_info.hwi_num = num;
783 784 entry_info.hwi_max = max;
784 785 entry_info.hwi_prealloc = num_prealloc;
785 786 entry_info.hwi_rsrcpool = rsrc_pool;
786 787 entry_info.hwi_rsrcname = rsrc_name;
787 788 status = tavor_rsrc_hw_entries_init(state, &entry_info);
788 789 if (status != DDI_SUCCESS) {
789 790 tavor_rsrc_fini(state, cleanup);
790 791 /* Set "status" and "errormsg" and goto failure */
791 792 TAVOR_TNF_FAIL(DDI_FAILURE, "CQC table");
792 793 goto rsrcinitp2_fail;
793 794 }
794 795 cleanup = TAVOR_RSRC_CLEANUP_LEVEL10;
795 796
796 797 /*
797 798 * Initialize the resource pool for the Extended QPC table entries.
798 799 * Notice that the number of EQPCs must be the same as the number
799 800 * of QP contexts. So the initialization is constructed in a
800 801 * similar way as above (for TAVOR_QPC). One notable difference
801 802 * here, however, is that by setting the rsrc_quantum field to
802 803 * zero (indicating a zero-sized object) we indicate that the
803 804 * object is not allocatable. The EQPC table is, in fact, managed
804 805 * internally by the hardware and it is, therefore, unnecessary to
805 806 * initialize an additional vmem_arena for this type of object.
806 807 */
807 808 num = ((uint64_t)1 << cfgprof->cp_log_num_qp);
808 809 max = ((uint64_t)1 << state->ts_devlim.log_max_qp);
809 810 num_prealloc = 0;
810 811 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQPC];
811 812 rsrc_pool->rsrc_type = TAVOR_EQPC;
812 813 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
813 814 rsrc_pool->rsrc_pool_size = (TAVOR_EQPC_SIZE * num);
814 815 rsrc_pool->rsrc_shift = 0;
815 816 rsrc_pool->rsrc_quantum = 0;
816 817 rsrc_pool->rsrc_align = TAVOR_EQPC_SIZE;
817 818 rsrc_pool->rsrc_state = state;
818 819 rsrc_pool->rsrc_start = NULL;
819 820 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_EQPC_VMEM);
820 821 entry_info.hwi_num = num;
821 822 entry_info.hwi_max = max;
822 823 entry_info.hwi_prealloc = num_prealloc;
823 824 entry_info.hwi_rsrcpool = rsrc_pool;
824 825 entry_info.hwi_rsrcname = rsrc_name;
825 826 status = tavor_rsrc_hw_entries_init(state, &entry_info);
826 827 if (status != DDI_SUCCESS) {
827 828 tavor_rsrc_fini(state, cleanup);
828 829 /* Set "status" and "errormsg" and goto failure */
829 830 TAVOR_TNF_FAIL(DDI_FAILURE, "Extended QPC table");
830 831 goto rsrcinitp2_fail;
831 832 }
832 833 cleanup = TAVOR_RSRC_CLEANUP_LEVEL11;
833 834
834 835 /*
835 836 * Initialize the resource pool for the UD address vector table
836 837 * entries. Notice that the number of UDAVs is configurable. The
837 838 * configured value must be less that the maximum value (obtained
838 839 * from the QUERY_DEV_LIM command) or the initialization will fail.
839 840 */
840 841 num = ((uint64_t)1 << cfgprof->cp_log_num_ah);
841 842 max = ((uint64_t)1 << state->ts_devlim.log_max_av);
842 843 num_prealloc = 0;
843 844 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UDAV];
844 845 rsrc_pool->rsrc_type = TAVOR_UDAV;
845 846 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
846 847 rsrc_pool->rsrc_pool_size = (TAVOR_UDAV_SIZE * num);
847 848 rsrc_pool->rsrc_shift = TAVOR_UDAV_SIZE_SHIFT;
848 849 rsrc_pool->rsrc_quantum = TAVOR_UDAV_SIZE;
849 850 rsrc_pool->rsrc_align = TAVOR_UDAV_SIZE;
850 851 rsrc_pool->rsrc_state = state;
851 852 rsrc_pool->rsrc_start = NULL;
852 853 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_UDAV_VMEM);
853 854 entry_info.hwi_num = num;
854 855 entry_info.hwi_max = max;
855 856 entry_info.hwi_prealloc = num_prealloc;
856 857 entry_info.hwi_rsrcpool = rsrc_pool;
857 858 entry_info.hwi_rsrcname = rsrc_name;
858 859 status = tavor_rsrc_hw_entries_init(state, &entry_info);
859 860 if (status != DDI_SUCCESS) {
860 861 tavor_rsrc_fini(state, cleanup);
861 862 /* Set "status" and "errormsg" and goto failure */
862 863 TAVOR_TNF_FAIL(DDI_FAILURE, "UDAV table");
863 864 goto rsrcinitp2_fail;
864 865 }
865 866 cleanup = TAVOR_RSRC_CLEANUP_LEVEL12;
866 867
867 868 /*
868 869 * Initialize the resource pool for the UAR scratch table entries.
869 870 * Notice that the number of UARSCRs is configurable. The configured
870 871 * value must be less that the maximum value (obtained from the
871 872 * QUERY_DEV_LIM command) or the initialization will fail.
872 873 * Like the EQPCs above, UARSCR objects are not allocatable. The
873 874 * UARSCR table is also managed internally by the hardware and it
874 875 * is, therefore, unnecessary to initialize an additional vmem_arena
875 876 * for this type of object. We indicate this by setting the
876 877 * rsrc_quantum field to zero (indicating a zero-sized object).
877 878 */
878 879 uarscr_size = state->ts_devlim.uarscr_entry_sz;
879 880 num = ((uint64_t)1 << cfgprof->cp_log_num_uar);
880 881 max = ((uint64_t)1 << (state->ts_devlim.log_max_uar_sz + 20 -
881 882 PAGESHIFT));
882 883 num_prealloc = 0;
883 884 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UAR_SCR];
884 885 rsrc_pool->rsrc_type = TAVOR_UAR_SCR;
885 886 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
886 887 rsrc_pool->rsrc_pool_size = (uarscr_size * num);
887 888 rsrc_pool->rsrc_shift = 0;
888 889 rsrc_pool->rsrc_quantum = 0;
889 890 rsrc_pool->rsrc_align = uarscr_size;
890 891 rsrc_pool->rsrc_state = state;
891 892 rsrc_pool->rsrc_start = NULL;
892 893 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_UARSCR_VMEM);
893 894 entry_info.hwi_num = num;
894 895 entry_info.hwi_max = max;
895 896 entry_info.hwi_prealloc = num_prealloc;
896 897 entry_info.hwi_rsrcpool = rsrc_pool;
897 898 entry_info.hwi_rsrcname = rsrc_name;
898 899 status = tavor_rsrc_hw_entries_init(state, &entry_info);
899 900 if (status != DDI_SUCCESS) {
900 901 tavor_rsrc_fini(state, cleanup);
901 902 /* Set "status" and "errormsg" and goto failure */
902 903 TAVOR_TNF_FAIL(DDI_FAILURE, "UAR scratch table");
903 904 goto rsrcinitp2_fail;
904 905 }
905 906 cleanup = TAVOR_RSRC_CLEANUP_LEVEL13;
906 907
907 908 /*
908 909 * Initialize the resource pool for the SRQC table entries. Notice
909 910 * that the number of SRQs is configurable. The configured value must
910 911 * be less that the maximum value (obtained from the QUERY_DEV_LIM
911 912 * command) or the initialization will fail. Note also that a certain
912 913 * number of SRQ contexts must be set aside for Tavor firmware use.
913 914 *
914 915 * Note: We only allocate these resources if SRQ is enabled in the
915 916 * config profile; see below.
916 917 */
917 918 num = ((uint64_t)1 << cfgprof->cp_log_num_srq);
918 919 max = ((uint64_t)1 << state->ts_devlim.log_max_srq);
919 920 num_prealloc = ((uint64_t)1 << state->ts_devlim.log_rsvd_srq);
920 921
921 922 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_SRQC];
922 923 rsrc_pool->rsrc_type = TAVOR_SRQC;
923 924 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
924 925 rsrc_pool->rsrc_pool_size = (TAVOR_SRQC_SIZE * num);
925 926 rsrc_pool->rsrc_shift = TAVOR_SRQC_SIZE_SHIFT;
926 927 rsrc_pool->rsrc_quantum = TAVOR_SRQC_SIZE;
927 928 rsrc_pool->rsrc_align = (TAVOR_SRQC_SIZE * num);
928 929 rsrc_pool->rsrc_state = state;
929 930 rsrc_pool->rsrc_start = NULL;
930 931 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_SRQC_VMEM);
931 932 entry_info.hwi_num = num;
932 933 entry_info.hwi_max = max;
933 934 entry_info.hwi_prealloc = num_prealloc;
934 935 entry_info.hwi_rsrcpool = rsrc_pool;
935 936 entry_info.hwi_rsrcname = rsrc_name;
936 937
937 938 /*
938 939 * SRQ support is configurable. Only if SRQ is enabled (the default)
939 940 * do we actually try to configure these resources. Otherwise, we
940 941 * simply set the cleanup level and continue on to the next resource
941 942 */
942 943 if (state->ts_cfg_profile->cp_srq_enable != 0) {
943 944 status = tavor_rsrc_hw_entries_init(state, &entry_info);
944 945 if (status != DDI_SUCCESS) {
945 946 tavor_rsrc_fini(state, cleanup);
946 947 /* Set "status" and "errormsg" and goto failure */
947 948 TAVOR_TNF_FAIL(DDI_FAILURE, "SRQC table");
948 949 goto rsrcinitp2_fail;
949 950 }
950 951 }
951 952 cleanup = TAVOR_RSRC_CLEANUP_LEVEL14;
952 953
953 954 /*
954 955 * Initialize the resource pool for "In" mailboxes. Notice that
955 956 * the number of "In" mailboxes, their size, and their location
956 957 * (DDR or system memory) is configurable. By default, however,
957 958 * all "In" mailboxes are located in system memory only (because
958 959 * they are primarily written to and rarely read from)
959 960 */
960 961 num = ((uint64_t)1 << cfgprof->cp_log_num_inmbox);
961 962 size = ((uint64_t)1 << cfgprof->cp_log_inmbox_size);
962 963 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_IN_MBOX];
963 964 rsrc_pool->rsrc_type = TAVOR_IN_MBOX;
964 965 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
965 966 rsrc_pool->rsrc_pool_size = (size * num);
966 967 rsrc_pool->rsrc_shift = cfgprof->cp_log_inmbox_size;
967 968 rsrc_pool->rsrc_quantum = size;
968 969 rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN;
969 970 rsrc_pool->rsrc_state = state;
970 971 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INMBOX_VMEM);
971 972 mbox_info.mbi_num = num;
972 973 mbox_info.mbi_size = size;
973 974 mbox_info.mbi_rsrcpool = rsrc_pool;
974 975 mbox_info.mbi_rsrcname = rsrc_name;
975 976 status = tavor_rsrc_mbox_init(state, &mbox_info);
976 977 if (status != DDI_SUCCESS) {
977 978 tavor_rsrc_fini(state, cleanup);
978 979 /* Set "status" and "errormsg" and goto failure */
979 980 TAVOR_TNF_FAIL(DDI_FAILURE, "in mailboxes");
980 981 goto rsrcinitp2_fail;
981 982 }
982 983 cleanup = TAVOR_RSRC_CLEANUP_LEVEL15;
983 984
984 985 /*
985 986 * Initialize the Tavor "In" mailbox list. This step actually uses
986 987 * the tavor_rsrc_alloc() for TAVOR_IN_MBOX to preallocate the
987 988 * "In" mailboxes, bind them for DMA access, and arrange them into
988 989 * an easily accessed fast-allocation mechanism (see tavor_cmd.c
989 990 * for more details)
990 991 */
991 992 status = tavor_inmbox_list_init(state);
992 993 if (status != DDI_SUCCESS) {
993 994 tavor_rsrc_fini(state, cleanup);
994 995 /* Set "status" and "errormsg" and goto failure */
995 996 TAVOR_TNF_FAIL(DDI_FAILURE, "in mailbox list");
996 997 goto rsrcinitp2_fail;
997 998 }
998 999 cleanup = TAVOR_RSRC_CLEANUP_LEVEL16;
999 1000
1000 1001 /*
1001 1002 * Initialize the resource pool for interrupt "In" mailboxes. Notice
1002 1003 * that the number of interrupt "In" mailboxes, their size, and their
1003 1004 * location (DDR or system memory) is configurable. By default,
1004 1005 * however, all interrupt "In" mailboxes are located in system memory
1005 1006 * only (because they are primarily written to and rarely read from)
1006 1007 */
1007 1008 num = ((uint64_t)1 << cfgprof->cp_log_num_intr_inmbox);
1008 1009 size = ((uint64_t)1 << cfgprof->cp_log_inmbox_size);
1009 1010 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_INTR_IN_MBOX];
1010 1011 rsrc_pool->rsrc_type = TAVOR_INTR_IN_MBOX;
1011 1012 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
1012 1013 rsrc_pool->rsrc_pool_size = (size * num);
1013 1014 rsrc_pool->rsrc_shift = cfgprof->cp_log_inmbox_size;
1014 1015 rsrc_pool->rsrc_quantum = size;
1015 1016 rsrc_pool->rsrc_align = TAVOR_MBOX_ALIGN;
1016 1017 rsrc_pool->rsrc_state = state;
1017 1018 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_INTR_INMBOX_VMEM);
1018 1019 mbox_info.mbi_num = num;
1019 1020 mbox_info.mbi_size = size;
1020 1021 mbox_info.mbi_rsrcpool = rsrc_pool;
1021 1022 mbox_info.mbi_rsrcname = rsrc_name;
1022 1023 status = tavor_rsrc_mbox_init(state, &mbox_info);
1023 1024 if (status != DDI_SUCCESS) {
1024 1025 tavor_rsrc_fini(state, cleanup);
1025 1026 /* Set "status" and "errormsg" and goto failure */
1026 1027 TAVOR_TNF_FAIL(DDI_FAILURE, "in intr mailboxes");
1027 1028 goto rsrcinitp2_fail;
1028 1029 }
1029 1030 cleanup = TAVOR_RSRC_CLEANUP_LEVEL17;
1030 1031
1031 1032 /*
1032 1033 * Initialize the Tavor interrupt "In" mailbox list. This step
1033 1034 * actually uses the tavor_rsrc_alloc() for TAVOR_IN_MBOX to
1034 1035 * preallocate the interrupt "In" mailboxes, bind them for DMA access,
1035 1036 * and arrange them into an easily accessed fast-allocation mechanism
1036 1037 * (see tavor_cmd.c for more details)
1037 1038 */
1038 1039 status = tavor_intr_inmbox_list_init(state);
1039 1040 if (status != DDI_SUCCESS) {
1040 1041 tavor_rsrc_fini(state, cleanup);
1041 1042 /* Set "status" and "errormsg" and goto failure */
1042 1043 TAVOR_TNF_FAIL(DDI_FAILURE, "in intr mailbox list");
1043 1044 goto rsrcinitp2_fail;
1044 1045 }
1045 1046 cleanup = TAVOR_RSRC_CLEANUP_LEVEL18;
1046 1047
1047 1048 /*
1048 1049 * Initialize the Tavor command handling interfaces. This step
1049 1050 * sets up the outstanding command tracking mechanism for easy access
1050 1051 * and fast allocation (see tavor_cmd.c for more details).
1051 1052 */
1052 1053 status = tavor_outstanding_cmdlist_init(state);
1053 1054 if (status != DDI_SUCCESS) {
1054 1055 tavor_rsrc_fini(state, cleanup);
1055 1056 /* Set "status" and "errormsg" and goto failure */
1056 1057 TAVOR_TNF_FAIL(DDI_FAILURE, "outstanding cmd list");
1057 1058 goto rsrcinitp2_fail;
1058 1059 }
1059 1060 cleanup = TAVOR_RSRC_CLEANUP_LEVEL19;
1060 1061
1061 1062 /*
1062 1063 * Calculate (and validate) the size of Multicast Group (MCG) entries
1063 1064 */
1064 1065 status = tavor_rsrc_mcg_entry_get_size(state, &mcg_size_shift);
1065 1066 if (status != DDI_SUCCESS) {
1066 1067 tavor_rsrc_fini(state, cleanup);
1067 1068 /* Set "status" and "errormsg" and goto failure */
1068 1069 TAVOR_TNF_FAIL(DDI_FAILURE, "failed get MCG size");
1069 1070 goto rsrcinitp2_fail;
1070 1071 }
1071 1072 mcg_size = TAVOR_MCGMEM_SZ(state);
1072 1073
1073 1074 /*
1074 1075 * Initialize the resource pool for the MCG table entries. Notice
1075 1076 * that the number of MCGs is configurable. The configured value must
1076 1077 * be less that the maximum value (obtained from the QUERY_DEV_LIM
1077 1078 * command) or the initialization will fail. Note also that a certain
1078 1079 * number of MCGs must be set aside for Tavor firmware use (they
1079 1080 * correspond to the number of MCGs used by the internal hash
1080 1081 * function.
1081 1082 */
1082 1083 num = ((uint64_t)1 << cfgprof->cp_log_num_mcg);
1083 1084 max = ((uint64_t)1 << state->ts_devlim.log_max_mcg);
1084 1085 num_prealloc = ((uint64_t)1 << cfgprof->cp_log_num_mcg_hash);
1085 1086 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MCG];
1086 1087 rsrc_pool->rsrc_type = TAVOR_MCG;
1087 1088 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
1088 1089 rsrc_pool->rsrc_pool_size = (mcg_size * num);
1089 1090 rsrc_pool->rsrc_shift = mcg_size_shift;
1090 1091 rsrc_pool->rsrc_quantum = mcg_size;
1091 1092 rsrc_pool->rsrc_align = mcg_size;
1092 1093 rsrc_pool->rsrc_state = state;
1093 1094 rsrc_pool->rsrc_start = NULL;
1094 1095 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_MCG_VMEM);
1095 1096 entry_info.hwi_num = num;
1096 1097 entry_info.hwi_max = max;
1097 1098 entry_info.hwi_prealloc = num_prealloc;
1098 1099 entry_info.hwi_rsrcpool = rsrc_pool;
1099 1100 entry_info.hwi_rsrcname = rsrc_name;
1100 1101 status = tavor_rsrc_hw_entries_init(state, &entry_info);
1101 1102 if (status != DDI_SUCCESS) {
1102 1103 tavor_rsrc_fini(state, cleanup);
1103 1104 /* Set "status" and "errormsg" and goto failure */
1104 1105 TAVOR_TNF_FAIL(DDI_FAILURE, "MCG table");
1105 1106 goto rsrcinitp2_fail;
1106 1107 }
1107 1108 cleanup = TAVOR_RSRC_CLEANUP_LEVEL20;
1108 1109
1109 1110 /*
1110 1111 * Initialize the resource pool for the EQC table entries. Notice
1111 1112 * that the number of EQs is hardcoded. The hardcoded value should
1112 1113 * be less that the maximum value (obtained from the QUERY_DEV_LIM
1113 1114 * command) or the initialization will fail.
1114 1115 */
1115 1116 num = TAVOR_NUM_EQ;
1116 1117 max = ((uint64_t)1 << state->ts_devlim.log_max_eq);
1117 1118 num_prealloc = 0;
1118 1119 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQC];
1119 1120 rsrc_pool->rsrc_type = TAVOR_EQC;
1120 1121 rsrc_pool->rsrc_loc = TAVOR_IN_DDR;
1121 1122 rsrc_pool->rsrc_pool_size = (TAVOR_EQC_SIZE * num);
1122 1123 rsrc_pool->rsrc_shift = TAVOR_EQC_SIZE_SHIFT;
1123 1124 rsrc_pool->rsrc_quantum = TAVOR_EQC_SIZE;
1124 1125 rsrc_pool->rsrc_align = (TAVOR_EQC_SIZE * num);
1125 1126 rsrc_pool->rsrc_state = state;
1126 1127 rsrc_pool->rsrc_start = NULL;
1127 1128 TAVOR_RSRC_NAME(rsrc_name, TAVOR_DDR_EQC_VMEM);
1128 1129 entry_info.hwi_num = num;
1129 1130 entry_info.hwi_max = max;
1130 1131 entry_info.hwi_prealloc = num_prealloc;
1131 1132 entry_info.hwi_rsrcpool = rsrc_pool;
1132 1133 entry_info.hwi_rsrcname = rsrc_name;
1133 1134 status = tavor_rsrc_hw_entries_init(state, &entry_info);
1134 1135 if (status != DDI_SUCCESS) {
1135 1136 tavor_rsrc_fini(state, cleanup);
1136 1137 /* Set "status" and "errormsg" and goto failure */
1137 1138 TAVOR_TNF_FAIL(DDI_FAILURE, "EQC table");
1138 1139 goto rsrcinitp2_fail;
1139 1140 }
1140 1141 cleanup = TAVOR_RSRC_CLEANUP_LEVEL21;
1141 1142
1142 1143 /*
1143 1144 * Initialize the resource pools for all objects that exist in
1144 1145 * system memory. This includes PD handles, MR handle, EQ handles,
1145 1146 * QP handles, etc. These objects are almost entirely managed using
1146 1147 * kmem_cache routines. (See comment above for more detail)
1147 1148 */
1148 1149
1149 1150 /*
1150 1151 * Initialize the resource pool for the PD handles. Notice
1151 1152 * that the number of PDHDLs is configurable. The configured value
1152 1153 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1153 1154 * command) or the initialization will fail. Note also that the PD
1154 1155 * handle has constructor and destructor methods associated with it.
1155 1156 */
1156 1157 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_PDHDL];
1157 1158 rsrc_pool->rsrc_type = TAVOR_PDHDL;
1158 1159 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1159 1160 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_pd_s);
1160 1161 rsrc_pool->rsrc_state = state;
1161 1162 TAVOR_RSRC_NAME(rsrc_name, TAVOR_PDHDL_CACHE);
1162 1163 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_pd);
1163 1164 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_pd);
1164 1165 hdl_info.swi_rsrcpool = rsrc_pool;
1165 1166 hdl_info.swi_constructor = tavor_rsrc_pdhdl_constructor;
1166 1167 hdl_info.swi_destructor = tavor_rsrc_pdhdl_destructor;
1167 1168 hdl_info.swi_rsrcname = rsrc_name;
1168 1169 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT;
1169 1170 status = tavor_rsrc_pd_handles_init(state, &hdl_info);
1170 1171 if (status != DDI_SUCCESS) {
1171 1172 tavor_rsrc_fini(state, cleanup);
1172 1173 /* Set "status" and "errormsg" and goto failure */
1173 1174 TAVOR_TNF_FAIL(DDI_FAILURE, "PD handle");
1174 1175 goto rsrcinitp2_fail;
1175 1176 }
1176 1177 cleanup = TAVOR_RSRC_CLEANUP_LEVEL22;
1177 1178
1178 1179 /*
1179 1180 * Initialize the resource pool for the MR handles. Notice
1180 1181 * that the number of MRHDLs is configurable. The configured value
1181 1182 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1182 1183 * command) or the initialization will fail.
1183 1184 */
1184 1185 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MRHDL];
1185 1186 rsrc_pool->rsrc_type = TAVOR_MRHDL;
1186 1187 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1187 1188 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_mr_s);
1188 1189 rsrc_pool->rsrc_state = state;
1189 1190 TAVOR_RSRC_NAME(rsrc_name, TAVOR_MRHDL_CACHE);
1190 1191 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mpt);
1191 1192 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mpt);
1192 1193 hdl_info.swi_rsrcpool = rsrc_pool;
1193 1194 hdl_info.swi_constructor = tavor_rsrc_mrhdl_constructor;
1194 1195 hdl_info.swi_destructor = tavor_rsrc_mrhdl_destructor;
1195 1196 hdl_info.swi_rsrcname = rsrc_name;
1196 1197 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT;
1197 1198 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1198 1199 if (status != DDI_SUCCESS) {
1199 1200 tavor_rsrc_fini(state, cleanup);
1200 1201 /* Set "status" and "errormsg" and goto failure */
1201 1202 TAVOR_TNF_FAIL(DDI_FAILURE, "MR handle");
1202 1203 goto rsrcinitp2_fail;
1203 1204 }
1204 1205 cleanup = TAVOR_RSRC_CLEANUP_LEVEL23;
1205 1206
1206 1207 /*
1207 1208 * Initialize the resource pool for the EQ handles. Notice
1208 1209 * that the number of EQHDLs is hardcoded. The hardcoded value
1209 1210 * should be less that the maximum value (obtained from the
1210 1211 * QUERY_DEV_LIM command) or the initialization will fail.
1211 1212 */
1212 1213 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQHDL];
1213 1214 rsrc_pool->rsrc_type = TAVOR_EQHDL;
1214 1215 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1215 1216 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_eq_s);
1216 1217 rsrc_pool->rsrc_state = state;
1217 1218 TAVOR_RSRC_NAME(rsrc_name, TAVOR_EQHDL_CACHE);
1218 1219 hdl_info.swi_num = TAVOR_NUM_EQ;
1219 1220 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_eq);
1220 1221 hdl_info.swi_rsrcpool = rsrc_pool;
1221 1222 hdl_info.swi_constructor = NULL;
1222 1223 hdl_info.swi_destructor = NULL;
1223 1224 hdl_info.swi_rsrcname = rsrc_name;
1224 1225 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT;
1225 1226 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1226 1227 if (status != DDI_SUCCESS) {
1227 1228 tavor_rsrc_fini(state, cleanup);
1228 1229 /* Set "status" and "errormsg" and goto failure */
1229 1230 TAVOR_TNF_FAIL(DDI_FAILURE, "EQ handle");
1230 1231 goto rsrcinitp2_fail;
1231 1232 }
1232 1233 cleanup = TAVOR_RSRC_CLEANUP_LEVEL24;
1233 1234
1234 1235 /*
1235 1236 * Initialize the resource pool for the CQ handles. Notice
1236 1237 * that the number of CQHDLs is configurable. The configured value
1237 1238 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1238 1239 * command) or the initialization will fail. Note also that the CQ
1239 1240 * handle has constructor and destructor methods associated with it.
1240 1241 */
1241 1242 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_CQHDL];
1242 1243 rsrc_pool->rsrc_type = TAVOR_CQHDL;
1243 1244 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1244 1245 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_cq_s);
1245 1246 rsrc_pool->rsrc_state = state;
1246 1247 TAVOR_RSRC_NAME(rsrc_name, TAVOR_CQHDL_CACHE);
1247 1248 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_cq);
1248 1249 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_cq);
1249 1250 hdl_info.swi_rsrcpool = rsrc_pool;
1250 1251 hdl_info.swi_constructor = tavor_rsrc_cqhdl_constructor;
1251 1252 hdl_info.swi_destructor = tavor_rsrc_cqhdl_destructor;
1252 1253 hdl_info.swi_rsrcname = rsrc_name;
1253 1254 hdl_info.swi_flags = (TAVOR_SWHDL_KMEMCACHE_INIT |
1254 1255 TAVOR_SWHDL_TABLE_INIT);
1255 1256 hdl_info.swi_prealloc_sz = sizeof (tavor_cqhdl_t);
1256 1257 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1257 1258 if (status != DDI_SUCCESS) {
1258 1259 tavor_rsrc_fini(state, cleanup);
1259 1260 /* Set "status" and "errormsg" and goto failure */
1260 1261 TAVOR_TNF_FAIL(DDI_FAILURE, "CQ handle");
1261 1262 goto rsrcinitp2_fail;
1262 1263 }
1263 1264
1264 1265 /*
1265 1266 * Save away the pointer to the central list of CQ handle pointers
1266 1267 * This this is used as a mechanism to enable fast CQnumber-to-CQhandle
1267 1268 * lookup during EQ event processing. The table is a list of
1268 1269 * tavor_cqhdl_t allocated by the above routine because of the
1269 1270 * TAVOR_SWHDL_TABLE_INIT flag. The table has as many tavor_cqhdl_t
1270 1271 * as the number of CQs.
1271 1272 */
1272 1273 state->ts_cqhdl = hdl_info.swi_table_ptr;
1273 1274 cleanup = TAVOR_RSRC_CLEANUP_LEVEL25;
1274 1275
1275 1276 /*
1276 1277 * Initialize the resource pool for the SRQ handles. Notice
1277 1278 * that the number of SRQHDLs is configurable. The configured value
1278 1279 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1279 1280 * command) or the initialization will fail. Note also that the SRQ
1280 1281 * handle has constructor and destructor methods associated with it.
1281 1282 *
1282 1283 * Note: We only allocate these resources if SRQ is enabled in the
1283 1284 * config profile; see below.
1284 1285 */
1285 1286 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_SRQHDL];
1286 1287 rsrc_pool->rsrc_type = TAVOR_SRQHDL;
1287 1288 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1288 1289 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_srq_s);
1289 1290 rsrc_pool->rsrc_state = state;
1290 1291 TAVOR_RSRC_NAME(rsrc_name, TAVOR_SRQHDL_CACHE);
1291 1292 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_srq);
1292 1293 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_srq);
1293 1294 hdl_info.swi_rsrcpool = rsrc_pool;
1294 1295 hdl_info.swi_constructor = tavor_rsrc_srqhdl_constructor;
1295 1296 hdl_info.swi_destructor = tavor_rsrc_srqhdl_destructor;
1296 1297 hdl_info.swi_rsrcname = rsrc_name;
1297 1298 hdl_info.swi_flags = (TAVOR_SWHDL_KMEMCACHE_INIT |
1298 1299 TAVOR_SWHDL_TABLE_INIT);
1299 1300 hdl_info.swi_prealloc_sz = sizeof (tavor_srqhdl_t);
1300 1301
1301 1302 /*
1302 1303 * SRQ support is configurable. Only if SRQ is enabled (the default)
1303 1304 * do we actually try to configure these resources. Otherwise, we
1304 1305 * simply set the cleanup level and continue on to the next resource
1305 1306 */
1306 1307 if (state->ts_cfg_profile->cp_srq_enable != 0) {
1307 1308 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1308 1309 if (status != DDI_SUCCESS) {
1309 1310 tavor_rsrc_fini(state, cleanup);
1310 1311 /* Set "status" and "errormsg" and goto failure */
1311 1312 TAVOR_TNF_FAIL(DDI_FAILURE, "SRQ handle");
1312 1313 goto rsrcinitp2_fail;
1313 1314 }
1314 1315
1315 1316 /*
1316 1317 * Save away the pointer to the central list of SRQ handle
1317 1318 * pointers This this is used as a mechanism to enable fast
1318 1319 * SRQnumber-to-SRQhandle lookup. The table is a list of
1319 1320 * tavor_srqhdl_t allocated by the above routine because of the
1320 1321 * TAVOR_SWHDL_TABLE_INIT flag. The table has as many
1321 1322 * tavor_srqhdl_t as the number of SRQs.
1322 1323 */
1323 1324 state->ts_srqhdl = hdl_info.swi_table_ptr;
1324 1325 }
1325 1326 cleanup = TAVOR_RSRC_CLEANUP_LEVEL26;
1326 1327
1327 1328 /*
1328 1329 * Initialize the resource pool for the address handles. Notice
1329 1330 * that the number of AHHDLs is configurable. The configured value
1330 1331 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1331 1332 * command) or the initialization will fail.
1332 1333 */
1333 1334 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_AHHDL];
1334 1335 rsrc_pool->rsrc_type = TAVOR_AHHDL;
1335 1336 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1336 1337 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_ah_s);
1337 1338 rsrc_pool->rsrc_state = state;
1338 1339 TAVOR_RSRC_NAME(rsrc_name, TAVOR_AHHDL_CACHE);
1339 1340 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_ah);
1340 1341 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_av);
1341 1342 hdl_info.swi_rsrcpool = rsrc_pool;
1342 1343 hdl_info.swi_constructor = tavor_rsrc_ahhdl_constructor;
1343 1344 hdl_info.swi_destructor = tavor_rsrc_ahhdl_destructor;
1344 1345 hdl_info.swi_rsrcname = rsrc_name;
1345 1346 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT;
1346 1347 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1347 1348 if (status != DDI_SUCCESS) {
1348 1349 tavor_rsrc_fini(state, cleanup);
1349 1350 /* Set "status" and "errormsg" and goto failure */
1350 1351 TAVOR_TNF_FAIL(DDI_FAILURE, "AH handle");
1351 1352 goto rsrcinitp2_fail;
1352 1353 }
1353 1354 cleanup = TAVOR_RSRC_CLEANUP_LEVEL27;
1354 1355
1355 1356 /*
1356 1357 * Initialize the resource pool for the QP handles. Notice
1357 1358 * that the number of QPHDLs is configurable. The configured value
1358 1359 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1359 1360 * command) or the initialization will fail. Note also that the QP
1360 1361 * handle has constructor and destructor methods associated with it.
1361 1362 */
1362 1363 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_QPHDL];
1363 1364 rsrc_pool->rsrc_type = TAVOR_QPHDL;
1364 1365 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1365 1366 rsrc_pool->rsrc_quantum = sizeof (struct tavor_sw_qp_s);
1366 1367 rsrc_pool->rsrc_state = state;
1367 1368 TAVOR_RSRC_NAME(rsrc_name, TAVOR_QPHDL_CACHE);
1368 1369 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_qp);
1369 1370 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_qp);
1370 1371 hdl_info.swi_rsrcpool = rsrc_pool;
1371 1372 hdl_info.swi_constructor = tavor_rsrc_qphdl_constructor;
1372 1373 hdl_info.swi_destructor = tavor_rsrc_qphdl_destructor;
1373 1374 hdl_info.swi_rsrcname = rsrc_name;
1374 1375 hdl_info.swi_flags = (TAVOR_SWHDL_KMEMCACHE_INIT |
1375 1376 TAVOR_SWHDL_TABLE_INIT);
1376 1377 hdl_info.swi_prealloc_sz = sizeof (tavor_qphdl_t);
1377 1378 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1378 1379 if (status != DDI_SUCCESS) {
1379 1380 tavor_rsrc_fini(state, cleanup);
1380 1381 /* Set "status" and "errormsg" and goto failure */
1381 1382 TAVOR_TNF_FAIL(DDI_FAILURE, "QP handle");
1382 1383 goto rsrcinitp2_fail;
1383 1384 }
1384 1385
1385 1386 /*
1386 1387 * Save away the pointer to the central list of QP handle pointers
1387 1388 * This this is used as a mechanism to enable fast QPnumber-to-QPhandle
1388 1389 * lookup during CQ event processing. The table is a list of
1389 1390 * tavor_qphdl_t allocated by the above routine because of the
1390 1391 * TAVOR_SWHDL_TABLE_INIT flag. The table has as many tavor_qphdl_t
1391 1392 * as the number of QPs.
1392 1393 */
1393 1394 state->ts_qphdl = hdl_info.swi_table_ptr;
1394 1395 cleanup = TAVOR_RSRC_CLEANUP_LEVEL28;
1395 1396
1396 1397 /*
1397 1398 * Initialize the resource pool for the reference count handles.
1398 1399 * Notice that the number of REFCNTs is configurable, but it's value
1399 1400 * is set to the number of MPTs. Since REFCNTs are used to support
1400 1401 * shared memory regions, it is possible that we might require as
1401 1402 * one REFCNT for every MPT.
1402 1403 */
1403 1404 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_REFCNT];
1404 1405 rsrc_pool->rsrc_type = TAVOR_REFCNT;
1405 1406 rsrc_pool->rsrc_loc = TAVOR_IN_SYSMEM;
1406 1407 rsrc_pool->rsrc_quantum = sizeof (tavor_sw_refcnt_t);
1407 1408 rsrc_pool->rsrc_state = state;
1408 1409 TAVOR_RSRC_NAME(rsrc_name, TAVOR_REFCNT_CACHE);
1409 1410 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mpt);
1410 1411 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mpt);
1411 1412 hdl_info.swi_rsrcpool = rsrc_pool;
1412 1413 hdl_info.swi_constructor = tavor_rsrc_refcnt_constructor;
1413 1414 hdl_info.swi_destructor = tavor_rsrc_refcnt_destructor;
1414 1415 hdl_info.swi_rsrcname = rsrc_name;
1415 1416 hdl_info.swi_flags = TAVOR_SWHDL_KMEMCACHE_INIT;
1416 1417 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1417 1418 if (status != DDI_SUCCESS) {
1418 1419 tavor_rsrc_fini(state, cleanup);
1419 1420 /* Set "status" and "errormsg" and goto failure */
1420 1421 TAVOR_TNF_FAIL(DDI_FAILURE, "reference count handle");
1421 1422 goto rsrcinitp2_fail;
1422 1423 }
1423 1424 cleanup = TAVOR_RSRC_CLEANUP_LEVEL29;
1424 1425
1425 1426 /*
1426 1427 * Initialize the resource pool for the MCG handles. Notice that for
1427 1428 * these MCG handles, we are allocating a table of structures (used to
1428 1429 * keep track of the MCG entries that are being written to hardware
1429 1430 * and to speed up multicast attach/detach operations).
1430 1431 */
1431 1432 hdl_info.swi_num = ((uint64_t)1 << cfgprof->cp_log_num_mcg);
1432 1433 hdl_info.swi_max = ((uint64_t)1 << state->ts_devlim.log_max_mcg);
1433 1434 hdl_info.swi_flags = TAVOR_SWHDL_TABLE_INIT;
1434 1435 hdl_info.swi_prealloc_sz = sizeof (struct tavor_sw_mcg_list_s);
1435 1436 status = tavor_rsrc_sw_handles_init(state, &hdl_info);
1436 1437 if (status != DDI_SUCCESS) {
1437 1438 tavor_rsrc_fini(state, cleanup);
1438 1439 /* Set "status" and "errormsg" and goto failure */
1439 1440 TAVOR_TNF_FAIL(DDI_FAILURE, "MCG handle");
1440 1441 goto rsrcinitp2_fail;
1441 1442 }
1442 1443 state->ts_mcghdl = hdl_info.swi_table_ptr;
1443 1444 cleanup = TAVOR_RSRC_CLEANUP_LEVEL30;
1444 1445
1445 1446 /*
1446 1447 * Initialize the resource pools for all objects that exist in
1447 1448 * UAR memory. The only objects that are allocated from UAR memory
1448 1449 * are the UAR pages which are used for holding Tavor hardware's
1449 1450 * doorbell registers.
1450 1451 */
1451 1452
1452 1453 /*
1453 1454 * Initialize the resource pool for the UAR pages. Notice
1454 1455 * that the number of UARPGs is configurable. The configured value
1455 1456 * must be less that the maximum value (obtained from the QUERY_DEV_LIM
1456 1457 * command) or the initialization will fail. Note also that by
1457 1458 * specifying the rsrc_start parameter in advance, we direct the
1458 1459 * initialization routine not to attempt to allocated space from the
1459 1460 * Tavor DDR vmem_arena.
1460 1461 */
1461 1462 num = ((uint64_t)1 << cfgprof->cp_log_num_uar);
1462 1463 max = ((uint64_t)1 << (state->ts_devlim.log_max_uar_sz + 20 -
1463 1464 PAGESHIFT));
1464 1465 num_prealloc = 0;
1465 1466 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UARPG];
1466 1467 rsrc_pool->rsrc_type = TAVOR_UARPG;
1467 1468 rsrc_pool->rsrc_loc = TAVOR_IN_UAR;
1468 1469 rsrc_pool->rsrc_pool_size = (num << PAGESHIFT);
1469 1470 rsrc_pool->rsrc_shift = PAGESHIFT;
1470 1471 rsrc_pool->rsrc_quantum = PAGESIZE;
1471 1472 rsrc_pool->rsrc_align = PAGESIZE;
1472 1473 rsrc_pool->rsrc_state = state;
1473 1474 rsrc_pool->rsrc_start = (void *)state->ts_reg_uar_baseaddr;
1474 1475 TAVOR_RSRC_NAME(rsrc_name, TAVOR_UAR_VMEM);
1475 1476 entry_info.hwi_num = num;
1476 1477 entry_info.hwi_max = max;
1477 1478 entry_info.hwi_prealloc = num_prealloc;
1478 1479 entry_info.hwi_rsrcpool = rsrc_pool;
1479 1480 entry_info.hwi_rsrcname = rsrc_name;
1480 1481 status = tavor_rsrc_hw_entries_init(state, &entry_info);
1481 1482 if (status != DDI_SUCCESS) {
1482 1483 tavor_rsrc_fini(state, cleanup);
1483 1484 /* Set "status" and "errormsg" and goto failure */
1484 1485 TAVOR_TNF_FAIL(DDI_FAILURE, "UAR page table");
1485 1486 goto rsrcinitp2_fail;
1486 1487 }
1487 1488 cleanup = TAVOR_RSRC_CLEANUP_ALL;
1488 1489
1489 1490 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN);
1490 1491 TAVOR_TNF_EXIT(tavor_rsrc_init_phase2);
1491 1492 return (DDI_SUCCESS);
1492 1493
1493 1494 rsrcinitp2_fail:
1494 1495 kmem_free(rsrc_name, TAVOR_RSRC_NAME_MAXLEN);
1495 1496 TNF_PROBE_1(tavor_rsrc_init_phase2_fail, TAVOR_TNF_ERROR, "",
1496 1497 tnf_string, msg, errormsg);
1497 1498 TAVOR_TNF_EXIT(tavor_rsrc_init_phase2);
1498 1499 return (status);
1499 1500 }
1500 1501
1501 1502
1502 1503 /*
1503 1504 * tavor_rsrc_fini()
1504 1505 * Context: Only called from attach() and/or detach() path contexts
1505 1506 */
1506 1507 void
1507 1508 tavor_rsrc_fini(tavor_state_t *state, tavor_rsrc_cleanup_level_t clean)
1508 1509 {
1509 1510 tavor_rsrc_sw_hdl_info_t hdl_info;
1510 1511 tavor_rsrc_hw_entry_info_t entry_info;
1511 1512 tavor_rsrc_mbox_info_t mbox_info;
1512 1513 tavor_cfg_profile_t *cfgprof;
1513 1514
1514 1515 TAVOR_TNF_ENTER(tavor_rsrc_fini);
1515 1516
1516 1517 ASSERT(state != NULL);
1517 1518
1518 1519 cfgprof = state->ts_cfg_profile;
1519 1520
1520 1521 switch (clean) {
1521 1522 /*
1522 1523 * If we add more resources that need to be cleaned up here, we should
1523 1524 * ensure that TAVOR_RSRC_CLEANUP_ALL is still the first entry (i.e.
1524 1525 * corresponds to the last resource allocated).
1525 1526 */
1526 1527 case TAVOR_RSRC_CLEANUP_ALL:
1527 1528 /* Cleanup the UAR page resource pool */
1528 1529 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UARPG];
1529 1530 tavor_rsrc_hw_entries_fini(state, &entry_info);
1530 1531 /* FALLTHROUGH */
1531 1532
1532 1533 case TAVOR_RSRC_CLEANUP_LEVEL30:
1533 1534 /* Cleanup the central MCG handle pointers list */
1534 1535 hdl_info.swi_rsrcpool = NULL;
1535 1536 hdl_info.swi_table_ptr = state->ts_mcghdl;
1536 1537 hdl_info.swi_num =
1537 1538 ((uint64_t)1 << cfgprof->cp_log_num_mcg);
1538 1539 hdl_info.swi_prealloc_sz = sizeof (struct tavor_sw_mcg_list_s);
1539 1540 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1540 1541 /* FALLTHROUGH */
1541 1542
1542 1543 case TAVOR_RSRC_CLEANUP_LEVEL29:
1543 1544 /* Cleanup the reference count resource pool */
1544 1545 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_REFCNT];
1545 1546 hdl_info.swi_table_ptr = NULL;
1546 1547 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1547 1548 /* FALLTHROUGH */
1548 1549
1549 1550 case TAVOR_RSRC_CLEANUP_LEVEL28:
1550 1551 /* Cleanup the QP handle resource pool */
1551 1552 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_QPHDL];
1552 1553 hdl_info.swi_table_ptr = state->ts_qphdl;
1553 1554 hdl_info.swi_num =
1554 1555 ((uint64_t)1 << cfgprof->cp_log_num_qp);
1555 1556 hdl_info.swi_prealloc_sz = sizeof (tavor_qphdl_t);
1556 1557 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1557 1558 /* FALLTHROUGH */
1558 1559
1559 1560 case TAVOR_RSRC_CLEANUP_LEVEL27:
1560 1561 /* Cleanup the address handle resource pool */
1561 1562 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_AHHDL];
1562 1563 hdl_info.swi_table_ptr = NULL;
1563 1564 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1564 1565 /* FALLTHROUGH */
1565 1566
1566 1567 case TAVOR_RSRC_CLEANUP_LEVEL26:
1567 1568 /*
1568 1569 * Cleanup the SRQ handle resource pool.
1569 1570 *
1570 1571 * Note: We only clean up if SRQ is enabled. Otherwise we
1571 1572 * simply fallthrough to the next resource cleanup.
1572 1573 */
1573 1574 if (state->ts_cfg_profile->cp_srq_enable != 0) {
1574 1575 hdl_info.swi_rsrcpool =
1575 1576 &state->ts_rsrc_hdl[TAVOR_SRQHDL];
1576 1577 hdl_info.swi_table_ptr = state->ts_srqhdl;
1577 1578 hdl_info.swi_num =
1578 1579 ((uint64_t)1 << cfgprof->cp_log_num_srq);
1579 1580 hdl_info.swi_prealloc_sz = sizeof (tavor_srqhdl_t);
1580 1581 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1581 1582 }
1582 1583 /* FALLTHROUGH */
1583 1584
1584 1585 case TAVOR_RSRC_CLEANUP_LEVEL25:
1585 1586 /* Cleanup the CQ handle resource pool */
1586 1587 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_CQHDL];
1587 1588 hdl_info.swi_table_ptr = state->ts_cqhdl;
1588 1589 hdl_info.swi_num =
1589 1590 ((uint64_t)1 << cfgprof->cp_log_num_cq);
1590 1591 hdl_info.swi_prealloc_sz = sizeof (tavor_cqhdl_t);
1591 1592 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1592 1593 /* FALLTHROUGH */
1593 1594
1594 1595 case TAVOR_RSRC_CLEANUP_LEVEL24:
1595 1596 /* Cleanup the EQ handle resource pool */
1596 1597 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_EQHDL];
1597 1598 hdl_info.swi_table_ptr = NULL;
1598 1599 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1599 1600 /* FALLTHROUGH */
1600 1601
1601 1602 case TAVOR_RSRC_CLEANUP_LEVEL23:
1602 1603 /* Cleanup the MR handle resource pool */
1603 1604 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MRHDL];
1604 1605 hdl_info.swi_table_ptr = NULL;
1605 1606 tavor_rsrc_sw_handles_fini(state, &hdl_info);
1606 1607 /* FALLTHROUGH */
1607 1608
1608 1609 case TAVOR_RSRC_CLEANUP_LEVEL22:
1609 1610 /* Cleanup the PD handle resource pool */
1610 1611 hdl_info.swi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_PDHDL];
1611 1612 hdl_info.swi_table_ptr = NULL;
1612 1613 tavor_rsrc_pd_handles_fini(state, &hdl_info);
1613 1614 /* FALLTHROUGH */
1614 1615
1615 1616 case TAVOR_RSRC_CLEANUP_LEVEL21:
1616 1617 /* Cleanup the EQC table resource pool */
1617 1618 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_EQC];
1618 1619 tavor_rsrc_hw_entries_fini(state, &entry_info);
1619 1620 /* FALLTHROUGH */
1620 1621
1621 1622 case TAVOR_RSRC_CLEANUP_LEVEL20:
1622 1623 /* Cleanup the MCG table resource pool */
1623 1624 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MCG];
1624 1625 tavor_rsrc_hw_entries_fini(state, &entry_info);
1625 1626 /* FALLTHROUGH */
1626 1627
1627 1628 case TAVOR_RSRC_CLEANUP_LEVEL19:
1628 1629 /* Cleanup the outstanding command list */
1629 1630 tavor_outstanding_cmdlist_fini(state);
1630 1631 /* FALLTHROUGH */
1631 1632
1632 1633 case TAVOR_RSRC_CLEANUP_LEVEL18:
1633 1634 /* Cleanup the "In" mailbox list */
1634 1635 tavor_intr_inmbox_list_fini(state);
1635 1636 /* FALLTHROUGH */
1636 1637
1637 1638 case TAVOR_RSRC_CLEANUP_LEVEL17:
1638 1639 /* Cleanup the interrupt "In" mailbox resource pool */
1639 1640 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[
1640 1641 TAVOR_INTR_IN_MBOX];
1641 1642 tavor_rsrc_mbox_fini(state, &mbox_info);
1642 1643 /* FALLTHROUGH */
1643 1644
1644 1645 case TAVOR_RSRC_CLEANUP_LEVEL16:
1645 1646 /* Cleanup the "In" mailbox list */
1646 1647 tavor_inmbox_list_fini(state);
1647 1648 /* FALLTHROUGH */
1648 1649
1649 1650 case TAVOR_RSRC_CLEANUP_LEVEL15:
1650 1651 /* Cleanup the "In" mailbox resource pool */
1651 1652 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_IN_MBOX];
1652 1653 tavor_rsrc_mbox_fini(state, &mbox_info);
1653 1654 /* FALLTHROUGH */
1654 1655
1655 1656 case TAVOR_RSRC_CLEANUP_LEVEL14:
1656 1657 /*
1657 1658 * Cleanup the SRQC table resource pool.
1658 1659 *
1659 1660 * Note: We only clean up if SRQ is enabled. Otherwise we
1660 1661 * simply fallthrough to the next resource cleanup.
1661 1662 */
1662 1663 if (state->ts_cfg_profile->cp_srq_enable != 0) {
1663 1664 entry_info.hwi_rsrcpool =
1664 1665 &state->ts_rsrc_hdl[TAVOR_SRQC];
1665 1666 tavor_rsrc_hw_entries_fini(state, &entry_info);
1666 1667 }
1667 1668 /* FALLTHROUGH */
1668 1669
1669 1670 case TAVOR_RSRC_CLEANUP_LEVEL13:
1670 1671 /* Cleanup the UAR scratch table resource pool */
1671 1672 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UAR_SCR];
1672 1673 tavor_rsrc_hw_entries_fini(state, &entry_info);
1673 1674 /* FALLTHROUGH */
1674 1675
1675 1676 case TAVOR_RSRC_CLEANUP_LEVEL12:
1676 1677 /* Cleanup the UDAV table resource pool */
1677 1678 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_UDAV];
1678 1679 tavor_rsrc_hw_entries_fini(state, &entry_info);
1679 1680 /* FALLTHROUGH */
1680 1681
1681 1682 case TAVOR_RSRC_CLEANUP_LEVEL11:
1682 1683 /* Cleanup the EQPC table resource pool */
1683 1684 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_EQPC];
1684 1685 tavor_rsrc_hw_entries_fini(state, &entry_info);
1685 1686 /* FALLTHROUGH */
1686 1687
1687 1688 case TAVOR_RSRC_CLEANUP_LEVEL10:
1688 1689 /* Cleanup the CQC table resource pool */
1689 1690 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_CQC];
1690 1691 tavor_rsrc_hw_entries_fini(state, &entry_info);
1691 1692 /* FALLTHROUGH */
1692 1693
1693 1694 case TAVOR_RSRC_CLEANUP_LEVEL9:
1694 1695 /* Cleanup the RDB table resource pool */
1695 1696 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_RDB];
1696 1697 tavor_rsrc_hw_entries_fini(state, &entry_info);
1697 1698 /* FALLTHROUGH */
1698 1699
1699 1700 case TAVOR_RSRC_CLEANUP_LEVEL8:
1700 1701 /* Cleanup the QPC table resource pool */
1701 1702 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_QPC];
1702 1703 tavor_rsrc_hw_entries_fini(state, &entry_info);
1703 1704 /* FALLTHROUGH */
1704 1705
1705 1706 case TAVOR_RSRC_CLEANUP_LEVEL7:
1706 1707 /* Cleanup the MTT table resource pool */
1707 1708 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MTT];
1708 1709 tavor_rsrc_hw_entries_fini(state, &entry_info);
1709 1710 /* FALLTHROUGH */
1710 1711
1711 1712 case TAVOR_RSRC_CLEANUP_LEVEL6:
1712 1713 /* Cleanup the MPT table resource pool */
1713 1714 entry_info.hwi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_MPT];
1714 1715 tavor_rsrc_hw_entries_fini(state, &entry_info);
1715 1716 /* FALLTHROUGH */
1716 1717
1717 1718 case TAVOR_RSRC_CLEANUP_LEVEL5:
1718 1719 /* Destroy the vmem arena for DDR memory */
1719 1720 vmem_destroy(state->ts_ddrvmem);
1720 1721 break;
1721 1722
1722 1723 /*
1723 1724 * The cleanup below comes from the "Phase 1" initialization step.
1724 1725 * (see tavor_rsrc_init_phase1() above)
1725 1726 */
1726 1727 case TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE:
1727 1728 /* Cleanup the interrupt "Out" mailbox list */
1728 1729 tavor_intr_outmbox_list_fini(state);
1729 1730 /* FALLTHROUGH */
1730 1731
1731 1732 case TAVOR_RSRC_CLEANUP_LEVEL3:
1732 1733 /* Cleanup the "Out" mailbox resource pool */
1733 1734 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[
1734 1735 TAVOR_INTR_OUT_MBOX];
1735 1736 tavor_rsrc_mbox_fini(state, &mbox_info);
1736 1737 /* FALLTHROUGH */
1737 1738
1738 1739 case TAVOR_RSRC_CLEANUP_LEVEL2:
1739 1740 /* Cleanup the "Out" mailbox list */
1740 1741 tavor_outmbox_list_fini(state);
1741 1742 /* FALLTHROUGH */
1742 1743
1743 1744 case TAVOR_RSRC_CLEANUP_LEVEL1:
1744 1745 /* Cleanup the "Out" mailbox resource pool */
1745 1746 mbox_info.mbi_rsrcpool = &state->ts_rsrc_hdl[TAVOR_OUT_MBOX];
1746 1747 tavor_rsrc_mbox_fini(state, &mbox_info);
1747 1748 /* FALLTHROUGH */
1748 1749
1749 1750 case TAVOR_RSRC_CLEANUP_LEVEL0:
1750 1751 /* Free the array of tavor_rsrc_pool_info_t's */
1751 1752 kmem_free(state->ts_rsrc_hdl, TAVOR_NUM_RESOURCES *
1752 1753 sizeof (tavor_rsrc_pool_info_t));
1753 1754 kmem_cache_destroy(state->ts_rsrc_cache);
1754 1755 break;
1755 1756
1756 1757 default:
1757 1758 TAVOR_WARNING(state, "unexpected resource cleanup level");
1758 1759 TNF_PROBE_0(tavor_rsrc_fini_default_fail, TAVOR_TNF_ERROR, "");
1759 1760 TAVOR_TNF_EXIT(tavor_rsrc_fini);
1760 1761 return;
1761 1762 }
1762 1763
1763 1764 TAVOR_TNF_EXIT(tavor_rsrc_fini);
1764 1765 }
1765 1766
1766 1767
1767 1768 /*
1768 1769 * tavor_rsrc_mbox_init()
1769 1770 * Context: Only called from attach() path context
1770 1771 */
1771 1772 static int
1772 1773 tavor_rsrc_mbox_init(tavor_state_t *state, tavor_rsrc_mbox_info_t *info)
1773 1774 {
1774 1775 tavor_rsrc_pool_info_t *rsrc_pool;
1775 1776 tavor_rsrc_priv_mbox_t *priv;
1776 1777 vmem_t *vmp;
1777 1778 uint64_t offset;
1778 1779 uint_t dma_xfer_mode;
1779 1780
1780 1781 TAVOR_TNF_ENTER(tavor_rsrc_mbox_init);
1781 1782
1782 1783 ASSERT(state != NULL);
1783 1784 ASSERT(info != NULL);
1784 1785
1785 1786 rsrc_pool = info->mbi_rsrcpool;
1786 1787 ASSERT(rsrc_pool != NULL);
1787 1788
1788 1789 dma_xfer_mode = state->ts_cfg_profile->cp_streaming_consistent;
1789 1790
1790 1791 /* Allocate and initialize mailbox private structure */
1791 1792 priv = kmem_zalloc(sizeof (tavor_rsrc_priv_mbox_t), KM_SLEEP);
1792 1793 priv->pmb_dip = state->ts_dip;
1793 1794 priv->pmb_acchdl = state->ts_reg_ddrhdl;
1794 1795 priv->pmb_devaccattr = state->ts_reg_accattr;
1795 1796 priv->pmb_xfer_mode = dma_xfer_mode;
1796 1797
1797 1798 /*
1798 1799 * Initialize many of the default DMA attributes. Then set alignment
1799 1800 * and scatter-gather restrictions specific for mailbox memory.
1800 1801 */
1801 1802 tavor_dma_attr_init(&priv->pmb_dmaattr);
1802 1803 priv->pmb_dmaattr.dma_attr_align = TAVOR_MBOX_ALIGN;
1803 1804 priv->pmb_dmaattr.dma_attr_sgllen = 1;
1804 1805
1805 1806 rsrc_pool->rsrc_private = priv;
1806 1807
1807 1808 /* Is object in DDR memory or system memory? */
1808 1809 if (rsrc_pool->rsrc_loc == TAVOR_IN_DDR) {
1809 1810 rsrc_pool->rsrc_ddr_offset = vmem_xalloc(state->ts_ddrvmem,
1810 1811 rsrc_pool->rsrc_pool_size, rsrc_pool->rsrc_align,
1811 1812 0, 0, NULL, NULL, VM_SLEEP);
1812 1813 if (rsrc_pool->rsrc_ddr_offset == NULL) {
1813 1814 /* Unable to alloc space for mailboxes */
1814 1815 kmem_free(priv, sizeof (tavor_rsrc_priv_mbox_t));
1815 1816 TNF_PROBE_0(tavor_rsrc_mbox_init_vma_fail,
1816 1817 TAVOR_TNF_ERROR, "");
1817 1818 TAVOR_TNF_EXIT(tavor_rsrc_mbox_init);
1818 1819 return (DDI_FAILURE);
1819 1820 }
1820 1821
1821 1822 /* Calculate offset and starting point (in DDR) */
1822 1823 offset = ((uintptr_t)rsrc_pool->rsrc_ddr_offset -
1823 1824 state->ts_ddr.ddr_baseaddr);
1824 1825 rsrc_pool->rsrc_start =
1825 1826 (void *)(uintptr_t)((uintptr_t)state->ts_reg_ddr_baseaddr +
1826 1827 offset);
1827 1828
1828 1829 /* Create new vmem arena for the mailboxes */
1829 1830 vmp = vmem_create(info->mbi_rsrcname,
1830 1831 rsrc_pool->rsrc_ddr_offset, rsrc_pool->rsrc_pool_size,
1831 1832 rsrc_pool->rsrc_quantum, NULL, NULL, NULL, 0, VM_SLEEP);
1832 1833 if (vmp == NULL) {
1833 1834 /* Unable to create vmem arena */
1834 1835 vmem_xfree(state->ts_ddrvmem,
1835 1836 rsrc_pool->rsrc_ddr_offset,
1836 1837 rsrc_pool->rsrc_pool_size);
1837 1838 kmem_free(priv, sizeof (tavor_rsrc_priv_mbox_t));
1838 1839 TNF_PROBE_0(tavor_rsrc_mbox_init_vmem_create_fail,
1839 1840 TAVOR_TNF_ERROR, "");
1840 1841 TAVOR_TNF_EXIT(tavor_rsrc_mbox_init);
1841 1842 return (DDI_FAILURE);
1842 1843 }
1843 1844 rsrc_pool->rsrc_vmp = vmp;
1844 1845 } else {
1845 1846 rsrc_pool->rsrc_ddr_offset = NULL;
1846 1847 rsrc_pool->rsrc_start = NULL;
1847 1848 rsrc_pool->rsrc_vmp = NULL;
1848 1849 }
1849 1850
1850 1851 TAVOR_TNF_EXIT(tavor_rsrc_mbox_init);
1851 1852 return (DDI_SUCCESS);
1852 1853 }
1853 1854
1854 1855
1855 1856 /*
1856 1857 * tavor_rsrc_mbox_fini()
1857 1858 * Context: Only called from attach() and/or detach() path contexts
1858 1859 */
1859 1860 static void
1860 1861 tavor_rsrc_mbox_fini(tavor_state_t *state, tavor_rsrc_mbox_info_t *info)
1861 1862 {
1862 1863 tavor_rsrc_pool_info_t *rsrc_pool;
1863 1864
1864 1865 TAVOR_TNF_ENTER(tavor_rsrc_mbox_fini);
1865 1866
1866 1867 ASSERT(state != NULL);
1867 1868 ASSERT(info != NULL);
1868 1869
1869 1870 rsrc_pool = info->mbi_rsrcpool;
1870 1871 ASSERT(rsrc_pool != NULL);
1871 1872
1872 1873 /* If mailboxes are DDR memory, then destroy and free up vmem */
1873 1874 if (rsrc_pool->rsrc_loc == TAVOR_IN_DDR) {
1874 1875
1875 1876 /* Destroy the specially created mbox vmem arena */
1876 1877 vmem_destroy(rsrc_pool->rsrc_vmp);
1877 1878
1878 1879 /* Free up the region from the ddr_vmem arena */
1879 1880 vmem_xfree(state->ts_ddrvmem, rsrc_pool->rsrc_ddr_offset,
1880 1881 rsrc_pool->rsrc_pool_size);
1881 1882 }
1882 1883
1883 1884 /* Free up the private struct */
1884 1885 kmem_free(rsrc_pool->rsrc_private, sizeof (tavor_rsrc_priv_mbox_t));
1885 1886
1886 1887 TAVOR_TNF_EXIT(tavor_rsrc_mbox_fini);
1887 1888 }
1888 1889
1889 1890
1890 1891 /*
1891 1892 * tavor_rsrc_hw_entries_init()
1892 1893 * Context: Only called from attach() path context
1893 1894 */
1894 1895 static int
1895 1896 tavor_rsrc_hw_entries_init(tavor_state_t *state,
1896 1897 tavor_rsrc_hw_entry_info_t *info)
1897 1898 {
1898 1899 tavor_rsrc_pool_info_t *rsrc_pool;
1899 1900 tavor_rsrc_t *rsvd_rsrc = NULL;
1900 1901 vmem_t *vmp;
1901 1902 uint64_t num_hwentry, max_hwentry, num_prealloc;
1902 1903 uint64_t offset;
1903 1904 int status;
1904 1905
1905 1906 TAVOR_TNF_ENTER(tavor_rsrc_hw_entries_init);
1906 1907
1907 1908 ASSERT(state != NULL);
1908 1909 ASSERT(info != NULL);
1909 1910
1910 1911 rsrc_pool = info->hwi_rsrcpool;
1911 1912 ASSERT(rsrc_pool != NULL);
1912 1913 num_hwentry = info->hwi_num;
1913 1914 max_hwentry = info->hwi_max;
1914 1915 num_prealloc = info->hwi_prealloc;
1915 1916
1916 1917 /* Make sure number of HW entries makes sense */
1917 1918 if (num_hwentry > max_hwentry) {
1918 1919 TNF_PROBE_2(tavor_rsrc_hw_entries_init_toomany_fail,
1919 1920 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of HW "
1920 1921 "entries exceeds device maximum", tnf_uint, maxhw,
1921 1922 max_hwentry);
1922 1923 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init);
1923 1924 return (DDI_FAILURE);
1924 1925 }
1925 1926
1926 1927 /*
1927 1928 * Determine if we need to allocate DDR space to set up the
1928 1929 * "rsrc_start" pointer. Not necessary if "rsrc_start" has already
1929 1930 * been initialized (as is the case for the UAR page init).
1930 1931 */
1931 1932 if (rsrc_pool->rsrc_start == NULL) {
1932 1933 /* Make sure HW entries table is aligned as specified */
1933 1934 rsrc_pool->rsrc_ddr_offset = vmem_xalloc(state->ts_ddrvmem,
1934 1935 rsrc_pool->rsrc_pool_size, rsrc_pool->rsrc_align,
1935 1936 0, 0, NULL, NULL, VM_NOSLEEP | VM_FIRSTFIT);
1936 1937 if (rsrc_pool->rsrc_ddr_offset == NULL) {
1937 1938 /* Unable to alloc space for aligned HW table */
1938 1939 TNF_PROBE_0(tavor_rsrc_hw_entry_table_vmxalloc_fail,
1939 1940 TAVOR_TNF_ERROR, "");
1940 1941 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init);
1941 1942 return (DDI_FAILURE);
1942 1943 }
1943 1944
1944 1945 /* Calculate offset and starting point (in DDR) */
1945 1946 offset = ((uintptr_t)rsrc_pool->rsrc_ddr_offset -
1946 1947 state->ts_ddr.ddr_baseaddr);
1947 1948 rsrc_pool->rsrc_start =
1948 1949 (void *)(uintptr_t)((uintptr_t)state->ts_reg_ddr_baseaddr +
1949 1950 offset);
1950 1951 } else {
1951 1952 rsrc_pool->rsrc_ddr_offset = rsrc_pool->rsrc_start;
1952 1953 }
1953 1954
1954 1955 /*
1955 1956 * Create new vmem arena for the HW entries table (if rsrc_quantum
1956 1957 * is non-zero). Otherwise if rsrc_quantum is zero, then these HW
1957 1958 * entries are not going to be dynamically allocatable (i.e. they
1958 1959 * won't be allocated/freed through tavor_rsrc_alloc/free). This
1959 1960 * latter option is used for EQPC and UARSCR resource which are, in
1960 1961 * fact, managed by the Tavor hardware.
1961 1962 */
1962 1963 if (rsrc_pool->rsrc_quantum != 0) {
1963 1964 vmp = vmem_create(info->hwi_rsrcname,
1964 1965 rsrc_pool->rsrc_ddr_offset, rsrc_pool->rsrc_pool_size,
1965 1966 rsrc_pool->rsrc_quantum, NULL, NULL, NULL, 0, VM_SLEEP);
1966 1967 if (vmp == NULL) {
1967 1968 /* Unable to create vmem arena */
1968 1969 if (rsrc_pool->rsrc_ddr_offset !=
1969 1970 rsrc_pool->rsrc_start) {
1970 1971 vmem_xfree(state->ts_ddrvmem,
1971 1972 rsrc_pool->rsrc_ddr_offset,
1972 1973 rsrc_pool->rsrc_pool_size);
1973 1974 }
1974 1975 TNF_PROBE_0(tavor_rsrc_hw_entries_init_vmemcreate_fail,
1975 1976 TAVOR_TNF_ERROR, "");
1976 1977 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init);
1977 1978 return (DDI_FAILURE);
1978 1979 }
1979 1980 rsrc_pool->rsrc_vmp = vmp;
1980 1981 } else {
1981 1982 rsrc_pool->rsrc_vmp = NULL;
1982 1983 }
1983 1984
1984 1985 /* The first HW entries may be reserved by Tavor firmware */
1985 1986 if (num_prealloc != 0) {
1986 1987 status = tavor_rsrc_alloc(state, rsrc_pool->rsrc_type,
1987 1988 num_prealloc, TAVOR_SLEEP, &rsvd_rsrc);
1988 1989 if (status != DDI_SUCCESS) {
1989 1990 /* Unable to preallocate the reserved HW entries */
1990 1991 if (rsrc_pool->rsrc_vmp != NULL) {
1991 1992 vmem_destroy(rsrc_pool->rsrc_vmp);
1992 1993 }
1993 1994 if (rsrc_pool->rsrc_ddr_offset !=
1994 1995 rsrc_pool->rsrc_start) {
1995 1996 vmem_xfree(state->ts_ddrvmem,
1996 1997 rsrc_pool->rsrc_ddr_offset,
1997 1998 rsrc_pool->rsrc_pool_size);
1998 1999 }
1999 2000 TNF_PROBE_0(tavor_rsrc_hw_entries_init_pre_fail,
2000 2001 TAVOR_TNF_ERROR, "");
2001 2002 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init);
2002 2003 return (DDI_FAILURE);
2003 2004 }
2004 2005 }
2005 2006 rsrc_pool->rsrc_private = rsvd_rsrc;
2006 2007
2007 2008 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_init);
2008 2009 return (DDI_SUCCESS);
2009 2010 }
2010 2011
2011 2012
2012 2013 /*
2013 2014 * tavor_rsrc_hw_entries_fini()
2014 2015 * Context: Only called from attach() and/or detach() path contexts
2015 2016 */
2016 2017 static void
2017 2018 tavor_rsrc_hw_entries_fini(tavor_state_t *state,
2018 2019 tavor_rsrc_hw_entry_info_t *info)
2019 2020 {
2020 2021 tavor_rsrc_pool_info_t *rsrc_pool;
2021 2022 tavor_rsrc_t *rsvd_rsrc;
2022 2023
2023 2024 TAVOR_TNF_ENTER(tavor_rsrc_hw_entries_fini);
2024 2025
2025 2026 ASSERT(state != NULL);
2026 2027 ASSERT(info != NULL);
2027 2028
2028 2029 rsrc_pool = info->hwi_rsrcpool;
2029 2030 ASSERT(rsrc_pool != NULL);
2030 2031
2031 2032 /* Free up any "reserved" (i.e. preallocated) HW entries */
2032 2033 rsvd_rsrc = (tavor_rsrc_t *)rsrc_pool->rsrc_private;
2033 2034 if (rsvd_rsrc != NULL) {
2034 2035 tavor_rsrc_free(state, &rsvd_rsrc);
2035 2036 }
2036 2037
2037 2038 /*
2038 2039 * If we've actually setup a vmem arena for the HW entries, then
2039 2040 * destroy it now
2040 2041 */
2041 2042 if (rsrc_pool->rsrc_vmp != NULL) {
2042 2043 vmem_destroy(rsrc_pool->rsrc_vmp);
2043 2044 }
2044 2045
2045 2046 /*
2046 2047 * Determine if a region was allocated from the tavor_ddr_vmem
2047 2048 * arena (and free it up if necessary)
2048 2049 */
2049 2050 if (rsrc_pool->rsrc_ddr_offset != rsrc_pool->rsrc_start) {
2050 2051 vmem_xfree(state->ts_ddrvmem, rsrc_pool->rsrc_ddr_offset,
2051 2052 rsrc_pool->rsrc_pool_size);
2052 2053 }
2053 2054
2054 2055 TAVOR_TNF_EXIT(tavor_rsrc_hw_entries_fini);
2055 2056 }
2056 2057
2057 2058
2058 2059 /*
2059 2060 * tavor_rsrc_sw_handles_init()
2060 2061 * Context: Only called from attach() path context
2061 2062 */
2062 2063 /* ARGSUSED */
2063 2064 static int
2064 2065 tavor_rsrc_sw_handles_init(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info)
2065 2066 {
2066 2067 tavor_rsrc_pool_info_t *rsrc_pool;
2067 2068 uint64_t num_swhdl, max_swhdl, prealloc_sz;
2068 2069
2069 2070 TAVOR_TNF_ENTER(tavor_rsrc_sw_handles_init);
2070 2071
2071 2072 ASSERT(state != NULL);
2072 2073 ASSERT(info != NULL);
2073 2074
2074 2075 rsrc_pool = info->swi_rsrcpool;
2075 2076 ASSERT(rsrc_pool != NULL);
2076 2077 num_swhdl = info->swi_num;
2077 2078 max_swhdl = info->swi_max;
2078 2079 prealloc_sz = info->swi_prealloc_sz;
2079 2080
2080 2081 /* Make sure number of SW handles makes sense */
2081 2082 if (num_swhdl > max_swhdl) {
2082 2083 TNF_PROBE_2(tavor_rsrc_sw_handles_init_toomany_fail,
2083 2084 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of SW "
2084 2085 "handles exceeds maximum", tnf_uint, maxsw, max_swhdl);
2085 2086 TAVOR_TNF_EXIT(tavor_rsrc_sw_handles_init);
2086 2087 return (DDI_FAILURE);
2087 2088 }
2088 2089
2089 2090 /*
2090 2091 * Depending on the flags parameter, create a kmem_cache for some
2091 2092 * number of software handle structures. Note: kmem_cache_create()
2092 2093 * will SLEEP until successful.
2093 2094 */
2094 2095 if (info->swi_flags & TAVOR_SWHDL_KMEMCACHE_INIT) {
2095 2096 rsrc_pool->rsrc_private = kmem_cache_create(
2096 2097 info->swi_rsrcname, rsrc_pool->rsrc_quantum, 0,
2097 2098 info->swi_constructor, info->swi_destructor, NULL,
2098 2099 rsrc_pool->rsrc_state, NULL, 0);
2099 2100 }
2100 2101
2101 2102 /* Allocate the central list of SW handle pointers */
2102 2103 if (info->swi_flags & TAVOR_SWHDL_TABLE_INIT) {
2103 2104 info->swi_table_ptr = kmem_zalloc(num_swhdl * prealloc_sz,
2104 2105 KM_SLEEP);
2105 2106 }
2106 2107
2107 2108 TAVOR_TNF_EXIT(tavor_rsrc_sw_handles_init);
2108 2109 return (DDI_SUCCESS);
2109 2110 }
2110 2111
2111 2112
2112 2113 /*
2113 2114 * tavor_rsrc_sw_handles_fini()
2114 2115 * Context: Only called from attach() and/or detach() path contexts
2115 2116 */
2116 2117 /* ARGSUSED */
2117 2118 static void
2118 2119 tavor_rsrc_sw_handles_fini(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info)
2119 2120 {
2120 2121 tavor_rsrc_pool_info_t *rsrc_pool;
2121 2122 uint64_t num_swhdl, prealloc_sz;
2122 2123
2123 2124 TAVOR_TNF_ENTER(tavor_rsrc_sw_handles_fini);
2124 2125
2125 2126 ASSERT(state != NULL);
2126 2127 ASSERT(info != NULL);
2127 2128
2128 2129 rsrc_pool = info->swi_rsrcpool;
2129 2130 num_swhdl = info->swi_num;
2130 2131 prealloc_sz = info->swi_prealloc_sz;
2131 2132
2132 2133 /*
2133 2134 * If a "software handle" kmem_cache exists for this resource, then
2134 2135 * destroy it now
2135 2136 */
2136 2137 if (rsrc_pool != NULL) {
2137 2138 kmem_cache_destroy(rsrc_pool->rsrc_private);
2138 2139 }
2139 2140
2140 2141 /* Free up this central list of SW handle pointers */
2141 2142 if (info->swi_table_ptr != NULL) {
2142 2143 kmem_free(info->swi_table_ptr, num_swhdl * prealloc_sz);
2143 2144 }
2144 2145
2145 2146 TAVOR_TNF_EXIT(tavor_rsrc_sw_handles_fini);
2146 2147 }
2147 2148
2148 2149
2149 2150 /*
2150 2151 * tavor_rsrc_pd_handles_init()
2151 2152 * Context: Only called from attach() path context
2152 2153 */
2153 2154 static int
2154 2155 tavor_rsrc_pd_handles_init(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info)
2155 2156 {
2156 2157 tavor_rsrc_pool_info_t *rsrc_pool;
2157 2158 vmem_t *vmp;
2158 2159 char vmem_name[TAVOR_RSRC_NAME_MAXLEN];
2159 2160 int status;
2160 2161
2161 2162 TAVOR_TNF_ENTER(tavor_rsrc_pd_handles_init);
2162 2163
2163 2164 ASSERT(state != NULL);
2164 2165 ASSERT(info != NULL);
2165 2166
2166 2167 rsrc_pool = info->swi_rsrcpool;
2167 2168 ASSERT(rsrc_pool != NULL);
2168 2169
2169 2170 /* Initialize the resource pool for software handle table */
2170 2171 status = tavor_rsrc_sw_handles_init(state, info);
2171 2172 if (status != DDI_SUCCESS) {
2172 2173 TNF_PROBE_0(tavor_rsrc_pdhdl_alloc_fail, TAVOR_TNF_ERROR, "");
2173 2174 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc);
2174 2175 return (DDI_FAILURE);
2175 2176 }
2176 2177
2177 2178 /* Build vmem arena name from Tavor instance */
2178 2179 TAVOR_RSRC_NAME(vmem_name, TAVOR_PDHDL_VMEM);
2179 2180
2180 2181 /* Create new vmem arena for PD numbers */
2181 2182 vmp = vmem_create(vmem_name, (caddr_t)1, info->swi_num, 1, NULL,
2182 2183 NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER);
2183 2184 if (vmp == NULL) {
2184 2185 /* Unable to create vmem arena */
2185 2186 info->swi_table_ptr = NULL;
2186 2187 tavor_rsrc_sw_handles_fini(state, info);
2187 2188 TNF_PROBE_0(tavor_rsrc_pd_handles_init_vmem_create_fail,
2188 2189 TAVOR_TNF_ERROR, "");
2189 2190 TAVOR_TNF_EXIT(tavor_rsrc_pd_handles_init);
2190 2191 return (DDI_FAILURE);
2191 2192 }
2192 2193 rsrc_pool->rsrc_vmp = vmp;
2193 2194
2194 2195 TAVOR_TNF_EXIT(tavor_rsrc_pd_handles_init);
2195 2196 return (DDI_SUCCESS);
2196 2197 }
2197 2198
2198 2199
2199 2200 /*
2200 2201 * tavor_rsrc_pd_handles_fini()
2201 2202 * Context: Only called from attach() and/or detach() path contexts
2202 2203 */
2203 2204 static void
2204 2205 tavor_rsrc_pd_handles_fini(tavor_state_t *state, tavor_rsrc_sw_hdl_info_t *info)
2205 2206 {
2206 2207 tavor_rsrc_pool_info_t *rsrc_pool;
2207 2208
2208 2209 TAVOR_TNF_ENTER(tavor_rsrc_pd_handles_fini);
2209 2210
2210 2211 ASSERT(state != NULL);
2211 2212 ASSERT(info != NULL);
2212 2213
2213 2214 rsrc_pool = info->swi_rsrcpool;
2214 2215
2215 2216 /* Destroy the specially created UAR scratch table vmem arena */
2216 2217 vmem_destroy(rsrc_pool->rsrc_vmp);
2217 2218
2218 2219 /* Destroy the "tavor_sw_pd_t" kmem_cache */
2219 2220 tavor_rsrc_sw_handles_fini(state, info);
2220 2221
2221 2222 TAVOR_TNF_EXIT(tavor_rsrc_pd_handles_fini);
2222 2223 }
2223 2224
2224 2225
2225 2226 /*
2226 2227 * tavor_rsrc_mbox_alloc()
2227 2228 * Context: Only called from attach() path context
2228 2229 */
2229 2230 static int
2230 2231 tavor_rsrc_mbox_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t num,
2231 2232 tavor_rsrc_t *hdl)
2232 2233 {
2233 2234 tavor_rsrc_priv_mbox_t *priv;
2234 2235 void *addr;
2235 2236 caddr_t kaddr;
2236 2237 uint64_t offset;
2237 2238 size_t real_len, temp_len;
2238 2239 int status;
2239 2240
2240 2241 TAVOR_TNF_ENTER(tavor_rsrc_mbox_alloc);
2241 2242
2242 2243 ASSERT(pool_info != NULL);
2243 2244 ASSERT(hdl != NULL);
2244 2245
2245 2246 /* Get the private pointer for the mailboxes */
2246 2247 priv = pool_info->rsrc_private;
2247 2248 ASSERT(priv != NULL);
2248 2249
2249 2250 /*
2250 2251 * Allocate a DMA handle for the mailbox. This will be used for
2251 2252 * two purposes (potentially). First, it could be used below in
2252 2253 * the call to ddi_dma_mem_alloc() - if the mailbox is to come from
2253 2254 * system memory. Second, it is definitely used later to bind
2254 2255 * the mailbox for DMA access from/by the hardware.
2255 2256 */
2256 2257 status = ddi_dma_alloc_handle(priv->pmb_dip, &priv->pmb_dmaattr,
2257 2258 DDI_DMA_SLEEP, NULL, &hdl->tr_dmahdl);
2258 2259 if (status != DDI_SUCCESS) {
2259 2260 TNF_PROBE_1(tavor_rsrc_mbox_alloc_dmahdl_fail, TAVOR_TNF_ERROR,
2260 2261 "", tnf_uint, status, status);
2261 2262 TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc);
2262 2263 return (DDI_FAILURE);
2263 2264 }
2264 2265
2265 2266 /* Is mailbox in DDR memory or system memory? */
2266 2267 if (pool_info->rsrc_loc == TAVOR_IN_DDR) {
2267 2268 /* Use vmem_alloc() to get DDR address of mbox */
2268 2269 hdl->tr_len = (num * pool_info->rsrc_quantum);
2269 2270 addr = vmem_alloc(pool_info->rsrc_vmp, hdl->tr_len,
2270 2271 VM_SLEEP);
2271 2272 if (addr == NULL) {
2272 2273 /* No more DDR available for mailbox entries */
2273 2274 ddi_dma_free_handle(&hdl->tr_dmahdl);
2274 2275 TNF_PROBE_0(tavor_rsrc_mbox_alloc_vma_fail,
2275 2276 TAVOR_TNF_ERROR, "");
2276 2277 TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc);
2277 2278 return (DDI_FAILURE);
2278 2279 }
2279 2280 hdl->tr_acchdl = priv->pmb_acchdl;
2280 2281
2281 2282 /* Calculate kernel virtual address (from the DDR offset) */
2282 2283 offset = ((uintptr_t)addr -
2283 2284 (uintptr_t)pool_info->rsrc_ddr_offset);
2284 2285 hdl->tr_addr = (void *)(uintptr_t)(offset +
2285 2286 (uintptr_t)pool_info->rsrc_start);
2286 2287
2287 2288 } else { /* TAVOR_IN_SYSMEM */
2288 2289
2289 2290 /* Use ddi_dma_mem_alloc() to get memory for mailbox */
2290 2291 temp_len = (num * pool_info->rsrc_quantum);
2291 2292 status = ddi_dma_mem_alloc(hdl->tr_dmahdl, temp_len,
2292 2293 &priv->pmb_devaccattr, priv->pmb_xfer_mode, DDI_DMA_SLEEP,
2293 2294 NULL, &kaddr, &real_len, &hdl->tr_acchdl);
2294 2295 if (status != DDI_SUCCESS) {
2295 2296 /* No more sys memory available for mailbox entries */
2296 2297 ddi_dma_free_handle(&hdl->tr_dmahdl);
2297 2298 TNF_PROBE_0(tavor_rsrc_mbox_alloc_dma_memalloc_fail,
2298 2299 TAVOR_TNF_ERROR, "");
2299 2300 TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc);
2300 2301 return (DDI_FAILURE);
2301 2302 }
2302 2303 hdl->tr_addr = (void *)kaddr;
2303 2304 hdl->tr_len = real_len;
2304 2305 }
2305 2306
2306 2307 TAVOR_TNF_EXIT(tavor_rsrc_mbox_alloc);
2307 2308 return (DDI_SUCCESS);
2308 2309 }
2309 2310
2310 2311
2311 2312 /*
2312 2313 * tavor_rsrc_mbox_free()
2313 2314 * Context: Can be called from interrupt or base context.
2314 2315 */
2315 2316 static void
2316 2317 tavor_rsrc_mbox_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl)
2317 2318 {
2318 2319 void *addr;
2319 2320 uint64_t offset;
2320 2321
2321 2322 TAVOR_TNF_ENTER(tavor_rsrc_mbox_free);
2322 2323
2323 2324 ASSERT(pool_info != NULL);
2324 2325 ASSERT(hdl != NULL);
2325 2326
2326 2327 /* Is mailbox in DDR memory or system memory? */
2327 2328 if (pool_info->rsrc_loc == TAVOR_IN_DDR) {
2328 2329
2329 2330 /* Calculate the allocated address (the mbox's DDR offset) */
2330 2331 offset = ((uintptr_t)hdl->tr_addr -
2331 2332 (uintptr_t)pool_info->rsrc_start);
2332 2333 addr = (void *)(uintptr_t)(offset +
2333 2334 (uintptr_t)pool_info->rsrc_ddr_offset);
2334 2335
2335 2336 /* Use vmem_free() to free up DDR memory for mailbox */
2336 2337 vmem_free(pool_info->rsrc_vmp, addr, hdl->tr_len);
2337 2338
2338 2339 } else { /* TAVOR_IN_SYSMEM */
2339 2340
2340 2341 /* Use ddi_dma_mem_free() to free up sys memory for mailbox */
2341 2342 ddi_dma_mem_free(&hdl->tr_acchdl);
2342 2343 }
2343 2344
2344 2345 /* Free the DMA handle for the mailbox */
2345 2346 ddi_dma_free_handle(&hdl->tr_dmahdl);
2346 2347
2347 2348 TAVOR_TNF_EXIT(tavor_rsrc_mbox_free);
2348 2349 }
2349 2350
2350 2351
2351 2352 /*
2352 2353 * tavor_rsrc_hw_entry_alloc()
2353 2354 * Context: Can be called from interrupt or base context.
2354 2355 */
2355 2356 static int
2356 2357 tavor_rsrc_hw_entry_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t num,
2357 2358 uint_t num_align, ddi_acc_handle_t acc_handle, uint_t sleepflag,
2358 2359 tavor_rsrc_t *hdl)
2359 2360 {
2360 2361 void *addr;
2361 2362 uint64_t offset;
2362 2363 uint32_t align;
2363 2364 int flag;
2364 2365
2365 2366 TAVOR_TNF_ENTER(tavor_rsrc_hw_entry_alloc);
2366 2367
2367 2368 ASSERT(pool_info != NULL);
2368 2369 ASSERT(hdl != NULL);
2369 2370
2370 2371 /*
2371 2372 * Tavor hardware entries (QPC, CQC, EQC, MPT, MTT, etc.) do not
2372 2373 * use dma_handle (because they are in Tavor locally attached DDR
2373 2374 * memory) and, generally, don't use the acc_handle (because the
2374 2375 * entries are not directly accessed by software). The exceptions
2375 2376 * to this rule are the UARPG and UDAV entries.
2376 2377 */
2377 2378
2378 2379 /*
2379 2380 * Use vmem_xalloc() to get a properly aligned pointer (based on
2380 2381 * the number requested) to the HW entry(ies). This handles the
2381 2382 * cases (for special QPCs and for RDB entries) where we need more
2382 2383 * than one and need to ensure that they are properly aligned.
2383 2384 */
2384 2385 flag = (sleepflag == TAVOR_SLEEP) ? VM_SLEEP : VM_NOSLEEP;
2385 2386 hdl->tr_len = (num * pool_info->rsrc_quantum);
2386 2387 align = (num_align * pool_info->rsrc_quantum);
2387 2388 addr = vmem_xalloc(pool_info->rsrc_vmp, hdl->tr_len,
2388 2389 align, 0, 0, NULL, NULL, flag | VM_FIRSTFIT);
2389 2390 if (addr == NULL) {
2390 2391 /* No more HW entries available */
2391 2392 TNF_PROBE_0(tavor_rsrc_hw_entry_alloc_vmxa_fail,
2392 2393 TAVOR_TNF_ERROR, "");
2393 2394 TAVOR_TNF_EXIT(tavor_rsrc_hw_entry_alloc);
2394 2395 return (DDI_FAILURE);
2395 2396 }
2396 2397
2397 2398 /* If an access handle was provided, fill it in */
2398 2399 if (acc_handle != 0) {
2399 2400 hdl->tr_acchdl = acc_handle;
2400 2401 }
2401 2402
2402 2403 /* Calculate vaddr and HW table index (from the DDR offset) */
2403 2404 offset = ((uintptr_t)addr - (uintptr_t)pool_info->rsrc_ddr_offset);
2404 2405 hdl->tr_addr = (void *)(uintptr_t)(offset +
2405 2406 (uintptr_t)pool_info->rsrc_start);
2406 2407 hdl->tr_indx = (offset >> pool_info->rsrc_shift);
2407 2408
2408 2409 TAVOR_TNF_EXIT(tavor_rsrc_hw_entry_alloc);
2409 2410 return (DDI_SUCCESS);
2410 2411 }
2411 2412
2412 2413
2413 2414 /*
2414 2415 * tavor_rsrc_hw_entry_free()
2415 2416 * Context: Can be called from interrupt or base context.
2416 2417 */
2417 2418 static void
2418 2419 tavor_rsrc_hw_entry_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl)
2419 2420 {
2420 2421 void *addr;
2421 2422 uint64_t offset;
2422 2423
2423 2424 TAVOR_TNF_ENTER(tavor_rsrc_hw_entry_free);
2424 2425
2425 2426 ASSERT(pool_info != NULL);
2426 2427 ASSERT(hdl != NULL);
2427 2428
2428 2429 /* Calculate the allocated address (the entry's DDR offset) */
2429 2430 offset = ((uintptr_t)hdl->tr_addr - (uintptr_t)pool_info->rsrc_start);
2430 2431 addr = (void *)(uintptr_t)(offset +
2431 2432 (uintptr_t)pool_info->rsrc_ddr_offset);
2432 2433
2433 2434 /* Use vmem_xfree() to free up the HW table entry */
2434 2435 vmem_xfree(pool_info->rsrc_vmp, addr, hdl->tr_len);
2435 2436
2436 2437 TAVOR_TNF_EXIT(tavor_rsrc_hw_entry_free);
2437 2438 }
2438 2439
2439 2440
2440 2441 /*
2441 2442 * tavor_rsrc_swhdl_alloc()
2442 2443 * Context: Can be called from interrupt or base context.
2443 2444 */
2444 2445 static int
2445 2446 tavor_rsrc_swhdl_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t sleepflag,
2446 2447 tavor_rsrc_t *hdl)
2447 2448 {
2448 2449 void *addr;
2449 2450 int flag;
2450 2451
2451 2452 TAVOR_TNF_ENTER(tavor_rsrc_swhdl_alloc);
2452 2453
2453 2454 ASSERT(pool_info != NULL);
2454 2455 ASSERT(hdl != NULL);
2455 2456
2456 2457 /* Allocate the software handle structure */
2457 2458 flag = (sleepflag == TAVOR_SLEEP) ? KM_SLEEP : KM_NOSLEEP;
2458 2459 addr = kmem_cache_alloc(pool_info->rsrc_private, flag);
2459 2460 if (addr == NULL) {
2460 2461 TNF_PROBE_0(tavor_rsrc_swhdl_alloc_kmca_fail, TAVOR_TNF_ERROR,
2461 2462 "");
2462 2463 TAVOR_TNF_EXIT(tavor_rsrc_swhdl_alloc);
2463 2464 return (DDI_FAILURE);
2464 2465 }
2465 2466 hdl->tr_len = pool_info->rsrc_quantum;
2466 2467 hdl->tr_addr = addr;
2467 2468
2468 2469 TAVOR_TNF_EXIT(tavor_rsrc_swhdl_alloc);
2469 2470 return (DDI_SUCCESS);
2470 2471 }
2471 2472
2472 2473
2473 2474 /*
2474 2475 * tavor_rsrc_swhdl_free()
2475 2476 * Context: Can be called from interrupt or base context.
2476 2477 */
2477 2478 static void
2478 2479 tavor_rsrc_swhdl_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl)
2479 2480 {
2480 2481 TAVOR_TNF_ENTER(tavor_rsrc_swhdl_free);
2481 2482
2482 2483 ASSERT(pool_info != NULL);
2483 2484 ASSERT(hdl != NULL);
2484 2485
2485 2486 /* Free the software handle structure */
2486 2487 kmem_cache_free(pool_info->rsrc_private, hdl->tr_addr);
2487 2488
2488 2489 TAVOR_TNF_EXIT(tavor_rsrc_swhdl_free);
2489 2490 }
2490 2491
2491 2492
2492 2493 /*
2493 2494 * tavor_rsrc_pdhdl_alloc()
2494 2495 * Context: Can be called from interrupt or base context.
2495 2496 */
2496 2497 static int
2497 2498 tavor_rsrc_pdhdl_alloc(tavor_rsrc_pool_info_t *pool_info, uint_t sleepflag,
2498 2499 tavor_rsrc_t *hdl)
2499 2500 {
2500 2501 tavor_pdhdl_t addr;
2501 2502 void *tmpaddr;
2502 2503 int flag, status;
2503 2504
2504 2505 TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_alloc);
2505 2506
2506 2507 ASSERT(pool_info != NULL);
2507 2508 ASSERT(hdl != NULL);
2508 2509
2509 2510 /* Allocate the software handle */
2510 2511 status = tavor_rsrc_swhdl_alloc(pool_info, sleepflag, hdl);
2511 2512 if (status != DDI_SUCCESS) {
2512 2513 TNF_PROBE_0(tavor_rsrc_pdhdl_alloc_fail, TAVOR_TNF_ERROR, "");
2513 2514 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc);
2514 2515 return (DDI_FAILURE);
2515 2516 }
2516 2517 addr = (tavor_pdhdl_t)hdl->tr_addr;
2517 2518 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*addr))
2518 2519
2519 2520 /* Allocate a PD number for the handle */
2520 2521 flag = (sleepflag == TAVOR_SLEEP) ? VM_SLEEP : VM_NOSLEEP;
2521 2522 tmpaddr = vmem_alloc(pool_info->rsrc_vmp, 1, flag);
2522 2523 if (tmpaddr == NULL) {
2523 2524 /* No more PD number entries available */
2524 2525 tavor_rsrc_swhdl_free(pool_info, hdl);
2525 2526 TNF_PROBE_0(tavor_rsrc_pdhdl_alloc_vma_fail,
2526 2527 TAVOR_TNF_ERROR, "");
2527 2528 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc);
2528 2529 return (DDI_FAILURE);
2529 2530 }
2530 2531 addr->pd_pdnum = (uint32_t)(uintptr_t)tmpaddr;
2531 2532 addr->pd_rsrcp = hdl;
2532 2533 hdl->tr_indx = addr->pd_pdnum;
2533 2534
2534 2535 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_alloc);
2535 2536 return (DDI_SUCCESS);
2536 2537 }
2537 2538
2538 2539
2539 2540 /*
2540 2541 * tavor_rsrc_pdhdl_free()
2541 2542 * Context: Can be called from interrupt or base context.
2542 2543 */
2543 2544 static void
2544 2545 tavor_rsrc_pdhdl_free(tavor_rsrc_pool_info_t *pool_info, tavor_rsrc_t *hdl)
2545 2546 {
2546 2547 TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_free);
2547 2548
2548 2549 ASSERT(pool_info != NULL);
2549 2550 ASSERT(hdl != NULL);
2550 2551
2551 2552 /* Use vmem_free() to free up the PD number */
2552 2553 vmem_free(pool_info->rsrc_vmp, (void *)(uintptr_t)hdl->tr_indx, 1);
2553 2554
2554 2555 /* Free the software handle structure */
2555 2556 tavor_rsrc_swhdl_free(pool_info, hdl);
2556 2557
2557 2558 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_free);
2558 2559 }
2559 2560
2560 2561
2561 2562 /*
2562 2563 * tavor_rsrc_pdhdl_constructor()
2563 2564 * Context: Can be called from interrupt or base context.
2564 2565 */
2565 2566 /* ARGSUSED */
2566 2567 static int
2567 2568 tavor_rsrc_pdhdl_constructor(void *pd, void *priv, int flags)
2568 2569 {
2569 2570 tavor_pdhdl_t pdhdl;
2570 2571 tavor_state_t *state;
2571 2572
2572 2573 TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_constructor);
2573 2574
2574 2575 pdhdl = (tavor_pdhdl_t)pd;
2575 2576 state = (tavor_state_t *)priv;
2576 2577
2577 2578 mutex_init(&pdhdl->pd_lock, NULL, MUTEX_DRIVER,
2578 2579 DDI_INTR_PRI(state->ts_intrmsi_pri));
2579 2580
2580 2581 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_constructor);
2581 2582 return (DDI_SUCCESS);
2582 2583 }
2583 2584
2584 2585
2585 2586 /*
2586 2587 * tavor_rsrc_pdhdl_destructor()
2587 2588 * Context: Can be called from interrupt or base context.
2588 2589 */
2589 2590 /* ARGSUSED */
2590 2591 static void
2591 2592 tavor_rsrc_pdhdl_destructor(void *pd, void *priv)
2592 2593 {
2593 2594 tavor_pdhdl_t pdhdl;
2594 2595
2595 2596 TAVOR_TNF_ENTER(tavor_rsrc_pdhdl_destructor);
2596 2597
2597 2598 pdhdl = (tavor_pdhdl_t)pd;
2598 2599
2599 2600 mutex_destroy(&pdhdl->pd_lock);
2600 2601
2601 2602 TAVOR_TNF_EXIT(tavor_rsrc_pdhdl_destructor);
2602 2603 }
2603 2604
2604 2605
2605 2606 /*
2606 2607 * tavor_rsrc_cqhdl_constructor()
2607 2608 * Context: Can be called from interrupt or base context.
2608 2609 */
2609 2610 /* ARGSUSED */
2610 2611 static int
2611 2612 tavor_rsrc_cqhdl_constructor(void *cq, void *priv, int flags)
2612 2613 {
2613 2614 tavor_cqhdl_t cqhdl;
2614 2615 tavor_state_t *state;
2615 2616
2616 2617 TAVOR_TNF_ENTER(tavor_rsrc_cqhdl_constructor);
2617 2618
2618 2619 cqhdl = (tavor_cqhdl_t)cq;
2619 2620 state = (tavor_state_t *)priv;
2620 2621
2621 2622 mutex_init(&cqhdl->cq_lock, NULL, MUTEX_DRIVER,
2622 2623 DDI_INTR_PRI(state->ts_intrmsi_pri));
2623 2624 mutex_init(&cqhdl->cq_wrid_wqhdr_lock, NULL, MUTEX_DRIVER,
2624 2625 DDI_INTR_PRI(state->ts_intrmsi_pri));
2625 2626
2626 2627 TAVOR_TNF_EXIT(tavor_rsrc_cqhdl_constructor);
2627 2628 return (DDI_SUCCESS);
2628 2629 }
2629 2630
2630 2631
2631 2632 /*
2632 2633 * tavor_rsrc_cqhdl_destructor()
2633 2634 * Context: Can be called from interrupt or base context.
2634 2635 */
2635 2636 /* ARGSUSED */
2636 2637 static void
2637 2638 tavor_rsrc_cqhdl_destructor(void *cq, void *priv)
2638 2639 {
2639 2640 tavor_cqhdl_t cqhdl;
2640 2641
2641 2642 TAVOR_TNF_ENTER(tavor_rsrc_cqhdl_destructor);
2642 2643
2643 2644 cqhdl = (tavor_cqhdl_t)cq;
2644 2645
2645 2646 mutex_destroy(&cqhdl->cq_wrid_wqhdr_lock);
2646 2647 mutex_destroy(&cqhdl->cq_lock);
2647 2648
2648 2649 TAVOR_TNF_EXIT(tavor_rsrc_cqhdl_destructor);
2649 2650 }
2650 2651
2651 2652
2652 2653 /*
2653 2654 * tavor_rsrc_qphdl_constructor()
2654 2655 * Context: Can be called from interrupt or base context.
2655 2656 */
2656 2657 /* ARGSUSED */
2657 2658 static int
2658 2659 tavor_rsrc_qphdl_constructor(void *qp, void *priv, int flags)
2659 2660 {
2660 2661 tavor_qphdl_t qphdl;
2661 2662 tavor_state_t *state;
2662 2663
2663 2664 TAVOR_TNF_ENTER(tavor_rsrc_qphdl_constructor);
2664 2665
2665 2666 qphdl = (tavor_qphdl_t)qp;
2666 2667 state = (tavor_state_t *)priv;
2667 2668
2668 2669 mutex_init(&qphdl->qp_lock, NULL, MUTEX_DRIVER,
2669 2670 DDI_INTR_PRI(state->ts_intrmsi_pri));
2670 2671
2671 2672 TAVOR_TNF_EXIT(tavor_rsrc_qphdl_constructor);
2672 2673 return (DDI_SUCCESS);
2673 2674 }
2674 2675
2675 2676
2676 2677 /*
2677 2678 * tavor_rsrc_qphdl_destructor()
2678 2679 * Context: Can be called from interrupt or base context.
2679 2680 */
2680 2681 /* ARGSUSED */
2681 2682 static void
2682 2683 tavor_rsrc_qphdl_destructor(void *qp, void *priv)
2683 2684 {
2684 2685 tavor_qphdl_t qphdl;
2685 2686
2686 2687 TAVOR_TNF_ENTER(tavor_rsrc_qphdl_destructor);
2687 2688
2688 2689 qphdl = (tavor_qphdl_t)qp;
2689 2690
2690 2691 mutex_destroy(&qphdl->qp_lock);
2691 2692
2692 2693 TAVOR_TNF_ENTER(tavor_rsrc_qphdl_destructor);
2693 2694 }
2694 2695
2695 2696
2696 2697 /*
2697 2698 * tavor_rsrc_srqhdl_constructor()
2698 2699 * Context: Can be called from interrupt or base context.
2699 2700 */
2700 2701 /* ARGSUSED */
2701 2702 static int
2702 2703 tavor_rsrc_srqhdl_constructor(void *srq, void *priv, int flags)
2703 2704 {
2704 2705 tavor_srqhdl_t srqhdl;
2705 2706 tavor_state_t *state;
2706 2707
2707 2708 TAVOR_TNF_ENTER(tavor_rsrc_srqhdl_constructor);
2708 2709
2709 2710 srqhdl = (tavor_srqhdl_t)srq;
2710 2711 state = (tavor_state_t *)priv;
2711 2712
2712 2713 mutex_init(&srqhdl->srq_lock, NULL, MUTEX_DRIVER,
2713 2714 DDI_INTR_PRI(state->ts_intrmsi_pri));
2714 2715
2715 2716 TAVOR_TNF_EXIT(tavor_rsrc_srqhdl_constructor);
2716 2717 return (DDI_SUCCESS);
2717 2718 }
2718 2719
2719 2720
2720 2721 /*
2721 2722 * tavor_rsrc_srqhdl_destructor()
2722 2723 * Context: Can be called from interrupt or base context.
2723 2724 */
2724 2725 /* ARGSUSED */
2725 2726 static void
2726 2727 tavor_rsrc_srqhdl_destructor(void *srq, void *priv)
2727 2728 {
2728 2729 tavor_srqhdl_t srqhdl;
2729 2730
2730 2731 TAVOR_TNF_ENTER(tavor_rsrc_srqhdl_destructor);
2731 2732
2732 2733 srqhdl = (tavor_srqhdl_t)srq;
2733 2734
2734 2735 mutex_destroy(&srqhdl->srq_lock);
2735 2736
2736 2737 TAVOR_TNF_EXIT(tavor_rsrc_srqhdl_destructor);
2737 2738 }
2738 2739
2739 2740
2740 2741 /*
2741 2742 * tavor_rsrc_refcnt_constructor()
2742 2743 * Context: Can be called from interrupt or base context.
2743 2744 */
2744 2745 /* ARGSUSED */
2745 2746 static int
2746 2747 tavor_rsrc_refcnt_constructor(void *rc, void *priv, int flags)
2747 2748 {
2748 2749 tavor_sw_refcnt_t *refcnt;
2749 2750 tavor_state_t *state;
2750 2751
2751 2752 TAVOR_TNF_ENTER(tavor_rsrc_refcnt_constructor);
2752 2753
2753 2754 refcnt = (tavor_sw_refcnt_t *)rc;
2754 2755 state = (tavor_state_t *)priv;
2755 2756
2756 2757 mutex_init(&refcnt->swrc_lock, NULL, MUTEX_DRIVER,
2757 2758 DDI_INTR_PRI(state->ts_intrmsi_pri));
2758 2759
2759 2760 TAVOR_TNF_EXIT(tavor_rsrc_refcnt_constructor);
2760 2761 return (DDI_SUCCESS);
2761 2762 }
2762 2763
2763 2764
2764 2765 /*
2765 2766 * tavor_rsrc_refcnt_destructor()
2766 2767 * Context: Can be called from interrupt or base context.
2767 2768 */
2768 2769 /* ARGSUSED */
2769 2770 static void
2770 2771 tavor_rsrc_refcnt_destructor(void *rc, void *priv)
2771 2772 {
2772 2773 tavor_sw_refcnt_t *refcnt;
2773 2774
2774 2775 TAVOR_TNF_ENTER(tavor_rsrc_refcnt_destructor);
2775 2776
2776 2777 refcnt = (tavor_sw_refcnt_t *)rc;
2777 2778
2778 2779 mutex_destroy(&refcnt->swrc_lock);
2779 2780
2780 2781 TAVOR_TNF_ENTER(tavor_rsrc_refcnt_destructor);
2781 2782 }
2782 2783
2783 2784
2784 2785 /*
2785 2786 * tavor_rsrc_ahhdl_constructor()
2786 2787 * Context: Can be called from interrupt or base context.
2787 2788 */
2788 2789 /* ARGSUSED */
2789 2790 static int
2790 2791 tavor_rsrc_ahhdl_constructor(void *ah, void *priv, int flags)
2791 2792 {
2792 2793 tavor_ahhdl_t ahhdl;
2793 2794 tavor_state_t *state;
2794 2795
2795 2796 TAVOR_TNF_ENTER(tavor_rsrc_ahhdl_constructor);
2796 2797
2797 2798 ahhdl = (tavor_ahhdl_t)ah;
2798 2799 state = (tavor_state_t *)priv;
2799 2800
2800 2801 mutex_init(&ahhdl->ah_lock, NULL, MUTEX_DRIVER,
2801 2802 DDI_INTR_PRI(state->ts_intrmsi_pri));
2802 2803
2803 2804 TAVOR_TNF_EXIT(tavor_rsrc_ahhdl_constructor);
2804 2805 return (DDI_SUCCESS);
2805 2806 }
2806 2807
2807 2808
2808 2809 /*
2809 2810 * tavor_rsrc_ahhdl_destructor()
2810 2811 * Context: Can be called from interrupt or base context.
2811 2812 */
2812 2813 /* ARGSUSED */
2813 2814 static void
2814 2815 tavor_rsrc_ahhdl_destructor(void *ah, void *priv)
2815 2816 {
2816 2817 tavor_ahhdl_t ahhdl;
2817 2818
2818 2819 TAVOR_TNF_ENTER(tavor_rsrc_ahhdl_destructor);
2819 2820
2820 2821 ahhdl = (tavor_ahhdl_t)ah;
2821 2822
2822 2823 mutex_destroy(&ahhdl->ah_lock);
2823 2824
2824 2825 TAVOR_TNF_ENTER(tavor_rsrc_ahhdl_destructor);
2825 2826 }
2826 2827
2827 2828
2828 2829 /*
2829 2830 * tavor_rsrc_mrhdl_constructor()
2830 2831 * Context: Can be called from interrupt or base context.
2831 2832 */
2832 2833 /* ARGSUSED */
2833 2834 static int
2834 2835 tavor_rsrc_mrhdl_constructor(void *mr, void *priv, int flags)
2835 2836 {
2836 2837 tavor_mrhdl_t mrhdl;
2837 2838 tavor_state_t *state;
2838 2839
2839 2840 TAVOR_TNF_ENTER(tavor_rsrc_mrhdl_constructor);
2840 2841
2841 2842 mrhdl = (tavor_mrhdl_t)mr;
2842 2843 state = (tavor_state_t *)priv;
2843 2844
2844 2845 mutex_init(&mrhdl->mr_lock, NULL, MUTEX_DRIVER,
2845 2846 DDI_INTR_PRI(state->ts_intrmsi_pri));
2846 2847
2847 2848 TAVOR_TNF_EXIT(tavor_rsrc_mrhdl_constructor);
2848 2849 return (DDI_SUCCESS);
2849 2850 }
2850 2851
2851 2852
2852 2853 /*
2853 2854 * tavor_rsrc_mrhdl_destructor()
2854 2855 * Context: Can be called from interrupt or base context.
2855 2856 */
2856 2857 /* ARGSUSED */
2857 2858 static void
2858 2859 tavor_rsrc_mrhdl_destructor(void *mr, void *priv)
2859 2860 {
2860 2861 tavor_mrhdl_t mrhdl;
2861 2862
2862 2863 TAVOR_TNF_ENTER(tavor_rsrc_mrhdl_destructor);
2863 2864
2864 2865 mrhdl = (tavor_mrhdl_t)mr;
2865 2866
2866 2867 mutex_destroy(&mrhdl->mr_lock);
2867 2868
2868 2869 TAVOR_TNF_ENTER(tavor_rsrc_mrhdl_destructor);
2869 2870 }
2870 2871
2871 2872
2872 2873 /*
2873 2874 * tavor_rsrc_mcg_entry_get_size()
2874 2875 */
2875 2876 static int
2876 2877 tavor_rsrc_mcg_entry_get_size(tavor_state_t *state, uint_t *mcg_size_shift)
2877 2878 {
↓ open down ↓ |
2832 lines elided |
↑ open up ↑ |
2878 2879 uint_t num_qp_per_mcg, max_qp_per_mcg, log2;
2879 2880
2880 2881 TAVOR_TNF_ENTER(tavor_rsrc_mcg_entry_get_size);
2881 2882
2882 2883 /*
2883 2884 * Round the configured number of QP per MCG to next larger
2884 2885 * power-of-2 size and update.
2885 2886 */
2886 2887 num_qp_per_mcg = state->ts_cfg_profile->cp_num_qp_per_mcg + 8;
2887 2888 log2 = highbit(num_qp_per_mcg);
2888 - if ((num_qp_per_mcg & (num_qp_per_mcg - 1)) == 0) {
2889 + if (ISP2(num_qp_per_mcg)) {
2889 2890 log2 = log2 - 1;
2890 2891 }
2891 2892 state->ts_cfg_profile->cp_num_qp_per_mcg = (1 << log2) - 8;
2892 2893
2893 2894 /* Now make sure number of QP per MCG makes sense */
2894 2895 num_qp_per_mcg = state->ts_cfg_profile->cp_num_qp_per_mcg;
2895 2896 max_qp_per_mcg = (1 << state->ts_devlim.log_max_qp_mcg);
2896 2897 if (num_qp_per_mcg > max_qp_per_mcg) {
2897 2898 TNF_PROBE_1(tavor_rsrc_mcg_getsz_toomany_qppermcg_fail,
2898 2899 TAVOR_TNF_ERROR, "", tnf_uint, maxqpmcg, max_qp_per_mcg);
2899 2900 TAVOR_TNF_EXIT(tavor_rsrc_mcg_entry_get_size);
2900 2901 return (DDI_FAILURE);
2901 2902 }
2902 2903
2903 2904 /* Return the (shift) size of an individual MCG HW entry */
2904 2905 *mcg_size_shift = log2 + 2;
2905 2906
2906 2907 TAVOR_TNF_EXIT(tavor_rsrc_mcg_entry_get_size);
2907 2908 return (DDI_SUCCESS);
2908 2909 }
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX