1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26 
  27 /*
  28  * tavor_qpmod.c
  29  *    Tavor Queue Pair Modify Routines
  30  *
  31  *    This contains all the routines necessary to implement the Tavor
  32  *    ModifyQP() verb.  This includes all the code for legal transitions to
  33  *    and from Reset, Init, RTR, RTS, SQD, SQErr, and Error.
  34  */
  35 
  36 #include <sys/sysmacros.h>
  37 #include <sys/types.h>
  38 #include <sys/conf.h>
  39 #include <sys/ddi.h>
  40 #include <sys/sunddi.h>
  41 #include <sys/modctl.h>
  42 #include <sys/bitmap.h>
  43 
  44 #include <sys/ib/adapters/tavor/tavor.h>
  45 #include <sys/ib/ib_pkt_hdrs.h>
  46 
  47 static int tavor_qp_reset2init(tavor_state_t *state, tavor_qphdl_t qp,
  48     ibt_qp_info_t *info_p);
  49 static int tavor_qp_init2init(tavor_state_t *state, tavor_qphdl_t qp,
  50     ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p);
  51 static int tavor_qp_init2rtr(tavor_state_t *state, tavor_qphdl_t qp,
  52     ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p);
  53 static int tavor_qp_rtr2rts(tavor_state_t *state, tavor_qphdl_t qp,
  54     ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p);
  55 static int tavor_qp_rts2rts(tavor_state_t *state, tavor_qphdl_t qp,
  56     ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p);
  57 static int tavor_qp_rts2sqd(tavor_state_t *state, tavor_qphdl_t qp,
  58     ibt_cep_modify_flags_t flags);
  59 static int tavor_qp_sqd2rts(tavor_state_t *state, tavor_qphdl_t qp,
  60     ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p);
  61 static int tavor_qp_sqd2sqd(tavor_state_t *state, tavor_qphdl_t qp,
  62     ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p);
  63 static int tavor_qp_sqerr2rts(tavor_state_t *state, tavor_qphdl_t qp,
  64     ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p);
  65 static int tavor_qp_to_error(tavor_state_t *state, tavor_qphdl_t qp);
  66 static int tavor_qp_reset2err(tavor_state_t *state, tavor_qphdl_t qp);
  67 
  68 static uint_t tavor_check_rdma_enable_flags(ibt_cep_modify_flags_t flags,
  69     ibt_qp_info_t *info_p, tavor_hw_qpc_t *qpc);
  70 static int tavor_qp_validate_resp_rsrc(tavor_state_t *state,
  71     ibt_qp_rc_attr_t *rc, uint_t *rra_max);
  72 static int tavor_qp_validate_init_depth(tavor_state_t *state,
  73     ibt_qp_rc_attr_t *rc, uint_t *sra_max);
  74 static int tavor_qp_validate_mtu(tavor_state_t *state, uint_t mtu);
  75 
  76 /*
  77  * tavor_qp_modify()
  78  *    Context: Can be called from interrupt or base context.
  79  */
  80 /* ARGSUSED */
  81 int
  82 tavor_qp_modify(tavor_state_t *state, tavor_qphdl_t qp,
  83     ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p,
  84     ibt_queue_sizes_t *actual_sz)
  85 {
  86         ibt_cep_state_t         cur_state, mod_state;
  87         ibt_cep_modify_flags_t  okflags;
  88         int                     status;
  89         char                    *errormsg;
  90 
  91         TAVOR_TNF_ENTER(tavor_qp_modify);
  92 
  93         /*
  94          * Lock the QP so that we can modify it atomically.  After grabbing
  95          * the lock, get the current QP state.  We will use this current QP
  96          * state to determine the legal transitions (and the checks that need
  97          * to be performed.)
  98          * Below you will find a case for every possible QP state.  In each
  99          * case we check that no flags are set which are not valid for the
 100          * possible transitions from that state.  If these tests pass (and if
 101          * the state transition we are attempting is legal), then we call
 102          * one of the helper functions.  Each of these functions does some
 103          * additional setup before posting a Tavor firmware command for the
 104          * appropriate state transition.
 105          */
 106         mutex_enter(&qp->qp_lock);
 107 
 108         /*
 109          * Verify that the transport type matches between the serv_type and the
 110          * qp_trans.  A caller to IBT must specify the qp_trans field as
 111          * IBT_UD_SRV, IBT_RC_SRV, or IBT_UC_SRV, depending on the QP.  We
 112          * check here that the correct value was specified, based on our
 113          * understanding of the QP serv type.
 114          *
 115          * Because callers specify part of a 'union' based on what QP type they
 116          * think they're working with, this ensures that we do not pickup bogus
 117          * data if the caller thought they were working with a different QP
 118          * type.
 119          */
 120         if (!(TAVOR_QP_TYPE_VALID(info_p->qp_trans, qp->qp_serv_type))) {
 121                 mutex_exit(&qp->qp_lock);
 122                 TNF_PROBE_1(tavor_qp_modify_inv_qp_trans_fail,
 123                     TAVOR_TNF_ERROR, "", tnf_uint, qptrans,
 124                     info_p->qp_trans);
 125                 TAVOR_TNF_EXIT(tavor_qp_modify);
 126                 return (IBT_QP_SRV_TYPE_INVALID);
 127         }
 128 
 129         /*
 130          * If this is a transition to RTS (which is valid from RTR, RTS,
 131          * SQError, and SQ Drain) then we should honor the "current QP state"
 132          * specified by the consumer.  This means converting the IBTF QP state
 133          * in "info_p->qp_current_state" to a Tavor QP state.  Otherwise, we
 134          * assume that we already know the current state (i.e. whatever it was
 135          * last modified to or queried as - in "qp->qp_state").
 136          */
 137         mod_state = info_p->qp_state;
 138 
 139         if (flags & IBT_CEP_SET_RTR_RTS) {
 140                 cur_state = TAVOR_QP_RTR;               /* Ready to Receive */
 141 
 142         } else if ((flags & IBT_CEP_SET_STATE) &&
 143             (mod_state == IBT_STATE_RTS)) {
 144 
 145                 /* Convert the current IBTF QP state to a Tavor QP state */
 146                 switch (info_p->qp_current_state) {
 147                 case IBT_STATE_RTR:
 148                         cur_state = TAVOR_QP_RTR;       /* Ready to Receive */
 149                         break;
 150                 case IBT_STATE_RTS:
 151                         cur_state = TAVOR_QP_RTS;       /* Ready to Send */
 152                         break;
 153                 case IBT_STATE_SQE:
 154                         cur_state = TAVOR_QP_SQERR;     /* Send Queue Error */
 155                         break;
 156                 case IBT_STATE_SQD:
 157                         cur_state = TAVOR_QP_SQD;       /* SQ Drained */
 158                         break;
 159                 default:
 160                         mutex_exit(&qp->qp_lock);
 161                         TNF_PROBE_1(tavor_qp_modify_inv_currqpstate_fail,
 162                             TAVOR_TNF_ERROR, "", tnf_uint, qpstate,
 163                             info_p->qp_current_state);
 164                         TAVOR_TNF_EXIT(tavor_qp_modify);
 165                         return (IBT_QP_STATE_INVALID);
 166                 }
 167         } else {
 168                 cur_state = qp->qp_state;
 169         }
 170 
 171         switch (cur_state) {
 172         case TAVOR_QP_RESET:
 173                 okflags = (IBT_CEP_SET_STATE | IBT_CEP_SET_RESET_INIT |
 174                     IBT_CEP_SET_RDMA_R | IBT_CEP_SET_RDMA_W |
 175                     IBT_CEP_SET_ATOMIC | IBT_CEP_SET_PKEY_IX |
 176                     IBT_CEP_SET_PORT | IBT_CEP_SET_QKEY);
 177 
 178                 /*
 179                  * Check for attempts to modify invalid attributes from the
 180                  * "Reset" state
 181                  */
 182                 if (flags & ~okflags) {
 183                         mutex_exit(&qp->qp_lock);
 184                         /* Set "status" and "errormsg" and goto failure */
 185                         TAVOR_TNF_FAIL(IBT_QP_ATTR_RO, "reset: invalid flag");
 186                         goto qpmod_fail;
 187                 }
 188 
 189                 /*
 190                  * Verify state transition is to either "Init", back to
 191                  * "Reset", or to "Error".
 192                  */
 193                 if ((flags & IBT_CEP_SET_RESET_INIT) &&
 194                     (flags & IBT_CEP_SET_STATE) &&
 195                     (mod_state != IBT_STATE_INIT)) {
 196                         /* Invalid transition - ambiguous flags */
 197                         mutex_exit(&qp->qp_lock);
 198                         /* Set "status" and "errormsg" and goto failure */
 199                         TAVOR_TNF_FAIL(IBT_QP_STATE_INVALID,
 200                             "reset: ambiguous flags");
 201                         goto qpmod_fail;
 202 
 203                 } else if ((flags & IBT_CEP_SET_RESET_INIT) ||
 204                     ((flags & IBT_CEP_SET_STATE) &&
 205                     (mod_state == IBT_STATE_INIT))) {
 206                         /*
 207                          * Attempt to transition from "Reset" to "Init"
 208                          */
 209                         status = tavor_qp_reset2init(state, qp, info_p);
 210                         if (status != DDI_SUCCESS) {
 211                                 mutex_exit(&qp->qp_lock);
 212                                 /* Set "status"/"errormsg", goto failure */
 213                                 TAVOR_TNF_FAIL(status, "reset to init");
 214                                 goto qpmod_fail;
 215                         }
 216                         qp->qp_state = TAVOR_QP_INIT;
 217 
 218                 } else if ((flags & IBT_CEP_SET_STATE) &&
 219                     (mod_state == IBT_STATE_RESET)) {
 220                         /*
 221                          * Attempt to transition from "Reset" back to "Reset"
 222                          *    Nothing to do here really... just drop the lock
 223                          *    and return success.  The qp->qp_state should
 224                          *    already be set to TAVOR_QP_RESET.
 225                          *
 226                          * Note: We return here because we do not want to fall
 227                          *    through to the tavor_wrid_from_reset_handling()
 228                          *    routine below (since we are not really moving
 229                          *    _out_ of the "Reset" state.
 230                          */
 231                         mutex_exit(&qp->qp_lock);
 232                         TNF_PROBE_0_DEBUG(tavor_qp_modify_rst2rst,
 233                             TAVOR_TNF_TRACE, "");
 234                         TAVOR_TNF_EXIT(tavor_qp_modify);
 235                         return (DDI_SUCCESS);
 236 
 237                 } else if ((flags & IBT_CEP_SET_STATE) &&
 238                     (mod_state == IBT_STATE_ERROR)) {
 239                         /*
 240                          * Attempt to transition from "Reset" to "Error"
 241                          */
 242                         status = tavor_qp_reset2err(state, qp);
 243                         if (status != DDI_SUCCESS) {
 244                                 mutex_exit(&qp->qp_lock);
 245                                 /* Set "status"/"errormsg", goto failure */
 246                                 TAVOR_TNF_FAIL(status, "reset to error");
 247                                 goto qpmod_fail;
 248                         }
 249                         qp->qp_state = TAVOR_QP_ERR;
 250 
 251                 } else {
 252                         /* Invalid transition - return error */
 253                         mutex_exit(&qp->qp_lock);
 254                         /* Set "status" and "errormsg" and goto failure */
 255                         TAVOR_TNF_FAIL(IBT_QP_STATE_INVALID,
 256                             "reset: invalid transition");
 257                         goto qpmod_fail;
 258                 }
 259 
 260                 /*
 261                  * Do any additional handling necessary here for the transition
 262                  * from the "Reset" state (e.g. re-initialize the workQ WRID
 263                  * lists).  Note: If tavor_wrid_from_reset_handling() fails,
 264                  * then we attempt to transition the QP back to the "Reset"
 265                  * state.  If that fails, then it is an indication of a serious
 266                  * problem (either HW or SW).  So we print out a warning
 267                  * message and return failure.
 268                  */
 269                 status = tavor_wrid_from_reset_handling(state, qp);
 270                 if (status != DDI_SUCCESS) {
 271                         if (tavor_qp_to_reset(state, qp) != DDI_SUCCESS) {
 272                                 TAVOR_WARNING(state, "failed to reset QP");
 273                         }
 274                         qp->qp_state = TAVOR_QP_RESET;
 275 
 276                         mutex_exit(&qp->qp_lock);
 277                         /* Set "status" and "errormsg" and goto failure */
 278                         TAVOR_TNF_FAIL(status, "reset: wrid_from_reset hdl");
 279                         goto qpmod_fail;
 280                 }
 281                 break;
 282 
 283         case TAVOR_QP_INIT:
 284                 okflags = (IBT_CEP_SET_STATE | IBT_CEP_SET_INIT_RTR |
 285                     IBT_CEP_SET_ADDS_VECT | IBT_CEP_SET_RDMARA_IN |
 286                     IBT_CEP_SET_MIN_RNR_NAK | IBT_CEP_SET_ALT_PATH |
 287                     IBT_CEP_SET_RDMA_R | IBT_CEP_SET_RDMA_W |
 288                     IBT_CEP_SET_ATOMIC | IBT_CEP_SET_PKEY_IX |
 289                     IBT_CEP_SET_QKEY | IBT_CEP_SET_PORT);
 290 
 291                 /*
 292                  * Check for attempts to modify invalid attributes from the
 293                  * "Init" state
 294                  */
 295                 if (flags & ~okflags) {
 296                         mutex_exit(&qp->qp_lock);
 297                         /* Set "status" and "errormsg" and goto failure */
 298                         TAVOR_TNF_FAIL(IBT_QP_ATTR_RO, "init: invalid flag");
 299                         goto qpmod_fail;
 300                 }
 301 
 302                 /*
 303                  * Verify state transition is to either "RTR", back to "Init",
 304                  * to "Reset", or to "Error"
 305                  */
 306                 if ((flags & IBT_CEP_SET_INIT_RTR) &&
 307                     (flags & IBT_CEP_SET_STATE) &&
 308                     (mod_state != IBT_STATE_RTR)) {
 309                         /* Invalid transition - ambiguous flags */
 310                         mutex_exit(&qp->qp_lock);
 311                         /* Set "status" and "errormsg" and goto failure */
 312                         TAVOR_TNF_FAIL(IBT_QP_STATE_INVALID,
 313                             "init: ambiguous flags");
 314                         goto qpmod_fail;
 315 
 316                 } else if ((flags & IBT_CEP_SET_INIT_RTR) ||
 317                     ((flags & IBT_CEP_SET_STATE) &&
 318                     (mod_state == IBT_STATE_RTR))) {
 319                         /*
 320                          * Attempt to transition from "Init" to "RTR"
 321                          */
 322                         status = tavor_qp_init2rtr(state, qp, flags, info_p);
 323                         if (status != DDI_SUCCESS) {
 324                                 mutex_exit(&qp->qp_lock);
 325                                 /* Set "status"/"errormsg", goto failure */
 326                                 TAVOR_TNF_FAIL(status, "init to rtr");
 327                                 goto qpmod_fail;
 328                         }
 329                         qp->qp_state = TAVOR_QP_RTR;
 330 
 331                 } else if ((flags & IBT_CEP_SET_STATE) &&
 332                     (mod_state == IBT_STATE_INIT)) {
 333                         /*
 334                          * Attempt to transition from "Init" to "Init"
 335                          */
 336                         status = tavor_qp_init2init(state, qp, flags, info_p);
 337                         if (status != DDI_SUCCESS) {
 338                                 mutex_exit(&qp->qp_lock);
 339                                 /* Set "status"/"errormsg", goto failure */
 340                                 TAVOR_TNF_FAIL(status, "init to init");
 341                                 goto qpmod_fail;
 342                         }
 343                         qp->qp_state = TAVOR_QP_INIT;
 344 
 345                 } else if ((flags & IBT_CEP_SET_STATE) &&
 346                     (mod_state == IBT_STATE_RESET)) {
 347                         /*
 348                          * Attempt to transition from "Init" to "Reset"
 349                          */
 350                         status = tavor_qp_to_reset(state, qp);
 351                         if (status != DDI_SUCCESS) {
 352                                 mutex_exit(&qp->qp_lock);
 353                                 /* Set "status"/"errormsg", goto failure */
 354                                 TAVOR_TNF_FAIL(status, "init to reset");
 355                                 goto qpmod_fail;
 356                         }
 357                         qp->qp_state = TAVOR_QP_RESET;
 358 
 359                         /*
 360                          * Do any additional handling necessary for the
 361                          * transition _to_ the "Reset" state (e.g. update the
 362                          * workQ WRID lists)
 363                          */
 364                         tavor_wrid_to_reset_handling(state, qp);
 365 
 366                 } else if ((flags & IBT_CEP_SET_STATE) &&
 367                     (mod_state == IBT_STATE_ERROR)) {
 368                         /*
 369                          * Attempt to transition from "Init" to "Error"
 370                          */
 371                         status = tavor_qp_to_error(state, qp);
 372                         if (status != DDI_SUCCESS) {
 373                                 mutex_exit(&qp->qp_lock);
 374                                 /* Set "status"/"errormsg", goto failure */
 375                                 TAVOR_TNF_FAIL(status, "init to error");
 376                                 goto qpmod_fail;
 377                         }
 378                         qp->qp_state = TAVOR_QP_ERR;
 379 
 380                 } else {
 381                         /* Invalid transition - return error */
 382                         mutex_exit(&qp->qp_lock);
 383                         /* Set "status" and "errormsg" and goto failure */
 384                         TAVOR_TNF_FAIL(IBT_QP_STATE_INVALID,
 385                             "init: invalid transition");
 386                         goto qpmod_fail;
 387                 }
 388                 break;
 389 
 390         case TAVOR_QP_RTR:
 391                 okflags = (IBT_CEP_SET_STATE | IBT_CEP_SET_RTR_RTS |
 392                     IBT_CEP_SET_TIMEOUT | IBT_CEP_SET_RETRY |
 393                     IBT_CEP_SET_RNR_NAK_RETRY | IBT_CEP_SET_RDMARA_OUT |
 394                     IBT_CEP_SET_RDMA_R | IBT_CEP_SET_RDMA_W |
 395                     IBT_CEP_SET_ATOMIC | IBT_CEP_SET_QKEY |
 396                     IBT_CEP_SET_ALT_PATH | IBT_CEP_SET_MIG |
 397                     IBT_CEP_SET_MIN_RNR_NAK);
 398 
 399                 /*
 400                  * Check for attempts to modify invalid attributes from the
 401                  * "RTR" state
 402                  */
 403                 if (flags & ~okflags) {
 404                         mutex_exit(&qp->qp_lock);
 405                         /* Set "status" and "errormsg" and goto failure */
 406                         TAVOR_TNF_FAIL(IBT_QP_ATTR_RO, "rtr: invalid flag");
 407                         goto qpmod_fail;
 408                 }
 409 
 410                 /*
 411                  * Verify state transition is to either "RTS", "Reset",
 412                  * or "Error"
 413                  */
 414                 if ((flags & IBT_CEP_SET_RTR_RTS) &&
 415                     (flags & IBT_CEP_SET_STATE) &&
 416                     (mod_state != IBT_STATE_RTS)) {
 417                         /* Invalid transition - ambiguous flags */
 418                         mutex_exit(&qp->qp_lock);
 419                         /* Set "status" and "errormsg" and goto failure */
 420                         TAVOR_TNF_FAIL(IBT_QP_STATE_INVALID,
 421                             "reset: ambiguous flags");
 422                         goto qpmod_fail;
 423 
 424                 } else if ((flags & IBT_CEP_SET_RTR_RTS) ||
 425                     ((flags & IBT_CEP_SET_STATE) &&
 426                     (mod_state == IBT_STATE_RTS))) {
 427                         /*
 428                          * Attempt to transition from "RTR" to "RTS"
 429                          */
 430                         status = tavor_qp_rtr2rts(state, qp, flags, info_p);
 431                         if (status != DDI_SUCCESS) {
 432                                 mutex_exit(&qp->qp_lock);
 433                                 /* Set "status"/"errormsg", goto failure */
 434                                 TAVOR_TNF_FAIL(status, "rtr to rts");
 435                                 goto qpmod_fail;
 436                         }
 437                         qp->qp_state = TAVOR_QP_RTS;
 438 
 439                 } else if ((flags & IBT_CEP_SET_STATE) &&
 440                     (mod_state == IBT_STATE_RESET)) {
 441                         /*
 442                          * Attempt to transition from "RTR" to "Reset"
 443                          */
 444                         status = tavor_qp_to_reset(state, qp);
 445                         if (status != DDI_SUCCESS) {
 446                                 mutex_exit(&qp->qp_lock);
 447                                 /* Set "status"/"errormsg", goto failure */
 448                                 TAVOR_TNF_FAIL(status, "rtr to reset");
 449                                 goto qpmod_fail;
 450                         }
 451                         qp->qp_state = TAVOR_QP_RESET;
 452 
 453                         /*
 454                          * Do any additional handling necessary for the
 455                          * transition _to_ the "Reset" state (e.g. update the
 456                          * workQ WRID lists)
 457                          */
 458                         tavor_wrid_to_reset_handling(state, qp);
 459 
 460                 } else if ((flags & IBT_CEP_SET_STATE) &&
 461                     (mod_state == IBT_STATE_ERROR)) {
 462                         /*
 463                          * Attempt to transition from "RTR" to "Error"
 464                          */
 465                         status = tavor_qp_to_error(state, qp);
 466                         if (status != DDI_SUCCESS) {
 467                                 mutex_exit(&qp->qp_lock);
 468                                 /* Set "status"/"errormsg", goto failure */
 469                                 TAVOR_TNF_FAIL(status, "rtr to error");
 470                                 goto qpmod_fail;
 471                         }
 472                         qp->qp_state = TAVOR_QP_ERR;
 473 
 474                 } else {
 475                         /* Invalid transition - return error */
 476                         mutex_exit(&qp->qp_lock);
 477                         /* Set "status" and "errormsg" and goto failure */
 478                         TAVOR_TNF_FAIL(IBT_QP_STATE_INVALID,
 479                             "rtr: invalid transition");
 480                         goto qpmod_fail;
 481                 }
 482                 break;
 483 
 484         case TAVOR_QP_RTS:
 485                 okflags = (IBT_CEP_SET_STATE | IBT_CEP_SET_RDMA_R |
 486                     IBT_CEP_SET_RDMA_W | IBT_CEP_SET_ATOMIC |
 487                     IBT_CEP_SET_QKEY | IBT_CEP_SET_ALT_PATH |
 488                     IBT_CEP_SET_MIG | IBT_CEP_SET_MIN_RNR_NAK |
 489                     IBT_CEP_SET_SQD_EVENT);
 490 
 491                 /*
 492                  * Check for attempts to modify invalid attributes from the
 493                  * "RTS" state
 494                  */
 495                 if (flags & ~okflags) {
 496                         mutex_exit(&qp->qp_lock);
 497                         /* Set "status" and "errormsg" and goto failure */
 498                         TAVOR_TNF_FAIL(IBT_QP_ATTR_RO, "rts: invalid flag");
 499                         goto qpmod_fail;
 500                 }
 501 
 502                 /*
 503                  * Verify state transition is to either "RTS", "SQD", "Reset",
 504                  * or "Error"
 505                  */
 506                 if ((flags & IBT_CEP_SET_STATE) &&
 507                     (mod_state == IBT_STATE_RTS)) {
 508                         /*
 509                          * Attempt to transition from "RTS" to "RTS"
 510                          */
 511                         status = tavor_qp_rts2rts(state, qp, flags, info_p);
 512                         if (status != DDI_SUCCESS) {
 513                                 mutex_exit(&qp->qp_lock);
 514                                 /* Set "status"/"errormsg", goto failure */
 515                                 TAVOR_TNF_FAIL(status, "rts to rts");
 516                                 goto qpmod_fail;
 517                         }
 518                         /* qp->qp_state = TAVOR_QP_RTS; */
 519 
 520                 } else if ((flags & IBT_CEP_SET_STATE) &&
 521                     (mod_state == IBT_STATE_SQD)) {
 522                         /*
 523                          * Attempt to transition from "RTS" to "SQD"
 524                          */
 525                         status = tavor_qp_rts2sqd(state, qp, flags);
 526                         if (status != DDI_SUCCESS) {
 527                                 mutex_exit(&qp->qp_lock);
 528                                 /* Set "status"/"errormsg", goto failure */
 529                                 TAVOR_TNF_FAIL(status, "rts to sqd");
 530                                 goto qpmod_fail;
 531                         }
 532                         qp->qp_state = TAVOR_QP_SQD;
 533 
 534                 } else if ((flags & IBT_CEP_SET_STATE) &&
 535                     (mod_state == IBT_STATE_RESET)) {
 536                         /*
 537                          * Attempt to transition from "RTS" to "Reset"
 538                          */
 539                         status = tavor_qp_to_reset(state, qp);
 540                         if (status != DDI_SUCCESS) {
 541                                 mutex_exit(&qp->qp_lock);
 542                                 /* Set "status"/"errormsg", goto failure */
 543                                 TAVOR_TNF_FAIL(status, "rts to reset");
 544                                 goto qpmod_fail;
 545                         }
 546                         qp->qp_state = TAVOR_QP_RESET;
 547 
 548                         /*
 549                          * Do any additional handling necessary for the
 550                          * transition _to_ the "Reset" state (e.g. update the
 551                          * workQ WRID lists)
 552                          */
 553                         tavor_wrid_to_reset_handling(state, qp);
 554 
 555                 } else if ((flags & IBT_CEP_SET_STATE) &&
 556                     (mod_state == IBT_STATE_ERROR)) {
 557                         /*
 558                          * Attempt to transition from "RTS" to "Error"
 559                          */
 560                         status = tavor_qp_to_error(state, qp);
 561                         if (status != DDI_SUCCESS) {
 562                                 mutex_exit(&qp->qp_lock);
 563                                 /* Set "status"/"errormsg", goto failure */
 564                                 TAVOR_TNF_FAIL(status, "rts to error");
 565                                 goto qpmod_fail;
 566                         }
 567                         qp->qp_state = TAVOR_QP_ERR;
 568 
 569                 } else {
 570                         /* Invalid transition - return error */
 571                         mutex_exit(&qp->qp_lock);
 572                         /* Set "status" and "errormsg" and goto failure */
 573                         TAVOR_TNF_FAIL(IBT_QP_STATE_INVALID,
 574                             "rts: invalid transition");
 575                         goto qpmod_fail;
 576                 }
 577                 break;
 578 
 579         case TAVOR_QP_SQERR:
 580                 okflags = (IBT_CEP_SET_STATE | IBT_CEP_SET_RDMA_R |
 581                     IBT_CEP_SET_RDMA_W | IBT_CEP_SET_ATOMIC |
 582                     IBT_CEP_SET_QKEY | IBT_CEP_SET_MIN_RNR_NAK);
 583 
 584                 /*
 585                  * Check for attempts to modify invalid attributes from the
 586                  * "SQErr" state
 587                  */
 588                 if (flags & ~okflags) {
 589                         mutex_exit(&qp->qp_lock);
 590                         /* Set "status" and "errormsg" and goto failure */
 591                         TAVOR_TNF_FAIL(IBT_QP_ATTR_RO, "sqerr: invalid flag");
 592                         goto qpmod_fail;
 593                 }
 594 
 595                 /*
 596                  * Verify state transition is to either "RTS", "Reset", or
 597                  * "Error"
 598                  */
 599                 if ((flags & IBT_CEP_SET_STATE) &&
 600                     (mod_state == IBT_STATE_RTS)) {
 601                         /*
 602                          * Attempt to transition from "SQErr" to "RTS"
 603                          */
 604                         status = tavor_qp_sqerr2rts(state, qp, flags, info_p);
 605                         if (status != DDI_SUCCESS) {
 606                                 mutex_exit(&qp->qp_lock);
 607                                 /* Set "status"/"errormsg", goto failure */
 608                                 TAVOR_TNF_FAIL(status, "sqerr to rts");
 609                                 goto qpmod_fail;
 610                         }
 611                         qp->qp_state = TAVOR_QP_RTS;
 612 
 613                 } else if ((flags & IBT_CEP_SET_STATE) &&
 614                     (mod_state == IBT_STATE_RESET)) {
 615                         /*
 616                          * Attempt to transition from "SQErr" to "Reset"
 617                          */
 618                         status = tavor_qp_to_reset(state, qp);
 619                         if (status != DDI_SUCCESS) {
 620                                 mutex_exit(&qp->qp_lock);
 621                                 /* Set "status"/"errormsg", goto failure */
 622                                 TAVOR_TNF_FAIL(status, "sqerr to reset");
 623                                 goto qpmod_fail;
 624                         }
 625                         qp->qp_state = TAVOR_QP_RESET;
 626 
 627                         /*
 628                          * Do any additional handling necessary for the
 629                          * transition _to_ the "Reset" state (e.g. update the
 630                          * workQ WRID lists)
 631                          */
 632                         tavor_wrid_to_reset_handling(state, qp);
 633 
 634                 } else if ((flags & IBT_CEP_SET_STATE) &&
 635                     (mod_state == IBT_STATE_ERROR)) {
 636                         /*
 637                          * Attempt to transition from "SQErr" to "Error"
 638                          */
 639                         status = tavor_qp_to_error(state, qp);
 640                         if (status != DDI_SUCCESS) {
 641                                 mutex_exit(&qp->qp_lock);
 642                                 /* Set "status"/"errormsg", goto failure */
 643                                 TAVOR_TNF_FAIL(status, "sqerr to error");
 644                                 goto qpmod_fail;
 645                         }
 646                         qp->qp_state = TAVOR_QP_ERR;
 647 
 648                 } else {
 649                         /* Invalid transition - return error */
 650                         mutex_exit(&qp->qp_lock);
 651                         /* Set "status" and "errormsg" and goto failure */
 652                         TAVOR_TNF_FAIL(IBT_QP_STATE_INVALID,
 653                             "sqerr: invalid transition");
 654                         goto qpmod_fail;
 655                 }
 656                 break;
 657 
 658         case TAVOR_QP_SQD:
 659                 okflags = (IBT_CEP_SET_STATE | IBT_CEP_SET_ADDS_VECT |
 660                     IBT_CEP_SET_ALT_PATH | IBT_CEP_SET_MIG |
 661                     IBT_CEP_SET_RDMARA_OUT | IBT_CEP_SET_RDMARA_IN |
 662                     IBT_CEP_SET_QKEY | IBT_CEP_SET_PKEY_IX |
 663                     IBT_CEP_SET_TIMEOUT | IBT_CEP_SET_RETRY |
 664                     IBT_CEP_SET_RNR_NAK_RETRY | IBT_CEP_SET_PORT |
 665                     IBT_CEP_SET_MIN_RNR_NAK | IBT_CEP_SET_RDMA_R |
 666                     IBT_CEP_SET_RDMA_W | IBT_CEP_SET_ATOMIC);
 667 
 668                 /*
 669                  * Check for attempts to modify invalid attributes from the
 670                  * "SQD" state
 671                  */
 672                 if (flags & ~okflags) {
 673                         mutex_exit(&qp->qp_lock);
 674                         /* Set "status" and "errormsg" and goto failure */
 675                         TAVOR_TNF_FAIL(IBT_QP_ATTR_RO, "sqd: invalid flag");
 676                         goto qpmod_fail;
 677                 }
 678 
 679                 /*
 680                  * Verify state transition is to either "SQD", "RTS", "Reset",
 681                  * or "Error"
 682                  */
 683 
 684                 if ((flags & IBT_CEP_SET_STATE) &&
 685                     (mod_state == IBT_STATE_SQD)) {
 686                         /*
 687                          * Attempt to transition from "SQD" to "SQD"
 688                          */
 689                         status = tavor_qp_sqd2sqd(state, qp, flags, info_p);
 690                         if (status != DDI_SUCCESS) {
 691                                 mutex_exit(&qp->qp_lock);
 692                                 /* Set "status"/"errormsg", goto failure */
 693                                 TAVOR_TNF_FAIL(status, "sqd to sqd");
 694                                 goto qpmod_fail;
 695                         }
 696                         qp->qp_state = TAVOR_QP_SQD;
 697 
 698                 } else if ((flags & IBT_CEP_SET_STATE) &&
 699                     (mod_state == IBT_STATE_RTS)) {
 700                         /*
 701                          * If still draining SQ, then fail transition attempt
 702                          * to RTS.
 703                          */
 704                         if (qp->qp_sqd_still_draining) {
 705                                 mutex_exit(&qp->qp_lock);
 706                                 /* Set "status"/"errormsg", goto failure */
 707                                 status = IBT_QP_STATE_INVALID;
 708                                 TAVOR_TNF_FAIL(status, "sqd to rts; draining");
 709                                 goto qpmod_fail;
 710                         }
 711 
 712                         /*
 713                          * Attempt to transition from "SQD" to "RTS"
 714                          */
 715                         status = tavor_qp_sqd2rts(state, qp, flags, info_p);
 716                         if (status != DDI_SUCCESS) {
 717                                 mutex_exit(&qp->qp_lock);
 718                                 /* Set "status"/"errormsg", goto failure */
 719                                 TAVOR_TNF_FAIL(status, "sqd to rts");
 720                                 goto qpmod_fail;
 721                         }
 722                         qp->qp_state = TAVOR_QP_RTS;
 723 
 724                 } else if ((flags & IBT_CEP_SET_STATE) &&
 725                     (mod_state == IBT_STATE_RESET)) {
 726                         /*
 727                          * Attempt to transition from "SQD" to "Reset"
 728                          */
 729                         status = tavor_qp_to_reset(state, qp);
 730                         if (status != DDI_SUCCESS) {
 731                                 mutex_exit(&qp->qp_lock);
 732                                 /* Set "status"/"errormsg", goto failure */
 733                                 TAVOR_TNF_FAIL(status, "sqd to reset");
 734                                 goto qpmod_fail;
 735                         }
 736                         qp->qp_state = TAVOR_QP_RESET;
 737 
 738                         /*
 739                          * Do any additional handling necessary for the
 740                          * transition _to_ the "Reset" state (e.g. update the
 741                          * workQ WRID lists)
 742                          */
 743                         tavor_wrid_to_reset_handling(state, qp);
 744 
 745                 } else if ((flags & IBT_CEP_SET_STATE) &&
 746                     (mod_state == IBT_STATE_ERROR)) {
 747                         /*
 748                          * Attempt to transition from "SQD" to "Error"
 749                          */
 750                         status = tavor_qp_to_error(state, qp);
 751                         if (status != DDI_SUCCESS) {
 752                                 mutex_exit(&qp->qp_lock);
 753                                 /* Set "status"/"errormsg", goto failure */
 754                                 TAVOR_TNF_FAIL(status, "sqd to error");
 755                                 goto qpmod_fail;
 756                         }
 757                         qp->qp_state = TAVOR_QP_ERR;
 758 
 759                 } else {
 760                         /* Invalid transition - return error */
 761                         mutex_exit(&qp->qp_lock);
 762                         /* Set "status" and "errormsg" and goto failure */
 763                         TAVOR_TNF_FAIL(IBT_QP_STATE_INVALID,
 764                             "sqd: invalid transition");
 765                         goto qpmod_fail;
 766                 }
 767                 break;
 768 
 769         case TAVOR_QP_ERR:
 770                 /*
 771                  * Verify state transition is to either "Reset" or back to
 772                  * "Error"
 773                  */
 774                 if ((flags & IBT_CEP_SET_STATE) &&
 775                     (mod_state == IBT_STATE_RESET)) {
 776                         /*
 777                          * Attempt to transition from "Error" to "Reset"
 778                          */
 779                         status = tavor_qp_to_reset(state, qp);
 780                         if (status != DDI_SUCCESS) {
 781                                 mutex_exit(&qp->qp_lock);
 782                                 /* Set "status"/"errormsg", goto failure */
 783                                 TAVOR_TNF_FAIL(status, "error to reset");
 784                                 goto qpmod_fail;
 785                         }
 786                         qp->qp_state = TAVOR_QP_RESET;
 787 
 788                         /*
 789                          * Do any additional handling necessary for the
 790                          * transition _to_ the "Reset" state (e.g. update the
 791                          * workQ WRID lists)
 792                          */
 793                         tavor_wrid_to_reset_handling(state, qp);
 794 
 795                 } else if ((flags & IBT_CEP_SET_STATE) &&
 796                     (mod_state == IBT_STATE_ERROR)) {
 797                         /*
 798                          * Attempt to transition from "Error" back to "Error"
 799                          *    Nothing to do here really... just drop the lock
 800                          *    and return success.  The qp->qp_state should
 801                          *    already be set to TAVOR_QP_ERR.
 802                          *
 803                          */
 804                         mutex_exit(&qp->qp_lock);
 805                         TNF_PROBE_0_DEBUG(tavor_qp_modify_err2err,
 806                             TAVOR_TNF_TRACE, "");
 807                         TAVOR_TNF_EXIT(tavor_qp_modify);
 808                         return (DDI_SUCCESS);
 809 
 810                 } else {
 811                         /* Invalid transition - return error */
 812                         mutex_exit(&qp->qp_lock);
 813                         /* Set "status" and "errormsg" and goto failure */
 814                         TAVOR_TNF_FAIL(IBT_QP_STATE_INVALID,
 815                             "error: invalid transition");
 816                         goto qpmod_fail;
 817                 }
 818                 break;
 819 
 820         default:
 821                 /*
 822                  * Invalid QP state.  If we got here then it's a warning of
 823                  * a probably serious problem.  So print a message and return
 824                  * failure
 825                  */
 826                 mutex_exit(&qp->qp_lock);
 827                 TAVOR_WARNING(state, "unknown QP state in modify");
 828                 /* Set "status" and "errormsg" and goto failure */
 829                 TAVOR_TNF_FAIL(IBT_QP_STATE_INVALID, "invalid curr QP state");
 830                 goto qpmod_fail;
 831         }
 832 
 833         mutex_exit(&qp->qp_lock);
 834         TAVOR_TNF_EXIT(tavor_qp_modify);
 835         return (DDI_SUCCESS);
 836 
 837 qpmod_fail:
 838         TNF_PROBE_1(tavor_qp_modify_fail, TAVOR_TNF_ERROR, "",
 839             tnf_string, msg, errormsg);
 840         TAVOR_TNF_EXIT(tavor_qp_modify);
 841         return (status);
 842 }
 843 
 844 
 845 /*
 846  * tavor_qp_reset2init()
 847  *    Context: Can be called from interrupt or base context.
 848  */
 849 static int
 850 tavor_qp_reset2init(tavor_state_t *state, tavor_qphdl_t qp,
 851     ibt_qp_info_t *info_p)
 852 {
 853         tavor_hw_qpc_t          *qpc;
 854         ibt_qp_rc_attr_t        *rc;
 855         ibt_qp_ud_attr_t        *ud;
 856         ibt_qp_uc_attr_t        *uc;
 857         uint_t                  portnum, pkeyindx;
 858         int                     status;
 859 
 860         TAVOR_TNF_ENTER(tavor_qp_reset2init);
 861 
 862         ASSERT(MUTEX_HELD(&qp->qp_lock));
 863 
 864         /*
 865          * Grab the temporary QPC entry from QP software state
 866          */
 867         qpc = &qp->qpc;
 868 
 869         /*
 870          * Fill in the common and/or Tavor-specific fields in the QPC
 871          */
 872         if (qp->qp_is_special) {
 873                 qpc->serv_type       = TAVOR_QP_MLX;
 874         } else {
 875                 qpc->serv_type       = qp->qp_serv_type;
 876         }
 877         qpc->pm_state                = TAVOR_QP_PMSTATE_MIGRATED;
 878         qpc->de                      = TAVOR_QP_DESC_EVT_ENABLED;
 879         qpc->sched_q         = TAVOR_QP_SCHEDQ_GET(qp->qp_qpnum);
 880         if (qp->qp_is_umap) {
 881                 qpc->usr_page = qp->qp_uarpg;
 882         } else {
 883                 qpc->usr_page = 0;
 884         }
 885         qpc->pd                      = qp->qp_pdhdl->pd_pdnum;
 886         qpc->wqe_baseaddr    = 0;
 887         qpc->wqe_lkey                = qp->qp_mrhdl->mr_lkey;
 888         qpc->ssc             = qp->qp_sq_sigtype;
 889         qpc->cqn_snd         = qp->qp_sq_cqhdl->cq_cqnum;
 890         qpc->rsc             = TAVOR_QP_RQ_ALL_SIGNALED;
 891         qpc->cqn_rcv         = qp->qp_rq_cqhdl->cq_cqnum;
 892         qpc->srq_en          = qp->qp_srq_en;
 893 
 894         if (qp->qp_srq_en == TAVOR_QP_SRQ_ENABLED) {
 895                 qpc->srq_number      = qp->qp_srqhdl->srq_srqnum;
 896         } else {
 897                 qpc->srq_number = 0;
 898         }
 899 
 900         /*
 901          * Now fill in the QPC fields which are specific to transport type
 902          */
 903         if (qp->qp_serv_type == TAVOR_QP_UD) {
 904                 ud = &info_p->qp_transport.ud;
 905 
 906                 /* Set the QKey */
 907                 qpc->qkey = ud->ud_qkey;
 908 
 909                 /* Check for valid port number and fill it in */
 910                 portnum = ud->ud_port;
 911                 if (tavor_portnum_is_valid(state, portnum)) {
 912                         qpc->pri_addr_path.portnum = portnum;
 913                 } else {
 914                         TNF_PROBE_1(tavor_qp_reset2init_inv_port_fail,
 915                             TAVOR_TNF_ERROR, "", tnf_uint, port, portnum);
 916                         TAVOR_TNF_EXIT(tavor_qp_reset2init);
 917                         return (IBT_HCA_PORT_INVALID);
 918                 }
 919 
 920                 /* Check for valid PKey index and fill it in */
 921                 pkeyindx = ud->ud_pkey_ix;
 922                 if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
 923                         qpc->pri_addr_path.pkey_indx = pkeyindx;
 924                         qp->qp_pkeyindx = pkeyindx;
 925                 } else {
 926                         TNF_PROBE_1(tavor_qp_reset2init_inv_pkey_fail,
 927                             TAVOR_TNF_ERROR, "", tnf_uint, pkeyindx, pkeyindx);
 928                         TAVOR_TNF_EXIT(tavor_qp_reset2init);
 929                         return (IBT_PKEY_IX_ILLEGAL);
 930                 }
 931 
 932         } else if (qp->qp_serv_type == TAVOR_QP_RC) {
 933                 rc = &info_p->qp_transport.rc;
 934 
 935                 /* Set the RDMA (recv) enable/disable flags */
 936                 qpc->rre = (info_p->qp_flags & IBT_CEP_RDMA_RD) ? 1 : 0;
 937                 qpc->rwe = (info_p->qp_flags & IBT_CEP_RDMA_WR) ? 1 : 0;
 938                 qpc->rae = (info_p->qp_flags & IBT_CEP_ATOMIC)  ? 1 : 0;
 939 
 940                 /* Check for valid port number and fill it in */
 941                 portnum = rc->rc_path.cep_hca_port_num;
 942                 if (tavor_portnum_is_valid(state, portnum)) {
 943                         qpc->pri_addr_path.portnum = portnum;
 944                 } else {
 945                         TNF_PROBE_1(tavor_qp_reset2init_inv_port_fail,
 946                             TAVOR_TNF_ERROR, "", tnf_uint, port, portnum);
 947                         TAVOR_TNF_EXIT(tavor_qp_reset2init);
 948                         return (IBT_HCA_PORT_INVALID);
 949                 }
 950 
 951                 /* Check for valid PKey index and fill it in */
 952                 pkeyindx = rc->rc_path.cep_pkey_ix;
 953                 if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
 954                         qpc->pri_addr_path.pkey_indx = pkeyindx;
 955                 } else {
 956                         TNF_PROBE_1(tavor_qp_reset2init_inv_pkey_fail,
 957                             TAVOR_TNF_ERROR, "", tnf_uint, pkeyindx, pkeyindx);
 958                         TAVOR_TNF_EXIT(tavor_qp_reset2init);
 959                         return (IBT_PKEY_IX_ILLEGAL);
 960                 }
 961 
 962         } else if (qp->qp_serv_type == TAVOR_QP_UC) {
 963                 uc = &info_p->qp_transport.uc;
 964 
 965                 /*
 966                  * Set the RDMA (recv) enable/disable flags.  Note: RDMA Read
 967                  * and Atomic are ignored by default.
 968                  */
 969                 qpc->rwe = (info_p->qp_flags & IBT_CEP_RDMA_WR) ? 1 : 0;
 970 
 971                 /* Check for valid port number and fill it in */
 972                 portnum = uc->uc_path.cep_hca_port_num;
 973                 if (tavor_portnum_is_valid(state, portnum)) {
 974                         qpc->pri_addr_path.portnum = portnum;
 975                 } else {
 976                         TNF_PROBE_1(tavor_qp_reset2init_inv_port_fail,
 977                             TAVOR_TNF_ERROR, "", tnf_uint, port, portnum);
 978                         TAVOR_TNF_EXIT(tavor_qp_reset2init);
 979                         return (IBT_HCA_PORT_INVALID);
 980                 }
 981 
 982                 /* Check for valid PKey index and fill it in */
 983                 pkeyindx = uc->uc_path.cep_pkey_ix;
 984                 if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
 985                         qpc->pri_addr_path.pkey_indx = pkeyindx;
 986                 } else {
 987                         TNF_PROBE_1(tavor_qp_reset2init_inv_pkey_fail,
 988                             TAVOR_TNF_ERROR, "", tnf_uint, pkeyindx, pkeyindx);
 989                         TAVOR_TNF_EXIT(tavor_qp_reset2init);
 990                         return (IBT_PKEY_IX_ILLEGAL);
 991                 }
 992         } else {
 993                 /*
 994                  * Invalid QP transport type. If we got here then it's a
 995                  * warning of a probably serious problem.  So print a message
 996                  * and return failure
 997                  */
 998                 TAVOR_WARNING(state, "unknown QP transport type in rst2init");
 999                 TNF_PROBE_0(tavor_qp_reset2init_inv_transtype_fail,
1000                     TAVOR_TNF_ERROR, "");
1001                 TAVOR_TNF_EXIT(tavor_qp_reset2init);
1002                 return (ibc_get_ci_failure(0));
1003         }
1004 
1005         /*
1006          * Post the RST2INIT_QP command to the Tavor firmware
1007          *
1008          * We do a TAVOR_NOSLEEP here because we are still holding the
1009          * "qp_lock".  If we got raised to interrupt level by priority
1010          * inversion, we do not want to block in this routine waiting for
1011          * success.
1012          */
1013         status = tavor_cmn_qp_cmd_post(state, RST2INIT_QP, qpc, qp->qp_qpnum,
1014             0, TAVOR_CMD_NOSLEEP_SPIN);
1015         if (status != TAVOR_CMD_SUCCESS) {
1016                 cmn_err(CE_CONT, "Tavor: RST2INIT_QP command failed: %08x\n",
1017                     status);
1018                 TNF_PROBE_1(tavor_qp_reset2init_cmd_fail, TAVOR_TNF_ERROR, "",
1019                     tnf_uint, status, status);
1020                 TAVOR_TNF_EXIT(tavor_qp_reset2init);
1021                 return (ibc_get_ci_failure(0));
1022         }
1023 
1024         TAVOR_TNF_EXIT(tavor_qp_reset2init);
1025         return (DDI_SUCCESS);
1026 }
1027 
1028 
1029 /*
1030  * tavor_qp_init2init()
1031  *    Context: Can be called from interrupt or base context.
1032  */
1033 static int
1034 tavor_qp_init2init(tavor_state_t *state, tavor_qphdl_t qp,
1035     ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p)
1036 {
1037         tavor_hw_qpc_t          *qpc;
1038         ibt_qp_rc_attr_t        *rc;
1039         ibt_qp_ud_attr_t        *ud;
1040         ibt_qp_uc_attr_t        *uc;
1041         uint_t                  portnum, pkeyindx;
1042         uint32_t                opmask = 0;
1043         int                     status;
1044 
1045         TAVOR_TNF_ENTER(tavor_qp_init2init);
1046 
1047         ASSERT(MUTEX_HELD(&qp->qp_lock));
1048 
1049         /*
1050          * Grab the temporary QPC entry from QP software state
1051          */
1052         qpc = &qp->qpc;
1053 
1054         /*
1055          * Since there are no common and/or Tavor-specific fields to be filled
1056          * in for this command, we begin with the QPC fields which are
1057          * specific to transport type.
1058          */
1059         if (qp->qp_serv_type == TAVOR_QP_UD) {
1060                 ud = &info_p->qp_transport.ud;
1061 
1062                 /*
1063                  * If we are attempting to modify the port for this QP, then
1064                  * check for valid port number and fill it in.  Also set the
1065                  * appropriate flag in the "opmask" parameter.
1066                  */
1067                 if (flags & IBT_CEP_SET_PORT) {
1068                         portnum = ud->ud_port;
1069                         if (tavor_portnum_is_valid(state, portnum)) {
1070                                 qpc->pri_addr_path.portnum = portnum;
1071                         } else {
1072                                 TNF_PROBE_1(tavor_qp_init2init_inv_port_fail,
1073                                     TAVOR_TNF_ERROR, "", tnf_uint, port,
1074                                     portnum);
1075                                 TAVOR_TNF_EXIT(tavor_qp_init2init);
1076                                 return (IBT_HCA_PORT_INVALID);
1077                         }
1078                         opmask |= TAVOR_CMD_OP_PRIM_PORT;
1079                 }
1080 
1081                 /*
1082                  * If we are attempting to modify the PKey index for this QP,
1083                  * then check for valid PKey index and fill it in.  Also set
1084                  * the appropriate flag in the "opmask" parameter.
1085                  */
1086                 if (flags & IBT_CEP_SET_PKEY_IX) {
1087                         pkeyindx = ud->ud_pkey_ix;
1088                         if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
1089                                 qpc->pri_addr_path.pkey_indx = pkeyindx;
1090                                 opmask |= TAVOR_CMD_OP_PKEYINDX;
1091                                 qp->qp_pkeyindx = pkeyindx;
1092                         } else {
1093                                 TNF_PROBE_1(tavor_qp_init2init_inv_pkey_fail,
1094                                     TAVOR_TNF_ERROR, "", tnf_uint, pkeyindx,
1095                                     pkeyindx);
1096                                 TAVOR_TNF_EXIT(tavor_qp_init2init);
1097                                 return (IBT_PKEY_IX_ILLEGAL);
1098                         }
1099                 }
1100 
1101                 /*
1102                  * If we are attempting to modify the QKey for this QP, then
1103                  * fill it in and set the appropriate flag in the "opmask"
1104                  * parameter.
1105                  */
1106                 if (flags & IBT_CEP_SET_QKEY) {
1107                         qpc->qkey = ud->ud_qkey;
1108                         opmask |= TAVOR_CMD_OP_QKEY;
1109                 }
1110 
1111         } else if (qp->qp_serv_type == TAVOR_QP_RC) {
1112                 rc = &info_p->qp_transport.rc;
1113 
1114                 /*
1115                  * If we are attempting to modify the port for this QP, then
1116                  * check for valid port number and fill it in.  Also set the
1117                  * appropriate flag in the "opmask" parameter.
1118                  */
1119                 if (flags & IBT_CEP_SET_PORT) {
1120                         portnum = rc->rc_path.cep_hca_port_num;
1121                         if (tavor_portnum_is_valid(state, portnum)) {
1122                                 qpc->pri_addr_path.portnum = portnum;
1123                         } else {
1124                                 TNF_PROBE_1(tavor_qp_init2init_inv_port_fail,
1125                                     TAVOR_TNF_ERROR, "", tnf_uint, port,
1126                                     portnum);
1127                                 TAVOR_TNF_EXIT(tavor_qp_init2init);
1128                                 return (IBT_HCA_PORT_INVALID);
1129                         }
1130                         opmask |= TAVOR_CMD_OP_PRIM_PORT;
1131                 }
1132 
1133                 /*
1134                  * If we are attempting to modify the PKey index for this QP,
1135                  * then check for valid PKey index and fill it in.  Also set
1136                  * the appropriate flag in the "opmask" parameter.
1137                  */
1138                 if (flags & IBT_CEP_SET_PKEY_IX) {
1139                         pkeyindx = rc->rc_path.cep_pkey_ix;
1140                         if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
1141                                 qpc->pri_addr_path.pkey_indx = pkeyindx;
1142                                 opmask |= TAVOR_CMD_OP_PKEYINDX;
1143                         } else {
1144                                 TNF_PROBE_1(tavor_qp_init2init_inv_pkey_fail,
1145                                     TAVOR_TNF_ERROR, "", tnf_uint, pkeyindx,
1146                                     pkeyindx);
1147                                 TAVOR_TNF_EXIT(tavor_qp_init2init);
1148                                 return (IBT_PKEY_IX_ILLEGAL);
1149                         }
1150                 }
1151 
1152                 /*
1153                  * Check if any of the flags indicate a change in the RDMA
1154                  * (recv) enable/disable flags and set the appropriate flag in
1155                  * the "opmask" parameter
1156                  */
1157                 opmask |= tavor_check_rdma_enable_flags(flags, info_p, qpc);
1158 
1159         } else if (qp->qp_serv_type == TAVOR_QP_UC) {
1160                 uc = &info_p->qp_transport.uc;
1161 
1162                 /*
1163                  * If we are attempting to modify the port for this QP, then
1164                  * check for valid port number and fill it in.  Also set the
1165                  * appropriate flag in the "opmask" parameter.
1166                  */
1167                 if (flags & IBT_CEP_SET_PORT) {
1168                         portnum = uc->uc_path.cep_hca_port_num;
1169                         if (tavor_portnum_is_valid(state, portnum)) {
1170                                 qpc->pri_addr_path.portnum = portnum;
1171                         } else {
1172                                 TNF_PROBE_1(tavor_qp_init2init_inv_port_fail,
1173                                     TAVOR_TNF_ERROR, "", tnf_uint, port,
1174                                     portnum);
1175                                 TAVOR_TNF_EXIT(tavor_qp_init2init);
1176                                 return (IBT_HCA_PORT_INVALID);
1177                         }
1178                         opmask |= TAVOR_CMD_OP_PRIM_PORT;
1179                 }
1180 
1181                 /*
1182                  * If we are attempting to modify the PKey index for this QP,
1183                  * then check for valid PKey index and fill it in.  Also set
1184                  * the appropriate flag in the "opmask" parameter.
1185                  */
1186                 if (flags & IBT_CEP_SET_PKEY_IX) {
1187                         pkeyindx = uc->uc_path.cep_pkey_ix;
1188                         if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
1189                                 qpc->pri_addr_path.pkey_indx = pkeyindx;
1190                                 opmask |= TAVOR_CMD_OP_PKEYINDX;
1191                         } else {
1192                                 TNF_PROBE_1(tavor_qp_init2init_inv_pkey_fail,
1193                                     TAVOR_TNF_ERROR, "", tnf_uint, pkeyindx,
1194                                     pkeyindx);
1195                                 TAVOR_TNF_EXIT(tavor_qp_init2init);
1196                                 return (IBT_PKEY_IX_ILLEGAL);
1197                         }
1198                 }
1199 
1200                 /*
1201                  * Check if any of the flags indicate a change in the RDMA
1202                  * Write (recv) enable/disable and set the appropriate flag
1203                  * in the "opmask" parameter. Note: RDMA Read and Atomic are
1204                  * not valid for UC transport.
1205                  */
1206                 if (flags & IBT_CEP_SET_RDMA_W) {
1207                         qpc->rwe = (info_p->qp_flags & IBT_CEP_RDMA_WR) ? 1 : 0;
1208                         opmask |= TAVOR_CMD_OP_RWE;
1209                 }
1210         } else {
1211                 /*
1212                  * Invalid QP transport type. If we got here then it's a
1213                  * warning of a probably serious problem.  So print a message
1214                  * and return failure
1215                  */
1216                 TAVOR_WARNING(state, "unknown QP transport type in init2init");
1217                 TNF_PROBE_0(tavor_qp_init2init_inv_transtype_fail,
1218                     TAVOR_TNF_ERROR, "");
1219                 TAVOR_TNF_EXIT(tavor_qp_init2init);
1220                 return (ibc_get_ci_failure(0));
1221         }
1222 
1223         /*
1224          * Post the INIT2INIT_QP command to the Tavor firmware
1225          *
1226          * We do a TAVOR_NOSLEEP here because we are still holding the
1227          * "qp_lock".  If we got raised to interrupt level by priority
1228          * inversion, we do not want to block in this routine waiting for
1229          * success.
1230          */
1231         status = tavor_cmn_qp_cmd_post(state, INIT2INIT_QP, qpc, qp->qp_qpnum,
1232             opmask, TAVOR_CMD_NOSLEEP_SPIN);
1233         if (status != TAVOR_CMD_SUCCESS) {
1234                 if (status != TAVOR_CMD_BAD_QP_STATE) {
1235                         cmn_err(CE_CONT, "Tavor: INIT2INIT_QP command failed: "
1236                             "%08x\n", status);
1237                         TNF_PROBE_1(tavor_qp_init2init_cmd_fail,
1238                             TAVOR_TNF_ERROR, "", tnf_uint, status, status);
1239                         TAVOR_TNF_EXIT(tavor_qp_init2init);
1240                         return (ibc_get_ci_failure(0));
1241                 } else {
1242                         TNF_PROBE_0(tavor_qp_init2init_inv_qpstate_fail,
1243                             TAVOR_TNF_ERROR, "");
1244                         TAVOR_TNF_EXIT(tavor_qp_init2init);
1245                         return (IBT_QP_STATE_INVALID);
1246                 }
1247         }
1248 
1249         TAVOR_TNF_EXIT(tavor_qp_init2init);
1250         return (DDI_SUCCESS);
1251 }
1252 
1253 
1254 /*
1255  * tavor_qp_init2rtr()
1256  *    Context: Can be called from interrupt or base context.
1257  */
1258 static int
1259 tavor_qp_init2rtr(tavor_state_t *state, tavor_qphdl_t qp,
1260     ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p)
1261 {
1262         tavor_hw_qpc_t          *qpc;
1263         ibt_qp_rc_attr_t        *rc;
1264         ibt_qp_ud_attr_t        *ud;
1265         ibt_qp_uc_attr_t        *uc;
1266         tavor_hw_addr_path_t    *qpc_path;
1267         ibt_adds_vect_t         *adds_vect;
1268         uint_t                  portnum, pkeyindx, rdma_ra_in, rra_max;
1269         uint_t                  mtu;
1270         uint32_t                opmask = 0;
1271         int                     status;
1272 
1273         TAVOR_TNF_ENTER(tavor_qp_init2rtr);
1274 
1275         ASSERT(MUTEX_HELD(&qp->qp_lock));
1276 
1277         /*
1278          * Grab the temporary QPC entry from QP software state
1279          */
1280         qpc = &qp->qpc;
1281 
1282         /*
1283          * Since there are no common and/or Tavor-specific fields to be filled
1284          * in for this command, we begin with the QPC fields which are
1285          * specific to transport type.
1286          */
1287         if (qp->qp_serv_type == TAVOR_QP_UD) {
1288                 ud = &info_p->qp_transport.ud;
1289 
1290                 /*
1291                  * If this UD QP is also a "special QP" (QP0 or QP1), then
1292                  * the MTU is 256 bytes.  However, Tavor HW requires us to
1293                  * set the MTU to 4 (which is the IB code for a 2K MTU).
1294                  * If this is not a special QP, then we set the MTU to the
1295                  * configured maximum (which defaults to 2K).  Note: the
1296                  * QPC "msg_max" must also be set so as to correspond with
1297                  * the specified MTU value.
1298                  */
1299                 if (qp->qp_is_special) {
1300                         qpc->mtu = 4;
1301                 } else {
1302                         qpc->mtu = state->ts_cfg_profile->cp_max_mtu;
1303                 }
1304                 qpc->msg_max = qpc->mtu + 7;  /* must equal MTU plus seven */
1305 
1306                 /*
1307                  * Save away the MTU value.  This is used in future sqd2sqd
1308                  * transitions, as the MTU must remain the same in future
1309                  * changes.
1310                  */
1311                 qp->qp_save_mtu = qpc->mtu;
1312 
1313                 /*
1314                  * If we are attempting to modify the PKey index for this QP,
1315                  * then check for valid PKey index and fill it in.  Also set
1316                  * the appropriate flag in the "opmask" parameter.
1317                  */
1318                 if (flags & IBT_CEP_SET_PKEY_IX) {
1319                         pkeyindx = ud->ud_pkey_ix;
1320                         if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
1321                                 qpc->pri_addr_path.pkey_indx = pkeyindx;
1322                                 opmask |= TAVOR_CMD_OP_PKEYINDX;
1323                                 qp->qp_pkeyindx = pkeyindx;
1324                         } else {
1325                                 TNF_PROBE_1(tavor_qp_init2rtr_inv_pkey_fail,
1326                                     TAVOR_TNF_ERROR, "", tnf_uint, pkeyindx,
1327                                     pkeyindx);
1328                                 TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1329                                 return (IBT_PKEY_IX_ILLEGAL);
1330                         }
1331                 }
1332 
1333                 /*
1334                  * If we are attempting to modify the QKey for this QP, then
1335                  * fill it in and set the appropriate flag in the "opmask"
1336                  * parameter.
1337                  */
1338                 if (flags & IBT_CEP_SET_QKEY) {
1339                         qpc->qkey = ud->ud_qkey;
1340                         opmask |= TAVOR_CMD_OP_QKEY;
1341                 }
1342 
1343         } else if (qp->qp_serv_type == TAVOR_QP_RC) {
1344                 rc = &info_p->qp_transport.rc;
1345                 qpc_path = &qpc->pri_addr_path;
1346                 adds_vect = &rc->rc_path.cep_adds_vect;
1347 
1348                 /*
1349                  * Set the common primary address path fields
1350                  */
1351                 status = tavor_set_addr_path(state, adds_vect, qpc_path,
1352                     TAVOR_ADDRPATH_QP, qp);
1353                 if (status != DDI_SUCCESS) {
1354                         TNF_PROBE_0(tavor_qp_init2rtr_setaddrpath_fail,
1355                             TAVOR_TNF_ERROR, "");
1356                         TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1357                         return (status);
1358                 }
1359 
1360                 /*
1361                  * The following values are apparently "required" here (as
1362                  * they are part of the IBA-defined "Remote Node Address
1363                  * Vector").  However, they are also going to be "required"
1364                  * later - at RTR2RTS_QP time.  Not sure why.  But we set
1365                  * them here anyway.
1366                  */
1367                 qpc_path->rnr_retry  = rc->rc_rnr_retry_cnt;
1368                 qpc->retry_cnt               = rc->rc_retry_cnt;
1369                 qpc_path->ack_timeout        = rc->rc_path.cep_timeout;
1370 
1371                 /*
1372                  * Setup the destination QP, recv PSN, MTU, max msg size,etc.
1373                  * Note max message size is defined to be the maximum IB
1374                  * allowed message size (which is 2^31 bytes).  Also max
1375                  * MTU is defined by HCA port properties.
1376                  */
1377                 qpc->rem_qpn   = rc->rc_dst_qpn;
1378                 qpc->next_rcv_psn = rc->rc_rq_psn;
1379                 qpc->msg_max   = TAVOR_QP_LOG_MAX_MSGSZ;
1380 
1381                 /*
1382                  * If this QP is using an SRQ, 'ric' must be set to 1.
1383                  */
1384                 qpc->ric = (qp->qp_srq_en == TAVOR_QP_SRQ_ENABLED) ? 1 : 0;
1385                 mtu = rc->rc_path_mtu;
1386                 if (tavor_qp_validate_mtu(state, mtu) != DDI_SUCCESS) {
1387                         TNF_PROBE_1(tavor_qp_init2rtr_inv_mtu_fail,
1388                             TAVOR_TNF_ERROR, "", tnf_uint, mtu, mtu);
1389                         TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1390                         return (IBT_HCA_PORT_MTU_EXCEEDED);
1391                 }
1392                 qpc->mtu = mtu;
1393 
1394                 /*
1395                  * Save away the MTU value.  This is used in future sqd2sqd
1396                  * transitions, as the MTU must remain the same in future
1397                  * changes.
1398                  */
1399                 qp->qp_save_mtu = qpc->mtu;
1400 
1401                 /*
1402                  * Though it is a "required" parameter, "min_rnr_nak" is
1403                  * optionally specifiable in Tavor.  So we hardcode the
1404                  * optional flag here.
1405                  */
1406                 qpc->min_rnr_nak = rc->rc_min_rnr_nak;
1407                 opmask |= TAVOR_CMD_OP_MINRNRNAK;
1408 
1409                 /*
1410                  * Check that the number of specified "incoming RDMA resources"
1411                  * is valid.  And if it is, then setup the "rra_max" and
1412                  * "ra_buf_index" fields in the QPC to point to the
1413                  * pre-allocated RDB resources (in DDR)
1414                  */
1415                 rdma_ra_in = rc->rc_rdma_ra_in;
1416                 if (tavor_qp_validate_resp_rsrc(state, rc, &rra_max) !=
1417                     DDI_SUCCESS) {
1418                         TNF_PROBE_1(tavor_qp_init2rtr_inv_rdma_in_fail,
1419                             TAVOR_TNF_ERROR, "", tnf_uint, rdma_ra_in,
1420                             rdma_ra_in);
1421                         TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1422                         return (IBT_INVALID_PARAM);
1423                 }
1424                 qpc->rra_max = rra_max;
1425                 qpc->ra_buff_indx = qp->qp_rdb_ddraddr >> TAVOR_RDB_SIZE_SHIFT;
1426 
1427                 /*
1428                  * If we are attempting to modify the PKey index for this QP,
1429                  * then check for valid PKey index and fill it in.  Also set
1430                  * the appropriate flag in the "opmask" parameter.
1431                  */
1432                 if (flags & IBT_CEP_SET_PKEY_IX) {
1433                         pkeyindx = rc->rc_path.cep_pkey_ix;
1434                         if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
1435                                 qpc->pri_addr_path.pkey_indx = pkeyindx;
1436                                 opmask |= TAVOR_CMD_OP_PKEYINDX;
1437                         } else {
1438                                 TNF_PROBE_1(tavor_qp_init2rtr_inv_pkey_fail,
1439                                     TAVOR_TNF_ERROR, "", tnf_uint, pkeyindx,
1440                                     pkeyindx);
1441                                 TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1442                                 return (IBT_PKEY_IX_ILLEGAL);
1443                         }
1444                 }
1445 
1446                 /*
1447                  * Check if any of the flags indicate a change in the RDMA
1448                  * (recv) enable/disable flags and set the appropriate flag in
1449                  * the "opmask" parameter
1450                  */
1451                 opmask |= tavor_check_rdma_enable_flags(flags, info_p, qpc);
1452 
1453                 /*
1454                  * Check for optional alternate path and fill in the
1455                  * appropriate QPC fields if one is specified
1456                  */
1457                 if (flags & IBT_CEP_SET_ALT_PATH) {
1458                         qpc_path = &qpc->alt_addr_path;
1459                         adds_vect = &rc->rc_alt_path.cep_adds_vect;
1460 
1461                         /* Set the common alternate address path fields */
1462                         status = tavor_set_addr_path(state, adds_vect, qpc_path,
1463                             TAVOR_ADDRPATH_QP, qp);
1464                         if (status != DDI_SUCCESS) {
1465                                 TNF_PROBE_0(tavor_qp_init2rtr_setaddrpath_fail,
1466                                     TAVOR_TNF_ERROR, "");
1467                                 TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1468                                 return (status);
1469                         }
1470                         qpc_path->ack_timeout = rc->rc_alt_path.cep_timeout;
1471 
1472                         /*
1473                          * Copy the "RNR Retry count" value from the primary
1474                          * path.  Just as we did above, we need to hardcode
1475                          * the optional flag here (see below).
1476                          */
1477                         qpc_path->rnr_retry = rc->rc_rnr_retry_cnt;
1478 
1479                         /*
1480                          * Check for valid alternate path port number and fill
1481                          * it in
1482                          */
1483                         portnum = rc->rc_alt_path.cep_hca_port_num;
1484                         if (tavor_portnum_is_valid(state, portnum)) {
1485                                 qpc->alt_addr_path.portnum = portnum;
1486                         } else {
1487                                 TNF_PROBE_1(tavor_qp_init2rtr_inv_altport_fail,
1488                                     TAVOR_TNF_ERROR, "", tnf_uint, altport,
1489                                     portnum);
1490                                 TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1491                                 return (IBT_HCA_PORT_INVALID);
1492                         }
1493 
1494                         /*
1495                          * Check for valid alternate path PKey index and fill
1496                          * it in
1497                          */
1498                         pkeyindx = rc->rc_alt_path.cep_pkey_ix;
1499                         if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
1500                                 qpc->alt_addr_path.pkey_indx = pkeyindx;
1501                         } else {
1502                                 TNF_PROBE_1(tavor_qp_init2rtr_inv_altpkey_fail,
1503                                     TAVOR_TNF_ERROR, "", tnf_uint, altpkeyindx,
1504                                     pkeyindx);
1505                                 TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1506                                 return (IBT_PKEY_IX_ILLEGAL);
1507                         }
1508                         opmask |= (TAVOR_CMD_OP_ALT_PATH |
1509                             TAVOR_CMD_OP_ALT_RNRRETRY);
1510                 }
1511 
1512         } else if (qp->qp_serv_type == TAVOR_QP_UC) {
1513                 uc = &info_p->qp_transport.uc;
1514                 qpc_path = &qpc->pri_addr_path;
1515                 adds_vect = &uc->uc_path.cep_adds_vect;
1516 
1517                 /*
1518                  * Set the common primary address path fields
1519                  */
1520                 status = tavor_set_addr_path(state, adds_vect, qpc_path,
1521                     TAVOR_ADDRPATH_QP, qp);
1522                 if (status != DDI_SUCCESS) {
1523                         TNF_PROBE_0(tavor_qp_init2rtr_setaddrpath_fail,
1524                             TAVOR_TNF_ERROR, "");
1525                         TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1526                         return (status);
1527                 }
1528 
1529                 /*
1530                  * Setup the destination QP, recv PSN, MTU, max msg size,etc.
1531                  * Note max message size is defined to be the maximum IB
1532                  * allowed message size (which is 2^31 bytes).  Also max
1533                  * MTU is defined by HCA port properties.
1534                  */
1535                 qpc->rem_qpn   = uc->uc_dst_qpn;
1536                 qpc->next_rcv_psn = uc->uc_rq_psn;
1537                 qpc->msg_max   = TAVOR_QP_LOG_MAX_MSGSZ;
1538                 mtu = uc->uc_path_mtu;
1539                 if (tavor_qp_validate_mtu(state, mtu) != DDI_SUCCESS) {
1540                         TNF_PROBE_1(tavor_qp_init2rtr_inv_mtu_fail,
1541                             TAVOR_TNF_ERROR, "", tnf_uint, mtu, mtu);
1542                         TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1543                         return (IBT_HCA_PORT_MTU_EXCEEDED);
1544                 }
1545                 qpc->mtu = mtu;
1546 
1547                 /*
1548                  * Save away the MTU value.  This is used in future sqd2sqd
1549                  * transitions, as the MTU must remain the same in future
1550                  * changes.
1551                  */
1552                 qp->qp_save_mtu = qpc->mtu;
1553 
1554                 /*
1555                  * If we are attempting to modify the PKey index for this QP,
1556                  * then check for valid PKey index and fill it in.  Also set
1557                  * the appropriate flag in the "opmask" parameter.
1558                  */
1559                 if (flags & IBT_CEP_SET_PKEY_IX) {
1560                         pkeyindx = uc->uc_path.cep_pkey_ix;
1561                         if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
1562                                 qpc->pri_addr_path.pkey_indx = pkeyindx;
1563                                 opmask |= TAVOR_CMD_OP_PKEYINDX;
1564                         } else {
1565                                 TNF_PROBE_1(tavor_qp_init2rtr_inv_pkey_fail,
1566                                     TAVOR_TNF_ERROR, "", tnf_uint, pkeyindx,
1567                                     pkeyindx);
1568                                 TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1569                                 return (IBT_PKEY_IX_ILLEGAL);
1570                         }
1571                 }
1572 
1573                 /*
1574                  * Check if any of the flags indicate a change in the RDMA
1575                  * Write (recv) enable/disable and set the appropriate flag
1576                  * in the "opmask" parameter. Note: RDMA Read and Atomic are
1577                  * not valid for UC transport.
1578                  */
1579                 if (flags & IBT_CEP_SET_RDMA_W) {
1580                         qpc->rwe = (info_p->qp_flags & IBT_CEP_RDMA_WR) ? 1 : 0;
1581                         opmask |= TAVOR_CMD_OP_RWE;
1582                 }
1583 
1584                 /*
1585                  * Check for optional alternate path and fill in the
1586                  * appropriate QPC fields if one is specified
1587                  */
1588                 if (flags & IBT_CEP_SET_ALT_PATH) {
1589                         qpc_path = &qpc->alt_addr_path;
1590                         adds_vect = &uc->uc_alt_path.cep_adds_vect;
1591 
1592                         /* Set the common alternate address path fields */
1593                         status = tavor_set_addr_path(state, adds_vect, qpc_path,
1594                             TAVOR_ADDRPATH_QP, qp);
1595                         if (status != DDI_SUCCESS) {
1596                                 TNF_PROBE_0(tavor_qp_init2rtr_setaddrpath_fail,
1597                                     TAVOR_TNF_ERROR, "");
1598                                 TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1599                                 return (status);
1600                         }
1601 
1602                         /*
1603                          * Check for valid alternate path port number and fill
1604                          * it in
1605                          */
1606                         portnum = uc->uc_alt_path.cep_hca_port_num;
1607                         if (tavor_portnum_is_valid(state, portnum)) {
1608                                 qpc->alt_addr_path.portnum = portnum;
1609                         } else {
1610                                 TNF_PROBE_1(tavor_qp_init2rtr_inv_altport_fail,
1611                                     TAVOR_TNF_ERROR, "", tnf_uint, altport,
1612                                     portnum);
1613                                 TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1614                                 return (IBT_HCA_PORT_INVALID);
1615                         }
1616 
1617                         /*
1618                          * Check for valid alternate path PKey index and fill
1619                          * it in
1620                          */
1621                         pkeyindx = uc->uc_alt_path.cep_pkey_ix;
1622                         if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
1623                                 qpc->alt_addr_path.pkey_indx = pkeyindx;
1624                         } else {
1625                                 TNF_PROBE_1(tavor_qp_init2rtr_inv_altpkey_fail,
1626                                     TAVOR_TNF_ERROR, "", tnf_uint, altpkeyindx,
1627                                     pkeyindx);
1628                                 TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1629                                 return (IBT_PKEY_IX_ILLEGAL);
1630                         }
1631                         opmask |= TAVOR_CMD_OP_ALT_PATH;
1632                 }
1633         } else {
1634                 /*
1635                  * Invalid QP transport type. If we got here then it's a
1636                  * warning of a probably serious problem.  So print a message
1637                  * and return failure
1638                  */
1639                 TAVOR_WARNING(state, "unknown QP transport type in init2rtr");
1640                 TNF_PROBE_0(tavor_qp_init2rtr_inv_transtype_fail,
1641                     TAVOR_TNF_ERROR, "");
1642                 TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1643                 return (ibc_get_ci_failure(0));
1644         }
1645 
1646         /*
1647          * Post the INIT2RTR_QP command to the Tavor firmware
1648          *
1649          * We do a TAVOR_NOSLEEP here because we are still holding the
1650          * "qp_lock".  If we got raised to interrupt level by priority
1651          * inversion, we do not want to block in this routine waiting for
1652          * success.
1653          */
1654         status = tavor_cmn_qp_cmd_post(state, INIT2RTR_QP, qpc, qp->qp_qpnum,
1655             opmask, TAVOR_CMD_NOSLEEP_SPIN);
1656         if (status != TAVOR_CMD_SUCCESS) {
1657                 if (status != TAVOR_CMD_BAD_QP_STATE) {
1658                         cmn_err(CE_CONT, "Tavor: INIT2RTR_QP command failed: "
1659                             "%08x\n", status);
1660                         TNF_PROBE_1(tavor_qp_init2rtr_cmd_fail,
1661                             TAVOR_TNF_ERROR, "", tnf_uint, status, status);
1662                         TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1663                         return (ibc_get_ci_failure(0));
1664                 } else {
1665                         TNF_PROBE_0(tavor_qp_init2rtr_inv_qpstate_fail,
1666                             TAVOR_TNF_ERROR, "");
1667                         TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1668                         return (IBT_QP_STATE_INVALID);
1669                 }
1670         }
1671 
1672         TAVOR_TNF_EXIT(tavor_qp_init2rtr);
1673         return (DDI_SUCCESS);
1674 }
1675 
1676 
1677 /*
1678  * tavor_qp_rtr2rts()
1679  *    Context: Can be called from interrupt or base context.
1680  */
1681 static int
1682 tavor_qp_rtr2rts(tavor_state_t *state, tavor_qphdl_t qp,
1683     ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p)
1684 {
1685         tavor_hw_qpc_t          *qpc;
1686         ibt_qp_rc_attr_t        *rc;
1687         ibt_qp_ud_attr_t        *ud;
1688         ibt_qp_uc_attr_t        *uc;
1689         tavor_hw_addr_path_t    *qpc_path;
1690         ibt_adds_vect_t         *adds_vect;
1691         uint_t                  portnum, pkeyindx, rdma_ra_out, sra_max;
1692         uint32_t                opmask = 0;
1693         int                     status;
1694 
1695         TAVOR_TNF_ENTER(tavor_qp_rtr2rts);
1696 
1697         ASSERT(MUTEX_HELD(&qp->qp_lock));
1698 
1699         /*
1700          * Grab the temporary QPC entry from QP software state
1701          */
1702         qpc = &qp->qpc;
1703 
1704         /*
1705          * Fill in the common and/or Tavor-specific fields in the QPC
1706          */
1707         qpc->flight_lim = TAVOR_QP_FLIGHT_LIM_UNLIMITED;
1708 
1709         /*
1710          * Now fill in the QPC fields which are specific to transport type
1711          */
1712         if (qp->qp_serv_type == TAVOR_QP_UD) {
1713                 ud = &info_p->qp_transport.ud;
1714 
1715                 /* Set the send PSN */
1716                 qpc->next_snd_psn = ud->ud_sq_psn;
1717 
1718                 /*
1719                  * If we are attempting to modify the QKey for this QP, then
1720                  * fill it in and set the appropriate flag in the "opmask"
1721                  * parameter.
1722                  */
1723                 if (flags & IBT_CEP_SET_QKEY) {
1724                         qpc->qkey = ud->ud_qkey;
1725                         opmask |= TAVOR_CMD_OP_QKEY;
1726                 }
1727 
1728         } else if (qp->qp_serv_type == TAVOR_QP_RC) {
1729                 rc = &info_p->qp_transport.rc;
1730                 qpc_path = &qpc->pri_addr_path;
1731 
1732                 /*
1733                  * Setup the send PSN, ACK timeout, and retry counts
1734                  */
1735                 qpc->next_snd_psn    = rc->rc_sq_psn;
1736                 qpc_path->ack_timeout        = rc->rc_path.cep_timeout;
1737                 qpc_path->rnr_retry  = rc->rc_rnr_retry_cnt;
1738                 qpc->retry_cnt               = rc->rc_retry_cnt;
1739 
1740                 /*
1741                  * Set "ack_req_freq" based on the configuration variable
1742                  */
1743                 qpc->ack_req_freq = state->ts_cfg_profile->cp_ackreq_freq;
1744 
1745                 /*
1746                  * Check that the number of specified "outgoing RDMA resources"
1747                  * is valid.  And if it is, then setup the "sra_max"
1748                  * appropriately
1749                  */
1750                 rdma_ra_out = rc->rc_rdma_ra_out;
1751                 if (tavor_qp_validate_init_depth(state, rc, &sra_max) !=
1752                     DDI_SUCCESS) {
1753                         TNF_PROBE_1(tavor_qp_rtr2rts_inv_rdma_out_fail,
1754                             TAVOR_TNF_ERROR, "", tnf_uint, rdma_ra_out,
1755                             rdma_ra_out);
1756                         TAVOR_TNF_EXIT(tavor_qp_rtr2rts);
1757                         return (IBT_INVALID_PARAM);
1758                 }
1759                 qpc->sra_max = sra_max;
1760 
1761                 /*
1762                  * Configure the QP to allow (sending of) all types of RC
1763                  * traffic.  Tavor hardware allows these bits to be set to
1764                  * zero (thereby disabling certain outgoing RDMA types), but
1765                  * we do not desire to do this.
1766                  */
1767                 qpc->sre = qpc->swe = qpc->sae = 1;
1768                 qpc->sic = 0;
1769 
1770                 /*
1771                  * Check if any of the flags indicate a change in the RDMA
1772                  * (recv) enable/disable flags and set the appropriate flag in
1773                  * the "opmask" parameter
1774                  */
1775                 opmask |= tavor_check_rdma_enable_flags(flags, info_p, qpc);
1776 
1777                 /*
1778                  * If we are attempting to modify the path migration state for
1779                  * this QP, then check for valid state and fill it in.  Also
1780                  * set the appropriate flag in the "opmask" parameter.
1781                  */
1782                 if (flags & IBT_CEP_SET_MIG) {
1783                         if (rc->rc_mig_state == IBT_STATE_MIGRATED) {
1784                                 qpc->pm_state = TAVOR_QP_PMSTATE_MIGRATED;
1785                         } else if (rc->rc_mig_state == IBT_STATE_REARMED) {
1786                                 qpc->pm_state = TAVOR_QP_PMSTATE_REARM;
1787                         } else {
1788                                 TNF_PROBE_1(tavor_qp_rtr2rts_inv_mig_state_fail,
1789                                     TAVOR_TNF_ERROR, "", tnf_uint, mig_state,
1790                                     rc->rc_mig_state);
1791                                 TAVOR_TNF_EXIT(tavor_qp_rtr2rts);
1792                                 return (IBT_QP_APM_STATE_INVALID);
1793                         }
1794                         opmask |= TAVOR_CMD_OP_PM_STATE;
1795                 }
1796 
1797                 /*
1798                  * If we are attempting to modify the "Minimum RNR NAK" value
1799                  * for this QP, then fill it in and set the appropriate flag
1800                  * in the "opmask" parameter.
1801                  */
1802                 if (flags & IBT_CEP_SET_MIN_RNR_NAK) {
1803                         qpc->min_rnr_nak = rc->rc_min_rnr_nak;
1804                         opmask |= TAVOR_CMD_OP_MINRNRNAK;
1805                 }
1806 
1807                 /*
1808                  * Check for optional alternate path and fill in the
1809                  * appropriate QPC fields if one is specified
1810                  */
1811                 if (flags & IBT_CEP_SET_ALT_PATH) {
1812                         qpc_path = &qpc->alt_addr_path;
1813                         adds_vect = &rc->rc_alt_path.cep_adds_vect;
1814 
1815                         /* Set the common alternate address path fields */
1816                         status = tavor_set_addr_path(state, adds_vect, qpc_path,
1817                             TAVOR_ADDRPATH_QP, qp);
1818                         if (status != DDI_SUCCESS) {
1819                                 TNF_PROBE_0(tavor_qp_rtr2rts_setaddrpath_fail,
1820                                     TAVOR_TNF_ERROR, "");
1821                                 TAVOR_TNF_EXIT(tavor_qp_rtr2rts);
1822                                 return (status);
1823                         }
1824 
1825                         qpc_path->ack_timeout = rc->rc_alt_path.cep_timeout;
1826 
1827                         /*
1828                          * Copy the "RNR Retry count" value from the primary
1829                          * path.  Just as we did above, we need to hardcode
1830                          * the optional flag here (see below).
1831                          */
1832                         qpc_path->rnr_retry = rc->rc_rnr_retry_cnt;
1833 
1834                         /*
1835                          * Check for valid alternate path port number and fill
1836                          * it in
1837                          */
1838                         portnum = rc->rc_alt_path.cep_hca_port_num;
1839                         if (tavor_portnum_is_valid(state, portnum)) {
1840                                 qpc->alt_addr_path.portnum = portnum;
1841                         } else {
1842                                 TNF_PROBE_1(tavor_qp_rtr2rts_inv_altport_fail,
1843                                     TAVOR_TNF_ERROR, "", tnf_uint, altport,
1844                                     portnum);
1845                                 TAVOR_TNF_EXIT(tavor_qp_rtr2rts);
1846                                 return (IBT_HCA_PORT_INVALID);
1847                         }
1848 
1849                         /*
1850                          * Check for valid alternate path PKey index and fill
1851                          * it in
1852                          */
1853                         pkeyindx = rc->rc_alt_path.cep_pkey_ix;
1854                         if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
1855                                 qpc->alt_addr_path.pkey_indx = pkeyindx;
1856                         } else {
1857                                 TNF_PROBE_1(tavor_qp_rtr2rts_inv_altpkey_fail,
1858                                     TAVOR_TNF_ERROR, "", tnf_uint, altpkeyindx,
1859                                     pkeyindx);
1860                                 TAVOR_TNF_EXIT(tavor_qp_rtr2rts);
1861                                 return (IBT_PKEY_IX_ILLEGAL);
1862                         }
1863                         opmask |= (TAVOR_CMD_OP_ALT_PATH |
1864                             TAVOR_CMD_OP_ALT_RNRRETRY);
1865                 }
1866 
1867         } else if (qp->qp_serv_type == TAVOR_QP_UC) {
1868                 uc = &info_p->qp_transport.uc;
1869 
1870                 /* Set the send PSN */
1871                 qpc->next_snd_psn = uc->uc_sq_psn;
1872 
1873                 /*
1874                  * Configure the QP to allow (sending of) all types of allowable
1875                  * UC traffic (i.e. RDMA Write).
1876                  */
1877                 qpc->swe = 1;
1878 
1879                 /*
1880                  * Check if any of the flags indicate a change in the RDMA
1881                  * Write (recv) enable/disable and set the appropriate flag
1882                  * in the "opmask" parameter. Note: RDMA Read and Atomic are
1883                  * not valid for UC transport.
1884                  */
1885                 if (flags & IBT_CEP_SET_RDMA_W) {
1886                         qpc->rwe = (info_p->qp_flags & IBT_CEP_RDMA_WR) ? 1 : 0;
1887                         opmask |= TAVOR_CMD_OP_RWE;
1888                 }
1889 
1890                 /*
1891                  * If we are attempting to modify the path migration state for
1892                  * this QP, then check for valid state and fill it in.  Also
1893                  * set the appropriate flag in the "opmask" parameter.
1894                  */
1895                 if (flags & IBT_CEP_SET_MIG) {
1896                         if (uc->uc_mig_state == IBT_STATE_MIGRATED) {
1897                                 qpc->pm_state = TAVOR_QP_PMSTATE_MIGRATED;
1898                         } else if (uc->uc_mig_state == IBT_STATE_REARMED) {
1899                                 qpc->pm_state = TAVOR_QP_PMSTATE_REARM;
1900                         } else {
1901                                 TNF_PROBE_1(tavor_qp_rtr2rts_inv_mig_state_fail,
1902                                     TAVOR_TNF_ERROR, "", tnf_uint, mig_state,
1903                                     uc->uc_mig_state);
1904                                 TAVOR_TNF_EXIT(tavor_qp_rtr2rts);
1905                                 return (IBT_QP_APM_STATE_INVALID);
1906                         }
1907                         opmask |= TAVOR_CMD_OP_PM_STATE;
1908                 }
1909 
1910                 /*
1911                  * Check for optional alternate path and fill in the
1912                  * appropriate QPC fields if one is specified
1913                  */
1914                 if (flags & IBT_CEP_SET_ALT_PATH) {
1915                         qpc_path = &qpc->alt_addr_path;
1916                         adds_vect = &uc->uc_alt_path.cep_adds_vect;
1917 
1918                         /* Set the common alternate address path fields */
1919                         status = tavor_set_addr_path(state, adds_vect, qpc_path,
1920                             TAVOR_ADDRPATH_QP, qp);
1921                         if (status != DDI_SUCCESS) {
1922                                 TNF_PROBE_0(tavor_qp_rtr2rts_setaddrpath_fail,
1923                                     TAVOR_TNF_ERROR, "");
1924                                 TAVOR_TNF_EXIT(tavor_qp_rtr2rts);
1925                                 return (status);
1926                         }
1927 
1928                         /*
1929                          * Check for valid alternate path port number and fill
1930                          * it in
1931                          */
1932                         portnum = uc->uc_alt_path.cep_hca_port_num;
1933                         if (tavor_portnum_is_valid(state, portnum)) {
1934                                 qpc->alt_addr_path.portnum = portnum;
1935                         } else {
1936                                 TNF_PROBE_1(tavor_qp_rtr2rts_inv_altport_fail,
1937                                     TAVOR_TNF_ERROR, "", tnf_uint, altport,
1938                                     portnum);
1939                                 TAVOR_TNF_EXIT(tavor_qp_rtr2rts);
1940                                 return (IBT_HCA_PORT_INVALID);
1941                         }
1942 
1943                         /*
1944                          * Check for valid alternate path PKey index and fill
1945                          * it in
1946                          */
1947                         pkeyindx = uc->uc_alt_path.cep_pkey_ix;
1948                         if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
1949                                 qpc->alt_addr_path.pkey_indx = pkeyindx;
1950                         } else {
1951                                 TNF_PROBE_1(tavor_qp_rtr2rts_inv_altpkey_fail,
1952                                     TAVOR_TNF_ERROR, "", tnf_uint, altpkeyindx,
1953                                     pkeyindx);
1954                                 TAVOR_TNF_EXIT(tavor_qp_rtr2rts);
1955                                 return (IBT_PKEY_IX_ILLEGAL);
1956                         }
1957                         opmask |= TAVOR_CMD_OP_ALT_PATH;
1958                 }
1959         } else {
1960                 /*
1961                  * Invalid QP transport type. If we got here then it's a
1962                  * warning of a probably serious problem.  So print a message
1963                  * and return failure
1964                  */
1965                 TAVOR_WARNING(state, "unknown QP transport type in rtr2rts");
1966                 TNF_PROBE_0(tavor_qp_rtr2rts_inv_transtype_fail,
1967                     TAVOR_TNF_ERROR, "");
1968                 TAVOR_TNF_EXIT(tavor_qp_rtr2rts);
1969                 return (ibc_get_ci_failure(0));
1970         }
1971 
1972         /*
1973          * Post the RTR2RTS_QP command to the Tavor firmware
1974          *
1975          * We do a TAVOR_NOSLEEP here because we are still holding the
1976          * "qp_lock".  If we got raised to interrupt level by priority
1977          * inversion, we do not want to block in this routine waiting for
1978          * success.
1979          */
1980         status = tavor_cmn_qp_cmd_post(state, RTR2RTS_QP, qpc, qp->qp_qpnum,
1981             opmask, TAVOR_CMD_NOSLEEP_SPIN);
1982         if (status != TAVOR_CMD_SUCCESS) {
1983                 if (status != TAVOR_CMD_BAD_QP_STATE) {
1984                         cmn_err(CE_CONT, "Tavor: RTR2RTS_QP command failed: "
1985                             "%08x\n", status);
1986                         TNF_PROBE_1(tavor_qp_rtr2rts_cmd_fail,
1987                             TAVOR_TNF_ERROR, "", tnf_uint, status, status);
1988                         TAVOR_TNF_EXIT(tavor_qp_rtr2rts);
1989                         return (ibc_get_ci_failure(0));
1990                 } else {
1991                         TNF_PROBE_0(tavor_qp_rtr2rts_inv_qpstate_fail,
1992                             TAVOR_TNF_ERROR, "");
1993                         TAVOR_TNF_EXIT(tavor_qp_rtr2rts);
1994                         return (IBT_QP_STATE_INVALID);
1995                 }
1996         }
1997 
1998         TAVOR_TNF_EXIT(tavor_qp_rtr2rts);
1999         return (DDI_SUCCESS);
2000 }
2001 
2002 
2003 /*
2004  * tavor_qp_rts2rts()
2005  *    Context: Can be called from interrupt or base context.
2006  */
2007 static int
2008 tavor_qp_rts2rts(tavor_state_t *state, tavor_qphdl_t qp,
2009     ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p)
2010 {
2011         tavor_hw_qpc_t          *qpc;
2012         ibt_qp_rc_attr_t        *rc;
2013         ibt_qp_ud_attr_t        *ud;
2014         ibt_qp_uc_attr_t        *uc;
2015         tavor_hw_addr_path_t    *qpc_path;
2016         ibt_adds_vect_t         *adds_vect;
2017         uint_t                  portnum, pkeyindx;
2018         uint32_t                opmask = 0;
2019         int                     status;
2020 
2021         TAVOR_TNF_ENTER(tavor_qp_rts2rts);
2022 
2023         ASSERT(MUTEX_HELD(&qp->qp_lock));
2024 
2025         /*
2026          * Grab the temporary QPC entry from QP software state
2027          */
2028         qpc = &qp->qpc;
2029 
2030         /*
2031          * Since there are no common and/or Tavor-specific fields to be filled
2032          * in for this command, we begin with the QPC fields which are
2033          * specific to transport type.
2034          */
2035         if (qp->qp_serv_type == TAVOR_QP_UD) {
2036                 ud = &info_p->qp_transport.ud;
2037 
2038                 /*
2039                  * If we are attempting to modify the QKey for this QP, then
2040                  * fill it in and set the appropriate flag in the "opmask"
2041                  * parameter.
2042                  */
2043                 if (flags & IBT_CEP_SET_QKEY) {
2044                         qpc->qkey = ud->ud_qkey;
2045                         opmask |= TAVOR_CMD_OP_QKEY;
2046                 }
2047 
2048         } else if (qp->qp_serv_type == TAVOR_QP_RC) {
2049                 rc = &info_p->qp_transport.rc;
2050 
2051                 /*
2052                  * Check if any of the flags indicate a change in the RDMA
2053                  * (recv) enable/disable flags and set the appropriate flag in
2054                  * the "opmask" parameter
2055                  */
2056                 opmask |= tavor_check_rdma_enable_flags(flags, info_p, qpc);
2057 
2058                 /*
2059                  * If we are attempting to modify the path migration state for
2060                  * this QP, then check for valid state and fill it in.  Also
2061                  * set the appropriate flag in the "opmask" parameter.
2062                  */
2063                 if (flags & IBT_CEP_SET_MIG) {
2064                         if (rc->rc_mig_state == IBT_STATE_MIGRATED) {
2065                                 qpc->pm_state = TAVOR_QP_PMSTATE_MIGRATED;
2066                         } else if (rc->rc_mig_state == IBT_STATE_REARMED) {
2067                                 qpc->pm_state = TAVOR_QP_PMSTATE_REARM;
2068                         } else {
2069                                 TNF_PROBE_1(tavor_qp_rts2rts_inv_mig_state_fail,
2070                                     TAVOR_TNF_ERROR, "", tnf_uint, mig_state,
2071                                     rc->rc_mig_state);
2072                                 TAVOR_TNF_EXIT(tavor_qp_rts2rts);
2073                                 return (IBT_QP_APM_STATE_INVALID);
2074                         }
2075                         opmask |= TAVOR_CMD_OP_PM_STATE;
2076                 }
2077 
2078                 /*
2079                  * If we are attempting to modify the "Minimum RNR NAK" value
2080                  * for this QP, then fill it in and set the appropriate flag
2081                  * in the "opmask" parameter.
2082                  */
2083                 if (flags & IBT_CEP_SET_MIN_RNR_NAK) {
2084                         qpc->min_rnr_nak = rc->rc_min_rnr_nak;
2085                         opmask |= TAVOR_CMD_OP_MINRNRNAK;
2086                 }
2087 
2088                 /*
2089                  * Check for optional alternate path and fill in the
2090                  * appropriate QPC fields if one is specified
2091                  */
2092                 if (flags & IBT_CEP_SET_ALT_PATH) {
2093                         qpc_path = &qpc->alt_addr_path;
2094                         adds_vect = &rc->rc_alt_path.cep_adds_vect;
2095 
2096                         /* Set the common alternate address path fields */
2097                         status = tavor_set_addr_path(state, adds_vect, qpc_path,
2098                             TAVOR_ADDRPATH_QP, qp);
2099                         if (status != DDI_SUCCESS) {
2100                                 TNF_PROBE_0(tavor_qp_rts2rts_setaddrpath_fail,
2101                                     TAVOR_TNF_ERROR, "");
2102                                 TAVOR_TNF_EXIT(tavor_qp_rts2rts);
2103                                 return (status);
2104                         }
2105                         qpc_path->ack_timeout = rc->rc_alt_path.cep_timeout;
2106 
2107                         /*
2108                          * Check for valid alternate path port number and fill
2109                          * it in
2110                          */
2111                         portnum = rc->rc_alt_path.cep_hca_port_num;
2112                         if (tavor_portnum_is_valid(state, portnum)) {
2113                                 qpc->alt_addr_path.portnum = portnum;
2114                         } else {
2115                                 TNF_PROBE_1(tavor_qp_rts2rts_inv_altport_fail,
2116                                     TAVOR_TNF_ERROR, "", tnf_uint, altport,
2117                                     portnum);
2118                                 TAVOR_TNF_EXIT(tavor_qp_rts2rts);
2119                                 return (IBT_HCA_PORT_INVALID);
2120                         }
2121 
2122                         /*
2123                          * Check for valid alternate path PKey index and fill
2124                          * it in
2125                          */
2126                         pkeyindx = rc->rc_alt_path.cep_pkey_ix;
2127                         if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
2128                                 qpc->alt_addr_path.pkey_indx = pkeyindx;
2129                         } else {
2130                                 TNF_PROBE_1(tavor_qp_rts2rts_inv_altpkey_fail,
2131                                     TAVOR_TNF_ERROR, "", tnf_uint, altpkeyindx,
2132                                     pkeyindx);
2133                                 TAVOR_TNF_EXIT(tavor_qp_rts2rts);
2134                                 return (IBT_PKEY_IX_ILLEGAL);
2135                         }
2136                         opmask |= TAVOR_CMD_OP_ALT_PATH;
2137                 }
2138 
2139         } else if (qp->qp_serv_type == TAVOR_QP_UC) {
2140                 uc = &info_p->qp_transport.uc;
2141 
2142                 /*
2143                  * Check if any of the flags indicate a change in the RDMA
2144                  * Write (recv) enable/disable and set the appropriate flag
2145                  * in the "opmask" parameter. Note: RDMA Read and Atomic are
2146                  * not valid for UC transport.
2147                  */
2148                 if (flags & IBT_CEP_SET_RDMA_W) {
2149                         qpc->rwe = (info_p->qp_flags & IBT_CEP_RDMA_WR) ? 1 : 0;
2150                         opmask |= TAVOR_CMD_OP_RWE;
2151                 }
2152 
2153                 /*
2154                  * If we are attempting to modify the path migration state for
2155                  * this QP, then check for valid state and fill it in.  Also
2156                  * set the appropriate flag in the "opmask" parameter.
2157                  */
2158                 if (flags & IBT_CEP_SET_MIG) {
2159                         if (uc->uc_mig_state == IBT_STATE_MIGRATED) {
2160                                 qpc->pm_state = TAVOR_QP_PMSTATE_MIGRATED;
2161                         } else if (uc->uc_mig_state == IBT_STATE_REARMED) {
2162                                 qpc->pm_state = TAVOR_QP_PMSTATE_REARM;
2163                         } else {
2164                                 TNF_PROBE_1(tavor_qp_rts2rts_inv_mig_state_fail,
2165                                     TAVOR_TNF_ERROR, "", tnf_uint, mig_state,
2166                                     uc->uc_mig_state);
2167                                 TAVOR_TNF_EXIT(tavor_qp_rts2rts);
2168                                 return (IBT_QP_APM_STATE_INVALID);
2169                         }
2170                         opmask |= TAVOR_CMD_OP_PM_STATE;
2171                 }
2172 
2173                 /*
2174                  * Check for optional alternate path and fill in the
2175                  * appropriate QPC fields if one is specified
2176                  */
2177                 if (flags & IBT_CEP_SET_ALT_PATH) {
2178                         qpc_path = &qpc->alt_addr_path;
2179                         adds_vect = &uc->uc_alt_path.cep_adds_vect;
2180 
2181                         /* Set the common alternate address path fields */
2182                         status = tavor_set_addr_path(state, adds_vect, qpc_path,
2183                             TAVOR_ADDRPATH_QP, qp);
2184                         if (status != DDI_SUCCESS) {
2185                                 TNF_PROBE_0(tavor_qp_rts2rts_setaddrpath_fail,
2186                                     TAVOR_TNF_ERROR, "");
2187                                 TAVOR_TNF_EXIT(tavor_qp_rts2rts);
2188                                 return (status);
2189                         }
2190 
2191                         /*
2192                          * Check for valid alternate path port number and fill
2193                          * it in
2194                          */
2195                         portnum = uc->uc_alt_path.cep_hca_port_num;
2196                         if (tavor_portnum_is_valid(state, portnum)) {
2197                                 qpc->alt_addr_path.portnum = portnum;
2198                         } else {
2199                                 TNF_PROBE_1(tavor_qp_rts2rts_inv_altport_fail,
2200                                     TAVOR_TNF_ERROR, "", tnf_uint, altport,
2201                                     portnum);
2202                                 TAVOR_TNF_EXIT(tavor_qp_rts2rts);
2203                                 return (IBT_HCA_PORT_INVALID);
2204                         }
2205 
2206                         /*
2207                          * Check for valid alternate path PKey index and fill
2208                          * it in
2209                          */
2210                         pkeyindx = uc->uc_alt_path.cep_pkey_ix;
2211                         if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
2212                                 qpc->alt_addr_path.pkey_indx = pkeyindx;
2213                         } else {
2214                                 TNF_PROBE_1(tavor_qp_rts2rts_inv_altpkey_fail,
2215                                     TAVOR_TNF_ERROR, "", tnf_uint, altpkeyindx,
2216                                     pkeyindx);
2217                                 TAVOR_TNF_EXIT(tavor_qp_rts2rts);
2218                                 return (IBT_PKEY_IX_ILLEGAL);
2219                         }
2220                         opmask |= TAVOR_CMD_OP_ALT_PATH;
2221                 }
2222         } else {
2223                 /*
2224                  * Invalid QP transport type. If we got here then it's a
2225                  * warning of a probably serious problem.  So print a message
2226                  * and return failure
2227                  */
2228                 TAVOR_WARNING(state, "unknown QP transport type in rts2rts");
2229                 TNF_PROBE_0(tavor_qp_rts2rts_inv_transtype_fail,
2230                     TAVOR_TNF_ERROR, "");
2231                 TAVOR_TNF_EXIT(tavor_qp_rts2rts);
2232                 return (ibc_get_ci_failure(0));
2233         }
2234 
2235         /*
2236          * Post the RTS2RTS_QP command to the Tavor firmware
2237          *
2238          * We do a TAVOR_NOSLEEP here because we are still holding the
2239          * "qp_lock".  If we got raised to interrupt level by priority
2240          * inversion, we do not want to block in this routine waiting for
2241          * success.
2242          */
2243         status = tavor_cmn_qp_cmd_post(state, RTS2RTS_QP, qpc, qp->qp_qpnum,
2244             opmask, TAVOR_CMD_NOSLEEP_SPIN);
2245         if (status != TAVOR_CMD_SUCCESS) {
2246                 if (status != TAVOR_CMD_BAD_QP_STATE) {
2247                         cmn_err(CE_CONT, "Tavor: RTS2RTS_QP command failed: "
2248                             "%08x\n", status);
2249                         TNF_PROBE_1(tavor_qp_rts2rts_cmd_fail,
2250                             TAVOR_TNF_ERROR, "", tnf_uint, status, status);
2251                         TAVOR_TNF_EXIT(tavor_qp_rts2rts);
2252                         return (ibc_get_ci_failure(0));
2253                 } else {
2254                         TNF_PROBE_0(tavor_qp_rts2rts_inv_qpstate_fail,
2255                             TAVOR_TNF_ERROR, "");
2256                         TAVOR_TNF_EXIT(tavor_qp_rts2rts);
2257                         return (IBT_QP_STATE_INVALID);
2258                 }
2259         }
2260 
2261         TAVOR_TNF_EXIT(tavor_qp_rts2rts);
2262         return (DDI_SUCCESS);
2263 }
2264 
2265 
2266 /*
2267  * tavor_qp_rts2sqd()
2268  *    Context: Can be called from interrupt or base context.
2269  */
2270 static int
2271 tavor_qp_rts2sqd(tavor_state_t *state, tavor_qphdl_t qp,
2272     ibt_cep_modify_flags_t flags)
2273 {
2274         int                     status;
2275 
2276         TAVOR_TNF_ENTER(tavor_qp_rts2sqd);
2277 
2278         ASSERT(MUTEX_HELD(&qp->qp_lock));
2279 
2280         /*
2281          * Set a flag to indicate whether or not the consumer is interested
2282          * in receiving the SQ drained event.  Since we are going to always
2283          * request hardware generation of the SQD event, we use the value in
2284          * "qp_forward_sqd_event" to determine whether or not to pass the event
2285          * to the IBTF or to silently consume it.
2286          */
2287         qp->qp_forward_sqd_event = (flags & IBT_CEP_SET_SQD_EVENT) ? 1 : 0;
2288 
2289         /*
2290          * Post the RTS2SQD_QP command to the Tavor firmware
2291          *
2292          * We do a TAVOR_NOSLEEP here because we are still holding the
2293          * "qp_lock".  If we got raised to interrupt level by priority
2294          * inversion, we do not want to block in this routine waiting for
2295          * success.
2296          */
2297         status = tavor_cmn_qp_cmd_post(state, RTS2SQD_QP, NULL, qp->qp_qpnum,
2298             0, TAVOR_CMD_NOSLEEP_SPIN);
2299         if (status != TAVOR_CMD_SUCCESS) {
2300                 if (status != TAVOR_CMD_BAD_QP_STATE) {
2301                         cmn_err(CE_CONT, "Tavor: RTS2SQD_QP command failed: "
2302                             "%08x\n", status);
2303                         TNF_PROBE_1(tavor_qp_rts2sqd_cmd_fail,
2304                             TAVOR_TNF_ERROR, "", tnf_uint, status, status);
2305                         TAVOR_TNF_EXIT(tavor_qp_rts2sqd);
2306                         return (ibc_get_ci_failure(0));
2307                 } else {
2308                         TNF_PROBE_0(tavor_qp_rts2sqd_inv_qpstate_fail,
2309                             TAVOR_TNF_ERROR, "");
2310                         TAVOR_TNF_EXIT(tavor_qp_rts2sqd);
2311                         return (IBT_QP_STATE_INVALID);
2312                 }
2313         }
2314 
2315         /*
2316          * Mark the current QP state as "SQ Draining".  This allows us to
2317          * distinguish between the two underlying states in SQD. (see QueryQP()
2318          * code in tavor_qp.c)
2319          */
2320         qp->qp_sqd_still_draining = 1;
2321 
2322         TAVOR_TNF_EXIT(tavor_qp_rts2sqd);
2323         return (DDI_SUCCESS);
2324 }
2325 
2326 
2327 /*
2328  * tavor_qp_sqd2rts()
2329  *    Context: Can be called from interrupt or base context.
2330  */
2331 static int
2332 tavor_qp_sqd2rts(tavor_state_t *state, tavor_qphdl_t qp,
2333     ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p)
2334 {
2335         tavor_hw_qpc_t          *qpc;
2336         ibt_qp_rc_attr_t        *rc;
2337         ibt_qp_ud_attr_t        *ud;
2338         ibt_qp_uc_attr_t        *uc;
2339         tavor_hw_addr_path_t    *qpc_path;
2340         ibt_adds_vect_t         *adds_vect;
2341         uint_t                  portnum, pkeyindx;
2342         uint32_t                opmask = 0;
2343         int                     status;
2344 
2345         TAVOR_TNF_ENTER(tavor_qp_sqd2rts);
2346 
2347         ASSERT(MUTEX_HELD(&qp->qp_lock));
2348 
2349         /*
2350          * Grab the temporary QPC entry from QP software state
2351          */
2352         qpc = &qp->qpc;
2353 
2354         /*
2355          * Since there are no common and/or Tavor-specific fields to be filled
2356          * in for this command, we begin with the QPC fields which are
2357          * specific to transport type.
2358          */
2359         if (qp->qp_serv_type == TAVOR_QP_UD) {
2360                 ud = &info_p->qp_transport.ud;
2361 
2362                 /*
2363                  * If we are attempting to modify the QKey for this QP, then
2364                  * fill it in and set the appropriate flag in the "opmask"
2365                  * parameter.
2366                  */
2367                 if (flags & IBT_CEP_SET_QKEY) {
2368                         qpc->qkey = ud->ud_qkey;
2369                         opmask |= TAVOR_CMD_OP_QKEY;
2370                 }
2371 
2372         } else if (qp->qp_serv_type == TAVOR_QP_RC) {
2373                 rc = &info_p->qp_transport.rc;
2374 
2375                 /*
2376                  * Check if any of the flags indicate a change in the RDMA
2377                  * (recv) enable/disable flags and set the appropriate flag in
2378                  * the "opmask" parameter
2379                  */
2380                 opmask |= tavor_check_rdma_enable_flags(flags, info_p, qpc);
2381 
2382                 /*
2383                  * If we are attempting to modify the path migration state for
2384                  * this QP, then check for valid state and fill it in.  Also
2385                  * set the appropriate flag in the "opmask" parameter.
2386                  */
2387                 if (flags & IBT_CEP_SET_MIG) {
2388                         if (rc->rc_mig_state == IBT_STATE_MIGRATED) {
2389                                 qpc->pm_state = TAVOR_QP_PMSTATE_MIGRATED;
2390                         } else if (rc->rc_mig_state == IBT_STATE_REARMED) {
2391                                 qpc->pm_state = TAVOR_QP_PMSTATE_REARM;
2392                         } else {
2393                                 TNF_PROBE_1(tavor_qp_sqd2rts_inv_mig_state_fail,
2394                                     TAVOR_TNF_ERROR, "", tnf_uint, mig_state,
2395                                     rc->rc_mig_state);
2396                                 TAVOR_TNF_EXIT(tavor_qp_sqd2rts);
2397                                 return (IBT_QP_APM_STATE_INVALID);
2398                         }
2399                         opmask |= TAVOR_CMD_OP_PM_STATE;
2400                 }
2401 
2402                 /*
2403                  * Check for optional alternate path and fill in the
2404                  * appropriate QPC fields if one is specified
2405                  */
2406                 if (flags & IBT_CEP_SET_ALT_PATH) {
2407                         qpc_path = &qpc->alt_addr_path;
2408                         adds_vect = &rc->rc_alt_path.cep_adds_vect;
2409 
2410                         /* Set the common alternate address path fields */
2411                         status = tavor_set_addr_path(state, adds_vect, qpc_path,
2412                             TAVOR_ADDRPATH_QP, qp);
2413                         if (status != DDI_SUCCESS) {
2414                                 TNF_PROBE_0(tavor_qp_sqd2rts_setaddrpath_fail,
2415                                     TAVOR_TNF_ERROR, "");
2416                                 TAVOR_TNF_EXIT(tavor_qp_sqd2rts);
2417                                 return (status);
2418                         }
2419                         qpc_path->ack_timeout = rc->rc_alt_path.cep_timeout;
2420 
2421                         /*
2422                          * Check for valid alternate path port number and fill
2423                          * it in
2424                          */
2425                         portnum = rc->rc_alt_path.cep_hca_port_num;
2426                         if (tavor_portnum_is_valid(state, portnum)) {
2427                                 qpc->alt_addr_path.portnum = portnum;
2428                         } else {
2429                                 TNF_PROBE_1(tavor_qp_sqd2rts_inv_altport_fail,
2430                                     TAVOR_TNF_ERROR, "", tnf_uint, altport,
2431                                     portnum);
2432                                 TAVOR_TNF_EXIT(tavor_qp_sqd2rts);
2433                                 return (IBT_HCA_PORT_INVALID);
2434                         }
2435 
2436                         /*
2437                          * Check for valid alternate path PKey index and fill
2438                          * it in
2439                          */
2440                         pkeyindx = rc->rc_alt_path.cep_pkey_ix;
2441                         if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
2442                                 qpc->alt_addr_path.pkey_indx = pkeyindx;
2443                         } else {
2444                                 TNF_PROBE_1(tavor_qp_sqd2rts_inv_altpkey_fail,
2445                                     TAVOR_TNF_ERROR, "", tnf_uint, altpkeyindx,
2446                                     pkeyindx);
2447                                 TAVOR_TNF_EXIT(tavor_qp_sqd2rts);
2448                                 return (IBT_PKEY_IX_ILLEGAL);
2449                         }
2450                         opmask |= TAVOR_CMD_OP_ALT_PATH;
2451                 }
2452 
2453                 /*
2454                  * If we are attempting to modify the "Minimum RNR NAK" value
2455                  * for this QP, then fill it in and set the appropriate flag
2456                  * in the "opmask" parameter.
2457                  */
2458                 if (flags & IBT_CEP_SET_MIN_RNR_NAK) {
2459                         qpc->min_rnr_nak = rc->rc_min_rnr_nak;
2460                         opmask |= TAVOR_CMD_OP_MINRNRNAK;
2461                 }
2462 
2463         } else if (qp->qp_serv_type == TAVOR_QP_UC) {
2464                 uc = &info_p->qp_transport.uc;
2465 
2466                 /*
2467                  * Check if any of the flags indicate a change in the RDMA
2468                  * Write (recv) enable/disable and set the appropriate flag
2469                  * in the "opmask" parameter. Note: RDMA Read and Atomic are
2470                  * not valid for UC transport.
2471                  */
2472                 if (flags & IBT_CEP_SET_RDMA_W) {
2473                         qpc->rwe = (info_p->qp_flags & IBT_CEP_RDMA_WR) ? 1 : 0;
2474                         opmask |= TAVOR_CMD_OP_RWE;
2475                 }
2476 
2477                 /*
2478                  * If we are attempting to modify the path migration state for
2479                  * this QP, then check for valid state and fill it in.  Also
2480                  * set the appropriate flag in the "opmask" parameter.
2481                  */
2482                 if (flags & IBT_CEP_SET_MIG) {
2483                         if (uc->uc_mig_state == IBT_STATE_MIGRATED) {
2484                                 qpc->pm_state = TAVOR_QP_PMSTATE_MIGRATED;
2485                         } else if (uc->uc_mig_state == IBT_STATE_REARMED) {
2486                                 qpc->pm_state = TAVOR_QP_PMSTATE_REARM;
2487                         } else {
2488                                 TNF_PROBE_1(tavor_qp_sqd2rts_inv_mig_state_fail,
2489                                     TAVOR_TNF_ERROR, "", tnf_uint, mig_state,
2490                                     uc->uc_mig_state);
2491                                 TAVOR_TNF_EXIT(tavor_qp_sqd2rts);
2492                                 return (IBT_QP_APM_STATE_INVALID);
2493                         }
2494                         opmask |= TAVOR_CMD_OP_PM_STATE;
2495                 }
2496 
2497                 /*
2498                  * Check for optional alternate path and fill in the
2499                  * appropriate QPC fields if one is specified
2500                  */
2501                 if (flags & IBT_CEP_SET_ALT_PATH) {
2502                         qpc_path = &qpc->alt_addr_path;
2503                         adds_vect = &uc->uc_alt_path.cep_adds_vect;
2504 
2505                         /* Set the common alternate address path fields */
2506                         status = tavor_set_addr_path(state, adds_vect, qpc_path,
2507                             TAVOR_ADDRPATH_QP, qp);
2508                         if (status != DDI_SUCCESS) {
2509                                 TNF_PROBE_0(tavor_qp_sqd2rts_setaddrpath_fail,
2510                                     TAVOR_TNF_ERROR, "");
2511                                 TAVOR_TNF_EXIT(tavor_qp_sqd2rts);
2512                                 return (status);
2513                         }
2514 
2515                         /*
2516                          * Check for valid alternate path port number and fill
2517                          * it in
2518                          */
2519                         portnum = uc->uc_alt_path.cep_hca_port_num;
2520                         if (tavor_portnum_is_valid(state, portnum)) {
2521                                 qpc->alt_addr_path.portnum = portnum;
2522                         } else {
2523                                 TNF_PROBE_1(tavor_qp_sqd2rts_inv_altport_fail,
2524                                     TAVOR_TNF_ERROR, "", tnf_uint, altport,
2525                                     portnum);
2526                                 TAVOR_TNF_EXIT(tavor_qp_sqd2rts);
2527                                 return (IBT_HCA_PORT_INVALID);
2528                         }
2529 
2530                         /*
2531                          * Check for valid alternate path PKey index and fill
2532                          * it in
2533                          */
2534                         pkeyindx = uc->uc_alt_path.cep_pkey_ix;
2535                         if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
2536                                 qpc->alt_addr_path.pkey_indx = pkeyindx;
2537                         } else {
2538                                 TNF_PROBE_1(tavor_qp_sqd2rts_inv_altpkey_fail,
2539                                     TAVOR_TNF_ERROR, "", tnf_uint, altpkeyindx,
2540                                     pkeyindx);
2541                                 TAVOR_TNF_EXIT(tavor_qp_sqd2rts);
2542                                 return (IBT_PKEY_IX_ILLEGAL);
2543                         }
2544                         opmask |= TAVOR_CMD_OP_ALT_PATH;
2545                 }
2546         } else {
2547                 /*
2548                  * Invalid QP transport type. If we got here then it's a
2549                  * warning of a probably serious problem.  So print a message
2550                  * and return failure
2551                  */
2552                 TAVOR_WARNING(state, "unknown QP transport type in sqd2rts");
2553                 TNF_PROBE_0(tavor_qp_sqd2rts_inv_transtype_fail,
2554                     TAVOR_TNF_ERROR, "");
2555                 TAVOR_TNF_EXIT(tavor_qp_sqd2rts);
2556                 return (ibc_get_ci_failure(0));
2557         }
2558 
2559         /*
2560          * Post the SQD2RTS_QP command to the Tavor firmware
2561          *
2562          * We do a TAVOR_NOSLEEP here because we are still holding the
2563          * "qp_lock".  If we got raised to interrupt level by priority
2564          * inversion, we do not want to block in this routine waiting for
2565          * success.
2566          */
2567         status = tavor_cmn_qp_cmd_post(state, SQD2RTS_QP, qpc, qp->qp_qpnum,
2568             opmask, TAVOR_CMD_NOSLEEP_SPIN);
2569         if (status != TAVOR_CMD_SUCCESS) {
2570                 if (status != TAVOR_CMD_BAD_QP_STATE) {
2571                         cmn_err(CE_CONT, "Tavor: SQD2RTS_QP command failed: "
2572                             "%08x\n", status);
2573                         TNF_PROBE_1(tavor_qp_sqd2rts_cmd_fail,
2574                             TAVOR_TNF_ERROR, "", tnf_uint, status, status);
2575                         TAVOR_TNF_EXIT(tavor_qp_sqd2rts);
2576                         return (ibc_get_ci_failure(0));
2577                 } else {
2578                         TNF_PROBE_0(tavor_qp_sqd2rts_inv_qpstate_fail,
2579                             TAVOR_TNF_ERROR, "");
2580                         TAVOR_TNF_EXIT(tavor_qp_sqd2rts);
2581                         return (IBT_QP_STATE_INVALID);
2582                 }
2583         }
2584 
2585         TAVOR_TNF_EXIT(tavor_qp_sqd2rts);
2586         return (DDI_SUCCESS);
2587 }
2588 
2589 
2590 /*
2591  * tavor_qp_sqd2sqd()
2592  *    Context: Can be called from interrupt or base context.
2593  */
2594 static int
2595 tavor_qp_sqd2sqd(tavor_state_t *state, tavor_qphdl_t qp,
2596     ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p)
2597 {
2598         tavor_hw_qpc_t          *qpc;
2599         ibt_qp_rc_attr_t        *rc;
2600         ibt_qp_ud_attr_t        *ud;
2601         ibt_qp_uc_attr_t        *uc;
2602         tavor_hw_addr_path_t    *qpc_path;
2603         ibt_adds_vect_t         *adds_vect;
2604         uint_t                  portnum, pkeyindx, rdma_ra_out, rdma_ra_in;
2605         uint_t                  rra_max, sra_max;
2606         uint32_t                opmask = 0;
2607         int                     status;
2608 
2609         TAVOR_TNF_ENTER(tavor_qp_sqd2sqd);
2610 
2611         ASSERT(MUTEX_HELD(&qp->qp_lock));
2612 
2613         /*
2614          * Grab the temporary QPC entry from QP software state
2615          */
2616         qpc = &qp->qpc;
2617 
2618         /*
2619          * Since there are no common and/or Tavor-specific fields to be filled
2620          * in for this command, we begin with the QPC fields which are
2621          * specific to transport type.
2622          */
2623         if (qp->qp_serv_type == TAVOR_QP_UD) {
2624                 ud = &info_p->qp_transport.ud;
2625 
2626                 /*
2627                  * If we are attempting to modify the PKey index for this QP,
2628                  * then check for valid PKey index and fill it in.  Also set
2629                  * the appropriate flag in the "opmask" parameter.
2630                  */
2631                 if (flags & IBT_CEP_SET_PKEY_IX) {
2632                         pkeyindx = ud->ud_pkey_ix;
2633                         if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
2634                                 qpc->pri_addr_path.pkey_indx = pkeyindx;
2635                                 opmask |= TAVOR_CMD_OP_PKEYINDX;
2636                                 qp->qp_pkeyindx = pkeyindx;
2637                         } else {
2638                                 TNF_PROBE_1(tavor_qp_sqd2sqd_inv_pkey_fail,
2639                                     TAVOR_TNF_ERROR, "", tnf_uint, pkeyindx,
2640                                     pkeyindx);
2641                                 TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
2642                                 return (IBT_PKEY_IX_ILLEGAL);
2643                         }
2644                 }
2645 
2646                 /*
2647                  * If we are attempting to modify the QKey for this QP, then
2648                  * fill it in and set the appropriate flag in the "opmask"
2649                  * parameter.
2650                  */
2651                 if (flags & IBT_CEP_SET_QKEY) {
2652                         qpc->qkey = ud->ud_qkey;
2653                         opmask |= TAVOR_CMD_OP_QKEY;
2654                 }
2655 
2656         } else if (qp->qp_serv_type == TAVOR_QP_RC) {
2657                 rc = &info_p->qp_transport.rc;
2658 
2659                 /*
2660                  * Check if any of the flags indicate a change in the RDMA
2661                  * (recv) enable/disable flags and set the appropriate flag in
2662                  * the "opmask" parameter
2663                  */
2664                 opmask |= tavor_check_rdma_enable_flags(flags, info_p, qpc);
2665 
2666                 /*
2667                  * Check for optional primary path and fill in the
2668                  * appropriate QPC fields if one is specified
2669                  */
2670                 if (flags & IBT_CEP_SET_ADDS_VECT) {
2671                         qpc_path = &qpc->pri_addr_path;
2672                         adds_vect = &rc->rc_path.cep_adds_vect;
2673 
2674                         /* Set the common primary address path fields */
2675                         status = tavor_set_addr_path(state, adds_vect, qpc_path,
2676                             TAVOR_ADDRPATH_QP, qp);
2677                         if (status != DDI_SUCCESS) {
2678                                 TNF_PROBE_0(tavor_qp_sqd2sqd_setaddrpath_fail,
2679                                     TAVOR_TNF_ERROR, "");
2680                                 TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
2681                                 return (status);
2682                         }
2683                         qpc_path->rnr_retry = rc->rc_rnr_retry_cnt;
2684                         qpc_path->ack_timeout = rc->rc_path.cep_timeout;
2685                         qpc->retry_cnt = rc->rc_retry_cnt;
2686 
2687                         /*
2688                          * MTU changes as part of sqd2sqd are not allowed.
2689                          * Simply keep the same MTU value here, stored in the
2690                          * qphdl from init2rtr time.
2691                          */
2692                         qpc->mtu = qp->qp_save_mtu;
2693 
2694                         opmask |= (TAVOR_CMD_OP_PRIM_PATH |
2695                             TAVOR_CMD_OP_RETRYCNT | TAVOR_CMD_OP_ACKTIMEOUT |
2696                             TAVOR_CMD_OP_PRIM_RNRRETRY);
2697                 }
2698 
2699                 /*
2700                  * If we are attempting to modify the path migration state for
2701                  * this QP, then check for valid state and fill it in.  Also
2702                  * set the appropriate flag in the "opmask" parameter.
2703                  */
2704                 if (flags & IBT_CEP_SET_MIG) {
2705                         if (rc->rc_mig_state == IBT_STATE_MIGRATED) {
2706                                 qpc->pm_state = TAVOR_QP_PMSTATE_MIGRATED;
2707                         } else if (rc->rc_mig_state == IBT_STATE_REARMED) {
2708                                 qpc->pm_state = TAVOR_QP_PMSTATE_REARM;
2709                         } else {
2710                                 TNF_PROBE_1(tavor_qp_sqd2sqd_inv_mig_state_fail,
2711                                     TAVOR_TNF_ERROR, "", tnf_uint, mig_state,
2712                                     rc->rc_mig_state);
2713                                 TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
2714                                 return (IBT_QP_APM_STATE_INVALID);
2715                         }
2716                         opmask |= TAVOR_CMD_OP_PM_STATE;
2717                 }
2718 
2719                 /*
2720                  * If we are attempting to modify the PKey index for this QP,
2721                  * then check for valid PKey index and fill it in.  Also set
2722                  * the appropriate flag in the "opmask" parameter.
2723                  */
2724                 if (flags & IBT_CEP_SET_PKEY_IX) {
2725                         pkeyindx = rc->rc_path.cep_pkey_ix;
2726                         if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
2727                                 qpc->pri_addr_path.pkey_indx = pkeyindx;
2728                                 opmask |= TAVOR_CMD_OP_PKEYINDX;
2729                         } else {
2730                                 TNF_PROBE_1(tavor_qp_sqd2sqd_inv_pkey_fail,
2731                                     TAVOR_TNF_ERROR, "", tnf_uint, pkeyindx,
2732                                     pkeyindx);
2733                                 TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
2734                                 return (IBT_PKEY_IX_ILLEGAL);
2735                         }
2736                 }
2737 
2738                 /*
2739                  * If we are attempting to modify the port for this QP, then
2740                  * check for valid port number and fill it in.  Also set the
2741                  * appropriate flag in the "opmask" parameter.
2742                  */
2743                 if (flags & IBT_CEP_SET_PORT) {
2744                         portnum = rc->rc_path.cep_hca_port_num;
2745                         if (tavor_portnum_is_valid(state, portnum)) {
2746                                 qpc->pri_addr_path.portnum = portnum;
2747                         } else {
2748                                 TNF_PROBE_1(tavor_qp_sqd2sqd_inv_port_fail,
2749                                     TAVOR_TNF_ERROR, "", tnf_uint, port,
2750                                     portnum);
2751                                 TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
2752                                 return (IBT_HCA_PORT_INVALID);
2753                         }
2754                         opmask |= TAVOR_CMD_OP_PRIM_PORT;
2755                 }
2756 
2757                 /*
2758                  * Check for optional alternate path and fill in the
2759                  * appropriate QPC fields if one is specified
2760                  */
2761                 if (flags & IBT_CEP_SET_ALT_PATH) {
2762                         qpc_path = &qpc->alt_addr_path;
2763                         adds_vect = &rc->rc_alt_path.cep_adds_vect;
2764 
2765                         /* Set the common alternate address path fields */
2766                         status = tavor_set_addr_path(state, adds_vect, qpc_path,
2767                             TAVOR_ADDRPATH_QP, qp);
2768                         if (status != DDI_SUCCESS) {
2769                                 TNF_PROBE_0(tavor_qp_sqd2sqd_setaddrpath_fail,
2770                                     TAVOR_TNF_ERROR, "");
2771                                 TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
2772                                 return (status);
2773                         }
2774                         qpc_path->ack_timeout = rc->rc_alt_path.cep_timeout;
2775 
2776                         /*
2777                          * Check for valid alternate path port number and fill
2778                          * it in
2779                          */
2780                         portnum = rc->rc_alt_path.cep_hca_port_num;
2781                         if (tavor_portnum_is_valid(state, portnum)) {
2782                                 qpc->alt_addr_path.portnum = portnum;
2783                         } else {
2784                                 TNF_PROBE_1(tavor_qp_sqd2sqd_inv_altport_fail,
2785                                     TAVOR_TNF_ERROR, "", tnf_uint, altport,
2786                                     portnum);
2787                                 TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
2788                                 return (IBT_HCA_PORT_INVALID);
2789                         }
2790 
2791                         /*
2792                          * Check for valid alternate path PKey index and fill
2793                          * it in
2794                          */
2795                         pkeyindx = rc->rc_alt_path.cep_pkey_ix;
2796                         if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
2797                                 qpc->alt_addr_path.pkey_indx = pkeyindx;
2798                         } else {
2799                                 TNF_PROBE_1(tavor_qp_sqd2sqd_inv_altpkey_fail,
2800                                     TAVOR_TNF_ERROR, "", tnf_uint, altpkeyindx,
2801                                     pkeyindx);
2802                                 TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
2803                                 return (IBT_PKEY_IX_ILLEGAL);
2804                         }
2805                         opmask |= TAVOR_CMD_OP_ALT_PATH;
2806                 }
2807 
2808                 /*
2809                  * If we are attempting to modify the number of "outgoing
2810                  * RDMA resources" for this QP, then check for valid value and
2811                  * fill it in.  Also set the appropriate flag in the "opmask"
2812                  * parameter.
2813                  */
2814                 if (flags & IBT_CEP_SET_RDMARA_OUT) {
2815                         rdma_ra_out = rc->rc_rdma_ra_out;
2816                         if (tavor_qp_validate_init_depth(state, rc,
2817                             &sra_max) != DDI_SUCCESS) {
2818                                 TNF_PROBE_1(tavor_qp_sqd2sqd_inv_rdma_out_fail,
2819                                     TAVOR_TNF_ERROR, "", tnf_uint, rdma_ra_out,
2820                                     rdma_ra_out);
2821                                 TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
2822                                 return (IBT_INVALID_PARAM);
2823                         }
2824                         qpc->sra_max = sra_max;
2825                         opmask |= TAVOR_CMD_OP_SRA_SET;
2826                 }
2827 
2828                 /*
2829                  * If we are attempting to modify the number of "incoming
2830                  * RDMA resources" for this QP, then check for valid value and
2831                  * update the "rra_max" and "ra_buf_index" fields in the QPC to
2832                  * point to the pre-allocated RDB resources (in DDR).  Also set
2833                  * the appropriate flag in the "opmask" parameter.
2834                  */
2835                 if (flags & IBT_CEP_SET_RDMARA_IN) {
2836                         rdma_ra_in = rc->rc_rdma_ra_in;
2837                         if (tavor_qp_validate_resp_rsrc(state, rc,
2838                             &rra_max) != DDI_SUCCESS) {
2839                                 TNF_PROBE_1(tavor_qp_sqd2sqd_inv_rdma_in_fail,
2840                                     TAVOR_TNF_ERROR, "", tnf_uint, rdma_ra_in,
2841                                     rdma_ra_in);
2842                                 TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
2843                                 return (IBT_INVALID_PARAM);
2844                         }
2845                         qpc->rra_max = rra_max;
2846                         qpc->ra_buff_indx = qp->qp_rdb_ddraddr >>
2847                             TAVOR_RDB_SIZE_SHIFT;
2848                         opmask |= TAVOR_CMD_OP_RRA_SET;
2849                 }
2850 
2851                 /*
2852                  * If we are attempting to modify the "Local Ack Timeout" value
2853                  * for this QP, then fill it in and set the appropriate flag in
2854                  * the "opmask" parameter.
2855                  */
2856                 if (flags & IBT_CEP_SET_TIMEOUT) {
2857                         qpc_path = &qpc->pri_addr_path;
2858                         qpc_path->ack_timeout = rc->rc_path.cep_timeout;
2859                         opmask |= TAVOR_CMD_OP_ACKTIMEOUT;
2860                 }
2861 
2862                 /*
2863                  * If we are attempting to modify the "Retry Count" for this QP,
2864                  * then fill it in and set the appropriate flag in the "opmask"
2865                  * parameter.
2866                  */
2867                 if (flags & IBT_CEP_SET_RETRY) {
2868                         qpc->retry_cnt = rc->rc_retry_cnt;
2869                         opmask |= TAVOR_CMD_OP_PRIM_RNRRETRY;
2870                 }
2871 
2872                 /*
2873                  * If we are attempting to modify the "RNR Retry Count" for this
2874                  * QP, then fill it in and set the appropriate flag in the
2875                  * "opmask" parameter.
2876                  */
2877                 if (flags & IBT_CEP_SET_RNR_NAK_RETRY) {
2878                         qpc_path = &qpc->pri_addr_path;
2879                         qpc_path->rnr_retry = rc->rc_rnr_retry_cnt;
2880                         opmask |= TAVOR_CMD_OP_RETRYCNT;
2881                 }
2882 
2883                 /*
2884                  * If we are attempting to modify the "Minimum RNR NAK" value
2885                  * for this QP, then fill it in and set the appropriate flag
2886                  * in the "opmask" parameter.
2887                  */
2888                 if (flags & IBT_CEP_SET_MIN_RNR_NAK) {
2889                         qpc->min_rnr_nak = rc->rc_min_rnr_nak;
2890                         opmask |= TAVOR_CMD_OP_MINRNRNAK;
2891                 }
2892 
2893         } else if (qp->qp_serv_type == TAVOR_QP_UC) {
2894                 uc = &info_p->qp_transport.uc;
2895 
2896                 /*
2897                  * Check if any of the flags indicate a change in the RDMA
2898                  * Write (recv) enable/disable and set the appropriate flag
2899                  * in the "opmask" parameter. Note: RDMA Read and Atomic are
2900                  * not valid for UC transport.
2901                  */
2902                 if (flags & IBT_CEP_SET_RDMA_W) {
2903                         qpc->rwe = (info_p->qp_flags & IBT_CEP_RDMA_WR) ? 1 : 0;
2904                         opmask |= TAVOR_CMD_OP_RWE;
2905                 }
2906 
2907                 /*
2908                  * Check for optional primary path and fill in the
2909                  * appropriate QPC fields if one is specified
2910                  */
2911                 if (flags & IBT_CEP_SET_ADDS_VECT) {
2912                         qpc_path = &qpc->pri_addr_path;
2913                         adds_vect = &uc->uc_path.cep_adds_vect;
2914 
2915                         /* Set the common primary address path fields */
2916                         status = tavor_set_addr_path(state, adds_vect, qpc_path,
2917                             TAVOR_ADDRPATH_QP, qp);
2918                         if (status != DDI_SUCCESS) {
2919                                 TNF_PROBE_0(tavor_qp_sqd2sqd_setaddrpath_fail,
2920                                     TAVOR_TNF_ERROR, "");
2921                                 TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
2922                                 return (status);
2923                         }
2924 
2925                         /*
2926                          * MTU changes as part of sqd2sqd are not allowed.
2927                          * Simply keep the same MTU value here, stored in the
2928                          * qphdl from init2rtr time.
2929                          */
2930                         qpc->mtu = qp->qp_save_mtu;
2931 
2932                         opmask |= TAVOR_CMD_OP_PRIM_PATH;
2933                 }
2934 
2935                 /*
2936                  * If we are attempting to modify the path migration state for
2937                  * this QP, then check for valid state and fill it in.  Also
2938                  * set the appropriate flag in the "opmask" parameter.
2939                  */
2940                 if (flags & IBT_CEP_SET_MIG) {
2941                         if (uc->uc_mig_state == IBT_STATE_MIGRATED) {
2942                                 qpc->pm_state = TAVOR_QP_PMSTATE_MIGRATED;
2943                         } else if (uc->uc_mig_state == IBT_STATE_REARMED) {
2944                                 qpc->pm_state = TAVOR_QP_PMSTATE_REARM;
2945                         } else {
2946                                 TNF_PROBE_1(tavor_qp_sqd2sqd_inv_mig_state_fail,
2947                                     TAVOR_TNF_ERROR, "", tnf_uint, mig_state,
2948                                     uc->uc_mig_state);
2949                                 TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
2950                                 return (IBT_QP_APM_STATE_INVALID);
2951                         }
2952                         opmask |= TAVOR_CMD_OP_PM_STATE;
2953                 }
2954 
2955                 /*
2956                  * If we are attempting to modify the PKey index for this QP,
2957                  * then check for valid PKey index and fill it in.  Also set
2958                  * the appropriate flag in the "opmask" parameter.
2959                  */
2960                 if (flags & IBT_CEP_SET_PKEY_IX) {
2961                         pkeyindx = uc->uc_path.cep_pkey_ix;
2962                         if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
2963                                 qpc->pri_addr_path.pkey_indx = pkeyindx;
2964                                 opmask |= TAVOR_CMD_OP_PKEYINDX;
2965                         } else {
2966                                 TNF_PROBE_1(tavor_qp_sqd2sqd_inv_pkey,
2967                                     TAVOR_TNF_ERROR, "", tnf_uint, pkeyindx,
2968                                     pkeyindx);
2969                                 TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
2970                                 return (IBT_PKEY_IX_ILLEGAL);
2971                         }
2972                 }
2973 
2974                 /*
2975                  * Check for optional alternate path and fill in the
2976                  * appropriate QPC fields if one is specified
2977                  */
2978                 if (flags & IBT_CEP_SET_ALT_PATH) {
2979                         qpc_path = &qpc->alt_addr_path;
2980                         adds_vect = &uc->uc_alt_path.cep_adds_vect;
2981 
2982                         /* Set the common alternate address path fields */
2983                         status = tavor_set_addr_path(state, adds_vect, qpc_path,
2984                             TAVOR_ADDRPATH_QP, qp);
2985                         if (status != DDI_SUCCESS) {
2986                                 TNF_PROBE_0(tavor_qp_sqd2sqd_setaddrpath_fail,
2987                                     TAVOR_TNF_ERROR, "");
2988                                 TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
2989                                 return (status);
2990                         }
2991 
2992                         /*
2993                          * Check for valid alternate path port number and fill
2994                          * it in
2995                          */
2996                         portnum = uc->uc_alt_path.cep_hca_port_num;
2997                         if (tavor_portnum_is_valid(state, portnum)) {
2998                                 qpc->alt_addr_path.portnum = portnum;
2999                         } else {
3000                                 TNF_PROBE_1(tavor_qp_sqd2sqd_inv_altport_fail,
3001                                     TAVOR_TNF_ERROR, "", tnf_uint, altport,
3002                                     portnum);
3003                                 TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
3004                                 return (IBT_HCA_PORT_INVALID);
3005                         }
3006 
3007                         /*
3008                          * Check for valid alternate path PKey index and fill
3009                          * it in
3010                          */
3011                         pkeyindx = uc->uc_alt_path.cep_pkey_ix;
3012                         if (tavor_pkeyindex_is_valid(state, pkeyindx)) {
3013                                 qpc->alt_addr_path.pkey_indx = pkeyindx;
3014                         } else {
3015                                 TNF_PROBE_1(tavor_qp_sqd2sqd_inv_altpkey_fail,
3016                                     TAVOR_TNF_ERROR, "", tnf_uint, altpkeyindx,
3017                                     pkeyindx);
3018                                 TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
3019                                 return (IBT_PKEY_IX_ILLEGAL);
3020                         }
3021                         opmask |= TAVOR_CMD_OP_ALT_PATH;
3022                 }
3023         } else {
3024                 /*
3025                  * Invalid QP transport type. If we got here then it's a
3026                  * warning of a probably serious problem.  So print a message
3027                  * and return failure
3028                  */
3029                 TAVOR_WARNING(state, "unknown QP transport type in sqd2sqd");
3030                 TNF_PROBE_0(tavor_qp_sqd2sqd_inv_transtype_fail,
3031                     TAVOR_TNF_ERROR, "");
3032                 TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
3033                 return (ibc_get_ci_failure(0));
3034         }
3035 
3036         /*
3037          * Post the SQD2SQD_QP command to the Tavor firmware
3038          *
3039          * We do a TAVOR_NOSLEEP here because we are still holding the
3040          * "qp_lock".  If we got raised to interrupt level by priority
3041          * inversion, we do not want to block in this routine waiting for
3042          * success.
3043          */
3044         status = tavor_cmn_qp_cmd_post(state, SQD2SQD_QP, qpc, qp->qp_qpnum,
3045             opmask, TAVOR_CMD_NOSLEEP_SPIN);
3046         if (status != TAVOR_CMD_SUCCESS) {
3047                 if (status != TAVOR_CMD_BAD_QP_STATE) {
3048                         cmn_err(CE_CONT, "Tavor: SQD2SQD_QP command failed: "
3049                             "%08x\n", status);
3050                         TNF_PROBE_1(tavor_qp_sqd2sqd_cmd_fail,
3051                             TAVOR_TNF_ERROR, "", tnf_uint, status, status);
3052                         TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
3053                         return (ibc_get_ci_failure(0));
3054                 } else {
3055                         TNF_PROBE_0(tavor_qp_sqd2sqd_inv_qpstate_fail,
3056                             TAVOR_TNF_ERROR, "");
3057                         TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
3058                         return (IBT_QP_STATE_INVALID);
3059                 }
3060         }
3061 
3062         TAVOR_TNF_EXIT(tavor_qp_sqd2sqd);
3063         return (DDI_SUCCESS);
3064 }
3065 
3066 
3067 /*
3068  * tavor_qp_sqerr2rts()
3069  *    Context: Can be called from interrupt or base context.
3070  */
3071 static int
3072 tavor_qp_sqerr2rts(tavor_state_t *state, tavor_qphdl_t qp,
3073     ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p)
3074 {
3075         tavor_hw_qpc_t          *qpc;
3076         ibt_qp_ud_attr_t        *ud;
3077         uint32_t                opmask = 0;
3078         int                     status;
3079 
3080         TAVOR_TNF_ENTER(tavor_qp_sqerr2rts);
3081 
3082         ASSERT(MUTEX_HELD(&qp->qp_lock));
3083 
3084         /*
3085          * Grab the temporary QPC entry from QP software state
3086          */
3087         qpc = &qp->qpc;
3088 
3089         /*
3090          * Since there are no common and/or Tavor-specific fields to be filled
3091          * in for this command, we begin with the QPC fields which are
3092          * specific to transport type.
3093          */
3094         if (qp->qp_serv_type == TAVOR_QP_UD) {
3095                 ud = &info_p->qp_transport.ud;
3096 
3097                 /*
3098                  * If we are attempting to modify the QKey for this QP, then
3099                  * fill it in and set the appropriate flag in the "opmask"
3100                  * parameter.
3101                  */
3102                 if (flags & IBT_CEP_SET_QKEY) {
3103                         qpc->qkey = ud->ud_qkey;
3104                         opmask |= TAVOR_CMD_OP_QKEY;
3105                 }
3106 
3107         } else if (qp->qp_serv_type == TAVOR_QP_UC) {
3108 
3109                 /*
3110                  * Check if any of the flags indicate a change in the RDMA
3111                  * Write (recv) enable/disable and set the appropriate flag
3112                  * in the "opmask" parameter. Note: RDMA Read and Atomic are
3113                  * not valid for UC transport.
3114                  */
3115                 if (flags & IBT_CEP_SET_RDMA_W) {
3116                         qpc->rwe = (info_p->qp_flags & IBT_CEP_RDMA_WR) ? 1 : 0;
3117                         opmask |= TAVOR_CMD_OP_RWE;
3118                 }
3119         } else {
3120                 /*
3121                  * Invalid QP transport type. If we got here then it's a
3122                  * warning of a probably serious problem.  So print a message
3123                  * and return failure
3124                  */
3125                 TAVOR_WARNING(state, "unknown QP transport type in sqerr2rts");
3126                 TNF_PROBE_0(tavor_qp_sqerr2rts_inv_transtype_fail,
3127                     TAVOR_TNF_ERROR, "");
3128                 TAVOR_TNF_EXIT(tavor_qp_sqerr2rts);
3129                 return (ibc_get_ci_failure(0));
3130         }
3131 
3132         /*
3133          * Post the SQERR2RTS_QP command to the Tavor firmware
3134          *
3135          * We do a TAVOR_NOSLEEP here because we are still holding the
3136          * "qp_lock".  If we got raised to interrupt level by priority
3137          * inversion, we do not want to block in this routine waiting for
3138          * success.
3139          */
3140         status = tavor_cmn_qp_cmd_post(state, SQERR2RTS_QP, qpc, qp->qp_qpnum,
3141             opmask, TAVOR_CMD_NOSLEEP_SPIN);
3142         if (status != TAVOR_CMD_SUCCESS) {
3143                 if (status != TAVOR_CMD_BAD_QP_STATE) {
3144                         cmn_err(CE_CONT, "Tavor: SQERR2RTS_QP command failed: "
3145                             "%08x\n", status);
3146                         TNF_PROBE_1(tavor_qp_sqerr2rts_cmd_fail,
3147                             TAVOR_TNF_ERROR, "", tnf_uint, status, status);
3148                         TAVOR_TNF_EXIT(tavor_qp_sqerr2rts);
3149                         return (ibc_get_ci_failure(0));
3150                 } else {
3151                         TNF_PROBE_0(tavor_qp_sqerr2rts_inv_qpstate_fail,
3152                             TAVOR_TNF_ERROR, "");
3153                         TAVOR_TNF_EXIT(tavor_qp_sqerr2rts);
3154                         return (IBT_QP_STATE_INVALID);
3155                 }
3156         }
3157 
3158         TAVOR_TNF_EXIT(tavor_qp_sqerr2rts);
3159         return (DDI_SUCCESS);
3160 }
3161 
3162 
3163 /*
3164  * tavor_qp_to_error()
3165  *    Context: Can be called from interrupt or base context.
3166  */
3167 static int
3168 tavor_qp_to_error(tavor_state_t *state, tavor_qphdl_t qp)
3169 {
3170         int     status;
3171 
3172         TAVOR_TNF_ENTER(tavor_qp_to_error);
3173 
3174         ASSERT(MUTEX_HELD(&qp->qp_lock));
3175 
3176         /*
3177          * Post the TOERR_QP command to the Tavor firmware
3178          *
3179          * We do a TAVOR_NOSLEEP here because we are still holding the
3180          * "qp_lock".  If we got raised to interrupt level by priority
3181          * inversion, we do not want to block in this routine waiting for
3182          * success.
3183          */
3184         status = tavor_cmn_qp_cmd_post(state, TOERR_QP, NULL, qp->qp_qpnum,
3185             0, TAVOR_CMD_NOSLEEP_SPIN);
3186         if (status != TAVOR_CMD_SUCCESS) {
3187                 cmn_err(CE_CONT, "Tavor: TOERR_QP command failed: %08x\n",
3188                     status);
3189                 TNF_PROBE_1(tavor_qp_to_error_cmd_fail,
3190                     TAVOR_TNF_ERROR, "", tnf_uint, status, status);
3191                 TAVOR_TNF_EXIT(tavor_qp_to_error);
3192                 return (ibc_get_ci_failure(0));
3193         }
3194 
3195         TAVOR_TNF_EXIT(tavor_qp_to_error);
3196         return (DDI_SUCCESS);
3197 }
3198 
3199 
3200 /*
3201  * tavor_qp_to_reset()
3202  *    Context: Can be called from interrupt or base context.
3203  */
3204 int
3205 tavor_qp_to_reset(tavor_state_t *state, tavor_qphdl_t qp)
3206 {
3207         tavor_hw_qpc_t  *qpc;
3208         int             status;
3209 
3210         TAVOR_TNF_ENTER(tavor_qp_to_reset);
3211 
3212         ASSERT(MUTEX_HELD(&qp->qp_lock));
3213 
3214         /*
3215          * Grab the temporary QPC entry from QP software state
3216          */
3217         qpc = &qp->qpc;
3218 
3219         /*
3220          * Post the TORST_QP command to the Tavor firmware
3221          *
3222          * We do a TAVOR_NOSLEEP here because we are still holding the
3223          * "qp_lock".  If we got raised to interrupt level by priority
3224          * inversion, we do not want to block in this routine waiting for
3225          * success.
3226          */
3227         status = tavor_cmn_qp_cmd_post(state, TORST_QP, qpc, qp->qp_qpnum,
3228             0, TAVOR_CMD_NOSLEEP_SPIN);
3229         if (status != TAVOR_CMD_SUCCESS) {
3230                 cmn_err(CE_CONT, "Tavor: TORST_QP command failed: %08x\n",
3231                     status);
3232                 TNF_PROBE_1(tavor_qp_to_reset_cmd_fail,
3233                     TAVOR_TNF_ERROR, "", tnf_uint, status, status);
3234                 TAVOR_TNF_EXIT(tavor_qp_to_reset);
3235                 return (ibc_get_ci_failure(0));
3236         }
3237 
3238         TAVOR_TNF_EXIT(tavor_qp_to_reset);
3239         return (DDI_SUCCESS);
3240 }
3241 
3242 
3243 /*
3244  * tavor_qp_reset2err()
3245  *    Context: Can be called from interrupt or base context.
3246  */
3247 static int
3248 tavor_qp_reset2err(tavor_state_t *state, tavor_qphdl_t qp)
3249 {
3250         tavor_hw_qpc_t  *qpc;
3251         int             status;
3252 
3253         TAVOR_TNF_ENTER(tavor_qp_reset2err);
3254 
3255         ASSERT(MUTEX_HELD(&qp->qp_lock));
3256 
3257         /*
3258          * In order to implement the transition from "Reset" directly to the
3259          * "Error" state, it is necessary to first give ownership of the QP
3260          * context to the Tavor hardware.  This is accomplished by transitioning
3261          * the QP to "Init" as an intermediate step and then, immediately
3262          * transitioning to "Error".
3263          *
3264          * When this function returns success, the QP context will be owned by
3265          * the Tavor hardware and will be in the "Error" state.
3266          */
3267 
3268         /*
3269          * Grab the temporary QPC entry from QP software state
3270          */
3271         qpc = &qp->qpc;
3272 
3273         /*
3274          * Fill in the common and/or Tavor-specific fields in the QPC
3275          */
3276         if (qp->qp_is_special) {
3277                 qpc->serv_type       = TAVOR_QP_MLX;
3278         } else {
3279                 qpc->serv_type       = qp->qp_serv_type;
3280         }
3281         qpc->pm_state                = TAVOR_QP_PMSTATE_MIGRATED;
3282         qpc->de                      = TAVOR_QP_DESC_EVT_ENABLED;
3283         qpc->sched_q         = TAVOR_QP_SCHEDQ_GET(qp->qp_qpnum);
3284         if (qp->qp_is_umap) {
3285                 qpc->usr_page = qp->qp_uarpg;
3286         } else {
3287                 qpc->usr_page = 0;
3288         }
3289         qpc->pd                      = qp->qp_pdhdl->pd_pdnum;
3290         qpc->wqe_baseaddr    = 0;
3291         qpc->wqe_lkey                = qp->qp_mrhdl->mr_lkey;
3292         qpc->ssc             = qp->qp_sq_sigtype;
3293         qpc->cqn_snd         = qp->qp_sq_cqhdl->cq_cqnum;
3294         qpc->rsc             = TAVOR_QP_RQ_ALL_SIGNALED;
3295         qpc->cqn_rcv         = qp->qp_rq_cqhdl->cq_cqnum;
3296         qpc->srq_en          = qp->qp_srq_en;
3297 
3298         if (qp->qp_srq_en == TAVOR_QP_SRQ_ENABLED) {
3299                 qpc->srq_number      = qp->qp_srqhdl->srq_srqnum;
3300         } else {
3301                 qpc->srq_number = 0;
3302         }
3303 
3304         /*
3305          * Now fill in the QPC fields which are specific to transport type
3306          */
3307         if (qp->qp_serv_type == TAVOR_QP_UD) {
3308                 /* Set the UD parameters to an invalid default */
3309                 qpc->qkey = 0;
3310                 qpc->pri_addr_path.portnum = 1;
3311                 qpc->pri_addr_path.pkey_indx = 0;
3312 
3313         } else if (qp->qp_serv_type == TAVOR_QP_RC) {
3314                 /* Set the RC parameters to invalid default */
3315                 qpc->rre = 0;
3316                 qpc->rwe = 0;
3317                 qpc->rae = 0;
3318                 qpc->pri_addr_path.portnum = 1;
3319                 qpc->pri_addr_path.pkey_indx = 0;
3320 
3321         } else if (qp->qp_serv_type == TAVOR_QP_UC) {
3322                 /* Set the UC parameters to invalid default */
3323                 qpc->rwe = 0;
3324                 qpc->pri_addr_path.portnum = 1;
3325                 qpc->pri_addr_path.pkey_indx = 0;
3326 
3327         } else {
3328                 /*
3329                  * Invalid QP transport type. If we got here then it's a
3330                  * warning of a probably serious problem.  So print a message
3331                  * and return failure
3332                  */
3333                 TAVOR_WARNING(state, "unknown QP transport type in rst2err");
3334                 TNF_PROBE_0(tavor_qp_reset2err_inv_transtype_fail,
3335                     TAVOR_TNF_ERROR, "");
3336                 TAVOR_TNF_EXIT(tavor_qp_reset2err);
3337                 return (ibc_get_ci_failure(0));
3338         }
3339 
3340         /*
3341          * Post the RST2INIT_QP command to the Tavor firmware
3342          *
3343          * We do a TAVOR_NOSLEEP here because we are still holding the
3344          * "qp_lock".  If we got raised to interrupt level by priority
3345          * inversion, we do not want to block in this routine waiting for
3346          * success.
3347          */
3348         status = tavor_cmn_qp_cmd_post(state, RST2INIT_QP, qpc, qp->qp_qpnum,
3349             0, TAVOR_CMD_NOSLEEP_SPIN);
3350         if (status != TAVOR_CMD_SUCCESS) {
3351                 cmn_err(CE_CONT, "Tavor: RST2INIT_QP command failed: %08x\n",
3352                     status);
3353                 TNF_PROBE_1(tavor_qp_reset2err_rst2init_cmd_fail,
3354                     TAVOR_TNF_ERROR, "", tnf_uint, status, status);
3355                 TAVOR_TNF_EXIT(tavor_qp_reset2err);
3356                 return (ibc_get_ci_failure(0));
3357         }
3358 
3359         /*
3360          * Now post the TOERR_QP command to the Tavor firmware
3361          *
3362          * We still do a TAVOR_NOSLEEP here because we are still holding the
3363          * "qp_lock".  Note:  If this fails (which it really never should),
3364          * it indicates a serious problem in the HW or SW.  We try to move
3365          * the QP back to the "Reset" state if possible and print a warning
3366          * message if not.  In any case, we return an error here.
3367          */
3368         status = tavor_cmn_qp_cmd_post(state, TOERR_QP, NULL, qp->qp_qpnum,
3369             0, TAVOR_CMD_NOSLEEP_SPIN);
3370         if (status != TAVOR_CMD_SUCCESS) {
3371                 cmn_err(CE_CONT, "Tavor: TOERR_QP command failed: %08x\n",
3372                     status);
3373                 if (tavor_qp_to_reset(state, qp) != DDI_SUCCESS) {
3374                         TAVOR_WARNING(state, "failed to reset QP context");
3375                 }
3376                 TNF_PROBE_1(tavor_qp_reset2err_toerr_cmd_fail,
3377                     TAVOR_TNF_ERROR, "", tnf_uint, status, status);
3378                 TAVOR_TNF_EXIT(tavor_qp_reset2err);
3379                 return (ibc_get_ci_failure(0));
3380         }
3381 
3382         TAVOR_TNF_EXIT(tavor_qp_reset2err);
3383         return (DDI_SUCCESS);
3384 }
3385 
3386 
3387 /*
3388  * tavor_check_rdma_enable_flags()
3389  *    Context: Can be called from interrupt or base context.
3390  */
3391 static uint_t
3392 tavor_check_rdma_enable_flags(ibt_cep_modify_flags_t flags,
3393     ibt_qp_info_t *info_p, tavor_hw_qpc_t *qpc)
3394 {
3395         uint_t  opmask = 0;
3396 
3397         if (flags & IBT_CEP_SET_RDMA_R) {
3398                 qpc->rre = (info_p->qp_flags & IBT_CEP_RDMA_RD) ? 1 : 0;
3399                 opmask |= TAVOR_CMD_OP_RRE;
3400         }
3401 
3402         if (flags & IBT_CEP_SET_RDMA_W) {
3403                 qpc->rwe = (info_p->qp_flags & IBT_CEP_RDMA_WR) ? 1 : 0;
3404                 opmask |= TAVOR_CMD_OP_RWE;
3405         }
3406 
3407         if (flags & IBT_CEP_SET_ATOMIC) {
3408                 qpc->rae = (info_p->qp_flags & IBT_CEP_ATOMIC) ? 1 : 0;
3409                 opmask |= TAVOR_CMD_OP_RAE;
3410         }
3411 
3412         return (opmask);
3413 }
3414 
3415 /*
3416  * tavor_qp_validate_resp_rsrc()
3417  *    Context: Can be called from interrupt or base context.
3418  */
3419 static int
3420 tavor_qp_validate_resp_rsrc(tavor_state_t *state, ibt_qp_rc_attr_t *rc,
3421     uint_t *rra_max)
3422 {
3423         uint_t  rdma_ra_in;
3424 
3425         rdma_ra_in = rc->rc_rdma_ra_in;
3426 
3427         /*
3428          * Check if number of responder resources is too large.  Return an
3429          * error if it is
3430          */
3431         if (rdma_ra_in > state->ts_cfg_profile->cp_hca_max_rdma_in_qp) {
3432                 return (IBT_INVALID_PARAM);
3433         }
3434 
3435         /*
3436          * If the number of responder resources is too small, round it up.
3437          * Then find the next highest power-of-2
3438          */
3439         if (rdma_ra_in == 0) {
3440                 rdma_ra_in = 1;
3441         }
3442         if (ISP2(rdma_ra_in)) {
3443                 *rra_max = highbit(rdma_ra_in) - 1;
3444         } else {
3445                 *rra_max = highbit(rdma_ra_in);
3446         }
3447         return (DDI_SUCCESS);
3448 }
3449 
3450 
3451 /*
3452  * tavor_qp_validate_init_depth()
3453  *    Context: Can be called from interrupt or base context.
3454  */
3455 static int
3456 tavor_qp_validate_init_depth(tavor_state_t *state, ibt_qp_rc_attr_t *rc,
3457     uint_t *sra_max)
3458 {
3459         uint_t  rdma_ra_out;
3460 
3461         rdma_ra_out = rc->rc_rdma_ra_out;
3462 
3463         /*
3464          * Check if requested initiator depth is too large.  Return an error
3465          * if it is
3466          */
3467         if (rdma_ra_out > state->ts_cfg_profile->cp_hca_max_rdma_out_qp) {
3468                 return (IBT_INVALID_PARAM);
3469         }
3470 
3471         /*
3472          * If the requested initiator depth is too small, round it up.
3473          * Then find the next highest power-of-2
3474          */
3475         if (rdma_ra_out == 0) {
3476                 rdma_ra_out = 1;
3477         }
3478         if (ISP2(rdma_ra_out)) {
3479                 *sra_max = highbit(rdma_ra_out) - 1;
3480         } else {
3481                 *sra_max = highbit(rdma_ra_out);
3482         }
3483         return (DDI_SUCCESS);
3484 }
3485 
3486 
3487 /*
3488  * tavor_qp_validate_mtu()
3489  *    Context: Can be called from interrupt or base context.
3490  */
3491 static int
3492 tavor_qp_validate_mtu(tavor_state_t *state, uint_t mtu)
3493 {
3494         /*
3495          * Check for invalid MTU values (i.e. zero or any value larger than
3496          * the HCA's port maximum).
3497          */
3498         if ((mtu == 0) || (mtu > state->ts_cfg_profile->cp_max_mtu)) {
3499                 return (IBT_HCA_PORT_MTU_EXCEEDED);
3500         }
3501         return (DDI_SUCCESS);
3502 }