1 /*
   2  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
   3  * Use is subject to license terms.
   4  */
   5 
   6 /*
   7  * Copyright (c) 1987 Regents of the University of California.
   8  * All rights reserved.
   9  *
  10  * Redistribution and use in source and binary forms are permitted
  11  * provided that the above copyright notice and this paragraph are
  12  * duplicated in all such forms and that any documentation,
  13  * advertising materials, and other materials related to such
  14  * distribution and use acknowledge that the software was developed
  15  * by the University of California, Berkeley. The name of the
  16  * University may not be used to endorse or promote products derived
  17  * from this software without specific prior written permission.
  18  * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
  19  * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
  20  * WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
  21  */
  22 
  23 #include "mpd_defs.h"
  24 #include "mpd_tables.h"
  25 
  26 /*
  27  * Probe types for probe()
  28  */
  29 #define PROBE_UNI       0x1234          /* Unicast probe packet */
  30 #define PROBE_MULTI     0x5678          /* Multicast probe packet */
  31 #define PROBE_RTT       0x9abc          /* RTT only probe packet */
  32 
  33 #define MSEC_PERMIN     (60 * MILLISEC) /* Number of milliseconds in a minute */
  34 
  35 /*
  36  * Format of probe / probe response packets. This is an ICMP Echo request
  37  * or ICMP Echo reply. Packet format is same for both IPv4 and IPv6
  38  */
  39 struct pr_icmp
  40 {
  41         uint8_t  pr_icmp_type;          /* type field */
  42         uint8_t  pr_icmp_code;          /* code field */
  43         uint16_t pr_icmp_cksum;         /* checksum field */
  44         uint16_t pr_icmp_id;            /* Identification */
  45         uint16_t pr_icmp_seq;           /* sequence number */
  46         uint64_t pr_icmp_timestamp;     /* Time stamp (in ns) */
  47         uint32_t pr_icmp_mtype;         /* Message type */
  48 };
  49 
  50 static struct in6_addr all_nodes_mcast_v6 = { { 0xff, 0x2, 0x0, 0x0,
  51                                     0x0, 0x0, 0x0, 0x0,
  52                                     0x0, 0x0, 0x0, 0x0,
  53                                     0x0, 0x0, 0x0, 0x1 } };
  54 
  55 static struct in_addr all_nodes_mcast_v4 = { { { 0xe0, 0x0, 0x0, 0x1 } } };
  56 
  57 static hrtime_t last_fdt_bumpup_time;   /* When FDT was bumped up last */
  58 
  59 static void             *find_ancillary(struct msghdr *msg, int cmsg_level,
  60     int cmsg_type);
  61 static void             pi_set_crtt(struct target *tg, int64_t m,
  62     boolean_t is_probe_uni);
  63 static void             incoming_echo_reply(struct phyint_instance *pii,
  64     struct pr_icmp *reply, struct in6_addr fromaddr, struct timeval *recv_tvp);
  65 static void             incoming_rtt_reply(struct phyint_instance *pii,
  66     struct pr_icmp *reply, struct in6_addr fromaddr);
  67 static void             incoming_mcast_reply(struct phyint_instance *pii,
  68     struct pr_icmp *reply, struct in6_addr fromaddr);
  69 
  70 static boolean_t        check_pg_crtt_improved(struct phyint_group *pg);
  71 static boolean_t        check_pii_crtt_improved(struct phyint_instance *pii);
  72 static boolean_t        check_exception_target(struct phyint_instance *pii,
  73     struct target *target);
  74 static void             probe_fail_info(struct phyint_instance *pii,
  75     struct target *cur_tg, struct probe_fail_count *pfinfo);
  76 static void             probe_success_info(struct phyint_instance *pii,
  77     struct target *cur_tg, struct probe_success_count *psinfo);
  78 static boolean_t        phyint_repaired(struct phyint *pi);
  79 
  80 static boolean_t        highest_ack_tg(uint16_t seq, struct target *tg);
  81 static int              in_cksum(ushort_t *addr, int len);
  82 static void             reset_snxt_basetimes(void);
  83 static int              ns2ms(int64_t ns);
  84 static int64_t          tv2ns(struct timeval *);
  85 
  86 /*
  87  * CRTT - Conservative Round Trip Time Estimate
  88  * Probe success - A matching probe reply received before CRTT ms has elapsed
  89  *      after sending the probe.
  90  * Probe failure - No probe reply received and more than CRTT ms has elapsed
  91  *      after sending the probe.
  92  *
  93  * TLS - Time last success. Most recent probe ack received at this time.
  94  * TFF - Time first fail. The time of the earliest probe failure in
  95  *      a consecutive series of probe failures.
  96  * NUM_PROBE_REPAIRS  - Number of consecutive successful probes required
  97  *      before declaring phyint repair.
  98  * NUM_PROBE_FAILS - Number of consecutive probe failures required to
  99  *      declare a phyint failure.
 100  *
 101  *                      Phyint state diagram
 102  *
 103  * The state of a phyint that is capable of being probed, is completely
 104  * specified by the 3-tuple <pi_state, pg_state, I>.
 105  *
 106  * A phyint starts in either PI_RUNNING or PI_OFFLINE, depending on whether
 107  * IFF_OFFLINE is set.  If the phyint is also configured with a test address
 108  * (the common case) and probe targets, then a phyint must also successfully
 109  * be able to send and receive probes in order to remain in the PI_RUNNING
 110  * state (otherwise, it transitions to PI_FAILED).
 111  *
 112  * Further, if a PI_RUNNING phyint is configured with a test address but is
 113  * unable to find any probe targets, it will transition to the PI_NOTARGETS
 114  * state, which indicates that the link is apparently functional but that
 115  * in.mpathd is unable to send probes to verify functionality (in this case,
 116  * in.mpathd makes the optimistic assumption that the interface is working
 117  * correctly and thus does not mark the interface FAILED, but reports it as
 118  * IPMP_IF_UNKNOWN through the async events and query interfaces).
 119  *
 120  * At any point, a phyint may be administratively marked offline via if_mpadm.
 121  * In this case, the interface always transitions to PI_OFFLINE, regardless
 122  * of its previous state.  When the interface is later brought back online,
 123  * in.mpathd acts as if the interface is new (and thus it transitions to
 124  * PI_RUNNING or PI_FAILED based on the status of the link and the result of
 125  * its probes, if probes are sent).
 126  *
 127  * pi_state -  PI_RUNNING or PI_FAILED
 128  *      PI_RUNNING: The failure detection logic says the phyint is good.
 129  *      PI_FAILED: The failure detection logic says the phyint has failed.
 130  *
 131  * pg_state  - PG_OK, PG_DEGRADED, or PG_FAILED.
 132  *      PG_OK: All interfaces in the group are OK.
 133  *      PG_DEGRADED: Some interfaces in the group are unusable.
 134  *      PG_FAILED: All interfaces in the group are unusable.
 135  *
 136  *      In the case of router targets, we assume that the current list of
 137  *      targets obtained from the routing table, is still valid, so the
 138  *      phyint stat is PI_FAILED. In the case of host targets, we delete the
 139  *      list of targets, and multicast to the all hosts, to reconstruct the
 140  *      target list. So the phyints are in the PI_NOTARGETS state.
 141  *
 142  * I -  value of (pi_flags & IFF_INACTIVE)
 143  *      IFF_INACTIVE: This phyint will not send or receive packets.
 144  *      Usually, inactive is tied to standby interfaces that are not yet
 145  *      needed (e.g., no non-standby interfaces in the group have failed).
 146  *      When failback has been disabled (FAILBACK=no configured), phyint can
 147  *      also be a non-STANDBY. In this case IFF_INACTIVE is set when phyint
 148  *      subsequently recovers after a failure.
 149  *
 150  * Not all 9 possible combinations of the above 3-tuple are possible.
 151  *
 152  * I is tracked by IP. pi_state is tracked by mpathd.
 153  *
 154  *                      pi_state state machine
 155  * ---------------------------------------------------------------------------
 156  *      Event                   State                   New State
 157  *                              Action:
 158  * ---------------------------------------------------------------------------
 159  *      IP interface failure    (PI_RUNNING, I == 0) -> (PI_FAILED, I == 0)
 160  *      detection               : set IFF_FAILED on this phyint
 161  *
 162  *      IP interface failure    (PI_RUNNING, I == 1) -> (PI_FAILED, I == 0)
 163  *      detection               : set IFF_FAILED on this phyint
 164  *
 165  *      IP interface repair     (PI_FAILED, I == 0, FAILBACK=yes)
 166  *      detection                                    -> (PI_RUNNING, I == 0)
 167  *                              : clear IFF_FAILED on this phyint
 168  *
 169  *      IP interface repair     (PI_FAILED, I == 0, FAILBACK=no)
 170  *      detection                                    ->      (PI_RUNNING, I == 1)
 171  *                              : clear IFF_FAILED on this phyint
 172  *                              : if failback is disabled set I == 1
 173  *
 174  *      Group failure           (perform on all phyints in the group)
 175  *      detection               PI_RUNNING              PI_FAILED
 176  *      (Router targets)        : set IFF_FAILED
 177  *
 178  *      Group failure           (perform on all phyints in the group)
 179  *      detection               PI_RUNNING              PI_NOTARGETS
 180  *      (Host targets)          : set IFF_FAILED
 181  *                              : delete the target list on all phyints
 182  * ---------------------------------------------------------------------------
 183  */
 184 
 185 struct probes_missed probes_missed;
 186 
 187 /*
 188  * Compose and transmit an ICMP ECHO REQUEST packet.  The IP header
 189  * will be added on by the kernel.  The id field identifies this phyint.
 190  * and the sequence number is an increasing (modulo 2^^16) integer. The data
 191  * portion holds the time value when the packet is sent. On echo this is
 192  * extracted to compute the round-trip time. Three different types of
 193  * probe packets are used.
 194  *
 195  * PROBE_UNI: This type is used to do failure detection / failure recovery
 196  *      and RTT calculation. PROBE_UNI probes are spaced apart in time,
 197  *      not less than the current CRTT. pii_probes[] stores data
 198  *      about these probes. These packets consume sequence number space.
 199  *
 200  * PROBE_RTT: This type is used to make only rtt measurements. Normally these
 201  *      are not used. Under heavy network load, the rtt may go up very high,
 202  *      due to a spike, or may appear to go high, due to extreme scheduling
 203  *      delays. Once the network stress is removed, mpathd takes long time to
 204  *      recover, because the probe_interval is already high, and it takes
 205  *      a long time to send out sufficient number of probes to bring down the
 206  *      rtt. To avoid this problem, PROBE_RTT probes are sent out every
 207  *      user_probe_interval ms. and will cause only rtt updates. These packets
 208  *      do not consume sequence number space nor is information about these
 209  *      packets stored in the pii_probes[]
 210  *
 211  * PROBE_MULTI: This type is only used to construct a list of targets, when
 212  *      no targets are known. The packet is multicast to the all hosts addr.
 213  */
 214 static void
 215 probe(struct phyint_instance *pii, uint_t probe_type, hrtime_t start_hrtime)
 216 {
 217         hrtime_t sent_hrtime;
 218         struct timeval sent_tv;
 219         struct pr_icmp probe_pkt;       /* Probe packet */
 220         struct sockaddr_storage targ;   /* target address */
 221         uint_t  targaddrlen;            /* targed address length */
 222         int     pr_ndx;                 /* probe index in pii->pii_probes[] */
 223         boolean_t sent = _B_FALSE;
 224         int     rval;
 225 
 226         if (debug & D_TARGET) {
 227                 logdebug("probe(%s %s %d %lld)\n", AF_STR(pii->pii_af),
 228                     pii->pii_name, probe_type, start_hrtime);
 229         }
 230 
 231         assert(pii->pii_probe_sock != -1);
 232         assert(probe_type == PROBE_UNI || probe_type == PROBE_MULTI ||
 233             probe_type == PROBE_RTT);
 234 
 235         probe_pkt.pr_icmp_type = (pii->pii_af == AF_INET) ?
 236             ICMP_ECHO_REQUEST : ICMP6_ECHO_REQUEST;
 237         probe_pkt.pr_icmp_code = 0;
 238         probe_pkt.pr_icmp_cksum = 0;
 239         probe_pkt.pr_icmp_seq = htons(pii->pii_snxt);
 240 
 241         /*
 242          * Since there is no need to do arithmetic on the icmpid,
 243          * (only equality check is done) pii_icmpid is stored in
 244          * network byte order at initialization itself.
 245          */
 246         probe_pkt.pr_icmp_id = pii->pii_icmpid;
 247         probe_pkt.pr_icmp_timestamp = htonll(start_hrtime);
 248         probe_pkt.pr_icmp_mtype = htonl(probe_type);
 249 
 250         /*
 251          * If probe_type is PROBE_MULTI, this packet will be multicast to
 252          * the all hosts address. Otherwise it is unicast to the next target.
 253          */
 254         assert(probe_type == PROBE_MULTI || ((pii->pii_target_next != NULL) &&
 255             pii->pii_rtt_target_next != NULL));
 256 
 257         bzero(&targ, sizeof (targ));
 258         targ.ss_family = pii->pii_af;
 259 
 260         if (pii->pii_af == AF_INET6) {
 261                 struct in6_addr *addr6;
 262 
 263                 addr6 = &((struct sockaddr_in6 *)&targ)->sin6_addr;
 264                 targaddrlen = sizeof (struct sockaddr_in6);
 265                 if (probe_type == PROBE_MULTI) {
 266                         *addr6 = all_nodes_mcast_v6;
 267                 } else if (probe_type == PROBE_UNI) {
 268                         *addr6 = pii->pii_target_next->tg_address;
 269                 } else { /* type is PROBE_RTT */
 270                         *addr6 = pii->pii_rtt_target_next->tg_address;
 271                 }
 272         } else {
 273                 struct in_addr *addr4;
 274 
 275                 addr4 = &((struct sockaddr_in *)&targ)->sin_addr;
 276                 targaddrlen = sizeof (struct sockaddr_in);
 277                 if (probe_type == PROBE_MULTI) {
 278                         *addr4 = all_nodes_mcast_v4;
 279                 } else if (probe_type == PROBE_UNI) {
 280                         IN6_V4MAPPED_TO_INADDR(
 281                             &pii->pii_target_next->tg_address, addr4);
 282                 } else { /* type is PROBE_RTT */
 283                         IN6_V4MAPPED_TO_INADDR(
 284                             &pii->pii_rtt_target_next->tg_address, addr4);
 285                 }
 286 
 287                 /*
 288                  * Compute the IPv4 icmp checksum. Does not cover the IP header.
 289                  */
 290                 probe_pkt.pr_icmp_cksum =
 291                     in_cksum((ushort_t *)&probe_pkt, (int)sizeof (probe_pkt));
 292         }
 293 
 294         /*
 295          * Use the current time as the time we sent.  Not atomic, but the best
 296          * we can do from here.
 297          */
 298         sent_hrtime = gethrtime();
 299         (void) gettimeofday(&sent_tv, NULL);
 300         rval = sendto(pii->pii_probe_sock, &probe_pkt, sizeof (probe_pkt), 0,
 301             (struct sockaddr *)&targ, targaddrlen);
 302         /*
 303          * If the send would block, this may either be transient or a hang in a
 304          * lower layer. We pretend the probe was actually sent, the daemon will
 305          * not see a reply to the probe and will fail the interface if normal
 306          * failure detection criteria are met.
 307          */
 308         if (rval == sizeof (probe_pkt) ||
 309             (rval == -1 && errno == EWOULDBLOCK)) {
 310                 sent = _B_TRUE;
 311         } else {
 312                 logperror_pii(pii, "probe: probe sendto");
 313         }
 314 
 315         /*
 316          * If this is a PROBE_UNI probe packet being unicast to a target, then
 317          * update our tables. We will need this info in processing the probe
 318          * response. PROBE_MULTI and PROBE_RTT packets are not used for
 319          * the purpose of failure or recovery detection. PROBE_MULTI packets
 320          * are only used to construct a list of targets. PROBE_RTT packets are
 321          * used only for updating the rtt and not for failure detection.
 322          */
 323         if (probe_type == PROBE_UNI && sent) {
 324                 pr_ndx = pii->pii_probe_next;
 325                 assert(pr_ndx >= 0 && pr_ndx < PROBE_STATS_COUNT);
 326 
 327                 /* Collect statistics, before we reuse the last slot. */
 328                 if (pii->pii_probes[pr_ndx].pr_status == PR_LOST)
 329                         pii->pii_cum_stats.lost++;
 330                 else if (pii->pii_probes[pr_ndx].pr_status == PR_ACKED)
 331                         pii->pii_cum_stats.acked++;
 332                 pii->pii_cum_stats.sent++;
 333 
 334                 pii->pii_probes[pr_ndx].pr_id = pii->pii_snxt;
 335                 pii->pii_probes[pr_ndx].pr_tv_sent = sent_tv;
 336                 pii->pii_probes[pr_ndx].pr_hrtime_sent = sent_hrtime;
 337                 pii->pii_probes[pr_ndx].pr_hrtime_start = start_hrtime;
 338                 pii->pii_probes[pr_ndx].pr_target = pii->pii_target_next;
 339                 probe_chstate(&pii->pii_probes[pr_ndx], pii, PR_UNACKED);
 340 
 341                 pii->pii_probe_next = PROBE_INDEX_NEXT(pii->pii_probe_next);
 342                 pii->pii_target_next = target_next(pii->pii_target_next);
 343                 assert(pii->pii_target_next != NULL);
 344                 /*
 345                  * If we have a single variable to denote the next target to
 346                  * probe for both rtt probes and failure detection probes, we
 347                  * could end up with a situation where the failure detection
 348                  * probe targets become disjoint from the rtt probe targets.
 349                  * Eg. if 2 targets and the actual fdt is double the user
 350                  * specified fdt. So we have 2 variables. In this scheme
 351                  * we also reset pii_rtt_target_next for every fdt probe,
 352                  * though that may not be necessary.
 353                  */
 354                 pii->pii_rtt_target_next = pii->pii_target_next;
 355                 pii->pii_snxt++;
 356         } else if (probe_type == PROBE_RTT) {
 357                 pii->pii_rtt_target_next =
 358                     target_next(pii->pii_rtt_target_next);
 359                 assert(pii->pii_rtt_target_next != NULL);
 360         }
 361 }
 362 
 363 /*
 364  * Incoming IPv4 data from wire, is received here. Called from main.
 365  */
 366 void
 367 in_data(struct phyint_instance *pii)
 368 {
 369         struct  sockaddr_in     from;
 370         struct  in6_addr        fromaddr;
 371         static uint64_t in_packet[(IP_MAXPACKET + 1)/8];
 372         static uint64_t ancillary_data[(IP_MAXPACKET + 1)/8];
 373         struct ip *ip;
 374         int     iphlen;
 375         int     len;
 376         char    abuf[INET_ADDRSTRLEN];
 377         struct msghdr msg;
 378         struct iovec iov;
 379         struct pr_icmp *reply;
 380         struct timeval *recv_tvp;
 381 
 382         if (debug & D_PROBE) {
 383                 logdebug("in_data(%s %s)\n",
 384                     AF_STR(pii->pii_af), pii->pii_name);
 385         }
 386 
 387         iov.iov_base = (char *)in_packet;
 388         iov.iov_len = sizeof (in_packet);
 389         msg.msg_iov = &iov;
 390         msg.msg_iovlen = 1;
 391         msg.msg_name = (struct sockaddr *)&from;
 392         msg.msg_namelen = sizeof (from);
 393         msg.msg_control = ancillary_data;
 394         msg.msg_controllen = sizeof (ancillary_data);
 395 
 396         /*
 397          * Poll has already told us that a message is waiting,
 398          * on this socket. Read it now. We should not block.
 399          */
 400         if ((len = recvmsg(pii->pii_probe_sock, &msg, 0)) < 0) {
 401                 logperror_pii(pii, "in_data: recvmsg");
 402                 return;
 403         }
 404 
 405         /*
 406          * If the datalink has indicated the link is down, don't go
 407          * any further.
 408          */
 409         if (LINK_DOWN(pii->pii_phyint))
 410                 return;
 411 
 412         /* Get the printable address for error reporting */
 413         (void) inet_ntop(AF_INET, &from.sin_addr, abuf, sizeof (abuf));
 414 
 415         /* Ignore packets > 64k or control buffers that don't fit */
 416         if (msg.msg_flags & (MSG_TRUNC|MSG_CTRUNC)) {
 417                 if (debug & D_PKTBAD) {
 418                         logdebug("Truncated message: msg_flags 0x%x from %s\n",
 419                             msg.msg_flags, abuf);
 420                 }
 421                 return;
 422         }
 423 
 424         /* Make sure packet contains at least minimum ICMP header */
 425         ip = (struct ip *)in_packet;
 426         iphlen = ip->ip_hl << 2;
 427         if (len < iphlen + ICMP_MINLEN) {
 428                 if (debug & D_PKTBAD) {
 429                         logdebug("in_data: packet too short (%d bytes)"
 430                             " from %s\n", len, abuf);
 431                 }
 432                 return;
 433         }
 434 
 435         /*
 436          * Subtract the IP hdr length, 'len' will be length of the probe
 437          * reply, starting from the icmp hdr.
 438          */
 439         len -= iphlen;
 440         /* LINTED */
 441         reply = (struct pr_icmp *)((char *)in_packet + iphlen);
 442 
 443         /* Probe replies are icmp echo replies. Ignore anything else */
 444         if (reply->pr_icmp_type != ICMP_ECHO_REPLY)
 445                 return;
 446 
 447         /*
 448          * The icmp id should match what we sent, which is stored
 449          * in pi_icmpid. The icmp code for reply must be 0.
 450          * The reply content must be a struct pr_icmp
 451          */
 452         if (reply->pr_icmp_id != pii->pii_icmpid) {
 453                 /* Not in response to our probe */
 454                 return;
 455         }
 456 
 457         if (reply->pr_icmp_code != 0) {
 458                 logtrace("probe reply code %d from %s on %s\n",
 459                     reply->pr_icmp_code, abuf, pii->pii_name);
 460                 return;
 461         }
 462 
 463         if (len < sizeof (struct pr_icmp)) {
 464                 logtrace("probe reply too short: %d bytes from %s on %s\n",
 465                     len, abuf, pii->pii_name);
 466                 return;
 467         }
 468 
 469         recv_tvp = find_ancillary(&msg, SOL_SOCKET, SCM_TIMESTAMP);
 470         if (recv_tvp == NULL) {
 471                 logtrace("message without timestamp from %s on %s\n",
 472                     abuf, pii->pii_name);
 473                 return;
 474         }
 475 
 476         IN6_INADDR_TO_V4MAPPED(&from.sin_addr, &fromaddr);
 477         if (reply->pr_icmp_mtype == htonl(PROBE_UNI))
 478                 /* Unicast probe reply */
 479                 incoming_echo_reply(pii, reply, fromaddr, recv_tvp);
 480         else if (reply->pr_icmp_mtype == htonl(PROBE_MULTI)) {
 481                 /* Multicast reply */
 482                 incoming_mcast_reply(pii, reply, fromaddr);
 483         } else if (reply->pr_icmp_mtype == htonl(PROBE_RTT)) {
 484                 incoming_rtt_reply(pii, reply, fromaddr);
 485         } else {
 486                 /* Probably not in response to our probe */
 487                 logtrace("probe reply type: %d from %s on %s\n",
 488                     reply->pr_icmp_mtype, abuf, pii->pii_name);
 489                 return;
 490         }
 491 }
 492 
 493 /*
 494  * Incoming IPv6 data from wire is received here. Called from main.
 495  */
 496 void
 497 in6_data(struct phyint_instance *pii)
 498 {
 499         struct sockaddr_in6 from;
 500         static uint64_t in_packet[(IP_MAXPACKET + 1)/8];
 501         static uint64_t ancillary_data[(IP_MAXPACKET + 1)/8];
 502         int len;
 503         char abuf[INET6_ADDRSTRLEN];
 504         struct msghdr msg;
 505         struct iovec iov;
 506         void    *opt;
 507         struct  pr_icmp *reply;
 508         struct  timeval *recv_tvp;
 509 
 510         if (debug & D_PROBE) {
 511                 logdebug("in6_data(%s %s)\n",
 512                     AF_STR(pii->pii_af), pii->pii_name);
 513         }
 514 
 515         iov.iov_base = (char *)in_packet;
 516         iov.iov_len = sizeof (in_packet);
 517         msg.msg_iov = &iov;
 518         msg.msg_iovlen = 1;
 519         msg.msg_name = (struct sockaddr *)&from;
 520         msg.msg_namelen = sizeof (from);
 521         msg.msg_control = ancillary_data;
 522         msg.msg_controllen = sizeof (ancillary_data);
 523 
 524         if ((len = recvmsg(pii->pii_probe_sock, &msg, 0)) < 0) {
 525                 logperror_pii(pii, "in6_data: recvmsg");
 526                 return;
 527         }
 528 
 529         /*
 530          * If the datalink has indicated that the link is down, don't go
 531          * any further.
 532          */
 533         if (LINK_DOWN(pii->pii_phyint))
 534                 return;
 535 
 536         /* Get the printable address for error reporting */
 537         (void) inet_ntop(AF_INET6, &from.sin6_addr, abuf, sizeof (abuf));
 538         if (len < ICMP_MINLEN) {
 539                 if (debug & D_PKTBAD) {
 540                         logdebug("Truncated message: msg_flags 0x%x from %s\n",
 541                             msg.msg_flags, abuf);
 542                 }
 543                 return;
 544         }
 545         /* Ignore packets > 64k or control buffers that don't fit */
 546         if (msg.msg_flags & (MSG_TRUNC|MSG_CTRUNC)) {
 547                 if (debug & D_PKTBAD) {
 548                         logdebug("Truncated message: msg_flags 0x%x from %s\n",
 549                             msg.msg_flags, abuf);
 550                 }
 551                 return;
 552         }
 553 
 554         reply = (struct pr_icmp *)in_packet;
 555         if (reply->pr_icmp_type != ICMP6_ECHO_REPLY)
 556                 return;
 557 
 558         if (reply->pr_icmp_id != pii->pii_icmpid) {
 559                 /* Not in response to our probe */
 560                 return;
 561         }
 562 
 563         /*
 564          * The kernel has already verified the the ICMP checksum.
 565          */
 566         if (!IN6_IS_ADDR_LINKLOCAL(&from.sin6_addr)) {
 567                 logtrace("ICMPv6 echo reply source address not linklocal from "
 568                     "%s on %s\n", abuf, pii->pii_name);
 569                 return;
 570         }
 571         opt = find_ancillary(&msg, IPPROTO_IPV6, IPV6_RTHDR);
 572         if (opt != NULL) {
 573                 /* Can't allow routing headers in probe replies  */
 574                 logtrace("message with routing header from %s on %s\n",
 575                     abuf, pii->pii_name);
 576                 return;
 577         }
 578 
 579         if (reply->pr_icmp_code != 0) {
 580                 logtrace("probe reply code: %d from %s on %s\n",
 581                     reply->pr_icmp_code, abuf, pii->pii_name);
 582                 return;
 583         }
 584         if (len < (sizeof (struct pr_icmp))) {
 585                 logtrace("probe reply too short: %d bytes from %s on %s\n",
 586                     len, abuf, pii->pii_name);
 587                 return;
 588         }
 589 
 590         recv_tvp = find_ancillary(&msg, SOL_SOCKET, SCM_TIMESTAMP);
 591         if (recv_tvp == NULL) {
 592                 logtrace("message without timestamp from %s on %s\n",
 593                     abuf, pii->pii_name);
 594                 return;
 595         }
 596 
 597         if (reply->pr_icmp_mtype == htonl(PROBE_UNI)) {
 598                 incoming_echo_reply(pii, reply, from.sin6_addr, recv_tvp);
 599         } else if (reply->pr_icmp_mtype == htonl(PROBE_MULTI)) {
 600                 incoming_mcast_reply(pii, reply, from.sin6_addr);
 601         } else if (reply->pr_icmp_mtype == htonl(PROBE_RTT)) {
 602                 incoming_rtt_reply(pii, reply, from.sin6_addr);
 603         } else  {
 604                 /* Probably not in response to our probe */
 605                 logtrace("probe reply type: %d from %s on %s\n",
 606                     reply->pr_icmp_mtype, abuf, pii->pii_name);
 607         }
 608 }
 609 
 610 /*
 611  * Process the incoming rtt reply, in response to our rtt probe.
 612  * Common for both IPv4 and IPv6. Unlike incoming_echo_reply() we don't
 613  * have any stored information about the probe we sent. So we don't log
 614  * any errors if we receive bad replies.
 615  */
 616 static void
 617 incoming_rtt_reply(struct phyint_instance *pii, struct pr_icmp *reply,
 618     struct in6_addr fromaddr)
 619 {
 620         int64_t m;              /* rtt measurement in ns */
 621         char    abuf[INET6_ADDRSTRLEN];
 622         struct  target  *target;
 623         struct  phyint_group *pg;
 624 
 625         /* Get the printable address for error reporting */
 626         (void) pr_addr(pii->pii_af, fromaddr, abuf, sizeof (abuf));
 627 
 628         if (debug & D_PROBE) {
 629                 logdebug("incoming_rtt_reply: %s %s %s\n",
 630                     AF_STR(pii->pii_af), pii->pii_name, abuf);
 631         }
 632 
 633         /* Do we know this target ? */
 634         target = target_lookup(pii, fromaddr);
 635         if (target == NULL)
 636                 return;
 637 
 638         m = (int64_t)(gethrtime() - ntohll(reply->pr_icmp_timestamp));
 639         /* Invalid rtt. It has wrapped around */
 640         if (m < 0)
 641                 return;
 642 
 643         /*
 644          * Don't update rtt until we see NUM_PROBE_REPAIRS probe responses
 645          * The initial few responses after the interface is repaired may
 646          * contain high rtt's because they could have been queued up waiting
 647          * for ARP/NDP resolution on a failed interface.
 648          */
 649         pg = pii->pii_phyint->pi_group;
 650         if ((pii->pii_state != PI_RUNNING) || GROUP_FAILED(pg))
 651                 return;
 652 
 653         /*
 654          * Update rtt only if the new rtt is lower than the current rtt.
 655          * (specified by the 3rd parameter to pi_set_crtt).
 656          * If a spike has caused the current probe_interval to be >
 657          * user_probe_interval, then this mechanism is used to bring down
 658          * the rtt rapidly once the network stress is removed.
 659          * If the new rtt is higher than the current rtt, we don't want to
 660          * update the rtt. We are having more than 1 outstanding probe and
 661          * the increase in rtt we are seeing is being unnecessarily weighted
 662          * many times. The regular rtt update will be handled by
 663          * incoming_echo_reply() and will take care of any rtt increase.
 664          */
 665         pi_set_crtt(target, m, _B_FALSE);
 666         if ((target->tg_crtt < (pg->pg_probeint / LOWER_FDT_TRIGGER)) &&
 667             (user_failure_detection_time < pg->pg_fdt) &&
 668             (last_fdt_bumpup_time + MIN_SETTLING_TIME < gethrtime())) {
 669                 /*
 670                  * If the crtt has now dropped by a factor of LOWER_FT_TRIGGER,
 671                  * investigate if we can improve the failure detection time to
 672                  * meet whatever the user specified.
 673                  */
 674                 if (check_pg_crtt_improved(pg)) {
 675                         pg->pg_fdt = MAX(pg->pg_fdt / NEXT_FDT_MULTIPLE,
 676                             user_failure_detection_time);
 677                         pg->pg_probeint = pg->pg_fdt / (NUM_PROBE_FAILS + 2);
 678                         if (pii->pii_phyint->pi_group != phyint_anongroup) {
 679                                 logerr("Improved failure detection time %d ms "
 680                                     "on (%s %s) for group \"%s\"\n",
 681                                     pg->pg_fdt, AF_STR(pii->pii_af),
 682                                     pii->pii_name,
 683                                     pii->pii_phyint->pi_group->pg_name);
 684                         }
 685                         if (user_failure_detection_time == pg->pg_fdt) {
 686                                 /* Avoid any truncation or rounding errors */
 687                                 pg->pg_probeint = user_probe_interval;
 688                                 /*
 689                                  * No more rtt probes will be sent. The actual
 690                                  * fdt has dropped to the user specified value.
 691                                  * pii_fd_snxt_basetime and pii_snxt_basetime
 692                                  * will be in sync henceforth.
 693                                  */
 694                                 reset_snxt_basetimes();
 695                         }
 696                 }
 697         }
 698 }
 699 
 700 /*
 701  * Process the incoming echo reply, in response to our unicast probe.
 702  * Common for both IPv4 and IPv6
 703  */
 704 static void
 705 incoming_echo_reply(struct phyint_instance *pii, struct pr_icmp *reply,
 706     struct in6_addr fromaddr, struct timeval *recv_tvp)
 707 {
 708         int64_t m;              /* rtt measurement in ns */
 709         hrtime_t cur_hrtime;    /* in ns from some arbitrary point */
 710         char    abuf[INET6_ADDRSTRLEN];
 711         int     pr_ndx;
 712         struct  target  *target;
 713         boolean_t exception;
 714         uint64_t pr_icmp_timestamp;
 715         uint16_t pr_icmp_seq;
 716         struct  probe_stats *pr_statp;
 717         struct  phyint_group *pg = pii->pii_phyint->pi_group;
 718 
 719         /* Get the printable address for error reporting */
 720         (void) pr_addr(pii->pii_af, fromaddr, abuf, sizeof (abuf));
 721 
 722         if (debug & D_PROBE) {
 723                 logdebug("incoming_echo_reply: %s %s %s seq %u recv_tvp %lld\n",
 724                     AF_STR(pii->pii_af), pii->pii_name, abuf,
 725                     ntohs(reply->pr_icmp_seq), tv2ns(recv_tvp));
 726         }
 727 
 728         pr_icmp_timestamp = ntohll(reply->pr_icmp_timestamp);
 729         pr_icmp_seq = ntohs(reply->pr_icmp_seq);
 730 
 731         /* Reject out of window probe replies */
 732         if (SEQ_GE(pr_icmp_seq, pii->pii_snxt) ||
 733             SEQ_LT(pr_icmp_seq, pii->pii_snxt - PROBE_STATS_COUNT)) {
 734                 logtrace("out of window probe seq %u snxt %u on %s from %s\n",
 735                     pr_icmp_seq, pii->pii_snxt, pii->pii_name, abuf);
 736                 pii->pii_cum_stats.unknown++;
 737                 return;
 738         }
 739 
 740         cur_hrtime = gethrtime();
 741         m = (int64_t)(cur_hrtime - pr_icmp_timestamp);
 742         if (m < 0) {
 743                 /*
 744                  * This is a ridiculously high value of rtt. rtt has wrapped
 745                  * around. Log a message, and ignore the rtt.
 746                  */
 747                 logerr("incoming_echo_reply: rtt wraparound cur_hrtime %lld "
 748                     "reply timestamp %lld\n", cur_hrtime, pr_icmp_timestamp);
 749         }
 750 
 751         /*
 752          * Get the probe index pr_ndx corresponding to the received icmp seq.
 753          * number in our pii->pii_probes[] array. The icmp sequence number
 754          * pii_snxt corresponds to the probe index pii->pii_probe_next
 755          */
 756         pr_ndx = MOD_SUB(pii->pii_probe_next,
 757             (uint16_t)(pii->pii_snxt - pr_icmp_seq), PROBE_STATS_COUNT);
 758 
 759         assert(PR_STATUS_VALID(pii->pii_probes[pr_ndx].pr_status));
 760 
 761         target = pii->pii_probes[pr_ndx].pr_target;
 762 
 763         /*
 764          * Perform sanity checks, whether this probe reply that we
 765          * have received is genuine
 766          */
 767         if (target != NULL) {
 768                 /*
 769                  * Compare the src. addr of the received ICMP or ICMPv6
 770                  * probe reply with the target address in our tables.
 771                  */
 772                 if (!IN6_ARE_ADDR_EQUAL(&target->tg_address, &fromaddr)) {
 773                         /*
 774                          * We don't have any record of having sent a probe to
 775                          * this target. This is a fake probe reply. Log an error
 776                          */
 777                         logtrace("probe status %d Fake probe reply seq %u "
 778                             "snxt %u on %s from %s\n",
 779                             pii->pii_probes[pr_ndx].pr_status,
 780                             pr_icmp_seq, pii->pii_snxt, pii->pii_name, abuf);
 781                         pii->pii_cum_stats.unknown++;
 782                         return;
 783                 } else if (pii->pii_probes[pr_ndx].pr_status == PR_ACKED) {
 784                         /*
 785                          * The address matches, but our tables indicate that
 786                          * this probe reply has been acked already. So this
 787                          * is a duplicate probe reply. Log an error
 788                          */
 789                         logtrace("probe status %d Duplicate probe reply seq %u "
 790                             "snxt %u on %s from %s\n",
 791                             pii->pii_probes[pr_ndx].pr_status,
 792                             pr_icmp_seq, pii->pii_snxt, pii->pii_name, abuf);
 793                         pii->pii_cum_stats.unknown++;
 794                         return;
 795                 }
 796         } else {
 797                 /*
 798                  * Target must not be NULL in the PR_UNACKED state
 799                  */
 800                 assert(pii->pii_probes[pr_ndx].pr_status != PR_UNACKED);
 801                 if (pii->pii_probes[pr_ndx].pr_status == PR_UNUSED) {
 802                         /*
 803                          * The probe stats slot is unused. So we didn't
 804                          * send out any probe to this target. This is a fake.
 805                          * Log an error.
 806                          */
 807                         logtrace("probe status %d Fake probe reply seq %u "
 808                             "snxt %u on %s from %s\n",
 809                             pii->pii_probes[pr_ndx].pr_status,
 810                             pr_icmp_seq, pii->pii_snxt, pii->pii_name, abuf);
 811                 }
 812                 pii->pii_cum_stats.unknown++;
 813                 return;
 814         }
 815 
 816         /*
 817          * If the rtt does not appear to be right, don't update the
 818          * rtt stats. This can happen if the system dropped into the
 819          * debugger, or the system was hung or too busy for a
 820          * substantial time that we didn't get a chance to run.
 821          */
 822         if ((m < 0) || (ns2ms(m) > PROBE_STATS_COUNT * pg->pg_probeint)) {
 823                 /*
 824                  * If the probe corresponding to this received response
 825                  * was truly sent 'm' ns. ago, then this response must
 826                  * have been rejected by the sequence number checks. The
 827                  * fact that it has passed the sequence number checks
 828                  * means that the measured rtt is wrong. We were probably
 829                  * scheduled long after the packet was received.
 830                  */
 831                 goto out;
 832         }
 833 
 834         /*
 835          * Don't update rtt until we see NUM_PROBE_REPAIRS probe responses
 836          * The initial few responses after the interface is repaired may
 837          * contain high rtt's because they could have been queued up waiting
 838          * for ARP/NDP resolution on a failed interface.
 839          */
 840         if ((pii->pii_state != PI_RUNNING) || GROUP_FAILED(pg))
 841                 goto out;
 842 
 843         /*
 844          * Don't update the Conservative Round Trip Time estimate for this
 845          * (phint, target) pair if this is the not the highest ack seq seen
 846          * thus far on this target.
 847          */
 848         if (!highest_ack_tg(pr_icmp_seq, target))
 849                 goto out;
 850 
 851         /*
 852          * Always update the rtt. This is a failure detection probe
 853          * and we want to measure both increase / decrease in rtt.
 854          */
 855         pi_set_crtt(target, m, _B_TRUE);
 856 
 857         /*
 858          * If the crtt exceeds the average time between probes,
 859          * investigate if this slow target is an exception. If so we
 860          * can avoid this target and still meet the failure detection
 861          * time. Otherwise we can't meet the failure detection time.
 862          */
 863         if (target->tg_crtt > pg->pg_probeint) {
 864                 exception = check_exception_target(pii, target);
 865                 if (exception) {
 866                         /*
 867                          * This target is exceptionally slow. Don't use it
 868                          * for future probes. check_exception_target() has
 869                          * made sure that we have at least MIN_PROBE_TARGETS
 870                          * other active targets
 871                          */
 872                         if (pii->pii_targets_are_routers) {
 873                                 /*
 874                                  * This is a slow router, mark it as slow
 875                                  * and don't use it for further probes. We
 876                                  * don't delete it, since it will be populated
 877                                  * again when we do a router scan. Hence we
 878                                  * need to maintain extra state (unlike the
 879                                  * host case below).  Mark it as TG_SLOW.
 880                                  */
 881                                 if (target->tg_status == TG_ACTIVE)
 882                                         pii->pii_ntargets--;
 883                                 target->tg_status = TG_SLOW;
 884                                 target->tg_latime = gethrtime();
 885                                 target->tg_rtt_sa = -1;
 886                                 target->tg_crtt = 0;
 887                                 target->tg_rtt_sd = 0;
 888                                 if (pii->pii_target_next == target) {
 889                                         pii->pii_target_next =
 890                                             target_next(target);
 891                                 }
 892                         } else {
 893                                 /*
 894                                  * the slow target is not a router, we can
 895                                  * just delete it. Send an icmp multicast and
 896                                  * pick the fastest responder that is not
 897                                  * already an active target. target_delete()
 898                                  * adjusts pii->pii_target_next
 899                                  */
 900                                 target_delete(target);
 901                                 probe(pii, PROBE_MULTI, cur_hrtime);
 902                         }
 903                 } else {
 904                         /*
 905                          * We can't meet the failure detection time.
 906                          * Log a message, and update the detection time to
 907                          * whatever we can achieve.
 908                          */
 909                         pg->pg_probeint = target->tg_crtt * NEXT_FDT_MULTIPLE;
 910                         pg->pg_fdt = pg->pg_probeint * (NUM_PROBE_FAILS + 2);
 911                         last_fdt_bumpup_time = gethrtime();
 912                         if (pg != phyint_anongroup) {
 913                                 logtrace("Cannot meet requested failure"
 914                                     " detection time of %d ms on (%s %s) new"
 915                                     " failure detection time for group \"%s\""
 916                                     " is %d ms\n", user_failure_detection_time,
 917                                     AF_STR(pii->pii_af), pii->pii_name,
 918                                     pg->pg_name, pg->pg_fdt);
 919                         }
 920                 }
 921         } else if ((target->tg_crtt < (pg->pg_probeint / LOWER_FDT_TRIGGER)) &&
 922             (user_failure_detection_time < pg->pg_fdt) &&
 923             (last_fdt_bumpup_time + MIN_SETTLING_TIME < gethrtime())) {
 924                 /*
 925                  * If the crtt has now dropped by a factor of LOWER_FDT_TRIGGER
 926                  * investigate if we can improve the failure detection time to
 927                  * meet whatever the user specified.
 928                  */
 929                 if (check_pg_crtt_improved(pg)) {
 930                         pg->pg_fdt = MAX(pg->pg_fdt / NEXT_FDT_MULTIPLE,
 931                             user_failure_detection_time);
 932                         pg->pg_probeint = pg->pg_fdt / (NUM_PROBE_FAILS + 2);
 933                         if (pg != phyint_anongroup) {
 934                                 logtrace("Improved failure detection time %d ms"
 935                                     " on (%s %s) for group \"%s\"\n",
 936                                     pg->pg_fdt, AF_STR(pii->pii_af),
 937                                     pii->pii_name, pg->pg_name);
 938                         }
 939                         if (user_failure_detection_time == pg->pg_fdt) {
 940                                 /* Avoid any truncation or rounding errors */
 941                                 pg->pg_probeint = user_probe_interval;
 942                                 /*
 943                                  * No more rtt probes will be sent. The actual
 944                                  * fdt has dropped to the user specified value.
 945                                  * pii_fd_snxt_basetime and pii_snxt_basetime
 946                                  * will be in sync henceforth.
 947                                  */
 948                                 reset_snxt_basetimes();
 949                         }
 950                 }
 951         }
 952 out:
 953         pr_statp = &pii->pii_probes[pr_ndx];
 954         pr_statp->pr_hrtime_ackproc = cur_hrtime;
 955         pr_statp->pr_hrtime_ackrecv = pr_statp->pr_hrtime_sent +
 956             (tv2ns(recv_tvp) - tv2ns(&pr_statp->pr_tv_sent));
 957 
 958         probe_chstate(pr_statp, pii, PR_ACKED);
 959 
 960         /*
 961          * Update pii->pii_rack, i.e. the sequence number of the last received
 962          * probe response, based on the echo reply we have received now, if
 963          * either of the following conditions are satisfied.
 964          * a. pii_rack is outside the current receive window of
 965          *    [pii->pii_snxt - PROBE_STATS_COUNT, pii->pii_snxt).
 966          *    This means we have not received probe responses for a
 967          *    long time, and the sequence number has wrapped around.
 968          * b. pii_rack is within the current receive window and this echo
 969          *    reply corresponds to the highest sequence number we have seen
 970          *    so far.
 971          */
 972         if (SEQ_GE(pii->pii_rack, pii->pii_snxt) ||
 973             SEQ_LT(pii->pii_rack, pii->pii_snxt - PROBE_STATS_COUNT) ||
 974             SEQ_GT(pr_icmp_seq, pii->pii_rack)) {
 975                 pii->pii_rack = pr_icmp_seq;
 976         }
 977 }
 978 
 979 /*
 980  * Returns true if seq is the highest unacknowledged seq for target tg
 981  * else returns false
 982  */
 983 static boolean_t
 984 highest_ack_tg(uint16_t seq, struct target *tg)
 985 {
 986         struct phyint_instance *pii;
 987         int      pr_ndx;
 988         uint16_t pr_seq;
 989 
 990         pii = tg->tg_phyint_inst;
 991 
 992         /*
 993          * Get the seq number of the most recent probe sent so far,
 994          * and also get the corresponding probe index in the probe stats
 995          * array.
 996          */
 997         pr_ndx = PROBE_INDEX_PREV(pii->pii_probe_next);
 998         pr_seq = pii->pii_snxt;
 999         pr_seq--;
1000 
1001         /*
1002          * Start from the most recent probe and walk back, trying to find
1003          * an acked probe corresponding to target tg.
1004          */
1005         for (; pr_ndx != pii->pii_probe_next;
1006             pr_ndx = PROBE_INDEX_PREV(pr_ndx), pr_seq--) {
1007                 if (pii->pii_probes[pr_ndx].pr_target == tg &&
1008                     pii->pii_probes[pr_ndx].pr_status == PR_ACKED) {
1009                         if (SEQ_GT(pr_seq, seq))
1010                                 return (_B_FALSE);
1011                 }
1012         }
1013         return (_B_TRUE);
1014 }
1015 
1016 /*
1017  * Check whether the crtt for the group has improved by a factor of
1018  * LOWER_FDT_TRIGGER.  Small crtt improvements are ignored to avoid failure
1019  * detection time flapping in the face of small crtt changes.
1020  */
1021 static boolean_t
1022 check_pg_crtt_improved(struct phyint_group *pg)
1023 {
1024         struct  phyint *pi;
1025 
1026         if (debug & D_PROBE)
1027                 logdebug("check_pg_crtt_improved()\n");
1028 
1029         /*
1030          * The crtt for the group is only improved if each phyint_instance
1031          * for both ipv4 and ipv6 is improved.
1032          */
1033         for (pi = pg->pg_phyint; pi != NULL; pi = pi->pi_pgnext) {
1034                 if (!check_pii_crtt_improved(pi->pi_v4) ||
1035                     !check_pii_crtt_improved(pi->pi_v6))
1036                         return (_B_FALSE);
1037         }
1038 
1039         return (_B_TRUE);
1040 }
1041 
1042 /*
1043  * Check whether the crtt has improved substantially on this phyint_instance.
1044  * Returns _B_TRUE if there's no crtt information available, because pii
1045  * is NULL or the phyint_instance is not capable of probing.
1046  */
1047 boolean_t
1048 check_pii_crtt_improved(struct phyint_instance *pii) {
1049         struct  target *tg;
1050 
1051         if (pii == NULL)
1052                 return (_B_TRUE);
1053 
1054         if (!PROBE_CAPABLE(pii) ||
1055             pii->pii_phyint->pi_state == PI_FAILED)
1056                 return (_B_TRUE);
1057 
1058         for (tg = pii->pii_targets; tg != NULL; tg = tg->tg_next) {
1059                 if (tg->tg_status != TG_ACTIVE)
1060                         continue;
1061                 if (tg->tg_crtt > (pii->pii_phyint->pi_group->pg_probeint /
1062                     LOWER_FDT_TRIGGER)) {
1063                         return (_B_FALSE);
1064                 }
1065         }
1066 
1067         return (_B_TRUE);
1068 }
1069 
1070 /*
1071  * This target responds very slowly to probes. The target's crtt exceeds
1072  * the probe interval of its group. Compare against other targets
1073  * and determine if this target is an exception, if so return true, else false
1074  */
1075 static boolean_t
1076 check_exception_target(struct phyint_instance *pii, struct target *target)
1077 {
1078         struct  target *tg;
1079         char abuf[INET6_ADDRSTRLEN];
1080 
1081         if (debug & D_PROBE) {
1082                 logdebug("check_exception_target(%s %s target %s)\n",
1083                     AF_STR(pii->pii_af), pii->pii_name,
1084                     pr_addr(pii->pii_af, target->tg_address,
1085                     abuf, sizeof (abuf)));
1086         }
1087 
1088         /*
1089          * We should have at least MIN_PROBE_TARGETS + 1 good targets now,
1090          * to make a good judgement. Otherwise don't drop this target.
1091          */
1092         if (pii->pii_ntargets <  MIN_PROBE_TARGETS + 1)
1093                 return (_B_FALSE);
1094 
1095         /*
1096          * Determine whether only this particular target is slow.
1097          * We know that this target's crtt exceeds the group's probe interval.
1098          * If all other active targets have a
1099          * crtt < (this group's probe interval) / EXCEPTION_FACTOR,
1100          * then this target is considered slow.
1101          */
1102         for (tg = pii->pii_targets; tg != NULL; tg = tg->tg_next) {
1103                 if (tg != target && tg->tg_status == TG_ACTIVE) {
1104                         if (tg->tg_crtt >
1105                             pii->pii_phyint->pi_group->pg_probeint /
1106                             EXCEPTION_FACTOR) {
1107                                 return (_B_FALSE);
1108                         }
1109                 }
1110         }
1111 
1112         return (_B_TRUE);
1113 }
1114 
1115 /*
1116  * Update the target list. The icmp all hosts multicast has given us
1117  * some host to which we can send probes. If we already have sufficient
1118  * targets, discard it.
1119  */
1120 static void
1121 incoming_mcast_reply(struct phyint_instance *pii, struct pr_icmp *reply,
1122     struct in6_addr fromaddr)
1123 /* ARGSUSED */
1124 {
1125         int af;
1126         char abuf[INET6_ADDRSTRLEN];
1127         struct phyint *pi;
1128 
1129         if (debug & D_PROBE) {
1130                 logdebug("incoming_mcast_reply(%s %s %s)\n",
1131                     AF_STR(pii->pii_af), pii->pii_name,
1132                     pr_addr(pii->pii_af, fromaddr, abuf, sizeof (abuf)));
1133         }
1134 
1135         /*
1136          * Using host targets is a fallback mechanism. If we have
1137          * found a router, don't add this host target. If we already
1138          * know MAX_PROBE_TARGETS, don't add another target.
1139          */
1140         assert(pii->pii_ntargets <= MAX_PROBE_TARGETS);
1141         if (pii->pii_targets != NULL) {
1142                 if (pii->pii_targets_are_routers ||
1143                     (pii->pii_ntargets == MAX_PROBE_TARGETS)) {
1144                         return;
1145                 }
1146         }
1147 
1148         if (IN6_IS_ADDR_UNSPECIFIED(&fromaddr) ||
1149             IN6_IS_ADDR_V4MAPPED_ANY(&fromaddr)) {
1150                 /*
1151                  * Guard against response from 0.0.0.0
1152                  * and ::. Log a trace message
1153                  */
1154                 logtrace("probe response from %s on %s\n",
1155                     pr_addr(pii->pii_af, fromaddr, abuf, sizeof (abuf)),
1156                     pii->pii_name);
1157                 return;
1158         }
1159 
1160         /*
1161          * This address is one of our own, so reject this address as a
1162          * valid probe target.
1163          */
1164         af = pii->pii_af;
1165         if (own_address(fromaddr))
1166                 return;
1167 
1168         /*
1169          * If the phyint is part a named group, then add the address to all
1170          * members of the group.  Otherwise, add the address only to the
1171          * phyint itself, since other phyints in the anongroup may not be on
1172          * the same subnet.
1173          */
1174         pi = pii->pii_phyint;
1175         if (pi->pi_group == phyint_anongroup) {
1176                 target_add(pii, fromaddr, _B_FALSE);
1177         } else {
1178                 pi = pi->pi_group->pg_phyint;
1179                 for (; pi != NULL; pi = pi->pi_pgnext)
1180                         target_add(PHYINT_INSTANCE(pi, af), fromaddr, _B_FALSE);
1181         }
1182 }
1183 
1184 /*
1185  * Compute CRTT given an existing scaled average, scaled deviation estimate
1186  * and a new rtt time.  The formula is from Jacobson and Karels'
1187  * "Congestion Avoidance and Control" in SIGCOMM '88.  The variable names
1188  * are the same as those in Appendix A.2 of that paper.
1189  *
1190  * m = new measurement
1191  * sa = scaled RTT average (8 * average estimates)
1192  * sv = scaled mean deviation (mdev) of RTT (4 * deviation estimates).
1193  * crtt = Conservative round trip time. Used to determine whether probe
1194  * has timed out.
1195  *
1196  * New scaled average and deviation are passed back via sap and svp
1197  */
1198 static int64_t
1199 compute_crtt(int64_t *sap, int64_t *svp, int64_t m)
1200 {
1201         int64_t sa = *sap;
1202         int64_t sv = *svp;
1203         int64_t crtt;
1204         int64_t saved_m = m;
1205 
1206         assert(*sap >= -1);
1207         assert(*svp >= 0);
1208 
1209         if (sa != -1) {
1210                 /*
1211                  * Update average estimator:
1212                  *      new rtt = old rtt + 1/8 Error
1213                  *          where Error = m - old rtt
1214                  *      i.e. 8 * new rtt = 8 * old rtt + Error
1215                  *      i.e. new sa =  old sa + Error
1216                  */
1217                 m -= sa >> 3;             /* m is now Error in estimate. */
1218                 if ((sa += m) < 0) {
1219                         /* Don't allow the smoothed average to be negative. */
1220                         sa = 0;
1221                 }
1222 
1223                 /*
1224                  * Update deviation estimator:
1225                  *      new mdev =  old mdev + 1/4 (abs(Error) - old mdev)
1226                  *      i.e. 4 * new mdev = 4 * old mdev +
1227                  *              (abs(Error) - old mdev)
1228                  *      i.e. new sv = old sv + (abs(Error) - old mdev)
1229                  */
1230                 if (m < 0)
1231                         m = -m;
1232                 m -= sv >> 2;
1233                 sv += m;
1234         } else {
1235                 /* Initialization. This is the first response received. */
1236                 sa = (m << 3);
1237                 sv = (m << 1);
1238         }
1239 
1240         crtt = (sa >> 3) + sv;
1241 
1242         if (debug & D_PROBE) {
1243                 logerr("compute_crtt: m = %lld sa = %lld, sv = %lld -> "
1244                     "crtt = %lld\n", saved_m, sa, sv, crtt);
1245         }
1246 
1247         *sap = sa;
1248         *svp = sv;
1249 
1250         /*
1251          * CRTT = average estimates  + 4 * deviation estimates
1252          *      = sa / 8 + sv
1253          */
1254         return (crtt);
1255 }
1256 
1257 static void
1258 pi_set_crtt(struct target *tg, int64_t m, boolean_t is_probe_uni)
1259 {
1260         struct phyint_instance *pii = tg->tg_phyint_inst;
1261         int probe_interval = pii->pii_phyint->pi_group->pg_probeint;
1262         int64_t sa = tg->tg_rtt_sa;
1263         int64_t sv = tg->tg_rtt_sd;
1264         int new_crtt;
1265         int i;
1266 
1267         if (debug & D_PROBE)
1268                 logdebug("pi_set_crtt: target -  m %lld\n", m);
1269 
1270         /* store the round trip time, in case we need to defer computation */
1271         tg->tg_deferred[tg->tg_num_deferred] = m;
1272 
1273         new_crtt = ns2ms(compute_crtt(&sa, &sv, m));
1274 
1275         /*
1276          * If this probe's round trip time would singlehandedly cause an
1277          * increase in the group's probe interval consider it suspect.
1278          */
1279         if ((new_crtt > probe_interval) && is_probe_uni) {
1280                 if (debug & D_PROBE) {
1281                         logdebug("Received a suspect probe on %s, new_crtt ="
1282                             " %d, probe_interval = %d, num_deferred = %d\n",
1283                             pii->pii_probe_logint->li_name, new_crtt,
1284                             probe_interval, tg->tg_num_deferred);
1285                 }
1286 
1287                 /*
1288                  * If we've deferred as many rtts as we plan on deferring, then
1289                  * assume the link really did slow down and process all queued
1290                  * rtts
1291                  */
1292                 if (tg->tg_num_deferred == MAXDEFERREDRTT) {
1293                         if (debug & D_PROBE) {
1294                                 logdebug("Received MAXDEFERREDRTT probes which "
1295                                     "would cause an increased probe_interval.  "
1296                                     "Integrating queued rtt data points.\n");
1297                         }
1298 
1299                         for (i = 0; i <= tg->tg_num_deferred; i++) {
1300                                 tg->tg_crtt = ns2ms(compute_crtt(&tg->tg_rtt_sa,
1301                                     &tg->tg_rtt_sd, tg->tg_deferred[i]));
1302                         }
1303 
1304                         tg->tg_num_deferred = 0;
1305                 } else {
1306                         tg->tg_num_deferred++;
1307                 }
1308                 return;
1309         }
1310 
1311         /*
1312          * If this is a normal probe, or an RTT probe that would lead to a
1313          * reduced CRTT, then update our CRTT data.  Further, if this was
1314          * a normal probe, pitch any deferred probes since our probes are
1315          * again being answered within our CRTT estimates.
1316          */
1317         if (is_probe_uni || new_crtt < tg->tg_crtt) {
1318                 tg->tg_rtt_sa = sa;
1319                 tg->tg_rtt_sd = sv;
1320                 tg->tg_crtt = new_crtt;
1321                 if (is_probe_uni)
1322                         tg->tg_num_deferred = 0;
1323         }
1324 }
1325 
1326 /*
1327  * Return a pointer to the specified option buffer.
1328  * If not found return NULL.
1329  */
1330 static void *
1331 find_ancillary(struct msghdr *msg, int cmsg_level, int cmsg_type)
1332 {
1333         struct cmsghdr *cmsg;
1334 
1335         for (cmsg = CMSG_FIRSTHDR(msg); cmsg != NULL;
1336             cmsg = CMSG_NXTHDR(msg, cmsg)) {
1337                 if (cmsg->cmsg_level == cmsg_level &&
1338                     cmsg->cmsg_type == cmsg_type) {
1339                         return (CMSG_DATA(cmsg));
1340                 }
1341         }
1342         return (NULL);
1343 }
1344 
1345 /*
1346  * Try to activate another INACTIVE interface in the same group as `pi'.
1347  * Prefer STANDBY INACTIVE to just INACTIVE.
1348  */
1349 void
1350 phyint_activate_another(struct phyint *pi)
1351 {
1352         struct phyint *pi2;
1353         struct phyint *inactivepi = NULL;
1354 
1355         if (pi->pi_group == phyint_anongroup)
1356                 return;
1357 
1358         for (pi2 = pi->pi_group->pg_phyint; pi2 != NULL; pi2 = pi2->pi_pgnext) {
1359                 if (pi == pi2 || !phyint_is_functioning(pi2) ||
1360                     !(pi2->pi_flags & IFF_INACTIVE))
1361                         continue;
1362 
1363                 inactivepi = pi2;
1364                 if (pi2->pi_flags & IFF_STANDBY)
1365                         break;
1366         }
1367 
1368         if (inactivepi != NULL)
1369                 (void) change_pif_flags(inactivepi, 0, IFF_INACTIVE);
1370 }
1371 
1372 /*
1373  * Transition a phyint to PI_RUNNING.  The caller must ensure that the
1374  * transition is appropriate.  Clears IFF_OFFLINE or IFF_FAILED if
1375  * appropriate.  Also sets IFF_INACTIVE on this or other interfaces as
1376  * appropriate (see comment below).  Finally, also updates the phyint's group
1377  * state to account for the change.
1378  */
1379 void
1380 phyint_transition_to_running(struct phyint *pi)
1381 {
1382         struct phyint *pi2;
1383         struct phyint *actstandbypi = NULL;
1384         uint_t nactive = 0, nnonstandby = 0;
1385         boolean_t onlining = (pi->pi_state == PI_OFFLINE);
1386         boolean_t initial = (pi->pi_state == PI_INIT);
1387         uint64_t set, clear;
1388 
1389         /*
1390          * The interface is running again, but should it or another interface
1391          * in the group end up INACTIVE?  There are three cases:
1392          *
1393          * 1. If it's a STANDBY interface, it should be end up INACTIVE if
1394          *    the group is operating at capacity (i.e., there are at least as
1395          *    many active interfaces as non-STANDBY interfaces in the group).
1396          *    No other interfaces should be changed.
1397          *
1398          * 2. If it's a non-STANDBY interface and we're onlining it or
1399          *    FAILBACK is enabled, then it should *not* end up INACTIVE.
1400          *    Further, if the group is above capacity as a result of this
1401          *    interface, then an active STANDBY interface in the group should
1402          *    end up INACTIVE.
1403          *
1404          * 3. If it's a non-STANDBY interface, we're repairing it, and
1405          *    FAILBACK is disabled, then it should end up INACTIVE *unless*
1406          *    the group was failed (in which case we have no choice but to
1407          *    use it).  No other interfaces should be changed.
1408          */
1409         if (pi->pi_group != phyint_anongroup) {
1410                 pi2 = pi->pi_group->pg_phyint;
1411                 for (; pi2 != NULL; pi2 = pi2->pi_pgnext) {
1412                         if (!(pi2->pi_flags & IFF_STANDBY))
1413                                 nnonstandby++;
1414 
1415                         if (phyint_is_functioning(pi2) &&
1416                             !(pi2->pi_flags & IFF_INACTIVE)) {
1417                                 nactive++;
1418                                 if (pi2->pi_flags & IFF_STANDBY)
1419                                         actstandbypi = pi2;
1420                         }
1421                 }
1422         }
1423 
1424         set = 0;
1425         clear = (onlining ? IFF_OFFLINE : IFF_FAILED);
1426 
1427         if (pi->pi_flags & IFF_STANDBY) {                        /* case 1 */
1428                 if (nactive >= nnonstandby)
1429                         set |= IFF_INACTIVE;
1430                 else
1431                         clear |= IFF_INACTIVE;
1432         } else if (onlining || failback_enabled) {              /* case 2 */
1433                 if (nactive >= nnonstandby && actstandbypi != NULL)
1434                         (void) change_pif_flags(actstandbypi, IFF_INACTIVE, 0);
1435         } else if (!initial && !GROUP_FAILED(pi->pi_group)) {        /* case 3 */
1436                 set |= IFF_INACTIVE;
1437         }
1438         (void) change_pif_flags(pi, set, clear);
1439 
1440         phyint_chstate(pi, PI_RUNNING);
1441 
1442         /*
1443          * Update the group state to account for the change.
1444          */
1445         phyint_group_refresh_state(pi->pi_group);
1446 }
1447 
1448 /*
1449  * Adjust IFF_INACTIVE on the provided `pi' to trend the group configuration
1450  * to have at least one active interface and as many active interfaces as
1451  * non-standby interfaces.
1452  */
1453 void
1454 phyint_standby_refresh_inactive(struct phyint *pi)
1455 {
1456         struct phyint *pi2;
1457         uint_t nactive = 0, nnonstandby = 0;
1458 
1459         /*
1460          * All phyints in the anonymous group are effectively in their own
1461          * group and thus active regardless of whether they're marked standby.
1462          */
1463         if (pi->pi_group == phyint_anongroup) {
1464                 (void) change_pif_flags(pi, 0, IFF_INACTIVE);
1465                 return;
1466         }
1467 
1468         /*
1469          * If the phyint isn't functioning we can't consider it.
1470          */
1471         if (!phyint_is_functioning(pi))
1472                 return;
1473 
1474         for (pi2 = pi->pi_group->pg_phyint; pi2 != NULL; pi2 = pi2->pi_pgnext) {
1475                 if (!(pi2->pi_flags & IFF_STANDBY))
1476                         nnonstandby++;
1477 
1478                 if (phyint_is_functioning(pi2) &&
1479                     !(pi2->pi_flags & IFF_INACTIVE))
1480                         nactive++;
1481         }
1482 
1483         if (nactive == 0 || nactive < nnonstandby)
1484                 (void) change_pif_flags(pi, 0, IFF_INACTIVE);
1485         else if (nactive > nnonstandby)
1486                 (void) change_pif_flags(pi, IFF_INACTIVE, 0);
1487 }
1488 
1489 /*
1490  * See if a previously failed interface has started working again.
1491  */
1492 void
1493 phyint_check_for_repair(struct phyint *pi)
1494 {
1495         if (!phyint_repaired(pi))
1496                 return;
1497 
1498         if (pi->pi_group == phyint_anongroup) {
1499                 logerr("IP interface repair detected on %s\n", pi->pi_name);
1500         } else {
1501                 logerr("IP interface repair detected on %s of group %s\n",
1502                     pi->pi_name, pi->pi_group->pg_name);
1503         }
1504 
1505         /*
1506          * If the interface is PI_OFFLINE, it can't be made PI_RUNNING yet.
1507          * So just clear IFF_OFFLINE and defer phyint_transition_to_running()
1508          * until it is brought back online.
1509          */
1510         if (pi->pi_state == PI_OFFLINE) {
1511                 (void) change_pif_flags(pi, 0, IFF_FAILED);
1512                 return;
1513         }
1514 
1515         phyint_transition_to_running(pi);       /* calls phyint_chstate() */
1516 }
1517 
1518 /*
1519  * See if an interface has failed, or if the whole group of interfaces has
1520  * failed.
1521  */
1522 static void
1523 phyint_inst_check_for_failure(struct phyint_instance *pii)
1524 {
1525         struct phyint   *pi = pii->pii_phyint;
1526         struct phyint   *pi2;
1527         boolean_t       was_active;
1528 
1529         switch (failure_state(pii)) {
1530         case PHYINT_FAILURE:
1531                 was_active = ((pi->pi_flags & IFF_INACTIVE) == 0);
1532 
1533                 (void) change_pif_flags(pi, IFF_FAILED, IFF_INACTIVE);
1534                 if (pi->pi_group == phyint_anongroup) {
1535                         logerr("IP interface failure detected on %s\n",
1536                             pii->pii_name);
1537                 } else {
1538                         logerr("IP interface failure detected on %s of group"
1539                             " %s\n", pii->pii_name, pi->pi_group->pg_name);
1540                 }
1541 
1542                 /*
1543                  * If the failed interface was active, activate another
1544                  * INACTIVE interface in the group if possible.
1545                  */
1546                 if (was_active)
1547                         phyint_activate_another(pi);
1548 
1549                 /*
1550                  * If the interface is offline, the state change will be
1551                  * noted when it comes back online.
1552                  */
1553                 if (pi->pi_state != PI_OFFLINE) {
1554                         phyint_chstate(pi, PI_FAILED);
1555                         reset_crtt_all(pi);
1556                 }
1557                 break;
1558 
1559         case GROUP_FAILURE:
1560                 pi2 = pi->pi_group->pg_phyint;
1561                 for (; pi2 != NULL; pi2 = pi2->pi_pgnext) {
1562                         (void) change_pif_flags(pi2, IFF_FAILED, IFF_INACTIVE);
1563                         if (pi2->pi_state == PI_OFFLINE) /* see comment above */
1564                                 continue;
1565 
1566                         reset_crtt_all(pi2);
1567                         /*
1568                          * In the case of host targets, we would have flushed
1569                          * the targets, and gone to PI_NOTARGETS state.
1570                          */
1571                         if (pi2->pi_state == PI_RUNNING)
1572                                 phyint_chstate(pi2, PI_FAILED);
1573                 }
1574                 break;
1575 
1576         default:
1577                 break;
1578         }
1579 }
1580 
1581 /*
1582  * Determines if any timeout event has occurred and returns the number of
1583  * milliseconds until the next timeout event for the phyint. Returns
1584  * TIMER_INFINITY for "never".
1585  */
1586 uint_t
1587 phyint_inst_timer(struct phyint_instance *pii)
1588 {
1589         int     pr_ndx;
1590         uint_t  timeout;
1591         struct  target  *cur_tg;
1592         struct  probe_stats *pr_statp;
1593         struct  phyint_instance *pii_other;
1594         struct  phyint *pi;
1595         int     valid_unack_count;
1596         int     i;
1597         int     interval;
1598         uint_t  check_time;
1599         uint_t  cur_time;
1600         hrtime_t cur_hrtime;
1601         int     probe_interval = pii->pii_phyint->pi_group->pg_probeint;
1602 
1603         cur_hrtime = gethrtime();
1604         cur_time = ns2ms(cur_hrtime);
1605 
1606         if (debug & D_TIMER) {
1607                 logdebug("phyint_inst_timer(%s %s)\n",
1608                     AF_STR(pii->pii_af), pii->pii_name);
1609         }
1610 
1611         pii_other = phyint_inst_other(pii);
1612         if (!PROBE_ENABLED(pii) && !PROBE_ENABLED(pii_other)) {
1613                 /*
1614                  * Check to see if we're here due to link up/down flapping; If
1615                  * enough time has passed, then try to bring the interface
1616                  * back up; otherwise, schedule a timer to bring it back up
1617                  * when enough time *has* elapsed.
1618                  */
1619                 pi = pii->pii_phyint;
1620                 if (pi->pi_state == PI_FAILED && LINK_UP(pi)) {
1621                         check_time = pi->pi_whenup[pi->pi_whendx] + MSEC_PERMIN;
1622                         if (check_time > cur_time)
1623                                 return (check_time - cur_time);
1624 
1625                         phyint_check_for_repair(pi);
1626                 }
1627         }
1628 
1629         /*
1630          * If probing is not enabled on this phyint instance, don't proceed.
1631          */
1632         if (!PROBE_ENABLED(pii))
1633                 return (TIMER_INFINITY);
1634 
1635         /*
1636          * If the timer has fired too soon, probably triggered
1637          * by some other phyint instance, return the remaining
1638          * time
1639          */
1640         if (TIME_LT(cur_time, pii->pii_snxt_time))
1641                 return (pii->pii_snxt_time - cur_time);
1642 
1643         /*
1644          * If the link is down, don't send any probes for now.
1645          */
1646         if (LINK_DOWN(pii->pii_phyint))
1647                 return (TIMER_INFINITY);
1648 
1649         /*
1650          * Randomize the next probe time, between MIN_RANDOM_FACTOR
1651          * and MAX_RANDOM_FACTOR with respect to the base probe time.
1652          * Base probe time is strictly periodic.
1653          */
1654         interval = GET_RANDOM(
1655             (int)(MIN_RANDOM_FACTOR * user_probe_interval),
1656             (int)(MAX_RANDOM_FACTOR * user_probe_interval));
1657         pii->pii_snxt_time = pii->pii_snxt_basetime + interval;
1658 
1659         /*
1660          * Check if the current time > next time to probe. If so, we missed
1661          * sending 1 or more probes, probably due to heavy system load. At least
1662          * 'MIN_RANDOM_FACTOR * user_probe_interval' ms has elapsed since we
1663          * were scheduled. Make adjustments to the times, in multiples of
1664          * user_probe_interval.
1665          */
1666         if (TIME_GT(cur_time, pii->pii_snxt_time)) {
1667                 int n;
1668 
1669                 n = (cur_time - pii->pii_snxt_time) / user_probe_interval;
1670                 pii->pii_snxt_time   += (n + 1) * user_probe_interval;
1671                 pii->pii_snxt_basetime       += (n + 1) * user_probe_interval;
1672                 logtrace("missed sending %d probes cur_time %u snxt_time %u"
1673                     " snxt_basetime %u\n", n + 1, cur_time, pii->pii_snxt_time,
1674                     pii->pii_snxt_basetime);
1675 
1676                 /* Collect statistics about missed probes */
1677                 probes_missed.pm_nprobes += n + 1;
1678                 probes_missed.pm_ntimes++;
1679         }
1680         pii->pii_snxt_basetime += user_probe_interval;
1681         interval = pii->pii_snxt_time - cur_time;
1682         if (debug & D_TARGET) {
1683                 logdebug("cur_time %u snxt_time %u snxt_basetime %u"
1684                     " interval %u\n", cur_time, pii->pii_snxt_time,
1685                     pii->pii_snxt_basetime, interval);
1686         }
1687 
1688         /*
1689          * If no targets are known, we need to send an ICMP multicast. The
1690          * probe type is PROBE_MULTI.  We'll check back in 'interval' msec
1691          * to see if we found a target.
1692          */
1693         if (pii->pii_target_next == NULL) {
1694                 assert(pii->pii_ntargets == 0);
1695                 pii->pii_fd_snxt_basetime = pii->pii_snxt_basetime;
1696                 probe(pii, PROBE_MULTI, cur_time);
1697                 return (interval);
1698         }
1699 
1700         if ((user_probe_interval != probe_interval) &&
1701             TIME_LT(pii->pii_snxt_time, pii->pii_fd_snxt_basetime)) {
1702                 /*
1703                  * the failure detection (fd) probe timer has not yet fired.
1704                  * Need to send only an rtt probe. The probe type is PROBE_RTT.
1705                  */
1706                 probe(pii, PROBE_RTT, cur_hrtime);
1707                 return (interval);
1708         }
1709         /*
1710          * the fd probe timer has fired. Need to do all failure
1711          * detection / recovery calculations, and then send an fd probe
1712          * of type PROBE_UNI.
1713          */
1714         if (user_probe_interval == probe_interval) {
1715                 /*
1716                  * We could have missed some probes, and then adjusted
1717                  * pii_snxt_basetime above. Otherwise we could have
1718                  * blindly added probe_interval to pii_fd_snxt_basetime.
1719                  */
1720                 pii->pii_fd_snxt_basetime = pii->pii_snxt_basetime;
1721         } else {
1722                 pii->pii_fd_snxt_basetime += probe_interval;
1723                 if (TIME_GT(cur_time, pii->pii_fd_snxt_basetime)) {
1724                         int n;
1725 
1726                         n = (cur_time - pii->pii_fd_snxt_basetime) /
1727                             probe_interval;
1728                         pii->pii_fd_snxt_basetime += (n + 1) * probe_interval;
1729                 }
1730         }
1731 
1732         /*
1733          * We can have at most, the latest 2 probes that we sent, in
1734          * the PR_UNACKED state. All previous probes sent, are either
1735          * PR_LOST or PR_ACKED. An unacknowledged probe is considered
1736          * timed out if the probe's time_start + the CRTT < currenttime.
1737          * For each of the last 2 probes, examine whether it has timed
1738          * out. If so, mark it PR_LOST. The probe stats is a circular array.
1739          */
1740         pr_ndx = PROBE_INDEX_PREV(pii->pii_probe_next);
1741         valid_unack_count = 0;
1742 
1743         for (i = 0; i < 2; i++) {
1744                 pr_statp = &pii->pii_probes[pr_ndx];
1745                 cur_tg = pii->pii_probes[pr_ndx].pr_target;
1746                 switch (pr_statp->pr_status) {
1747                 case PR_ACKED:
1748                         /*
1749                          * We received back an ACK, so the switch clearly
1750                          * is not dropping our traffic, and thus we can
1751                          * enable failure detection immediately.
1752                          */
1753                         if (pii->pii_fd_hrtime > gethrtime()) {
1754                                 if (debug & D_PROBE) {
1755                                         logdebug("successful probe on %s; "
1756                                             "ending quiet period\n",
1757                                             pii->pii_phyint->pi_name);
1758                                 }
1759                                 pii->pii_fd_hrtime = gethrtime();
1760                         }
1761                         break;
1762 
1763                 case PR_UNACKED:
1764                         assert(cur_tg != NULL);
1765                         /*
1766                          * The crtt could be zero for some reason,
1767                          * Eg. the phyint could be failed. If the crtt is
1768                          * not available use group's probe interval,
1769                          * which is a worst case estimate.
1770                          */
1771                         timeout = ns2ms(pr_statp->pr_hrtime_start);
1772                         if (cur_tg->tg_crtt != 0) {
1773                                 timeout += cur_tg->tg_crtt;
1774                         } else {
1775                                 timeout += probe_interval;
1776                         }
1777                         if (TIME_LT(timeout, cur_time)) {
1778                                 pr_statp->pr_time_lost = timeout;
1779                                 probe_chstate(pr_statp, pii, PR_LOST);
1780                         } else if (i == 1) {
1781                                 /*
1782                                  * We are forced to consider this probe
1783                                  * lost, as we can have at most 2 unack.
1784                                  * probes any time, and we will be sending a
1785                                  * probe at the end of this function.
1786                                  * Normally, we should not be here, but
1787                                  * this can happen if an incoming response
1788                                  * that was considered lost has increased
1789                                  * the crtt for this target, and also bumped
1790                                  * up the FDT. Note that we never cancel or
1791                                  * increase the current pii_time_left, so
1792                                  * when the timer fires, we find 2 valid
1793                                  * unacked probes, and they are yet to timeout
1794                                  */
1795                                 pr_statp->pr_time_lost = cur_time;
1796                                 probe_chstate(pr_statp, pii, PR_LOST);
1797                         } else {
1798                                 /*
1799                                  * Only the most recent probe can enter
1800                                  * this 'else' arm. The second most recent
1801                                  * probe must take either of the above arms,
1802                                  * if it is unacked.
1803                                  */
1804                                 valid_unack_count++;
1805                         }
1806                         break;
1807                 }
1808                 pr_ndx = PROBE_INDEX_PREV(pr_ndx);
1809         }
1810 
1811         /*
1812          * We send out 1 probe randomly in the interval between one half
1813          * and one probe interval for the group. Given that the CRTT is always
1814          * less than the group's probe interval, we can have at most 1
1815          * unacknowledged probe now.  All previous probes are either lost or
1816          * acked.
1817          */
1818         assert(valid_unack_count == 0 || valid_unack_count == 1);
1819 
1820         /*
1821          * The timer has fired. Take appropriate action depending
1822          * on the current state of the phyint.
1823          *
1824          * PI_RUNNING state     - Failure detection
1825          * PI_FAILED state      - Repair detection
1826          */
1827         switch (pii->pii_phyint->pi_state) {
1828         case PI_FAILED:
1829                 /*
1830                  * If the most recent probe (excluding unacked probes that
1831                  * are yet to time out) has been acked, check whether the
1832                  * phyint is now repaired.
1833                  */
1834                 if (pii->pii_rack + valid_unack_count + 1 == pii->pii_snxt) {
1835                         phyint_check_for_repair(pii->pii_phyint);
1836                 }
1837                 break;
1838 
1839         case PI_RUNNING:
1840                 /*
1841                  * It's possible our probes have been lost because of a
1842                  * spanning-tree mandated quiet period on the switch.  If so,
1843                  * ignore the lost probes.
1844                  */
1845                 if (pii->pii_fd_hrtime - cur_hrtime > 0)
1846                         break;
1847 
1848                 if (pii->pii_rack + valid_unack_count + 1 != pii->pii_snxt) {
1849                         /*
1850                          * We have 1 or more failed probes (excluding unacked
1851                          * probes that are yet to time out). Determine if the
1852                          * phyint has failed.
1853                          */
1854                         phyint_inst_check_for_failure(pii);
1855                 }
1856                 break;
1857 
1858         default:
1859                 logerr("phyint_inst_timer: invalid state %d\n",
1860                     pii->pii_phyint->pi_state);
1861                 abort();
1862         }
1863 
1864         /*
1865          * Start the next probe. probe() will also set pii->pii_probe_time_left
1866          * to the group's probe interval. If phyint_failed -> target_flush_hosts
1867          * was called, the target list may be empty.
1868          */
1869         if (pii->pii_target_next != NULL) {
1870                 probe(pii, PROBE_UNI, cur_hrtime);
1871                 /*
1872                  * If we have just the one probe target, and we're not using
1873                  * router targets, try to find another as we presently have
1874                  * no resilience.
1875                  */
1876                 if (!pii->pii_targets_are_routers && pii->pii_ntargets == 1)
1877                         probe(pii, PROBE_MULTI, cur_hrtime);
1878         } else {
1879                 probe(pii, PROBE_MULTI, cur_hrtime);
1880         }
1881         return (interval);
1882 }
1883 
1884 /*
1885  * Start the probe timer for an interface instance.
1886  */
1887 void
1888 start_timer(struct phyint_instance *pii)
1889 {
1890         uint32_t interval;
1891 
1892         /*
1893          * Spread the base probe times (pi_snxt_basetime) across phyints
1894          * uniformly over the (curtime..curtime + the group's probe_interval).
1895          * pi_snxt_basetime is strictly periodic with a frequency of
1896          * the group's probe interval. The actual probe time pi_snxt_time
1897          * adds some randomness to pi_snxt_basetime and happens in probe().
1898          * For the 1st probe on each phyint after the timer is started,
1899          * pi_snxt_time and pi_snxt_basetime are the same.
1900          */
1901         interval = GET_RANDOM(0,
1902             (int)pii->pii_phyint->pi_group->pg_probeint);
1903 
1904         pii->pii_snxt_basetime = getcurrenttime() + interval;
1905         pii->pii_fd_snxt_basetime = pii->pii_snxt_basetime;
1906         pii->pii_snxt_time = pii->pii_snxt_basetime;
1907         timer_schedule(interval);
1908 }
1909 
1910 /*
1911  * Restart the probe timer on an interface instance.
1912  */
1913 static void
1914 restart_timer(struct phyint_instance *pii)
1915 {
1916         /*
1917          * We don't need to restart the timer if it was never started in
1918          * the first place (pii->pii_basetime_inited not set), as the timer
1919          * won't have gone off yet.
1920          */
1921         if (pii->pii_basetime_inited != 0) {
1922 
1923                 if (debug & D_LINKNOTE)
1924                         logdebug("restart timer: restarting timer on %s, "
1925                             "address family %s\n", pii->pii_phyint->pi_name,
1926                             AF_STR(pii->pii_af));
1927 
1928                 start_timer(pii);
1929         }
1930 }
1931 
1932 static void
1933 process_link_state_down(struct phyint *pi)
1934 {
1935         logerr("The link has gone down on %s\n", pi->pi_name);
1936 
1937         /*
1938          * Clear the probe statistics arrays, we don't want the repair
1939          * detection logic relying on probes that were successful prior
1940          * to the link going down.
1941          */
1942         if (PROBE_CAPABLE(pi->pi_v4))
1943                 clear_pii_probe_stats(pi->pi_v4);
1944         if (PROBE_CAPABLE(pi->pi_v6))
1945                 clear_pii_probe_stats(pi->pi_v6);
1946         /*
1947          * Check for interface failure.  Although we know the interface
1948          * has failed, we don't know if all the other interfaces in the
1949          * group have failed as well.
1950          */
1951         if ((pi->pi_state == PI_RUNNING) ||
1952             (pi->pi_state != PI_FAILED && !GROUP_FAILED(pi->pi_group))) {
1953                 if (debug & D_LINKNOTE) {
1954                         logdebug("process_link_state_down:"
1955                             " checking for failure on %s\n", pi->pi_name);
1956                 }
1957 
1958                 if (pi->pi_v4 != NULL)
1959                         phyint_inst_check_for_failure(pi->pi_v4);
1960                 else if (pi->pi_v6 != NULL)
1961                         phyint_inst_check_for_failure(pi->pi_v6);
1962         }
1963 }
1964 
1965 static void
1966 process_link_state_up(struct phyint *pi)
1967 {
1968         logerr("The link has come up on %s\n", pi->pi_name);
1969 
1970         /*
1971          * We stopped any running timers on each instance when the link
1972          * went down, so restart them.
1973          */
1974         if (pi->pi_v4)
1975                 restart_timer(pi->pi_v4);
1976         if (pi->pi_v6)
1977                 restart_timer(pi->pi_v6);
1978 
1979         phyint_check_for_repair(pi);
1980 
1981         pi->pi_whenup[pi->pi_whendx++] = getcurrenttime();
1982         if (pi->pi_whendx == LINK_UP_PERMIN)
1983                 pi->pi_whendx = 0;
1984 }
1985 
1986 /*
1987  * Process any changes in link state passed up from the interfaces.
1988  */
1989 void
1990 process_link_state_changes(void)
1991 {
1992         struct phyint *pi;
1993 
1994         /* Look for interfaces where the link state has just changed */
1995 
1996         for (pi = phyints; pi != NULL; pi = pi->pi_next) {
1997                 boolean_t old_link_state_up = LINK_UP(pi);
1998 
1999                 /*
2000                  * Except when the "phyint" structure is created, this is
2001                  * the only place the link state is updated.  This allows
2002                  * this routine to detect changes in link state, rather
2003                  * than just the current state.
2004                  */
2005                 UPDATE_LINK_STATE(pi);
2006 
2007                 if (LINK_DOWN(pi)) {
2008                         /*
2009                          * Has link just gone down?
2010                          */
2011                         if (old_link_state_up)
2012                                 process_link_state_down(pi);
2013                 } else {
2014                         /*
2015                          * Has link just gone back up?
2016                          */
2017                         if (!old_link_state_up)
2018                                 process_link_state_up(pi);
2019                 }
2020         }
2021 }
2022 
2023 void
2024 reset_crtt_all(struct phyint *pi)
2025 {
2026         struct phyint_instance *pii;
2027         struct target *tg;
2028 
2029         pii = pi->pi_v4;
2030         if (pii != NULL) {
2031                 for (tg = pii->pii_targets; tg != NULL; tg = tg->tg_next) {
2032                         tg->tg_crtt = 0;
2033                         tg->tg_rtt_sa = -1;
2034                         tg->tg_rtt_sd = 0;
2035                 }
2036         }
2037 
2038         pii = pi->pi_v6;
2039         if (pii != NULL) {
2040                 for (tg = pii->pii_targets; tg != NULL; tg = tg->tg_next) {
2041                         tg->tg_crtt = 0;
2042                         tg->tg_rtt_sa = -1;
2043                         tg->tg_rtt_sd = 0;
2044                 }
2045         }
2046 }
2047 
2048 /*
2049  * Check if the phyint has failed the last NUM_PROBE_FAILS consecutive
2050  * probes on both instances IPv4 and IPv6.
2051  * If the interface has failed, return the time of the first probe failure
2052  * in "tff".
2053  */
2054 static int
2055 phyint_inst_probe_failure_state(struct phyint_instance *pii, uint_t *tff)
2056 {
2057         uint_t  pi_tff;
2058         struct  target *cur_tg;
2059         struct  probe_fail_count pfinfo;
2060         struct  phyint_instance *pii_other;
2061         int     pr_ndx;
2062 
2063         /*
2064          * Get the number of consecutive failed probes on
2065          * this phyint across all targets. Also get the number
2066          * of consecutive failed probes on this target only
2067          */
2068         pr_ndx = PROBE_INDEX_PREV(pii->pii_probe_next);
2069         cur_tg = pii->pii_probes[pr_ndx].pr_target;
2070         probe_fail_info(pii, cur_tg, &pfinfo);
2071 
2072         /* Get the time of first failure, for later use */
2073         pi_tff = pfinfo.pf_tff;
2074 
2075         /*
2076          * If the current target has not responded to the
2077          * last NUM_PROBE_FAILS probes, and other targets are
2078          * responding delete this target. Dead gateway detection
2079          * will eventually remove this target (if router) from the
2080          * routing tables. If that does not occur, we may end
2081          * up adding this to our list again.
2082          */
2083         if (pfinfo.pf_nfail < NUM_PROBE_FAILS &&
2084             pfinfo.pf_nfail_tg >= NUM_PROBE_FAILS) {
2085                 if (pii->pii_targets_are_routers) {
2086                         if (cur_tg->tg_status == TG_ACTIVE)
2087                                 pii->pii_ntargets--;
2088                         cur_tg->tg_status = TG_DEAD;
2089                         cur_tg->tg_crtt = 0;
2090                         cur_tg->tg_rtt_sa = -1;
2091                         cur_tg->tg_rtt_sd = 0;
2092                         if (pii->pii_target_next == cur_tg)
2093                                 pii->pii_target_next = target_next(cur_tg);
2094                 } else {
2095                         target_delete(cur_tg);
2096                         probe(pii, PROBE_MULTI, gethrtime());
2097                 }
2098                 return (PHYINT_OK);
2099         }
2100 
2101         /*
2102          * If the phyint has lost NUM_PROBE_FAILS or more
2103          * consecutive probes, on both IPv4 and IPv6 protocol
2104          * instances of the phyint, then trigger failure
2105          * detection, else return false
2106          */
2107         if (pfinfo.pf_nfail < NUM_PROBE_FAILS)
2108                 return (PHYINT_OK);
2109 
2110         pii_other = phyint_inst_other(pii);
2111         if (PROBE_CAPABLE(pii_other)) {
2112                 probe_fail_info(pii_other, NULL, &pfinfo);
2113                 if (pfinfo.pf_nfail >= NUM_PROBE_FAILS) {
2114                         /*
2115                          * We have NUM_PROBE_FAILS or more failures
2116                          * on both IPv4 and IPv6. Get the earliest
2117                          * time when failure was detected on this
2118                          * phyint across IPv4 and IPv6.
2119                          */
2120                         if (TIME_LT(pfinfo.pf_tff, pi_tff))
2121                                 pi_tff = pfinfo.pf_tff;
2122                 } else {
2123                         /*
2124                          * This instance has < NUM_PROBE_FAILS failure.
2125                          * So return false
2126                          */
2127                         return (PHYINT_OK);
2128                 }
2129         }
2130         *tff = pi_tff;
2131         return (PHYINT_FAILURE);
2132 }
2133 
2134 /*
2135  * Check if the link has gone down on this phyint, or it has failed the
2136  * last NUM_PROBE_FAILS consecutive probes on both instances IPv4 and IPv6.
2137  * Also look at other phyints of this group, for group failures.
2138  */
2139 int
2140 failure_state(struct phyint_instance *pii)
2141 {
2142         struct  probe_success_count psinfo;
2143         uint_t  pi2_tls;                /* time last success */
2144         uint_t  pi_tff;                 /* time first fail */
2145         struct  phyint *pi2;
2146         struct  phyint *pi;
2147         struct  phyint_instance *pii2;
2148         struct  phyint_group *pg;
2149         int     retval;
2150 
2151         if (debug & D_FAILREP)
2152                 logdebug("phyint_failed(%s)\n", pii->pii_name);
2153 
2154         pi = pii->pii_phyint;
2155         pg = pi->pi_group;
2156 
2157         if (LINK_UP(pi) && phyint_inst_probe_failure_state(pii, &pi_tff) ==
2158             PHYINT_OK)
2159                 return (PHYINT_OK);
2160 
2161         /*
2162          * At this point, the link is down, or the phyint is suspect, as it
2163          * has lost NUM_PROBE_FAILS or more probes. If the phyint does not
2164          * belong to any group, this is a PHYINT_FAILURE.  Otherwise, continue
2165          * on to determine whether this should be considered a PHYINT_FAILURE
2166          * or GROUP_FAILURE.
2167          */
2168         if (pg == phyint_anongroup)
2169                 return (PHYINT_FAILURE);
2170 
2171         /*
2172          * Need to compare against other phyints of the same group
2173          * to exclude group failures. If the failure was detected via
2174          * probing, then if the time of last success (tls) of any
2175          * phyint is more recent than the time of first fail (tff) of the
2176          * phyint in question, and the link is up on the phyint,
2177          * then it is a phyint failure. Otherwise it is a group failure.
2178          * If failure was detected via a link down notification sent from
2179          * the driver to IP, we see if any phyints in the group are still
2180          * running and haven't received a link down notification.  We
2181          * will usually be processing the link down notification shortly
2182          * after it was received, so there is no point looking at the tls
2183          * of other phyints.
2184          */
2185         retval = GROUP_FAILURE;
2186         for (pi2 = pg->pg_phyint; pi2 != NULL; pi2 = pi2->pi_pgnext) {
2187                 /* Exclude ourself from comparison */
2188                 if (pi2 == pi)
2189                         continue;
2190 
2191                 if (LINK_DOWN(pi)) {
2192                         /*
2193                          * We use FLAGS_TO_LINK_STATE() to test the flags
2194                          * directly, rather then LINK_UP() or LINK_DOWN(), as
2195                          * we may not have got round to processing the link
2196                          * state for the other phyints in the group yet.
2197                          *
2198                          * The check for PI_RUNNING and group failure handles
2199                          * the case when the group begins to recover.
2200                          * PI_RUNNING will be set, and group failure cleared
2201                          * only after receipt of NUM_PROBE_REPAIRS, by which
2202                          * time the other phyints should have received at
2203                          * least 1 packet, and so will not have NUM_PROBE_FAILS.
2204                          */
2205                         if ((pi2->pi_state == PI_RUNNING) &&
2206                             !GROUP_FAILED(pg) && FLAGS_TO_LINK_STATE(pi2)) {
2207                                 retval = PHYINT_FAILURE;
2208                                 break;
2209                         }
2210                         continue;
2211                 }
2212 
2213                 if (LINK_DOWN(pi2))
2214                         continue;
2215 
2216                 /*
2217                  * If there's no probe-based failure detection on this
2218                  * interface, and its link is still up, then it's still
2219                  * working and thus the group has not failed.
2220                  */
2221                 if (!PROBE_ENABLED(pi2->pi_v4) && !PROBE_ENABLED(pi2->pi_v6)) {
2222                         retval = PHYINT_FAILURE;
2223                         break;
2224                 }
2225 
2226                 /*
2227                  * Need to compare against both IPv4 and IPv6 instances.
2228                  */
2229                 pii2 = pi2->pi_v4;
2230                 if (pii2 != NULL) {
2231                         probe_success_info(pii2, NULL, &psinfo);
2232                         if (psinfo.ps_tls_valid) {
2233                                 pi2_tls = psinfo.ps_tls;
2234                                 /*
2235                                  * See comment above regarding check
2236                                  * for PI_RUNNING and group failure.
2237                                  */
2238                                 if (TIME_GT(pi2_tls, pi_tff) &&
2239                                     (pi2->pi_state == PI_RUNNING) &&
2240                                     !GROUP_FAILED(pg) &&
2241                                     FLAGS_TO_LINK_STATE(pi2)) {
2242                                         retval = PHYINT_FAILURE;
2243                                         break;
2244                                 }
2245                         }
2246                 }
2247 
2248                 pii2 = pi2->pi_v6;
2249                 if (pii2 != NULL) {
2250                         probe_success_info(pii2, NULL, &psinfo);
2251                         if (psinfo.ps_tls_valid) {
2252                                 pi2_tls = psinfo.ps_tls;
2253                                 /*
2254                                  * See comment above regarding check
2255                                  * for PI_RUNNING and group failure.
2256                                  */
2257                                 if (TIME_GT(pi2_tls, pi_tff) &&
2258                                     (pi2->pi_state == PI_RUNNING) &&
2259                                     !GROUP_FAILED(pg) &&
2260                                     FLAGS_TO_LINK_STATE(pi2)) {
2261                                         retval = PHYINT_FAILURE;
2262                                         break;
2263                                 }
2264                         }
2265                 }
2266         }
2267 
2268         /*
2269          * Update the group state to account for the changes.
2270          */
2271         phyint_group_refresh_state(pg);
2272         return (retval);
2273 }
2274 
2275 /*
2276  * Return the information associated with consecutive probe successes
2277  * starting with the most recent probe. At most the last 2 probes can be
2278  * in the unacknowledged state. All previous probes have either failed
2279  * or succeeded.
2280  */
2281 static void
2282 probe_success_info(struct phyint_instance *pii, struct target *cur_tg,
2283     struct probe_success_count *psinfo)
2284 {
2285         uint_t  i;
2286         struct probe_stats *pr_statp;
2287         uint_t most_recent;
2288         uint_t second_most_recent;
2289         boolean_t pi_found_failure = _B_FALSE;
2290         boolean_t tg_found_failure = _B_FALSE;
2291         uint_t now;
2292         uint_t timeout;
2293         struct target *tg;
2294 
2295         if (debug & D_FAILREP)
2296                 logdebug("probe_success_info(%s)\n", pii->pii_name);
2297 
2298         bzero(psinfo, sizeof (*psinfo));
2299         now = getcurrenttime();
2300 
2301         /*
2302          * Start with the most recent probe, and count the number
2303          * of consecutive probe successes. Latch the number of successes
2304          * on hitting a failure.
2305          */
2306         most_recent = PROBE_INDEX_PREV(pii->pii_probe_next);
2307         second_most_recent = PROBE_INDEX_PREV(most_recent);
2308 
2309         for (i = most_recent; i != pii->pii_probe_next;
2310             i = PROBE_INDEX_PREV(i)) {
2311                 pr_statp = &pii->pii_probes[i];
2312 
2313                 switch (pr_statp->pr_status) {
2314                 case PR_UNACKED:
2315                         /*
2316                          * Only the most recent 2 probes can be unacknowledged
2317                          */
2318                         assert(i == most_recent || i == second_most_recent);
2319 
2320                         tg = pr_statp->pr_target;
2321                         assert(tg != NULL);
2322                         /*
2323                          * The crtt could be zero for some reason,
2324                          * Eg. the phyint could be failed. If the crtt is
2325                          * not available use the value of the group's probe
2326                          * interval which is a worst case estimate.
2327                          */
2328                         timeout = ns2ms(pr_statp->pr_hrtime_start);
2329                         if (tg->tg_crtt != 0) {
2330                                 timeout += tg->tg_crtt;
2331                         } else {
2332                                 timeout +=
2333                                     pii->pii_phyint->pi_group->pg_probeint;
2334                         }
2335 
2336                         if (TIME_LT(timeout, now)) {
2337                                 /*
2338                                  * We hit a failure. Latch the total number of
2339                                  * recent consecutive successes.
2340                                  */
2341                                 pr_statp->pr_time_lost = timeout;
2342                                 probe_chstate(pr_statp, pii, PR_LOST);
2343                                 pi_found_failure = _B_TRUE;
2344                                 if (cur_tg != NULL && tg == cur_tg) {
2345                                         /*
2346                                          * We hit a failure for the desired
2347                                          * target. Latch the number of recent
2348                                          * consecutive successes for this target
2349                                          */
2350                                         tg_found_failure = _B_TRUE;
2351                                 }
2352                         }
2353                         break;
2354 
2355                 case PR_ACKED:
2356                         /*
2357                          * Bump up the count of probe successes, if we
2358                          * have not seen any failure so far.
2359                          */
2360                         if (!pi_found_failure)
2361                                 psinfo->ps_nsucc++;
2362 
2363                         if (cur_tg != NULL && pr_statp->pr_target == cur_tg &&
2364                             !tg_found_failure) {
2365                                 psinfo->ps_nsucc_tg++;
2366                         }
2367 
2368                         /*
2369                          * Record the time of last success, if this is
2370                          * the most recent probe success.
2371                          */
2372                         if (!psinfo->ps_tls_valid) {
2373                                 psinfo->ps_tls =
2374                                     ns2ms(pr_statp->pr_hrtime_ackproc);
2375                                 psinfo->ps_tls_valid = _B_TRUE;
2376                         }
2377                         break;
2378 
2379                 case PR_LOST:
2380                         /*
2381                          * We hit a failure. Latch the total number of
2382                          * recent consecutive successes.
2383                          */
2384                         pi_found_failure = _B_TRUE;
2385                         if (cur_tg != NULL && pr_statp->pr_target == cur_tg) {
2386                                 /*
2387                                  * We hit a failure for the desired target.
2388                                  * Latch the number of recent consecutive
2389                                  * successes for this target
2390                                  */
2391                                 tg_found_failure = _B_TRUE;
2392                         }
2393                         break;
2394 
2395                 default:
2396                         return;
2397 
2398                 }
2399         }
2400 }
2401 
2402 /*
2403  * Return the information associated with consecutive probe failures
2404  * starting with the most recent probe. Only the last 2 probes can be in the
2405  * unacknowledged state. All previous probes have either failed or succeeded.
2406  */
2407 static void
2408 probe_fail_info(struct phyint_instance *pii, struct target *cur_tg,
2409     struct probe_fail_count *pfinfo)
2410 {
2411         int     i;
2412         struct probe_stats *pr_statp;
2413         boolean_t       tg_found_success = _B_FALSE;
2414         boolean_t       pi_found_success = _B_FALSE;
2415         int     most_recent;
2416         int     second_most_recent;
2417         uint_t  now;
2418         uint_t  timeout;
2419         struct  target *tg;
2420 
2421         if (debug & D_FAILREP)
2422                 logdebug("probe_fail_info(%s)\n", pii->pii_name);
2423 
2424         bzero(pfinfo, sizeof (*pfinfo));
2425         now = getcurrenttime();
2426 
2427         /*
2428          * Start with the most recent probe, and count the number
2429          * of consecutive probe failures. Latch the number of failures
2430          * on hitting a probe success.
2431          */
2432         most_recent = PROBE_INDEX_PREV(pii->pii_probe_next);
2433         second_most_recent = PROBE_INDEX_PREV(most_recent);
2434 
2435         for (i = most_recent; i != pii->pii_probe_next;
2436             i = PROBE_INDEX_PREV(i)) {
2437                 pr_statp = &pii->pii_probes[i];
2438 
2439                 assert(PR_STATUS_VALID(pr_statp->pr_status));
2440 
2441                 switch (pr_statp->pr_status) {
2442                 case PR_UNACKED:
2443                         /*
2444                          * Only the most recent 2 probes can be unacknowledged
2445                          */
2446                         assert(i == most_recent || i == second_most_recent);
2447 
2448                         tg = pr_statp->pr_target;
2449                         /*
2450                          * Target is guaranteed to exist in the unack. state
2451                          */
2452                         assert(tg != NULL);
2453                         /*
2454                          * The crtt could be zero for some reason,
2455                          * Eg. the phyint could be failed. If the crtt is
2456                          * not available use the group's probe interval,
2457                          * which is a worst case estimate.
2458                          */
2459                         timeout = ns2ms(pr_statp->pr_hrtime_start);
2460                         if (tg->tg_crtt != 0) {
2461                                 timeout += tg->tg_crtt;
2462                         } else {
2463                                 timeout +=
2464                                     pii->pii_phyint->pi_group->pg_probeint;
2465                         }
2466 
2467                         if (TIME_GT(timeout, now))
2468                                 break;
2469 
2470                         pr_statp->pr_time_lost = timeout;
2471                         probe_chstate(pr_statp, pii, PR_LOST);
2472                         /* FALLTHRU */
2473 
2474                 case PR_LOST:
2475                         if (!pi_found_success) {
2476                                 pfinfo->pf_nfail++;
2477                                 pfinfo->pf_tff = pr_statp->pr_time_lost;
2478                         }
2479                         if (cur_tg != NULL && pr_statp->pr_target == cur_tg &&
2480                             !tg_found_success)  {
2481                                 pfinfo->pf_nfail_tg++;
2482                         }
2483                         break;
2484 
2485                 default:
2486                         /*
2487                          * We hit a success or unused slot. Latch the
2488                          * total number of recent consecutive failures.
2489                          */
2490                         pi_found_success = _B_TRUE;
2491                         if (cur_tg != NULL && pr_statp->pr_target == cur_tg) {
2492                                 /*
2493                                  * We hit a success for the desired target.
2494                                  * Latch the number of recent consecutive
2495                                  * failures for this target
2496                                  */
2497                                 tg_found_success = _B_TRUE;
2498                         }
2499                 }
2500         }
2501 }
2502 
2503 /*
2504  * Change the state of probe `pr' on phyint_instance `pii' to state `state'.
2505  */
2506 void
2507 probe_chstate(struct probe_stats *pr, struct phyint_instance *pii, int state)
2508 {
2509         if (pr->pr_status == state)
2510                 return;
2511 
2512         pr->pr_status = state;
2513         (void) probe_state_event(pr, pii);
2514 }
2515 
2516 /*
2517  * Check if the phyint has been repaired.  If no test address has been
2518  * configured, then consider the interface repaired if the link is up (unless
2519  * the link is flapping; see below).  Otherwise, look for proof of probes
2520  * being sent and received. If last NUM_PROBE_REPAIRS probes are fine on
2521  * either IPv4 or IPv6 instance, the phyint can be considered repaired.
2522  */
2523 static boolean_t
2524 phyint_repaired(struct phyint *pi)
2525 {
2526         struct  probe_success_count psinfo;
2527         struct  phyint_instance *pii;
2528         struct  target *cur_tg;
2529         int     pr_ndx;
2530         uint_t  cur_time;
2531 
2532         if (debug & D_FAILREP)
2533                 logdebug("phyint_repaired(%s)\n", pi->pi_name);
2534 
2535         if (LINK_DOWN(pi))
2536                 return (_B_FALSE);
2537 
2538         /*
2539          * If we don't have any test addresses and the link is up, then
2540          * consider the interface repaired, unless we've received more than
2541          * LINK_UP_PERMIN link up notifications in the last minute, in
2542          * which case we keep the link down until we drop back below
2543          * the threshold.
2544          */
2545         if (!PROBE_ENABLED(pi->pi_v4) && !PROBE_ENABLED(pi->pi_v6)) {
2546                 cur_time = getcurrenttime();
2547                 if ((pi->pi_whenup[pi->pi_whendx] == 0 ||
2548                     (cur_time - pi->pi_whenup[pi->pi_whendx]) > MSEC_PERMIN)) {
2549                         pi->pi_lfmsg_printed = 0;
2550                         return (_B_TRUE);
2551                 }
2552                 if (!pi->pi_lfmsg_printed) {
2553                         logerr("The link has come up on %s more than %d times "
2554                             "in the last minute; disabling repair until it "
2555                             "stabilizes\n", pi->pi_name, LINK_UP_PERMIN);
2556                         pi->pi_lfmsg_printed = 1;
2557                 }
2558 
2559                 return (_B_FALSE);
2560         }
2561 
2562         pii = pi->pi_v4;
2563         if (PROBE_CAPABLE(pii)) {
2564                 pr_ndx = PROBE_INDEX_PREV(pii->pii_probe_next);
2565                 cur_tg = pii->pii_probes[pr_ndx].pr_target;
2566                 probe_success_info(pii, cur_tg, &psinfo);
2567                 if (psinfo.ps_nsucc >= NUM_PROBE_REPAIRS ||
2568                     psinfo.ps_nsucc_tg >= NUM_PROBE_REPAIRS)
2569                         return (_B_TRUE);
2570         }
2571 
2572         pii = pi->pi_v6;
2573         if (PROBE_CAPABLE(pii)) {
2574                 pr_ndx = PROBE_INDEX_PREV(pii->pii_probe_next);
2575                 cur_tg = pii->pii_probes[pr_ndx].pr_target;
2576                 probe_success_info(pii, cur_tg, &psinfo);
2577                 if (psinfo.ps_nsucc >= NUM_PROBE_REPAIRS ||
2578                     psinfo.ps_nsucc_tg >= NUM_PROBE_REPAIRS)
2579                         return (_B_TRUE);
2580         }
2581 
2582         return (_B_FALSE);
2583 }
2584 
2585 /*
2586  * Used to set/clear phyint flags, by making a SIOCSLIFFLAGS call.
2587  */
2588 boolean_t
2589 change_pif_flags(struct phyint *pi, uint64_t set, uint64_t clear)
2590 {
2591         int ifsock;
2592         struct lifreq lifr;
2593         uint64_t old_flags;
2594 
2595         if (debug & D_FAILREP) {
2596                 logdebug("change_pif_flags(%s): set %llx clear %llx\n",
2597                     pi->pi_name, set, clear);
2598         }
2599 
2600         if (pi->pi_v4 != NULL)
2601                 ifsock = ifsock_v4;
2602         else
2603                 ifsock = ifsock_v6;
2604 
2605         /*
2606          * Get the current flags from the kernel, and set/clear the
2607          * desired phyint flags. Since we set only phyint flags, we can
2608          * do it on either IPv4 or IPv6 instance.
2609          */
2610         (void) strlcpy(lifr.lifr_name, pi->pi_name, sizeof (lifr.lifr_name));
2611 
2612         if (ioctl(ifsock, SIOCGLIFFLAGS, (char *)&lifr) < 0) {
2613                 if (errno != ENXIO)
2614                         logperror("change_pif_flags: ioctl (get flags)");
2615                 return (_B_FALSE);
2616         }
2617 
2618         old_flags = lifr.lifr_flags;
2619         lifr.lifr_flags |= set;
2620         lifr.lifr_flags &= ~clear;
2621 
2622         if (old_flags == lifr.lifr_flags) {
2623                 /* No change in the flags. No need to send ioctl */
2624                 return (_B_TRUE);
2625         }
2626 
2627         if (ioctl(ifsock, SIOCSLIFFLAGS, (char *)&lifr) < 0) {
2628                 if (errno != ENXIO)
2629                         logperror("change_pif_flags: ioctl (set flags)");
2630                 return (_B_FALSE);
2631         }
2632 
2633         /*
2634          * Keep pi_flags in synch. with actual flags. Assumes flags are
2635          * phyint flags.
2636          */
2637         pi->pi_flags |= set;
2638         pi->pi_flags &= ~clear;
2639 
2640         if (pi->pi_v4 != NULL)
2641                 pi->pi_v4->pii_flags = pi->pi_flags;
2642 
2643         if (pi->pi_v6 != NULL)
2644                 pi->pi_v6->pii_flags = pi->pi_flags;
2645 
2646         return (_B_TRUE);
2647 }
2648 
2649 /*
2650  * icmp cksum computation for IPv4.
2651  */
2652 static int
2653 in_cksum(ushort_t *addr, int len)
2654 {
2655         register int nleft = len;
2656         register ushort_t *w = addr;
2657         register ushort_t answer;
2658         ushort_t odd_byte = 0;
2659         register int sum = 0;
2660 
2661         /*
2662          *  Our algorithm is simple, using a 32 bit accumulator (sum),
2663          *  we add sequential 16 bit words to it, and at the end, fold
2664          *  back all the carry bits from the top 16 bits into the lower
2665          *  16 bits.
2666          */
2667         while (nleft > 1)  {
2668                 sum += *w++;
2669                 nleft -= 2;
2670         }
2671 
2672         /* mop up an odd byte, if necessary */
2673         if (nleft == 1) {
2674                 *(uchar_t *)(&odd_byte) = *(uchar_t *)w;
2675                 sum += odd_byte;
2676         }
2677 
2678         /*
2679          * add back carry outs from top 16 bits to low 16 bits
2680          */
2681         sum = (sum >> 16) + (sum & 0xffff);   /* add hi 16 to low 16 */
2682         sum += (sum >> 16);                       /* add carry */
2683         answer = ~sum;                          /* truncate to 16 bits */
2684         return (answer);
2685 }
2686 
2687 static void
2688 reset_snxt_basetimes(void)
2689 {
2690         struct phyint_instance *pii;
2691 
2692         for (pii = phyint_instances; pii != NULL; pii = pii->pii_next) {
2693                 pii->pii_fd_snxt_basetime = pii->pii_snxt_basetime;
2694         }
2695 }
2696 
2697 /*
2698  * Is the address one of our own addresses? Unfortunately,
2699  * we cannot check our phyint tables to determine if the address
2700  * is our own. This is because, we don't track interfaces that
2701  * are not part of any group. We have to either use a 'bind' or
2702  * get the complete list of all interfaces using SIOCGLIFCONF,
2703  * to do this check. We could also use SIOCTMYADDR.
2704  * Bind fails for the local zone address, so we might include local zone
2705  * address as target address. If local zone address is a target address
2706  * and it is up, it is not possible to detect the interface failure.
2707  * SIOCTMYADDR also doesn't consider local zone address as own address.
2708  * So, we choose to use SIOCGLIFCONF to collect the local addresses, and they
2709  * are stored in `localaddrs'
2710  */
2711 boolean_t
2712 own_address(struct in6_addr addr)
2713 {
2714         addrlist_t *addrp;
2715         struct sockaddr_storage ss;
2716         int af = IN6_IS_ADDR_V4MAPPED(&addr) ? AF_INET : AF_INET6;
2717 
2718         addr2storage(af, &addr, &ss);
2719         for (addrp = localaddrs; addrp != NULL; addrp = addrp->al_next) {
2720                 if (sockaddrcmp(&ss, &addrp->al_addr))
2721                         return (_B_TRUE);
2722         }
2723         return (_B_FALSE);
2724 }
2725 
2726 static int
2727 ns2ms(int64_t ns)
2728 {
2729         return (NSEC2MSEC(ns));
2730 }
2731 
2732 static int64_t
2733 tv2ns(struct timeval *tvp)
2734 {
2735         return (tvp->tv_sec * NANOSEC + tvp->tv_usec * 1000);
2736 }