1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25 
  26 #pragma ident   "%Z%%M% %I%     %E% SMI"
  27 
  28 #if defined(lint)
  29 #include <sys/types.h>
  30 #include <sys/thread.h>
  31 #else   /* lint */
  32 #include "assym.h"
  33 #endif  /* lint */
  34 
  35 #include <sys/asi.h>
  36 #include <sys/sun4asi.h>
  37 #include <sys/machasi.h>
  38 #include <sys/asm_linkage.h>
  39 #include <sys/pte.h>
  40 #include <sys/mmu.h>
  41 #include <sys/intreg.h>
  42 #include <sys/zulumod.h>
  43 #include <vm/hat_sfmmu.h>
  44 #include <sys/zulu_hat.h>
  45 #include <zuluvm_offsets.h>
  46 
  47 #ifdef lint
  48 void
  49 zuluvm_dmv_tlbmiss_tl1()
  50 {}
  51 
  52 #else   /* lint */
  53 
  54         DGDEF(zuluvm_base_pgsize)
  55         .word   0
  56 
  57         ENTRY_NP(zuluvm_dmv_tlbmiss_tl1)
  58 
  59         ! g1 - zuluvm_state_t pointer
  60         ! g2 - IRDR_0
  61         mov     UIII_IRDR_1, %g3
  62         ldxa    [%g3]ASI_INTR_RECEIVE, %g5
  63         stx     %g5, [%g1 + ZULUVM_ASM_TLB_ADDR]
  64         mov     UIII_IRDR_6, %g3
  65         ldxa    [%g3]ASI_INTR_RECEIVE, %g5
  66         stx     %g5, [%g1 + ZULUVM_ASM_TLB_TYPE]
  67 
  68         stxa    %g0, [%g0]ASI_INTR_RECEIVE_STATUS       ! clear the BUSY bit
  69         membar  #Sync
  70 
  71         mov     %g1, %g7
  72 
  73         ! check the fast tlb miss flag
  74         sethi   %hi(zuluvm_fast_tlb), %g6
  75         lduw    [%g6 + %lo(zuluvm_fast_tlb)], %g6
  76         brz,pn  %g6, send_intr1
  77           mov   ZULUVM_TTE_DELAY, %g1
  78 #if 1
  79         add     %g7, ZULUVM_STATE, %g4
  80         mov     ZULUVM_STATE_IDLE, %g1
  81         mov     ZULUVM_STATE_TLB_PENDING, %g6
  82         casa    [%g4]ASI_N, %g1, %g6
  83         cmp     %g6, %g1
  84         be,pt   %icc, 2f
  85           nop
  86 
  87         mov     ZULUVM_STATE_CANCELED, %g1
  88         cmp     %g6, %g1
  89         be,pt   %icc, 1f
  90           mov   ZULUVM_STATE_STOPPED, %g1
  91         retry
  92 1:
  93         st      %g1, [%g4]
  94 #ifdef ZULUVM_STATS
  95         lduw    [%g7 + ZULUVM_ST_TLBCANCEL], %g3
  96         add     %g3, 1, %g3
  97         stuw    %g3, [%g7 + ZULUVM_ST_TLBCANCEL]
  98 #endif
  99         retry
 100 
 101 2:
 102         ldx     [%g7 + ZULUVM_ASM_TLB_TYPE], %g4
 103         and     %g4, ZULUVM_DMA_MASK, %g4
 104 #ifdef ZULUVM_STATS
 105         cmp     %g4, ZULUVM_DMA2
 106         be,a,pn %icc, 1f
 107           add   %g7, ZULUVM_ST_DTLB2MISS, %g1
 108         cmp     %g4, ZULUVM_ITLB1
 109         be,a,pn %icc, 1f
 110           add   %g7, ZULUVM_ST_ITLB1MISS, %g1
 111         cmp     %g4, ZULUVM_ITLB2
 112         be,a,pn %icc, 1f
 113           add   %g7, ZULUVM_ST_ITLB2MISS, %g1
 114         add     %g7, ZULUVM_ST_DTLB1MISS, %g1
 115 1:
 116         lduw    [%g1], %g3
 117         add     %g3, 1, %g3
 118         stuw    %g3, [%g1]
 119 #endif
 120         /*
 121          * lookup the tte in the tsb
 122          * %g1 - vaddr[63:13], ctx[12:0]
 123          * %g2 - our trap level
 124          * %g3 - return address
 125          * %g7 - zulu data pointer (needs to be preserved)
 126          * return:
 127          * %g1 - flags [63..58] and pfn [31..0]
 128          * %g2 - status code if %g1 is null
 129          * %g7 - zulu data pointer
 130          */
 131         mov     1, %g2
 132         set     zulu_hat_tsb_lookup_tl1, %g3
 133         jmpl    %g3, %g3
 134           ldx   [%g7 + ZULUVM_ASM_TLB_ADDR], %g1        ! vaddr(tag)
 135 
 136         /*
 137          * did we find a tte ??
 138          * If not, %g2 has the error code
 139          */
 140         brgez,a,pt %g1, send_intr
 141         mov     %g2, %g1
 142 
 143         set     zulu_tsb_hit, %g6
 144         ldx     [%g6], %g3
 145         add     %g3, 1, %g3
 146         stx     %g3, [%g6]
 147 
 148         /*
 149          * get flags and pfn
 150          */
 151         sllx    %g1, 32, %g6
 152         srlx    %g6, 32, %g6                    ! %g6 pfn
 153         srlx    %g1, 59, %g3
 154         and     %g3, 0x7, %g2                   ! %g2 page size
 155         srlx    %g3, 3, %g4
 156         and     %g4, 1, %g4                     ! %g4 write perm
 157         mov     %g6, %g1 
 158 
 159         /*
 160          * check if this is a dtlb2 miss(no itlb, pgsz != 8k)
 161          * and if the current dtlb2 pgsz != tte pgsz
 162          */
 163         ldx     [%g7 + ZULUVM_ASM_TLB_TYPE], %g3
 164         and     %g3, 0x1, %g3
 165         brnz,pt %g3, 3f                         ! not 0 => itlb => handles
 166           nop
 167 
 168         ! check page size, base page size is always handled by dtlb1, so we
 169         ! only need to check against dtlb2
 170         sethi   %hi(zuluvm_base_pgsize), %g3
 171         lduw    [%g3 + %lo(zuluvm_base_pgsize)], %g3
 172         cmp     %g2, %g3
 173         be,pt   %icc, 2f
 174         cmp     %g2, ZULU_TTE4M
 175         be,pt   %icc, 2f                        ! TTE4M => dtlb2 => ok!
 176           nop
 177 
 178 #ifdef ZULUVM_STATS
 179         lduw    [%g7 + ZULUVM_ST_PAGESIZE], %g3
 180         add     %g3, 1, %g3
 181         stuw    %g3, [%g7 + ZULUVM_ST_PAGESIZE]
 182         add     %g7, ZULUVM_ST_MISS, %g3
 183         sll     %g2, 2, %g5
 184         add     %g5, %g3, %g5
 185         lduw    [%g5], %g3
 186         add     %g3, 1, %g3
 187         stuw    %g3, [%g5]
 188 #endif
 189         ! set tte size to ZULUVM_BASE_PGSZ
 190         sethi   %hi(zuluvm_base_pgsize), %g3
 191         lduw    [%g3 + %lo(zuluvm_base_pgsize)], %g3
 192         ba,pt   %icc, 3f
 193           mov   %g3, %g2
 194 2:
 195 
 196 #ifdef ZULUVM_STATS
 197         add     %g7, ZULUVM_ST_MISS, %g3
 198         sll     %g2, 2, %g5
 199         add     %g3, %g5, %g5
 200         lduw    [%g5], %g3
 201         add     %g3, 1, %g3
 202         stuw    %g3, [%g5]
 203 #endif
 204 
 205         ! we maintain data on the last pfns for the last 12 pfns that we
 206         ! processed
 207 3:
 208         lduw    [%g7 + ZULUVM_PFNCNT], %g5
 209         add     %g5, 4, %g3
 210         cmp     %g3, 48
 211         be,a,pn %icc, 1f
 212           mov   %g0, %g3
 213  
 214 1:                           
 215         stuw    %g3, [%g7 + ZULUVM_PFNCNT]
 216         sllx    %g5, 3, %g5
 217         add     %g7, ZULUVM_PFNBUF, %g3
 218         add     %g3, %g5, %g3
 219         stx     %g1, [%g3]
 220         stx     %g2, [%g3 + 8]
 221         stx     %g4, [%g3 + 16]
 222         ldx     [%g7 + ZULUVM_ASM_TLB_TYPE], %g5
 223         stx     %g5, [%g3 + 24]
 224 
 225         ldx     [%g7 + ZULUVM_ASM_TLB_TYPE], %g3
 226         and     %g3, 0x3, %g3                   ! tlbtype
 227         ldx     [%g7 + ZULUVM_ARG], %g6
 228 
 229         ! write tte to zulu mmu
 230         ! %g1 pfn
 231         ! %g2 tte size
 232         ! %g3 tlbtype
 233         ! %g4 tte wrperm
 234         ! %g6 zulu device driver arg
 235         ! %g7 devtab pointer
 236 
 237         sllx    %g1, ZULUVM_ZFB_MMU_TLB_D_PA_SHIFT, %g1
 238         mov     0x1, %g5
 239         sllx    %g5, 63, %g5                    ! ZFB_MMU_TLB_D_V_MASK
 240         or      %g1, %g5, %g1
 241         or      %g1, ZULUVM_ZFB_MMU_TLB_D_C_MASK, %g1
 242         sllx    %g2, ZULUVM_ZFB_MMU_TLB_D_SZ_SHIFT, %g2
 243 
 244         brz,pt  %g4, 3f                         ! write perm ??
 245           or    %g2, %g1, %g1
 246 
 247         or      %g1, ZULUVM_ZFB_MMU_TLB_D_W_MASK, %g1
 248 3:
 249         ! at this point %g1 is ready to be written to the corresponding
 250         ! data_in register, let's see which if it was itlb or dtlb...
 251         and     %g3, ZULUVM_ITLB_FLAG, %g3
 252                                                 ! assumption is that data miss
 253         brz,pt  %g3, 4f                         ! is more likely than instr miss
 254           ldx   [%g7 + ZULUVM_PAMMU], %g2       ! physical addr of zulu mmu regs
 255 
 256         ! instruction miss
 257         mov     ZULUVM_ZFB_MMU_TLB_CR_IMISS_MASK, %g5
 258         add     %g2, ZULUVM_ITLB_DATA_IN, %g4
 259         !stxa  %g1, [%g4]ASI_IO
 260         ba,pt   %xcc, 5f
 261           stxa  %g1, [%g4]ASI_IO
 262           !ldxa    [%g4]ASI_IO, %g4
 263 4:
 264         ! data miss
 265         mov     ZULUVM_ZFB_MMU_TLB_CR_DMISS_MASK, %g5
 266         add     %g2, ZULUVM_DTLB_DATA_IN, %g4
 267         stxa    %g1, [%g4]ASI_IO
 268         !ldxa    [%g4]ASI_IO, %g4
 269 5:
 270         add     %g7, ZULUVM_STATE, %g4
 271         mov     ZULUVM_STATE_TLB_PENDING, %g6
 272         mov     ZULUVM_STATE_IDLE, %g1
 273         casa    [%g4]ASI_N, %g6, %g1
 274         cmp     %g6, %g1
 275         bne,a,pn %icc, stopped
 276           mov   ZULUVM_STATE_STOPPED, %g3
 277 
 278         ldx     [%g7 + ZULUVM_PAMMU], %g2
 279         add     %g2, ZULUVM_TLB_CONTROL, %g2
 280         stxa    %g5, [%g2]ASI_IO
 281         !ldxa   [%g2]ASI_IO, %g3
 282         retry
 283 
 284 send_intr:      
 285         add     %g7, ZULUVM_STATE, %g4
 286         mov     ZULUVM_STATE_INTR_QUEUED, %g5
 287         mov     ZULUVM_STATE_TLB_PENDING, %g3
 288         casa    [%g4]ASI_N, %g3, %g5
 289         cmp     %g3, %g5
 290         be,pt  %icc, deliver_intr
 291         mov     ZULUVM_STATE_STOPPED, %g3
 292         ba,pt   %icc, stopped
 293           nop
 294 #endif
 295 
 296 send_intr1:
 297         add     %g7, ZULUVM_STATE, %g4
 298         mov     ZULUVM_STATE_IDLE, %g3
 299         mov     ZULUVM_STATE_INTR_QUEUED, %g5
 300         casa    [%g4]ASI_N, %g3, %g5
 301         cmp     %g3, %g5
 302         be,pt  %icc, deliver_intr
 303         mov     ZULUVM_STATE_STOPPED, %g3
 304 stopped:
 305         st      %g3, [%g4]
 306 #ifdef ZULUVM_STATS
 307         lduw    [%g7 + ZULUVM_ST_TLBCANCEL], %g3
 308         add     %g3, 1, %g3
 309         stuw    %g3, [%g7 + ZULUVM_ST_TLBCANCEL]
 310 #endif
 311         retry 
 312 
 313 deliver_intr:
 314         stx     %g1, [%g7 + ZULUVM_ASM_TLB_ERRCODE]     ! set the error field
 315         stx     %g6, [%g7 + ZULUVM_ASM_TLB_TTE] ! deliver tte in data_0
 316                                                 ! %g6 is invalid if error != SUCCESS
 317         ! setsoftint_tl1(uint64_t inum, uint64_t dummy)
 318         set     setsoftint_tl1, %g5
 319         jmp     %g5
 320         ldx     [%g7 + ZULUVM_INTRNUM], %g1
 321 
 322         SET_SIZE(zuluvm_dmv_tlbmiss_tl1)
 323 
 324 #endif  /* lint */
 325