Print this page
first pass

Split Close
Expand all
Collapse all
          --- old/usr/src/common/crypto/aes/amd64/aes_intel.s
          +++ new/usr/src/common/crypto/aes/amd64/aes_intel.s
↓ open down ↓ 75 lines elided ↑ open up ↑
  76   76   * OpenSolaris OS modifications
  77   77   *
  78   78   * This source originates as files aes-intel.S and eng_aesni_asm.pl, in
  79   79   * patches sent sent Dec. 9, 2008 and Dec. 24, 2008, respectively, by
  80   80   * Huang Ying of Intel to the openssl-dev mailing list under the subject
  81   81   * of "Add support to Intel AES-NI instruction set for x86_64 platform".
  82   82   *
  83   83   * This OpenSolaris version has these major changes from the original source:
  84   84   *
  85   85   * 1. Added OpenSolaris ENTRY_NP/SET_SIZE macros from
  86      - * /usr/include/sys/asm_linkage.h, lint(1B) guards, EXPORT DELETE START
  87      - * and EXPORT DELETE END markers, and dummy C function definitions for lint.
       86 + * /usr/include/sys/asm_linkage.h, lint(1B) guards, and dummy C function
       87 + * definitions for lint.
  88   88   *
  89   89   * 2. Formatted code, added comments, and added #includes and #defines.
  90   90   *
  91   91   * 3. If bit CR0.TS is set, clear and set the TS bit, after and before
  92   92   * calling kpreempt_disable() and kpreempt_enable().
  93   93   * If the TS bit is not set, Save and restore %xmm registers at the beginning
  94   94   * and end of function calls (%xmm* registers are not saved and restored by
  95   95   * during kernel thread preemption).
  96   96   *
  97   97   * 4. Renamed functions, reordered parameters, and changed return value
↓ open down ↓ 200 lines elided ↑ open up ↑
 298  298   * Helper functions called by rijndael_key_setup_inc_intel().
 299  299   * Also used indirectly by rijndael_key_setup_dec_intel().
 300  300   *
 301  301   * Input:
 302  302   * %xmm0        User-provided cipher key
 303  303   * %xmm1        Round constant
 304  304   * Output:
 305  305   * (%rcx)       AES key
 306  306   */
 307  307  
 308      -        /* EXPORT DELETE START */
 309  308  .align  16
 310  309  _key_expansion_128:
 311  310  _key_expansion_256a:
 312  311          pshufd  $0b11111111, %xmm1, %xmm1
 313  312          shufps  $0b00010000, %xmm0, %xmm4
 314  313          pxor    %xmm4, %xmm0
 315  314          shufps  $0b10001100, %xmm0, %xmm4
 316  315          pxor    %xmm4, %xmm0
 317  316          pxor    %xmm1, %xmm0
 318  317          movaps  %xmm0, (%rcx)
↓ open down ↓ 52 lines elided ↑ open up ↑
 371  370          pshufd  $0b10101010, %xmm1, %xmm1
 372  371          shufps  $0b00010000, %xmm2, %xmm4
 373  372          pxor    %xmm4, %xmm2
 374  373          shufps  $0b10001100, %xmm2, %xmm4
 375  374          pxor    %xmm4, %xmm2
 376  375          pxor    %xmm1, %xmm2
 377  376          movaps  %xmm2, (%rcx)
 378  377          add     $0x10, %rcx
 379  378          ret
 380  379          SET_SIZE(_key_expansion_256b)
 381      -        /* EXPORT DELETE END */
 382  380  
 383  381  
 384  382  /*
 385  383   * rijndael_key_setup_enc_intel()
 386  384   * Expand the cipher key into the encryption key schedule.
 387  385   *
 388  386   * For kernel code, caller is responsible for ensuring kpreempt_disable()
 389  387   * has been called.  This is because %xmm registers are not saved/restored.
 390  388   * Clear and set the CR0.TS bit on entry and exit, respectively,  if TS is set
 391  389   * on entry.  Otherwise, if TS is not set, save and restore %xmm registers
↓ open down ↓ 25 lines elided ↑ open up ↑
 417  415  #define KEYSIZE32               edx     /* P3, 32 bits */
 418  416  #define KEYSIZE64               rdx     /* P3, 64 bits */
 419  417  #endif  /* OPENSSL_INTERFACE */
 420  418  
 421  419  #define ROUNDS32                KEYSIZE32       /* temp */
 422  420  #define ROUNDS64                KEYSIZE64       /* temp */
 423  421  #define ENDAESKEY               USERCIPHERKEY   /* temp */
 424  422  
 425  423  
 426  424  ENTRY_NP(rijndael_key_setup_enc_intel)
 427      -        /* EXPORT DELETE START */
 428  425          CLEAR_TS_OR_PUSH_XMM0_TO_XMM6(%r10)
 429  426  
 430  427          / NULL pointer sanity check
 431  428          test    %USERCIPHERKEY, %USERCIPHERKEY
 432  429          jz      .Lenc_key_invalid_param
 433  430          test    %AESKEY, %AESKEY
 434  431          jz      .Lenc_key_invalid_param
 435  432  
 436  433          movups  (%USERCIPHERKEY), %xmm0 / user key (first 16 bytes)
 437  434          movaps  %xmm0, (%AESKEY)
↓ open down ↓ 135 lines elided ↑ open up ↑
 573  570  #endif  /* OPENSSL_INTERFACE */
 574  571  
 575  572  .Lenc_key_invalid_key_bits:
 576  573          SET_TS_OR_POP_XMM0_TO_XMM6(%r10)
 577  574  #ifdef  OPENSSL_INTERFACE
 578  575          mov     $-2, %rax       / keysize is invalid
 579  576  #else   /* Open Solaris Interface */
 580  577          xor     %rax, %rax      / a key pointer is NULL or invalid keysize
 581  578  #endif  /* OPENSSL_INTERFACE */
 582  579  
 583      -        /* EXPORT DELETE END */
 584  580          ret
 585  581          SET_SIZE(rijndael_key_setup_enc_intel)
 586  582  
 587  583  
 588  584  /*
 589  585   * rijndael_key_setup_dec_intel()
 590  586   * Expand the cipher key into the decryption key schedule.
 591  587   *
 592  588   * For kernel code, caller is responsible for ensuring kpreempt_disable()
 593  589   * has been called.  This is because %xmm registers are not saved/restored.
↓ open down ↓ 6 lines elided ↑ open up ↑
 600  596   *      uint64_t keyBits);
 601  597   * Return value is 0 on error, number of rounds on success.
 602  598   * P1->P2, P2->P3, P3->P1
 603  599   *
 604  600   * Original Intel OpenSSL interface:
 605  601   * int intel_AES_set_decrypt_key(const unsigned char *userKey,
 606  602   *      const int bits, AES_KEY *key);
 607  603   * Return value is non-zero on error, 0 on success.
 608  604   */
 609  605  ENTRY_NP(rijndael_key_setup_dec_intel)
 610      -        /* EXPORT DELETE START */
 611  606          / Generate round keys used for encryption
 612  607          call    rijndael_key_setup_enc_intel
 613  608          test    %rax, %rax
 614  609  #ifdef  OPENSSL_INTERFACE
 615  610          jnz     .Ldec_key_exit  / Failed if returned non-0
 616  611  #else   /* OpenSolaris Interface */
 617  612          jz      .Ldec_key_exit  / Failed if returned 0
 618  613  #endif  /* OPENSSL_INTERFACE */
 619  614  
 620  615          CLEAR_TS_OR_PUSH_XMM0_XMM1(%r10)
↓ open down ↓ 32 lines elided ↑ open up ↑
 653  648          movaps  %xmm1, (%rcx)
 654  649          lea     0x10(%rcx), %rcx
 655  650          cmp     %ENDAESKEY, %rcx
 656  651          jnz     .Ldec_key_inv_loop
 657  652  
 658  653          SET_TS_OR_POP_XMM0_XMM1(%r10)
 659  654  
 660  655  .Ldec_key_exit:
 661  656          / OpenSolaris: rax = # rounds (10, 12, or 14) or 0 for error
 662  657          / OpenSSL: rax = 0 for OK, or non-zero for error
 663      -        /* EXPORT DELETE END */
 664  658          ret
 665  659          SET_SIZE(rijndael_key_setup_dec_intel)
 666  660  
 667  661  
 668  662  /*
 669  663   * aes_encrypt_intel()
 670  664   * Encrypt a single block (in and out can overlap).
 671  665   *
 672  666   * For kernel code, caller is responsible for ensuring kpreempt_disable()
 673  667   * has been called.  This is because %xmm registers are not saved/restored.
↓ open down ↓ 30 lines elided ↑ open up ↑
 704  698  #define KEYP            rdi     /* P1, 64 bits */
 705  699  #define NROUNDS         esi     /* P2, 32 bits */
 706  700  #define INP             rdx     /* P3, 64 bits */
 707  701  #define OUTP            rcx     /* P4, 64 bits */
 708  702  #endif  /* OPENSSL_INTERFACE */
 709  703  
 710  704  #define STATE           xmm0    /* temporary, 128 bits */
 711  705  #define KEY             xmm1    /* temporary, 128 bits */
 712  706  
 713  707  ENTRY_NP(aes_encrypt_intel)
 714      -        /* EXPORT DELETE START */
 715  708          CLEAR_TS_OR_PUSH_XMM0_XMM1(%r10)
 716  709  
 717  710          movups  (%INP), %STATE                  / input
 718  711          movaps  (%KEYP), %KEY                   / key
 719  712  #ifdef  OPENSSL_INTERFACE
 720  713          mov     240(%KEYP), %NROUNDS32          / round count
 721  714  #else   /* OpenSolaris Interface */
 722  715          /* Round count is already present as P2 in %rsi/%esi */
 723  716  #endif  /* OPENSSL_INTERFACE */
 724  717  
↓ open down ↓ 38 lines elided ↑ open up ↑
 763  756          aesenc  %KEY, %STATE
 764  757          movaps  0x50(%KEYP), %KEY
 765  758          aesenc  %KEY, %STATE
 766  759          movaps  0x60(%KEYP), %KEY
 767  760          aesenc  %KEY, %STATE
 768  761          movaps  0x70(%KEYP), %KEY
 769  762          aesenclast       %KEY, %STATE           / last round
 770  763          movups  %STATE, (%OUTP)                 / output
 771  764  
 772  765          SET_TS_OR_POP_XMM0_XMM1(%r10)
 773      -        /* EXPORT DELETE END */
 774  766          ret
 775  767          SET_SIZE(aes_encrypt_intel)
 776  768  
 777  769  
 778  770  /*
 779  771   * aes_decrypt_intel()
 780  772   * Decrypt a single block (in and out can overlap).
 781  773   *
 782  774   * For kernel code, caller is responsible for ensuring kpreempt_disable()
 783  775   * has been called.  This is because %xmm registers are not saved/restored.
↓ open down ↓ 7 lines elided ↑ open up ↑
 791  783   *
 792  784   * Original OpenSolaris Interface:
 793  785   * void aes_decrypt_intel(const aes_ks_t *ks, int Nr,
 794  786   *      const uint32_t pt[4], uint32_t ct[4])/
 795  787   *
 796  788   * Original Intel OpenSSL Interface:
 797  789   * void intel_AES_decrypt(const unsigned char *in, unsigned char *out,
 798  790   *      const AES_KEY *key);
 799  791   */
 800  792  ENTRY_NP(aes_decrypt_intel)
 801      -        /* EXPORT DELETE START */
 802  793          CLEAR_TS_OR_PUSH_XMM0_XMM1(%r10)
 803  794  
 804  795          movups  (%INP), %STATE                  / input
 805  796          movaps  (%KEYP), %KEY                   / key
 806  797  #ifdef  OPENSSL_INTERFACE
 807  798          mov     240(%KEYP), %NROUNDS32          / round count
 808  799  #else   /* OpenSolaris Interface */
 809  800          /* Round count is already present as P2 in %rsi/%esi */
 810  801  #endif  /* OPENSSL_INTERFACE */
 811  802  
↓ open down ↓ 39 lines elided ↑ open up ↑
 851  842          movaps  0x50(%KEYP), %KEY
 852  843          aesdec  %KEY, %STATE
 853  844          movaps  0x60(%KEYP), %KEY
 854  845          aesdec  %KEY, %STATE
 855  846          movaps  0x70(%KEYP), %KEY
 856  847          aesdeclast      %KEY, %STATE            / last round
 857  848          movups  %STATE, (%OUTP)                 / output
 858  849  
 859  850          SET_TS_OR_POP_XMM0_XMM1(%r10)
 860  851          ret
 861      -        /* EXPORT DELETE END */
 862  852          SET_SIZE(aes_decrypt_intel)
 863  853  
 864  854  #endif  /* lint || __lint */
    
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX