disasm: Add basic AVX512 support

Disassembler can translate EVEX prefix, count up to 32 vector registers
and recognize new ZMM / opmask registers.

Signed-off-by: Jin Kyu Song <jin.kyu.song@intel.com>

3 files changed, 100 insertions, 6 deletions

diff --git a/disasm.c b/disasm.c
index ff6f2638..dbecde22 100644
--- a/disasm.c
+++ b/disasm.c
@@ -81,6 +81,7 @@ struct prefix_info {
     uint8_t vex_v;
     uint8_t vex_lp;     /* VEX.LP fields */
     uint32_t rex;       /* REX prefix present */
+    uint8_t evex[3];    /* EVEX prefix present */
 };
 
 #define getu8(x) (*(uint8_t *)(x))
@@ -133,12 +134,14 @@ static enum reg_enum whichreg(opflags_t regflags, int regval, int rex)
         {FPU0,    R_ST0},
         {XMM0,    R_XMM0},
         {YMM0,    R_YMM0},
+        {ZMM0,    R_ZMM0},
         {REG_ES,  R_ES},
         {REG_CS,  R_CS},
         {REG_SS,  R_SS},
         {REG_DS,  R_DS},
         {REG_FS,  R_FS},
-        {REG_GS,  R_GS}
+        {REG_GS,  R_GS},
+        {OPMASK0, R_K0},
     };
 
     if (!(regflags & (REGISTER|REGMEM)))
@@ -151,7 +154,7 @@ static enum reg_enum whichreg(opflags_t regflags, int regval, int rex)
             return specific_registers[i].reg;
 
     /* All the entries below look up regval in an 16-entry array */
-    if (regval < 0 || regval > 15)
+    if (regval < 0 || regval > (rex & REX_EV ? 31 : 15))
         return 0;
 
     if (!(REG8 & ~regflags)) {
@@ -185,6 +188,10 @@ static enum reg_enum whichreg(opflags_t regflags, int regval, int rex)
         return nasm_rd_xmmreg[regval];
     if (!(YMMREG & ~regflags))
         return nasm_rd_ymmreg[regval];
+    if (!(ZMMREG & ~regflags))
+        return nasm_rd_zmmreg[regval];
+    if (!(OPMASKREG & ~regflags))
+        return nasm_rd_opmaskreg[regval];
 
     return 0;
 }
@@ -198,7 +205,9 @@ static uint8_t *do_ea(uint8_t *data, int modrm, int asize,
 {
     int mod, rm, scale, index, base;
     int rex;
+    uint8_t *evex;
     uint8_t sib = 0;
+    bool is_evex = !!(ins->rex & REX_EV);
 
     mod = (modrm >> 6) & 03;
     rm = modrm & 07;
@@ -206,11 +215,15 @@ static uint8_t *do_ea(uint8_t *data, int modrm, int asize,
     if (mod != 3 && asize != 16 && rm == 4)
         sib = *data++;
 
-    rex = ins->rex;
+    rex  = ins->rex;
+    evex = ins->evex_p;
 
     if (mod == 3) {             /* pure register version */
         op->basereg = rm+(rex & REX_B ? 8 : 0);
         op->segment |= SEG_RMREG;
+        if (is_evex && segsize == 64) {
+            op->basereg += (evex[0] & EVEX_P0X ? 0 : 16);
+        }
         return data;
     }
 
@@ -564,6 +577,8 @@ static int matches(const struct itemplate *t, uint8_t *data,
             if (!data)
                 return false;
             opx->basereg = ((modrm >> 3) & 7) + (ins->rex & REX_R ? 8 : 0);
+            if ((ins->rex & REX_EV) && (segsize == 64))
+                opx->basereg += (ins->evex_p[0] & EVEX_P0RP ? 0 : 16);
             break;
         }
 
@@ -617,6 +632,55 @@ static int matches(const struct itemplate *t, uint8_t *data,
             break;
         }
 
+        case4(0240):
+        case 0250:
+        {
+            uint8_t evexm   = *r++;
+            uint8_t evexwlp = *r++;
+            ins->evex_tuple = *r++ - 0300;
+
+            ins->rex |= REX_EV;
+            if ((prefix->rex & (REX_EV|REX_V|REX_P)) != REX_EV)
+                return false;
+
+            if ((evexm & 0x1f) != prefix->vex_m)
+                return false;
+
+            switch (evexwlp & 060) {
+            case 000:
+                if (prefix->rex & REX_W)
+                    return false;
+                break;
+            case 020:
+                if (!(prefix->rex & REX_W))
+                    return false;
+                ins->rex |= REX_W;
+                break;
+            case 040:        /* VEX.W is a don't care */
+                ins->rex &= ~REX_W;
+                break;
+            case 060:
+                break;
+            }
+
+            /* If EVEX.b is set, EVEX.L'L can be rounding control bits */
+            if ((evexwlp ^ prefix->vex_lp) &
+                ((prefix->evex[2] & EVEX_P2B) ? 0x03 : 0x0f))
+                return false;
+
+            if (c == 0250) {
+                if ((prefix->vex_v != 0) || !(prefix->evex[2] & EVEX_P2VP))
+                    return false;
+            } else {
+                opx->segment |= SEG_RMREG;
+                opx->basereg = ((~prefix->evex[2] & EVEX_P2VP) << (4 - 3) ) |
+                                prefix->vex_v;
+            }
+            vex_ok = true;
+            memcpy(ins->evex_p, prefix->evex, 3);
+            break;
+        }
+
         case4(0260):
         case 0270:
         {
@@ -879,7 +943,7 @@ static int matches(const struct itemplate *t, uint8_t *data,
         }
     }
 
-    if (!vex_ok && (ins->rex & REX_V))
+    if (!vex_ok && (ins->rex & (REX_V | REX_EV)))
         return false;
 
     /* REX cannot be combined with VEX */
@@ -1045,6 +1109,31 @@ int32_t disasm(uint8_t *data, char *output, int outbufsize, int segsize,
             end_prefix = true;
             break;
 
+        case 0x62:
+        {
+            uint8_t evex_p0 = data[1] & 0x0f;
+            if (segsize == 64 ||
+                ((evex_p0 >= 0x01) && (evex_p0 <= 0x03))) {
+                data++;        /* 62h EVEX prefix */
+                prefix.evex[0] = *data++;
+                prefix.evex[1] = *data++;
+                prefix.evex[2] = *data++;
+
+                prefix.rex    = REX_EV;
+                prefix.vex_c  = RV_EVEX;
+                prefix.rex   |= (~prefix.evex[0] >> 5) & 7; /* REX_RXB */
+                prefix.rex   |= (prefix.evex[1] >> (7-3)) & REX_W;
+                prefix.vex_m  = prefix.evex[0] & EVEX_P0MM;
+                prefix.vex_v  = (~prefix.evex[1] & EVEX_P1VVVV) >> 3;
+                prefix.vex_lp = ((prefix.evex[2] & EVEX_P2LL) >> (5-2)) |
+                                (prefix.evex[1] & EVEX_P1PP);
+
+                ix = itable_vex[prefix.vex_c][prefix.vex_m][prefix.vex_lp & 3];
+            }
+            end_prefix = true;
+            break;
+        }
+
         case 0x8F:
             if ((data[1] & 030) != 0 &&
                     (segsize == 64 || (data[1] & 0xc0) == 0xc0)) {
diff --git a/insns.dat b/insns.dat
index 9435e03d..edf7db8b 100644
--- a/insns.dat
+++ b/insns.dat
@@ -3730,8 +3730,8 @@ VEXTRACTI32X4    mem128|mask,zmmreg,imm8                       [mri:t4:        e
 VEXTRACTI32X4    xmmreg|mask|z,zmmreg,imm8                     [mri:           evex.512.66.0f3a.w0 39 /r ib ]  AVX512,FUTURE
 VEXTRACTI64X4    ymmreg|mask|z,zmmreg,imm8                     [mri:           evex.512.66.0f3a.w1 3b /r ib ]  AVX512,FUTURE
 VEXTRACTI64X4    mem256|mask,zmmreg,imm8                       [mri:t4:        evex.512.66.0f3a.w1 3b /r ib ]  AVX512,FUTURE
-VEXTRACTPS       rm32,xmmreg,imm8                              [mri:t1s:      evex.128.66.0f3a.wig 17 /r ib ]  AVX512,FUTURE
 VEXTRACTPS       rm64,xmmreg,imm8                              [mri:t1s:       evex.128.66.0f3a.w1 17 /r ib ]  AVX512,FUTURE
+VEXTRACTPS       rm32,xmmreg,imm8                              [mri:t1s:      evex.128.66.0f3a.wig 17 /r ib ]  AVX512,FUTURE
 VFIXUPIMMPD      zmmreg|mask|z,zmmreg,zmmrm512|b64|sae,imm8    [rvmi:fv:   evex.nds.512.66.0f3a.w1 54 /r ib ]  AVX512,FUTURE
 VFIXUPIMMPS      zmmreg|mask|z,zmmreg,zmmrm512|b32|sae,imm8    [rvmi:fv:   evex.nds.512.66.0f3a.w0 54 /r ib ]  AVX512,FUTURE
 VFIXUPIMMSD      xmmreg|mask|z,xmmreg,xmmrm64|sae,imm8         [rvmi:t1s:  evex.nds.lig.66.0f3a.w1 55 /r ib ]  AVX512,FUTURE
diff --git a/nasm.h b/nasm.h
index b68a8ba8..cb786f80 100644
--- a/nasm.h
+++ b/nasm.h
@@ -509,8 +509,12 @@ static inline uint8_t get_cond_opcode(enum ccode c)
 /*
  * EVEX bit field
  */
+#define EVEX_P0MM       0x03        /* EVEX P[1:0] : Legacy escape        */
 #define EVEX_P0RP       0x10        /* EVEX P[4] : High-16 reg            */
 #define EVEX_P0X        0x40        /* EVEX P[6] : High-16 rm             */
+#define EVEX_P1PP       0x03        /* EVEX P[9:8] : Legacy prefix        */
+#define EVEX_P1VVVV     0x78        /* EVEX P[14:11] : NDS register       */
+#define EVEX_P1W        0x80        /* EVEX P[15] : Osize extension       */
 #define EVEX_P2AAA      0x07        /* EVEX P[18:16] : Embedded opmask    */
 #define EVEX_P2VP       0x08        /* EVEX P[19] : High-16 NDS reg       */
 #define EVEX_P2B        0x10        /* EVEX P[20] : Broadcast / RC / SAE  */
@@ -523,7 +527,8 @@ static inline uint8_t get_cond_opcode(enum ccode c)
  */
 enum vex_class {
     RV_VEX      = 0,    /* C4/C5 */
-    RV_XOP      = 1     /* 8F */
+    RV_XOP      = 1,    /* 8F */
+    RV_EVEX     = 2,    /* 62 */
 };
 
 /*