(Fix #341) SYSENTER instruction is not properly hooked with uc_hook_add in x86 emulation.

helper_sysenter in qemu/target-i386/seg_helper.c didn't check properly if a call interrupt callback was registred.
It has been fixed by copying the helper_syscall behavior.

qemu/target-i386/helper.h

@@ -49,7 +49,7 @@ DEF_HELPER_4(enter_level, void, env, int, int, tl)
 #ifdef TARGET_X86_64
 DEF_HELPER_4(enter64_level, void, env, int, int, tl)
 #endif
-DEF_HELPER_1(sysenter, void, env)
+DEF_HELPER_2(sysenter, void, env, int)
 DEF_HELPER_2(sysexit, void, env, int)
 #ifdef TARGET_X86_64
 DEF_HELPER_2(syscall, void, env, int)

qemu/target-i386/seg_helper.c

@@ -2301,8 +2301,18 @@ void helper_lret_protected(CPUX86State *env, int shift, int addend)
     helper_ret_protected(env, shift, 0, addend);
 }
 
-void helper_sysenter(CPUX86State *env)
+void helper_sysenter(CPUX86State *env, int next_eip_addend)
 {
+    // Unicorn: call interrupt callback if registered
+    struct uc_struct *uc = env->uc;
+    if (uc->hook_syscall_idx) {
+        ((uc_cb_insn_syscall_t)uc->hook_callbacks[uc->hook_syscall_idx].callback)(
+            uc, uc->hook_callbacks[uc->hook_syscall_idx].user_data);
+    }
+    env->eip += next_eip_addend;
+
+    return;
+
     if (env->sysenter_cs == 0) {
         raise_exception_err(env, EXCP0D_GPF, 0);
     }

qemu/target-i386/translate.c

@@ -7425,14 +7425,11 @@ static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
         /* For Intel SYSENTER is valid on 64-bit */
         if (CODE64(s) && env->cpuid_vendor1 != CPUID_VENDOR_INTEL_1)
             goto illegal_op;
-        if (!s->pe) {
+
-            gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
+        gen_update_cc_op(s);
-        } else {
+        gen_jmp_im(s, pc_start - s->cs_base);
-            gen_update_cc_op(s);
+        gen_helper_sysenter(tcg_ctx, cpu_env, tcg_const_i32(tcg_ctx, s->pc - pc_start));
-            gen_jmp_im(s, pc_start - s->cs_base);
+        gen_eob(s);
-            gen_helper_sysenter(tcg_ctx, cpu_env);
-            gen_eob(s);
-        }
         break;
     case 0x135: /* sysexit */
         /* For Intel SYSEXIT is valid on 64-bit */

tests/regress/sysenter_hook_x86.c

@@ -0,0 +1,60 @@
+#include <unicorn/unicorn.h>
+
+// code to be emulated
+#define X86_CODE32 "\x0F\x34" // SYSENTER
+
+// memory address where emulation starts
+#define ADDRESS 0x1000000
+
+int got_sysenter = 0;
+
+void sysenter (uc_engine *uc, void *user) {
+    printf ("SYSENTER hook called.\n");
+    got_sysenter = 1;
+}
+
+int main(int argc, char **argv, char **envp)
+{
+  uc_engine *uc;
+  uc_err err;
+  uc_hook sysenterHook;
+
+  // Initialize emulator in X86-32bit mode
+  err = uc_open(UC_ARCH_X86, UC_MODE_32, &uc);
+  if (err != UC_ERR_OK) {
+    printf("Failed on uc_open() with error returned: %u\n", err);
+    return -1;
+  }
+
+  // map 2MB memory for this emulation
+  uc_mem_map(uc, ADDRESS, 2 * 1024 * 1024, UC_PROT_ALL);
+
+  // write machine code to be emulated to memory
+  if (uc_mem_write(uc, ADDRESS, X86_CODE32, sizeof(X86_CODE32) - 1)) {
+    printf("Failed to write emulation code to memory, quit!\n");
+    return -1;
+  }
+
+  // Hook the SYSENTER instructions
+  if (uc_hook_add (uc, &sysenterHook, UC_HOOK_INSN, sysenter, NULL, UC_X86_INS_SYSENTER) != UC_ERR_OK) {
+      printf ("Cannot hook SYSENTER instruction\n.");
+      return -1;
+  }
+
+  // emulate code in infinite time & unlimited instructions
+  err=uc_emu_start(uc, ADDRESS, ADDRESS + sizeof(X86_CODE32) - 1, 0, 0);
+  if (err) {
+    printf("Failed on uc_emu_start() with error returned %u: %s\n",
+      err, uc_strerror(err));
+  }
+
+  printf("Emulation done.\n");
+  uc_close(uc);
+
+  if (!got_sysenter) {
+    printf ("[!] ERROR : SYSENTER hook not called.\n");
+    return -1;
+  }
+
+  return 0;
+}