Added an invalid instruction hook (#1132)

* first draft for an invalid instruction hook

* Fixed documentation on return value of invalid insn hook

include/uc_priv.h

@@ -107,6 +107,7 @@ enum uc_hook_idx {
     UC_HOOK_MEM_WRITE_IDX,
     UC_HOOK_MEM_FETCH_IDX,
     UC_HOOK_MEM_READ_AFTER_IDX,
+    UC_HOOK_INSN_INVALID_IDX,
 
     UC_HOOK_MAX,
 };

include/unicorn/unicorn.h

@@ -173,6 +173,15 @@ typedef void (*uc_cb_hookcode_t)(uc_engine *uc, uint64_t address, uint32_t size,
 */
 typedef void (*uc_cb_hookintr_t)(uc_engine *uc, uint32_t intno, void *user_data);
 
+/*
+  Callback function for tracing invalid instructions
+
+  @user_data: user data passed to tracing APIs.
+
+  @return: return true to continue, or false to stop program (due to invalid instruction).
+*/
+typedef bool (*uc_cb_hookinsn_invalid_t)(uc_engine *uc, void *user_data);
+
 /*
   Callback function for tracing IN instruction of X86
 
@@ -236,6 +245,8 @@ typedef enum uc_hook_type {
     // Hook memory read events, but only successful access.
     // The callback will be triggered after successful read.
     UC_HOOK_MEM_READ_AFTER = 1 << 13,
+    // Hook invalid instructions exceptions.
+    UC_HOOK_INSN_INVALID = 1 << 14,
 } uc_hook_type;
 
 // Hook type for all events of unmapped memory access