1 /* $NetBSD: cmd_parser.c,v 1.2 2021/12/18 23:45:31 riastradh Exp $ */ 2 3 /* 4 * Copyright(c) 2011-2016 Intel Corporation. All rights reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the next 14 * paragraph) shall be included in all copies or substantial portions of the 15 * Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 * SOFTWARE. 24 * 25 * Authors: 26 * Ke Yu 27 * Kevin Tian <kevin.tian@intel.com> 28 * Zhiyuan Lv <zhiyuan.lv@intel.com> 29 * 30 * Contributors: 31 * Min He <min.he@intel.com> 32 * Ping Gao <ping.a.gao@intel.com> 33 * Tina Zhang <tina.zhang@intel.com> 34 * Yulei Zhang <yulei.zhang@intel.com> 35 * Zhi Wang <zhi.a.wang@intel.com> 36 * 37 */ 38 39 #include <sys/cdefs.h> 40 __KERNEL_RCSID(0, "$NetBSD: cmd_parser.c,v 1.2 2021/12/18 23:45:31 riastradh Exp $"); 41 42 #include <linux/slab.h> 43 44 #include "i915_drv.h" 45 #include "gt/intel_ring.h" 46 #include "gvt.h" 47 #include "i915_pvinfo.h" 48 #include "trace.h" 49 50 #define INVALID_OP (~0U) 51 52 #define OP_LEN_MI 9 53 #define OP_LEN_2D 10 54 #define OP_LEN_3D_MEDIA 16 55 #define OP_LEN_MFX_VC 16 56 #define OP_LEN_VEBOX 16 57 58 #define CMD_TYPE(cmd) (((cmd) >> 29) & 7) 59 60 struct sub_op_bits { 61 int hi; 62 int low; 63 }; 64 struct decode_info { 65 const char *name; 66 int op_len; 67 int nr_sub_op; 68 const struct sub_op_bits *sub_op; 69 }; 70 71 #define MAX_CMD_BUDGET 0x7fffffff 72 #define MI_WAIT_FOR_PLANE_C_FLIP_PENDING (1<<15) 73 #define MI_WAIT_FOR_PLANE_B_FLIP_PENDING (1<<9) 74 #define MI_WAIT_FOR_PLANE_A_FLIP_PENDING (1<<1) 75 76 #define MI_WAIT_FOR_SPRITE_C_FLIP_PENDING (1<<20) 77 #define MI_WAIT_FOR_SPRITE_B_FLIP_PENDING (1<<10) 78 #define MI_WAIT_FOR_SPRITE_A_FLIP_PENDING (1<<2) 79 80 /* Render Command Map */ 81 82 /* MI_* command Opcode (28:23) */ 83 #define OP_MI_NOOP 0x0 84 #define OP_MI_SET_PREDICATE 0x1 /* HSW+ */ 85 #define OP_MI_USER_INTERRUPT 0x2 86 #define OP_MI_WAIT_FOR_EVENT 0x3 87 #define OP_MI_FLUSH 0x4 88 #define OP_MI_ARB_CHECK 0x5 89 #define OP_MI_RS_CONTROL 0x6 /* HSW+ */ 90 #define OP_MI_REPORT_HEAD 0x7 91 #define OP_MI_ARB_ON_OFF 0x8 92 #define OP_MI_URB_ATOMIC_ALLOC 0x9 /* HSW+ */ 93 #define OP_MI_BATCH_BUFFER_END 0xA 94 #define OP_MI_SUSPEND_FLUSH 0xB 95 #define OP_MI_PREDICATE 0xC /* IVB+ */ 96 #define OP_MI_TOPOLOGY_FILTER 0xD /* IVB+ */ 97 #define OP_MI_SET_APPID 0xE /* IVB+ */ 98 #define OP_MI_RS_CONTEXT 0xF /* HSW+ */ 99 #define OP_MI_LOAD_SCAN_LINES_INCL 0x12 /* HSW+ */ 100 #define OP_MI_DISPLAY_FLIP 0x14 101 #define OP_MI_SEMAPHORE_MBOX 0x16 102 #define OP_MI_SET_CONTEXT 0x18 103 #define OP_MI_MATH 0x1A 104 #define OP_MI_URB_CLEAR 0x19 105 #define OP_MI_SEMAPHORE_SIGNAL 0x1B /* BDW+ */ 106 #define OP_MI_SEMAPHORE_WAIT 0x1C /* BDW+ */ 107 108 #define OP_MI_STORE_DATA_IMM 0x20 109 #define OP_MI_STORE_DATA_INDEX 0x21 110 #define OP_MI_LOAD_REGISTER_IMM 0x22 111 #define OP_MI_UPDATE_GTT 0x23 112 #define OP_MI_STORE_REGISTER_MEM 0x24 113 #define OP_MI_FLUSH_DW 0x26 114 #define OP_MI_CLFLUSH 0x27 115 #define OP_MI_REPORT_PERF_COUNT 0x28 116 #define OP_MI_LOAD_REGISTER_MEM 0x29 /* HSW+ */ 117 #define OP_MI_LOAD_REGISTER_REG 0x2A /* HSW+ */ 118 #define OP_MI_RS_STORE_DATA_IMM 0x2B /* HSW+ */ 119 #define OP_MI_LOAD_URB_MEM 0x2C /* HSW+ */ 120 #define OP_MI_STORE_URM_MEM 0x2D /* HSW+ */ 121 #define OP_MI_2E 0x2E /* BDW+ */ 122 #define OP_MI_2F 0x2F /* BDW+ */ 123 #define OP_MI_BATCH_BUFFER_START 0x31 124 125 /* Bit definition for dword 0 */ 126 #define _CMDBIT_BB_START_IN_PPGTT (1UL << 8) 127 128 #define OP_MI_CONDITIONAL_BATCH_BUFFER_END 0x36 129 130 #define BATCH_BUFFER_ADDR_MASK ((1UL << 32) - (1U << 2)) 131 #define BATCH_BUFFER_ADDR_HIGH_MASK ((1UL << 16) - (1U)) 132 #define BATCH_BUFFER_ADR_SPACE_BIT(x) (((x) >> 8) & 1U) 133 #define BATCH_BUFFER_2ND_LEVEL_BIT(x) ((x) >> 22 & 1U) 134 135 /* 2D command: Opcode (28:22) */ 136 #define OP_2D(x) ((2<<7) | x) 137 138 #define OP_XY_SETUP_BLT OP_2D(0x1) 139 #define OP_XY_SETUP_CLIP_BLT OP_2D(0x3) 140 #define OP_XY_SETUP_MONO_PATTERN_SL_BLT OP_2D(0x11) 141 #define OP_XY_PIXEL_BLT OP_2D(0x24) 142 #define OP_XY_SCANLINES_BLT OP_2D(0x25) 143 #define OP_XY_TEXT_BLT OP_2D(0x26) 144 #define OP_XY_TEXT_IMMEDIATE_BLT OP_2D(0x31) 145 #define OP_XY_COLOR_BLT OP_2D(0x50) 146 #define OP_XY_PAT_BLT OP_2D(0x51) 147 #define OP_XY_MONO_PAT_BLT OP_2D(0x52) 148 #define OP_XY_SRC_COPY_BLT OP_2D(0x53) 149 #define OP_XY_MONO_SRC_COPY_BLT OP_2D(0x54) 150 #define OP_XY_FULL_BLT OP_2D(0x55) 151 #define OP_XY_FULL_MONO_SRC_BLT OP_2D(0x56) 152 #define OP_XY_FULL_MONO_PATTERN_BLT OP_2D(0x57) 153 #define OP_XY_FULL_MONO_PATTERN_MONO_SRC_BLT OP_2D(0x58) 154 #define OP_XY_MONO_PAT_FIXED_BLT OP_2D(0x59) 155 #define OP_XY_MONO_SRC_COPY_IMMEDIATE_BLT OP_2D(0x71) 156 #define OP_XY_PAT_BLT_IMMEDIATE OP_2D(0x72) 157 #define OP_XY_SRC_COPY_CHROMA_BLT OP_2D(0x73) 158 #define OP_XY_FULL_IMMEDIATE_PATTERN_BLT OP_2D(0x74) 159 #define OP_XY_FULL_MONO_SRC_IMMEDIATE_PATTERN_BLT OP_2D(0x75) 160 #define OP_XY_PAT_CHROMA_BLT OP_2D(0x76) 161 #define OP_XY_PAT_CHROMA_BLT_IMMEDIATE OP_2D(0x77) 162 163 /* 3D/Media Command: Pipeline Type(28:27) Opcode(26:24) Sub Opcode(23:16) */ 164 #define OP_3D_MEDIA(sub_type, opcode, sub_opcode) \ 165 ((3 << 13) | ((sub_type) << 11) | ((opcode) << 8) | (sub_opcode)) 166 167 #define OP_STATE_PREFETCH OP_3D_MEDIA(0x0, 0x0, 0x03) 168 169 #define OP_STATE_BASE_ADDRESS OP_3D_MEDIA(0x0, 0x1, 0x01) 170 #define OP_STATE_SIP OP_3D_MEDIA(0x0, 0x1, 0x02) 171 #define OP_3D_MEDIA_0_1_4 OP_3D_MEDIA(0x0, 0x1, 0x04) 172 173 #define OP_3DSTATE_VF_STATISTICS_GM45 OP_3D_MEDIA(0x1, 0x0, 0x0B) 174 175 #define OP_PIPELINE_SELECT OP_3D_MEDIA(0x1, 0x1, 0x04) 176 177 #define OP_MEDIA_VFE_STATE OP_3D_MEDIA(0x2, 0x0, 0x0) 178 #define OP_MEDIA_CURBE_LOAD OP_3D_MEDIA(0x2, 0x0, 0x1) 179 #define OP_MEDIA_INTERFACE_DESCRIPTOR_LOAD OP_3D_MEDIA(0x2, 0x0, 0x2) 180 #define OP_MEDIA_GATEWAY_STATE OP_3D_MEDIA(0x2, 0x0, 0x3) 181 #define OP_MEDIA_STATE_FLUSH OP_3D_MEDIA(0x2, 0x0, 0x4) 182 #define OP_MEDIA_POOL_STATE OP_3D_MEDIA(0x2, 0x0, 0x5) 183 184 #define OP_MEDIA_OBJECT OP_3D_MEDIA(0x2, 0x1, 0x0) 185 #define OP_MEDIA_OBJECT_PRT OP_3D_MEDIA(0x2, 0x1, 0x2) 186 #define OP_MEDIA_OBJECT_WALKER OP_3D_MEDIA(0x2, 0x1, 0x3) 187 #define OP_GPGPU_WALKER OP_3D_MEDIA(0x2, 0x1, 0x5) 188 189 #define OP_3DSTATE_CLEAR_PARAMS OP_3D_MEDIA(0x3, 0x0, 0x04) /* IVB+ */ 190 #define OP_3DSTATE_DEPTH_BUFFER OP_3D_MEDIA(0x3, 0x0, 0x05) /* IVB+ */ 191 #define OP_3DSTATE_STENCIL_BUFFER OP_3D_MEDIA(0x3, 0x0, 0x06) /* IVB+ */ 192 #define OP_3DSTATE_HIER_DEPTH_BUFFER OP_3D_MEDIA(0x3, 0x0, 0x07) /* IVB+ */ 193 #define OP_3DSTATE_VERTEX_BUFFERS OP_3D_MEDIA(0x3, 0x0, 0x08) 194 #define OP_3DSTATE_VERTEX_ELEMENTS OP_3D_MEDIA(0x3, 0x0, 0x09) 195 #define OP_3DSTATE_INDEX_BUFFER OP_3D_MEDIA(0x3, 0x0, 0x0A) 196 #define OP_3DSTATE_VF_STATISTICS OP_3D_MEDIA(0x3, 0x0, 0x0B) 197 #define OP_3DSTATE_VF OP_3D_MEDIA(0x3, 0x0, 0x0C) /* HSW+ */ 198 #define OP_3DSTATE_CC_STATE_POINTERS OP_3D_MEDIA(0x3, 0x0, 0x0E) 199 #define OP_3DSTATE_SCISSOR_STATE_POINTERS OP_3D_MEDIA(0x3, 0x0, 0x0F) 200 #define OP_3DSTATE_VS OP_3D_MEDIA(0x3, 0x0, 0x10) 201 #define OP_3DSTATE_GS OP_3D_MEDIA(0x3, 0x0, 0x11) 202 #define OP_3DSTATE_CLIP OP_3D_MEDIA(0x3, 0x0, 0x12) 203 #define OP_3DSTATE_SF OP_3D_MEDIA(0x3, 0x0, 0x13) 204 #define OP_3DSTATE_WM OP_3D_MEDIA(0x3, 0x0, 0x14) 205 #define OP_3DSTATE_CONSTANT_VS OP_3D_MEDIA(0x3, 0x0, 0x15) 206 #define OP_3DSTATE_CONSTANT_GS OP_3D_MEDIA(0x3, 0x0, 0x16) 207 #define OP_3DSTATE_CONSTANT_PS OP_3D_MEDIA(0x3, 0x0, 0x17) 208 #define OP_3DSTATE_SAMPLE_MASK OP_3D_MEDIA(0x3, 0x0, 0x18) 209 #define OP_3DSTATE_CONSTANT_HS OP_3D_MEDIA(0x3, 0x0, 0x19) /* IVB+ */ 210 #define OP_3DSTATE_CONSTANT_DS OP_3D_MEDIA(0x3, 0x0, 0x1A) /* IVB+ */ 211 #define OP_3DSTATE_HS OP_3D_MEDIA(0x3, 0x0, 0x1B) /* IVB+ */ 212 #define OP_3DSTATE_TE OP_3D_MEDIA(0x3, 0x0, 0x1C) /* IVB+ */ 213 #define OP_3DSTATE_DS OP_3D_MEDIA(0x3, 0x0, 0x1D) /* IVB+ */ 214 #define OP_3DSTATE_STREAMOUT OP_3D_MEDIA(0x3, 0x0, 0x1E) /* IVB+ */ 215 #define OP_3DSTATE_SBE OP_3D_MEDIA(0x3, 0x0, 0x1F) /* IVB+ */ 216 #define OP_3DSTATE_PS OP_3D_MEDIA(0x3, 0x0, 0x20) /* IVB+ */ 217 #define OP_3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP OP_3D_MEDIA(0x3, 0x0, 0x21) /* IVB+ */ 218 #define OP_3DSTATE_VIEWPORT_STATE_POINTERS_CC OP_3D_MEDIA(0x3, 0x0, 0x23) /* IVB+ */ 219 #define OP_3DSTATE_BLEND_STATE_POINTERS OP_3D_MEDIA(0x3, 0x0, 0x24) /* IVB+ */ 220 #define OP_3DSTATE_DEPTH_STENCIL_STATE_POINTERS OP_3D_MEDIA(0x3, 0x0, 0x25) /* IVB+ */ 221 #define OP_3DSTATE_BINDING_TABLE_POINTERS_VS OP_3D_MEDIA(0x3, 0x0, 0x26) /* IVB+ */ 222 #define OP_3DSTATE_BINDING_TABLE_POINTERS_HS OP_3D_MEDIA(0x3, 0x0, 0x27) /* IVB+ */ 223 #define OP_3DSTATE_BINDING_TABLE_POINTERS_DS OP_3D_MEDIA(0x3, 0x0, 0x28) /* IVB+ */ 224 #define OP_3DSTATE_BINDING_TABLE_POINTERS_GS OP_3D_MEDIA(0x3, 0x0, 0x29) /* IVB+ */ 225 #define OP_3DSTATE_BINDING_TABLE_POINTERS_PS OP_3D_MEDIA(0x3, 0x0, 0x2A) /* IVB+ */ 226 #define OP_3DSTATE_SAMPLER_STATE_POINTERS_VS OP_3D_MEDIA(0x3, 0x0, 0x2B) /* IVB+ */ 227 #define OP_3DSTATE_SAMPLER_STATE_POINTERS_HS OP_3D_MEDIA(0x3, 0x0, 0x2C) /* IVB+ */ 228 #define OP_3DSTATE_SAMPLER_STATE_POINTERS_DS OP_3D_MEDIA(0x3, 0x0, 0x2D) /* IVB+ */ 229 #define OP_3DSTATE_SAMPLER_STATE_POINTERS_GS OP_3D_MEDIA(0x3, 0x0, 0x2E) /* IVB+ */ 230 #define OP_3DSTATE_SAMPLER_STATE_POINTERS_PS OP_3D_MEDIA(0x3, 0x0, 0x2F) /* IVB+ */ 231 #define OP_3DSTATE_URB_VS OP_3D_MEDIA(0x3, 0x0, 0x30) /* IVB+ */ 232 #define OP_3DSTATE_URB_HS OP_3D_MEDIA(0x3, 0x0, 0x31) /* IVB+ */ 233 #define OP_3DSTATE_URB_DS OP_3D_MEDIA(0x3, 0x0, 0x32) /* IVB+ */ 234 #define OP_3DSTATE_URB_GS OP_3D_MEDIA(0x3, 0x0, 0x33) /* IVB+ */ 235 #define OP_3DSTATE_GATHER_CONSTANT_VS OP_3D_MEDIA(0x3, 0x0, 0x34) /* HSW+ */ 236 #define OP_3DSTATE_GATHER_CONSTANT_GS OP_3D_MEDIA(0x3, 0x0, 0x35) /* HSW+ */ 237 #define OP_3DSTATE_GATHER_CONSTANT_HS OP_3D_MEDIA(0x3, 0x0, 0x36) /* HSW+ */ 238 #define OP_3DSTATE_GATHER_CONSTANT_DS OP_3D_MEDIA(0x3, 0x0, 0x37) /* HSW+ */ 239 #define OP_3DSTATE_GATHER_CONSTANT_PS OP_3D_MEDIA(0x3, 0x0, 0x38) /* HSW+ */ 240 #define OP_3DSTATE_DX9_CONSTANTF_VS OP_3D_MEDIA(0x3, 0x0, 0x39) /* HSW+ */ 241 #define OP_3DSTATE_DX9_CONSTANTF_PS OP_3D_MEDIA(0x3, 0x0, 0x3A) /* HSW+ */ 242 #define OP_3DSTATE_DX9_CONSTANTI_VS OP_3D_MEDIA(0x3, 0x0, 0x3B) /* HSW+ */ 243 #define OP_3DSTATE_DX9_CONSTANTI_PS OP_3D_MEDIA(0x3, 0x0, 0x3C) /* HSW+ */ 244 #define OP_3DSTATE_DX9_CONSTANTB_VS OP_3D_MEDIA(0x3, 0x0, 0x3D) /* HSW+ */ 245 #define OP_3DSTATE_DX9_CONSTANTB_PS OP_3D_MEDIA(0x3, 0x0, 0x3E) /* HSW+ */ 246 #define OP_3DSTATE_DX9_LOCAL_VALID_VS OP_3D_MEDIA(0x3, 0x0, 0x3F) /* HSW+ */ 247 #define OP_3DSTATE_DX9_LOCAL_VALID_PS OP_3D_MEDIA(0x3, 0x0, 0x40) /* HSW+ */ 248 #define OP_3DSTATE_DX9_GENERATE_ACTIVE_VS OP_3D_MEDIA(0x3, 0x0, 0x41) /* HSW+ */ 249 #define OP_3DSTATE_DX9_GENERATE_ACTIVE_PS OP_3D_MEDIA(0x3, 0x0, 0x42) /* HSW+ */ 250 #define OP_3DSTATE_BINDING_TABLE_EDIT_VS OP_3D_MEDIA(0x3, 0x0, 0x43) /* HSW+ */ 251 #define OP_3DSTATE_BINDING_TABLE_EDIT_GS OP_3D_MEDIA(0x3, 0x0, 0x44) /* HSW+ */ 252 #define OP_3DSTATE_BINDING_TABLE_EDIT_HS OP_3D_MEDIA(0x3, 0x0, 0x45) /* HSW+ */ 253 #define OP_3DSTATE_BINDING_TABLE_EDIT_DS OP_3D_MEDIA(0x3, 0x0, 0x46) /* HSW+ */ 254 #define OP_3DSTATE_BINDING_TABLE_EDIT_PS OP_3D_MEDIA(0x3, 0x0, 0x47) /* HSW+ */ 255 256 #define OP_3DSTATE_VF_INSTANCING OP_3D_MEDIA(0x3, 0x0, 0x49) /* BDW+ */ 257 #define OP_3DSTATE_VF_SGVS OP_3D_MEDIA(0x3, 0x0, 0x4A) /* BDW+ */ 258 #define OP_3DSTATE_VF_TOPOLOGY OP_3D_MEDIA(0x3, 0x0, 0x4B) /* BDW+ */ 259 #define OP_3DSTATE_WM_CHROMAKEY OP_3D_MEDIA(0x3, 0x0, 0x4C) /* BDW+ */ 260 #define OP_3DSTATE_PS_BLEND OP_3D_MEDIA(0x3, 0x0, 0x4D) /* BDW+ */ 261 #define OP_3DSTATE_WM_DEPTH_STENCIL OP_3D_MEDIA(0x3, 0x0, 0x4E) /* BDW+ */ 262 #define OP_3DSTATE_PS_EXTRA OP_3D_MEDIA(0x3, 0x0, 0x4F) /* BDW+ */ 263 #define OP_3DSTATE_RASTER OP_3D_MEDIA(0x3, 0x0, 0x50) /* BDW+ */ 264 #define OP_3DSTATE_SBE_SWIZ OP_3D_MEDIA(0x3, 0x0, 0x51) /* BDW+ */ 265 #define OP_3DSTATE_WM_HZ_OP OP_3D_MEDIA(0x3, 0x0, 0x52) /* BDW+ */ 266 #define OP_3DSTATE_COMPONENT_PACKING OP_3D_MEDIA(0x3, 0x0, 0x55) /* SKL+ */ 267 268 #define OP_3DSTATE_DRAWING_RECTANGLE OP_3D_MEDIA(0x3, 0x1, 0x00) 269 #define OP_3DSTATE_SAMPLER_PALETTE_LOAD0 OP_3D_MEDIA(0x3, 0x1, 0x02) 270 #define OP_3DSTATE_CHROMA_KEY OP_3D_MEDIA(0x3, 0x1, 0x04) 271 #define OP_SNB_3DSTATE_DEPTH_BUFFER OP_3D_MEDIA(0x3, 0x1, 0x05) 272 #define OP_3DSTATE_POLY_STIPPLE_OFFSET OP_3D_MEDIA(0x3, 0x1, 0x06) 273 #define OP_3DSTATE_POLY_STIPPLE_PATTERN OP_3D_MEDIA(0x3, 0x1, 0x07) 274 #define OP_3DSTATE_LINE_STIPPLE OP_3D_MEDIA(0x3, 0x1, 0x08) 275 #define OP_3DSTATE_AA_LINE_PARAMS OP_3D_MEDIA(0x3, 0x1, 0x0A) 276 #define OP_3DSTATE_GS_SVB_INDEX OP_3D_MEDIA(0x3, 0x1, 0x0B) 277 #define OP_3DSTATE_SAMPLER_PALETTE_LOAD1 OP_3D_MEDIA(0x3, 0x1, 0x0C) 278 #define OP_3DSTATE_MULTISAMPLE_BDW OP_3D_MEDIA(0x3, 0x0, 0x0D) 279 #define OP_SNB_3DSTATE_STENCIL_BUFFER OP_3D_MEDIA(0x3, 0x1, 0x0E) 280 #define OP_SNB_3DSTATE_HIER_DEPTH_BUFFER OP_3D_MEDIA(0x3, 0x1, 0x0F) 281 #define OP_SNB_3DSTATE_CLEAR_PARAMS OP_3D_MEDIA(0x3, 0x1, 0x10) 282 #define OP_3DSTATE_MONOFILTER_SIZE OP_3D_MEDIA(0x3, 0x1, 0x11) 283 #define OP_3DSTATE_PUSH_CONSTANT_ALLOC_VS OP_3D_MEDIA(0x3, 0x1, 0x12) /* IVB+ */ 284 #define OP_3DSTATE_PUSH_CONSTANT_ALLOC_HS OP_3D_MEDIA(0x3, 0x1, 0x13) /* IVB+ */ 285 #define OP_3DSTATE_PUSH_CONSTANT_ALLOC_DS OP_3D_MEDIA(0x3, 0x1, 0x14) /* IVB+ */ 286 #define OP_3DSTATE_PUSH_CONSTANT_ALLOC_GS OP_3D_MEDIA(0x3, 0x1, 0x15) /* IVB+ */ 287 #define OP_3DSTATE_PUSH_CONSTANT_ALLOC_PS OP_3D_MEDIA(0x3, 0x1, 0x16) /* IVB+ */ 288 #define OP_3DSTATE_SO_DECL_LIST OP_3D_MEDIA(0x3, 0x1, 0x17) 289 #define OP_3DSTATE_SO_BUFFER OP_3D_MEDIA(0x3, 0x1, 0x18) 290 #define OP_3DSTATE_BINDING_TABLE_POOL_ALLOC OP_3D_MEDIA(0x3, 0x1, 0x19) /* HSW+ */ 291 #define OP_3DSTATE_GATHER_POOL_ALLOC OP_3D_MEDIA(0x3, 0x1, 0x1A) /* HSW+ */ 292 #define OP_3DSTATE_DX9_CONSTANT_BUFFER_POOL_ALLOC OP_3D_MEDIA(0x3, 0x1, 0x1B) /* HSW+ */ 293 #define OP_3DSTATE_SAMPLE_PATTERN OP_3D_MEDIA(0x3, 0x1, 0x1C) 294 #define OP_PIPE_CONTROL OP_3D_MEDIA(0x3, 0x2, 0x00) 295 #define OP_3DPRIMITIVE OP_3D_MEDIA(0x3, 0x3, 0x00) 296 297 /* VCCP Command Parser */ 298 299 /* 300 * Below MFX and VBE cmd definition is from vaapi intel driver project (BSD License) 301 * git://anongit.freedesktop.org/vaapi/intel-driver 302 * src/i965_defines.h 303 * 304 */ 305 306 #define OP_MFX(pipeline, op, sub_opa, sub_opb) \ 307 (3 << 13 | \ 308 (pipeline) << 11 | \ 309 (op) << 8 | \ 310 (sub_opa) << 5 | \ 311 (sub_opb)) 312 313 #define OP_MFX_PIPE_MODE_SELECT OP_MFX(2, 0, 0, 0) /* ALL */ 314 #define OP_MFX_SURFACE_STATE OP_MFX(2, 0, 0, 1) /* ALL */ 315 #define OP_MFX_PIPE_BUF_ADDR_STATE OP_MFX(2, 0, 0, 2) /* ALL */ 316 #define OP_MFX_IND_OBJ_BASE_ADDR_STATE OP_MFX(2, 0, 0, 3) /* ALL */ 317 #define OP_MFX_BSP_BUF_BASE_ADDR_STATE OP_MFX(2, 0, 0, 4) /* ALL */ 318 #define OP_2_0_0_5 OP_MFX(2, 0, 0, 5) /* ALL */ 319 #define OP_MFX_STATE_POINTER OP_MFX(2, 0, 0, 6) /* ALL */ 320 #define OP_MFX_QM_STATE OP_MFX(2, 0, 0, 7) /* IVB+ */ 321 #define OP_MFX_FQM_STATE OP_MFX(2, 0, 0, 8) /* IVB+ */ 322 #define OP_MFX_PAK_INSERT_OBJECT OP_MFX(2, 0, 2, 8) /* IVB+ */ 323 #define OP_MFX_STITCH_OBJECT OP_MFX(2, 0, 2, 0xA) /* IVB+ */ 324 325 #define OP_MFD_IT_OBJECT OP_MFX(2, 0, 1, 9) /* ALL */ 326 327 #define OP_MFX_WAIT OP_MFX(1, 0, 0, 0) /* IVB+ */ 328 #define OP_MFX_AVC_IMG_STATE OP_MFX(2, 1, 0, 0) /* ALL */ 329 #define OP_MFX_AVC_QM_STATE OP_MFX(2, 1, 0, 1) /* ALL */ 330 #define OP_MFX_AVC_DIRECTMODE_STATE OP_MFX(2, 1, 0, 2) /* ALL */ 331 #define OP_MFX_AVC_SLICE_STATE OP_MFX(2, 1, 0, 3) /* ALL */ 332 #define OP_MFX_AVC_REF_IDX_STATE OP_MFX(2, 1, 0, 4) /* ALL */ 333 #define OP_MFX_AVC_WEIGHTOFFSET_STATE OP_MFX(2, 1, 0, 5) /* ALL */ 334 #define OP_MFD_AVC_PICID_STATE OP_MFX(2, 1, 1, 5) /* HSW+ */ 335 #define OP_MFD_AVC_DPB_STATE OP_MFX(2, 1, 1, 6) /* IVB+ */ 336 #define OP_MFD_AVC_SLICEADDR OP_MFX(2, 1, 1, 7) /* IVB+ */ 337 #define OP_MFD_AVC_BSD_OBJECT OP_MFX(2, 1, 1, 8) /* ALL */ 338 #define OP_MFC_AVC_PAK_OBJECT OP_MFX(2, 1, 2, 9) /* ALL */ 339 340 #define OP_MFX_VC1_PRED_PIPE_STATE OP_MFX(2, 2, 0, 1) /* ALL */ 341 #define OP_MFX_VC1_DIRECTMODE_STATE OP_MFX(2, 2, 0, 2) /* ALL */ 342 #define OP_MFD_VC1_SHORT_PIC_STATE OP_MFX(2, 2, 1, 0) /* IVB+ */ 343 #define OP_MFD_VC1_LONG_PIC_STATE OP_MFX(2, 2, 1, 1) /* IVB+ */ 344 #define OP_MFD_VC1_BSD_OBJECT OP_MFX(2, 2, 1, 8) /* ALL */ 345 346 #define OP_MFX_MPEG2_PIC_STATE OP_MFX(2, 3, 0, 0) /* ALL */ 347 #define OP_MFX_MPEG2_QM_STATE OP_MFX(2, 3, 0, 1) /* ALL */ 348 #define OP_MFD_MPEG2_BSD_OBJECT OP_MFX(2, 3, 1, 8) /* ALL */ 349 #define OP_MFC_MPEG2_SLICEGROUP_STATE OP_MFX(2, 3, 2, 3) /* ALL */ 350 #define OP_MFC_MPEG2_PAK_OBJECT OP_MFX(2, 3, 2, 9) /* ALL */ 351 352 #define OP_MFX_2_6_0_0 OP_MFX(2, 6, 0, 0) /* IVB+ */ 353 #define OP_MFX_2_6_0_8 OP_MFX(2, 6, 0, 8) /* IVB+ */ 354 #define OP_MFX_2_6_0_9 OP_MFX(2, 6, 0, 9) /* IVB+ */ 355 356 #define OP_MFX_JPEG_PIC_STATE OP_MFX(2, 7, 0, 0) 357 #define OP_MFX_JPEG_HUFF_TABLE_STATE OP_MFX(2, 7, 0, 2) 358 #define OP_MFD_JPEG_BSD_OBJECT OP_MFX(2, 7, 1, 8) 359 360 #define OP_VEB(pipeline, op, sub_opa, sub_opb) \ 361 (3 << 13 | \ 362 (pipeline) << 11 | \ 363 (op) << 8 | \ 364 (sub_opa) << 5 | \ 365 (sub_opb)) 366 367 #define OP_VEB_SURFACE_STATE OP_VEB(2, 4, 0, 0) 368 #define OP_VEB_STATE OP_VEB(2, 4, 0, 2) 369 #define OP_VEB_DNDI_IECP_STATE OP_VEB(2, 4, 0, 3) 370 371 struct parser_exec_state; 372 373 typedef int (*parser_cmd_handler)(struct parser_exec_state *s); 374 375 #define GVT_CMD_HASH_BITS 7 376 377 /* which DWords need address fix */ 378 #define ADDR_FIX_1(x1) (1 << (x1)) 379 #define ADDR_FIX_2(x1, x2) (ADDR_FIX_1(x1) | ADDR_FIX_1(x2)) 380 #define ADDR_FIX_3(x1, x2, x3) (ADDR_FIX_1(x1) | ADDR_FIX_2(x2, x3)) 381 #define ADDR_FIX_4(x1, x2, x3, x4) (ADDR_FIX_1(x1) | ADDR_FIX_3(x2, x3, x4)) 382 #define ADDR_FIX_5(x1, x2, x3, x4, x5) (ADDR_FIX_1(x1) | ADDR_FIX_4(x2, x3, x4, x5)) 383 384 #define DWORD_FIELD(dword, end, start) \ 385 FIELD_GET(GENMASK(end, start), cmd_val(s, dword)) 386 387 #define OP_LENGTH_BIAS 2 388 #define CMD_LEN(value) (value + OP_LENGTH_BIAS) 389 390 static int gvt_check_valid_cmd_length(int len, int valid_len) 391 { 392 if (valid_len != len) { 393 gvt_err("len is not valid: len=%u valid_len=%u\n", 394 len, valid_len); 395 return -EFAULT; 396 } 397 return 0; 398 } 399 400 struct cmd_info { 401 const char *name; 402 u32 opcode; 403 404 #define F_LEN_MASK 3U 405 #define F_LEN_CONST 1U 406 #define F_LEN_VAR 0U 407 /* value is const although LEN maybe variable */ 408 #define F_LEN_VAR_FIXED (1<<1) 409 410 /* 411 * command has its own ip advance logic 412 * e.g. MI_BATCH_START, MI_BATCH_END 413 */ 414 #define F_IP_ADVANCE_CUSTOM (1<<2) 415 u32 flag; 416 417 #define R_RCS BIT(RCS0) 418 #define R_VCS1 BIT(VCS0) 419 #define R_VCS2 BIT(VCS1) 420 #define R_VCS (R_VCS1 | R_VCS2) 421 #define R_BCS BIT(BCS0) 422 #define R_VECS BIT(VECS0) 423 #define R_ALL (R_RCS | R_VCS | R_BCS | R_VECS) 424 /* rings that support this cmd: BLT/RCS/VCS/VECS */ 425 u16 rings; 426 427 /* devices that support this cmd: SNB/IVB/HSW/... */ 428 u16 devices; 429 430 /* which DWords are address that need fix up. 431 * bit 0 means a 32-bit non address operand in command 432 * bit 1 means address operand, which could be 32-bit 433 * or 64-bit depending on different architectures.( 434 * defined by "gmadr_bytes_in_cmd" in intel_gvt. 435 * No matter the address length, each address only takes 436 * one bit in the bitmap. 437 */ 438 u16 addr_bitmap; 439 440 /* flag == F_LEN_CONST : command length 441 * flag == F_LEN_VAR : length bias bits 442 * Note: length is in DWord 443 */ 444 u32 len; 445 446 parser_cmd_handler handler; 447 448 /* valid length in DWord */ 449 u32 valid_len; 450 }; 451 452 struct cmd_entry { 453 struct hlist_node hlist; 454 const struct cmd_info *info; 455 }; 456 457 enum { 458 RING_BUFFER_INSTRUCTION, 459 BATCH_BUFFER_INSTRUCTION, 460 BATCH_BUFFER_2ND_LEVEL, 461 }; 462 463 enum { 464 GTT_BUFFER, 465 PPGTT_BUFFER 466 }; 467 468 struct parser_exec_state { 469 struct intel_vgpu *vgpu; 470 int ring_id; 471 472 int buf_type; 473 474 /* batch buffer address type */ 475 int buf_addr_type; 476 477 /* graphics memory address of ring buffer start */ 478 unsigned long ring_start; 479 unsigned long ring_size; 480 unsigned long ring_head; 481 unsigned long ring_tail; 482 483 /* instruction graphics memory address */ 484 unsigned long ip_gma; 485 486 /* mapped va of the instr_gma */ 487 void *ip_va; 488 void *rb_va; 489 490 void *ret_bb_va; 491 /* next instruction when return from batch buffer to ring buffer */ 492 unsigned long ret_ip_gma_ring; 493 494 /* next instruction when return from 2nd batch buffer to batch buffer */ 495 unsigned long ret_ip_gma_bb; 496 497 /* batch buffer address type (GTT or PPGTT) 498 * used when ret from 2nd level batch buffer 499 */ 500 int saved_buf_addr_type; 501 bool is_ctx_wa; 502 503 const struct cmd_info *info; 504 505 struct intel_vgpu_workload *workload; 506 }; 507 508 #define gmadr_dw_number(s) \ 509 (s->vgpu->gvt->device_info.gmadr_bytes_in_cmd >> 2) 510 511 static unsigned long bypass_scan_mask = 0; 512 513 /* ring ALL, type = 0 */ 514 static const struct sub_op_bits sub_op_mi[] = { 515 {31, 29}, 516 {28, 23}, 517 }; 518 519 static const struct decode_info decode_info_mi = { 520 "MI", 521 OP_LEN_MI, 522 ARRAY_SIZE(sub_op_mi), 523 sub_op_mi, 524 }; 525 526 /* ring RCS, command type 2 */ 527 static const struct sub_op_bits sub_op_2d[] = { 528 {31, 29}, 529 {28, 22}, 530 }; 531 532 static const struct decode_info decode_info_2d = { 533 "2D", 534 OP_LEN_2D, 535 ARRAY_SIZE(sub_op_2d), 536 sub_op_2d, 537 }; 538 539 /* ring RCS, command type 3 */ 540 static const struct sub_op_bits sub_op_3d_media[] = { 541 {31, 29}, 542 {28, 27}, 543 {26, 24}, 544 {23, 16}, 545 }; 546 547 static const struct decode_info decode_info_3d_media = { 548 "3D_Media", 549 OP_LEN_3D_MEDIA, 550 ARRAY_SIZE(sub_op_3d_media), 551 sub_op_3d_media, 552 }; 553 554 /* ring VCS, command type 3 */ 555 static const struct sub_op_bits sub_op_mfx_vc[] = { 556 {31, 29}, 557 {28, 27}, 558 {26, 24}, 559 {23, 21}, 560 {20, 16}, 561 }; 562 563 static const struct decode_info decode_info_mfx_vc = { 564 "MFX_VC", 565 OP_LEN_MFX_VC, 566 ARRAY_SIZE(sub_op_mfx_vc), 567 sub_op_mfx_vc, 568 }; 569 570 /* ring VECS, command type 3 */ 571 static const struct sub_op_bits sub_op_vebox[] = { 572 {31, 29}, 573 {28, 27}, 574 {26, 24}, 575 {23, 21}, 576 {20, 16}, 577 }; 578 579 static const struct decode_info decode_info_vebox = { 580 "VEBOX", 581 OP_LEN_VEBOX, 582 ARRAY_SIZE(sub_op_vebox), 583 sub_op_vebox, 584 }; 585 586 static const struct decode_info *ring_decode_info[I915_NUM_ENGINES][8] = { 587 [RCS0] = { 588 &decode_info_mi, 589 NULL, 590 NULL, 591 &decode_info_3d_media, 592 NULL, 593 NULL, 594 NULL, 595 NULL, 596 }, 597 598 [VCS0] = { 599 &decode_info_mi, 600 NULL, 601 NULL, 602 &decode_info_mfx_vc, 603 NULL, 604 NULL, 605 NULL, 606 NULL, 607 }, 608 609 [BCS0] = { 610 &decode_info_mi, 611 NULL, 612 &decode_info_2d, 613 NULL, 614 NULL, 615 NULL, 616 NULL, 617 NULL, 618 }, 619 620 [VECS0] = { 621 &decode_info_mi, 622 NULL, 623 NULL, 624 &decode_info_vebox, 625 NULL, 626 NULL, 627 NULL, 628 NULL, 629 }, 630 631 [VCS1] = { 632 &decode_info_mi, 633 NULL, 634 NULL, 635 &decode_info_mfx_vc, 636 NULL, 637 NULL, 638 NULL, 639 NULL, 640 }, 641 }; 642 643 static inline u32 get_opcode(u32 cmd, int ring_id) 644 { 645 const struct decode_info *d_info; 646 647 d_info = ring_decode_info[ring_id][CMD_TYPE(cmd)]; 648 if (d_info == NULL) 649 return INVALID_OP; 650 651 return cmd >> (32 - d_info->op_len); 652 } 653 654 static inline const struct cmd_info *find_cmd_entry(struct intel_gvt *gvt, 655 unsigned int opcode, int ring_id) 656 { 657 struct cmd_entry *e; 658 659 hash_for_each_possible(gvt->cmd_table, e, hlist, opcode) { 660 if (opcode == e->info->opcode && e->info->rings & BIT(ring_id)) 661 return e->info; 662 } 663 return NULL; 664 } 665 666 static inline const struct cmd_info *get_cmd_info(struct intel_gvt *gvt, 667 u32 cmd, int ring_id) 668 { 669 u32 opcode; 670 671 opcode = get_opcode(cmd, ring_id); 672 if (opcode == INVALID_OP) 673 return NULL; 674 675 return find_cmd_entry(gvt, opcode, ring_id); 676 } 677 678 static inline u32 sub_op_val(u32 cmd, u32 hi, u32 low) 679 { 680 return (cmd >> low) & ((1U << (hi - low + 1)) - 1); 681 } 682 683 static inline void print_opcode(u32 cmd, int ring_id) 684 { 685 const struct decode_info *d_info; 686 int i; 687 688 d_info = ring_decode_info[ring_id][CMD_TYPE(cmd)]; 689 if (d_info == NULL) 690 return; 691 692 gvt_dbg_cmd("opcode=0x%x %s sub_ops:", 693 cmd >> (32 - d_info->op_len), d_info->name); 694 695 for (i = 0; i < d_info->nr_sub_op; i++) 696 pr_err("0x%x ", sub_op_val(cmd, d_info->sub_op[i].hi, 697 d_info->sub_op[i].low)); 698 699 pr_err("\n"); 700 } 701 702 static inline u32 *cmd_ptr(struct parser_exec_state *s, int index) 703 { 704 return s->ip_va + (index << 2); 705 } 706 707 static inline u32 cmd_val(struct parser_exec_state *s, int index) 708 { 709 return *cmd_ptr(s, index); 710 } 711 712 static void parser_exec_state_dump(struct parser_exec_state *s) 713 { 714 int cnt = 0; 715 int i; 716 717 gvt_dbg_cmd(" vgpu%d RING%d: ring_start(%08lx) ring_end(%08lx)" 718 " ring_head(%08lx) ring_tail(%08lx)\n", s->vgpu->id, 719 s->ring_id, s->ring_start, s->ring_start + s->ring_size, 720 s->ring_head, s->ring_tail); 721 722 gvt_dbg_cmd(" %s %s ip_gma(%08lx) ", 723 s->buf_type == RING_BUFFER_INSTRUCTION ? 724 "RING_BUFFER" : "BATCH_BUFFER", 725 s->buf_addr_type == GTT_BUFFER ? 726 "GTT" : "PPGTT", s->ip_gma); 727 728 if (s->ip_va == NULL) { 729 gvt_dbg_cmd(" ip_va(NULL)"); 730 return; 731 } 732 733 gvt_dbg_cmd(" ip_va=%p: %08x %08x %08x %08x\n", 734 s->ip_va, cmd_val(s, 0), cmd_val(s, 1), 735 cmd_val(s, 2), cmd_val(s, 3)); 736 737 print_opcode(cmd_val(s, 0), s->ring_id); 738 739 s->ip_va = (u32 *)((((u64)s->ip_va) >> 12) << 12); 740 741 while (cnt < 1024) { 742 gvt_dbg_cmd("ip_va=%p: ", s->ip_va); 743 for (i = 0; i < 8; i++) 744 gvt_dbg_cmd("%08x ", cmd_val(s, i)); 745 gvt_dbg_cmd("\n"); 746 747 s->ip_va += 8 * sizeof(u32); 748 cnt += 8; 749 } 750 } 751 752 static inline void update_ip_va(struct parser_exec_state *s) 753 { 754 unsigned long len = 0; 755 756 if (WARN_ON(s->ring_head == s->ring_tail)) 757 return; 758 759 if (s->buf_type == RING_BUFFER_INSTRUCTION) { 760 unsigned long ring_top = s->ring_start + s->ring_size; 761 762 if (s->ring_head > s->ring_tail) { 763 if (s->ip_gma >= s->ring_head && s->ip_gma < ring_top) 764 len = (s->ip_gma - s->ring_head); 765 else if (s->ip_gma >= s->ring_start && 766 s->ip_gma <= s->ring_tail) 767 len = (ring_top - s->ring_head) + 768 (s->ip_gma - s->ring_start); 769 } else 770 len = (s->ip_gma - s->ring_head); 771 772 s->ip_va = s->rb_va + len; 773 } else {/* shadow batch buffer */ 774 s->ip_va = s->ret_bb_va; 775 } 776 } 777 778 static inline int ip_gma_set(struct parser_exec_state *s, 779 unsigned long ip_gma) 780 { 781 WARN_ON(!IS_ALIGNED(ip_gma, 4)); 782 783 s->ip_gma = ip_gma; 784 update_ip_va(s); 785 return 0; 786 } 787 788 static inline int ip_gma_advance(struct parser_exec_state *s, 789 unsigned int dw_len) 790 { 791 s->ip_gma += (dw_len << 2); 792 793 if (s->buf_type == RING_BUFFER_INSTRUCTION) { 794 if (s->ip_gma >= s->ring_start + s->ring_size) 795 s->ip_gma -= s->ring_size; 796 update_ip_va(s); 797 } else { 798 s->ip_va += (dw_len << 2); 799 } 800 801 return 0; 802 } 803 804 static inline int get_cmd_length(const struct cmd_info *info, u32 cmd) 805 { 806 if ((info->flag & F_LEN_MASK) == F_LEN_CONST) 807 return info->len; 808 else 809 return (cmd & ((1U << info->len) - 1)) + 2; 810 return 0; 811 } 812 813 static inline int cmd_length(struct parser_exec_state *s) 814 { 815 return get_cmd_length(s->info, cmd_val(s, 0)); 816 } 817 818 /* do not remove this, some platform may need clflush here */ 819 #define patch_value(s, addr, val) do { \ 820 *addr = val; \ 821 } while (0) 822 823 static bool is_shadowed_mmio(unsigned int offset) 824 { 825 bool ret = false; 826 827 if ((offset == 0x2168) || /*BB current head register UDW */ 828 (offset == 0x2140) || /*BB current header register */ 829 (offset == 0x211c) || /*second BB header register UDW */ 830 (offset == 0x2114)) { /*second BB header register UDW */ 831 ret = true; 832 } 833 return ret; 834 } 835 836 static inline bool is_force_nonpriv_mmio(unsigned int offset) 837 { 838 return (offset >= 0x24d0 && offset < 0x2500); 839 } 840 841 static int force_nonpriv_reg_handler(struct parser_exec_state *s, 842 unsigned int offset, unsigned int index, char *cmd) 843 { 844 struct intel_gvt *gvt = s->vgpu->gvt; 845 unsigned int data; 846 u32 ring_base; 847 u32 nopid; 848 struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv; 849 850 if (!strcmp(cmd, "lri")) 851 data = cmd_val(s, index + 1); 852 else { 853 gvt_err("Unexpected forcenonpriv 0x%x write from cmd %s\n", 854 offset, cmd); 855 return -EINVAL; 856 } 857 858 ring_base = dev_priv->engine[s->ring_id]->mmio_base; 859 nopid = i915_mmio_reg_offset(RING_NOPID(ring_base)); 860 861 if (!intel_gvt_in_force_nonpriv_whitelist(gvt, data) && 862 data != nopid) { 863 gvt_err("Unexpected forcenonpriv 0x%x LRI write, value=0x%x\n", 864 offset, data); 865 patch_value(s, cmd_ptr(s, index), nopid); 866 return 0; 867 } 868 return 0; 869 } 870 871 static inline bool is_mocs_mmio(unsigned int offset) 872 { 873 return ((offset >= 0xc800) && (offset <= 0xcff8)) || 874 ((offset >= 0xb020) && (offset <= 0xb0a0)); 875 } 876 877 static int mocs_cmd_reg_handler(struct parser_exec_state *s, 878 unsigned int offset, unsigned int index) 879 { 880 if (!is_mocs_mmio(offset)) 881 return -EINVAL; 882 vgpu_vreg(s->vgpu, offset) = cmd_val(s, index + 1); 883 return 0; 884 } 885 886 static int cmd_reg_handler(struct parser_exec_state *s, 887 unsigned int offset, unsigned int index, char *cmd) 888 { 889 struct intel_vgpu *vgpu = s->vgpu; 890 struct intel_gvt *gvt = vgpu->gvt; 891 u32 ctx_sr_ctl; 892 893 if (offset + 4 > gvt->device_info.mmio_size) { 894 gvt_vgpu_err("%s access to (%x) outside of MMIO range\n", 895 cmd, offset); 896 return -EFAULT; 897 } 898 899 if (!intel_gvt_mmio_is_cmd_access(gvt, offset)) { 900 gvt_vgpu_err("%s access to non-render register (%x)\n", 901 cmd, offset); 902 return -EBADRQC; 903 } 904 905 if (is_shadowed_mmio(offset)) { 906 gvt_vgpu_err("found access of shadowed MMIO %x\n", offset); 907 return 0; 908 } 909 910 if (is_mocs_mmio(offset) && 911 mocs_cmd_reg_handler(s, offset, index)) 912 return -EINVAL; 913 914 if (is_force_nonpriv_mmio(offset) && 915 force_nonpriv_reg_handler(s, offset, index, cmd)) 916 return -EPERM; 917 918 if (offset == i915_mmio_reg_offset(DERRMR) || 919 offset == i915_mmio_reg_offset(FORCEWAKE_MT)) { 920 /* Writing to HW VGT_PVINFO_PAGE offset will be discarded */ 921 patch_value(s, cmd_ptr(s, index), VGT_PVINFO_PAGE); 922 } 923 924 /* TODO 925 * In order to let workload with inhibit context to generate 926 * correct image data into memory, vregs values will be loaded to 927 * hw via LRIs in the workload with inhibit context. But as 928 * indirect context is loaded prior to LRIs in workload, we don't 929 * want reg values specified in indirect context overwritten by 930 * LRIs in workloads. So, when scanning an indirect context, we 931 * update reg values in it into vregs, so LRIs in workload with 932 * inhibit context will restore with correct values 933 */ 934 if (IS_GEN(gvt->dev_priv, 9) && 935 intel_gvt_mmio_is_in_ctx(gvt, offset) && 936 !strncmp(cmd, "lri", 3)) { 937 intel_gvt_hypervisor_read_gpa(s->vgpu, 938 s->workload->ring_context_gpa + 12, &ctx_sr_ctl, 4); 939 /* check inhibit context */ 940 if (ctx_sr_ctl & 1) { 941 u32 data = cmd_val(s, index + 1); 942 943 if (intel_gvt_mmio_has_mode_mask(s->vgpu->gvt, offset)) 944 intel_vgpu_mask_mmio_write(vgpu, 945 offset, &data, 4); 946 else 947 vgpu_vreg(vgpu, offset) = data; 948 } 949 } 950 951 /* TODO: Update the global mask if this MMIO is a masked-MMIO */ 952 intel_gvt_mmio_set_cmd_accessed(gvt, offset); 953 return 0; 954 } 955 956 #define cmd_reg(s, i) \ 957 (cmd_val(s, i) & GENMASK(22, 2)) 958 959 #define cmd_reg_inhibit(s, i) \ 960 (cmd_val(s, i) & GENMASK(22, 18)) 961 962 #define cmd_gma(s, i) \ 963 (cmd_val(s, i) & GENMASK(31, 2)) 964 965 #define cmd_gma_hi(s, i) \ 966 (cmd_val(s, i) & GENMASK(15, 0)) 967 968 static int cmd_handler_lri(struct parser_exec_state *s) 969 { 970 int i, ret = 0; 971 int cmd_len = cmd_length(s); 972 struct intel_gvt *gvt = s->vgpu->gvt; 973 u32 valid_len = CMD_LEN(1); 974 975 /* 976 * Official intel docs are somewhat sloppy , check the definition of 977 * MI_LOAD_REGISTER_IMM. 978 */ 979 #define MAX_VALID_LEN 127 980 if ((cmd_len < valid_len) || (cmd_len > MAX_VALID_LEN)) { 981 gvt_err("len is not valid: len=%u valid_len=%u\n", 982 cmd_len, valid_len); 983 return -EFAULT; 984 } 985 986 for (i = 1; i < cmd_len; i += 2) { 987 if (IS_BROADWELL(gvt->dev_priv) && s->ring_id != RCS0) { 988 if (s->ring_id == BCS0 && 989 cmd_reg(s, i) == i915_mmio_reg_offset(DERRMR)) 990 ret |= 0; 991 else 992 ret |= cmd_reg_inhibit(s, i) ? -EBADRQC : 0; 993 } 994 if (ret) 995 break; 996 ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "lri"); 997 if (ret) 998 break; 999 } 1000 return ret; 1001 } 1002 1003 static int cmd_handler_lrr(struct parser_exec_state *s) 1004 { 1005 int i, ret = 0; 1006 int cmd_len = cmd_length(s); 1007 1008 for (i = 1; i < cmd_len; i += 2) { 1009 if (IS_BROADWELL(s->vgpu->gvt->dev_priv)) 1010 ret |= ((cmd_reg_inhibit(s, i) || 1011 (cmd_reg_inhibit(s, i + 1)))) ? 1012 -EBADRQC : 0; 1013 if (ret) 1014 break; 1015 ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "lrr-src"); 1016 if (ret) 1017 break; 1018 ret |= cmd_reg_handler(s, cmd_reg(s, i + 1), i, "lrr-dst"); 1019 if (ret) 1020 break; 1021 } 1022 return ret; 1023 } 1024 1025 static inline int cmd_address_audit(struct parser_exec_state *s, 1026 unsigned long guest_gma, int op_size, bool index_mode); 1027 1028 static int cmd_handler_lrm(struct parser_exec_state *s) 1029 { 1030 struct intel_gvt *gvt = s->vgpu->gvt; 1031 int gmadr_bytes = gvt->device_info.gmadr_bytes_in_cmd; 1032 unsigned long gma; 1033 int i, ret = 0; 1034 int cmd_len = cmd_length(s); 1035 1036 for (i = 1; i < cmd_len;) { 1037 if (IS_BROADWELL(gvt->dev_priv)) 1038 ret |= (cmd_reg_inhibit(s, i)) ? -EBADRQC : 0; 1039 if (ret) 1040 break; 1041 ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "lrm"); 1042 if (ret) 1043 break; 1044 if (cmd_val(s, 0) & (1 << 22)) { 1045 gma = cmd_gma(s, i + 1); 1046 if (gmadr_bytes == 8) 1047 gma |= (cmd_gma_hi(s, i + 2)) << 32; 1048 ret |= cmd_address_audit(s, gma, sizeof(u32), false); 1049 if (ret) 1050 break; 1051 } 1052 i += gmadr_dw_number(s) + 1; 1053 } 1054 return ret; 1055 } 1056 1057 static int cmd_handler_srm(struct parser_exec_state *s) 1058 { 1059 int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd; 1060 unsigned long gma; 1061 int i, ret = 0; 1062 int cmd_len = cmd_length(s); 1063 1064 for (i = 1; i < cmd_len;) { 1065 ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "srm"); 1066 if (ret) 1067 break; 1068 if (cmd_val(s, 0) & (1 << 22)) { 1069 gma = cmd_gma(s, i + 1); 1070 if (gmadr_bytes == 8) 1071 gma |= (cmd_gma_hi(s, i + 2)) << 32; 1072 ret |= cmd_address_audit(s, gma, sizeof(u32), false); 1073 if (ret) 1074 break; 1075 } 1076 i += gmadr_dw_number(s) + 1; 1077 } 1078 return ret; 1079 } 1080 1081 struct cmd_interrupt_event { 1082 int pipe_control_notify; 1083 int mi_flush_dw; 1084 int mi_user_interrupt; 1085 }; 1086 1087 static struct cmd_interrupt_event cmd_interrupt_events[] = { 1088 [RCS0] = { 1089 .pipe_control_notify = RCS_PIPE_CONTROL, 1090 .mi_flush_dw = INTEL_GVT_EVENT_RESERVED, 1091 .mi_user_interrupt = RCS_MI_USER_INTERRUPT, 1092 }, 1093 [BCS0] = { 1094 .pipe_control_notify = INTEL_GVT_EVENT_RESERVED, 1095 .mi_flush_dw = BCS_MI_FLUSH_DW, 1096 .mi_user_interrupt = BCS_MI_USER_INTERRUPT, 1097 }, 1098 [VCS0] = { 1099 .pipe_control_notify = INTEL_GVT_EVENT_RESERVED, 1100 .mi_flush_dw = VCS_MI_FLUSH_DW, 1101 .mi_user_interrupt = VCS_MI_USER_INTERRUPT, 1102 }, 1103 [VCS1] = { 1104 .pipe_control_notify = INTEL_GVT_EVENT_RESERVED, 1105 .mi_flush_dw = VCS2_MI_FLUSH_DW, 1106 .mi_user_interrupt = VCS2_MI_USER_INTERRUPT, 1107 }, 1108 [VECS0] = { 1109 .pipe_control_notify = INTEL_GVT_EVENT_RESERVED, 1110 .mi_flush_dw = VECS_MI_FLUSH_DW, 1111 .mi_user_interrupt = VECS_MI_USER_INTERRUPT, 1112 }, 1113 }; 1114 1115 static int cmd_handler_pipe_control(struct parser_exec_state *s) 1116 { 1117 int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd; 1118 unsigned long gma; 1119 bool index_mode = false; 1120 unsigned int post_sync; 1121 int ret = 0; 1122 u32 hws_pga, val; 1123 1124 post_sync = (cmd_val(s, 1) & PIPE_CONTROL_POST_SYNC_OP_MASK) >> 14; 1125 1126 /* LRI post sync */ 1127 if (cmd_val(s, 1) & PIPE_CONTROL_MMIO_WRITE) 1128 ret = cmd_reg_handler(s, cmd_reg(s, 2), 1, "pipe_ctrl"); 1129 /* post sync */ 1130 else if (post_sync) { 1131 if (post_sync == 2) 1132 ret = cmd_reg_handler(s, 0x2350, 1, "pipe_ctrl"); 1133 else if (post_sync == 3) 1134 ret = cmd_reg_handler(s, 0x2358, 1, "pipe_ctrl"); 1135 else if (post_sync == 1) { 1136 /* check ggtt*/ 1137 if ((cmd_val(s, 1) & PIPE_CONTROL_GLOBAL_GTT_IVB)) { 1138 gma = cmd_val(s, 2) & GENMASK(31, 3); 1139 if (gmadr_bytes == 8) 1140 gma |= (cmd_gma_hi(s, 3)) << 32; 1141 /* Store Data Index */ 1142 if (cmd_val(s, 1) & (1 << 21)) 1143 index_mode = true; 1144 ret |= cmd_address_audit(s, gma, sizeof(u64), 1145 index_mode); 1146 if (ret) 1147 return ret; 1148 if (index_mode) { 1149 hws_pga = s->vgpu->hws_pga[s->ring_id]; 1150 gma = hws_pga + gma; 1151 patch_value(s, cmd_ptr(s, 2), gma); 1152 val = cmd_val(s, 1) & (~(1 << 21)); 1153 patch_value(s, cmd_ptr(s, 1), val); 1154 } 1155 } 1156 } 1157 } 1158 1159 if (ret) 1160 return ret; 1161 1162 if (cmd_val(s, 1) & PIPE_CONTROL_NOTIFY) 1163 set_bit(cmd_interrupt_events[s->ring_id].pipe_control_notify, 1164 s->workload->pending_events); 1165 return 0; 1166 } 1167 1168 static int cmd_handler_mi_user_interrupt(struct parser_exec_state *s) 1169 { 1170 set_bit(cmd_interrupt_events[s->ring_id].mi_user_interrupt, 1171 s->workload->pending_events); 1172 patch_value(s, cmd_ptr(s, 0), MI_NOOP); 1173 return 0; 1174 } 1175 1176 static int cmd_advance_default(struct parser_exec_state *s) 1177 { 1178 return ip_gma_advance(s, cmd_length(s)); 1179 } 1180 1181 static int cmd_handler_mi_batch_buffer_end(struct parser_exec_state *s) 1182 { 1183 int ret; 1184 1185 if (s->buf_type == BATCH_BUFFER_2ND_LEVEL) { 1186 s->buf_type = BATCH_BUFFER_INSTRUCTION; 1187 ret = ip_gma_set(s, s->ret_ip_gma_bb); 1188 s->buf_addr_type = s->saved_buf_addr_type; 1189 } else { 1190 s->buf_type = RING_BUFFER_INSTRUCTION; 1191 s->buf_addr_type = GTT_BUFFER; 1192 if (s->ret_ip_gma_ring >= s->ring_start + s->ring_size) 1193 s->ret_ip_gma_ring -= s->ring_size; 1194 ret = ip_gma_set(s, s->ret_ip_gma_ring); 1195 } 1196 return ret; 1197 } 1198 1199 struct mi_display_flip_command_info { 1200 int pipe; 1201 int plane; 1202 int event; 1203 i915_reg_t stride_reg; 1204 i915_reg_t ctrl_reg; 1205 i915_reg_t surf_reg; 1206 u64 stride_val; 1207 u64 tile_val; 1208 u64 surf_val; 1209 bool async_flip; 1210 }; 1211 1212 struct plane_code_mapping { 1213 int pipe; 1214 int plane; 1215 int event; 1216 }; 1217 1218 static int gen8_decode_mi_display_flip(struct parser_exec_state *s, 1219 struct mi_display_flip_command_info *info) 1220 { 1221 struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv; 1222 struct plane_code_mapping gen8_plane_code[] = { 1223 [0] = {PIPE_A, PLANE_A, PRIMARY_A_FLIP_DONE}, 1224 [1] = {PIPE_B, PLANE_A, PRIMARY_B_FLIP_DONE}, 1225 [2] = {PIPE_A, PLANE_B, SPRITE_A_FLIP_DONE}, 1226 [3] = {PIPE_B, PLANE_B, SPRITE_B_FLIP_DONE}, 1227 [4] = {PIPE_C, PLANE_A, PRIMARY_C_FLIP_DONE}, 1228 [5] = {PIPE_C, PLANE_B, SPRITE_C_FLIP_DONE}, 1229 }; 1230 u32 dword0, dword1, dword2; 1231 u32 v; 1232 1233 dword0 = cmd_val(s, 0); 1234 dword1 = cmd_val(s, 1); 1235 dword2 = cmd_val(s, 2); 1236 1237 v = (dword0 & GENMASK(21, 19)) >> 19; 1238 if (WARN_ON(v >= ARRAY_SIZE(gen8_plane_code))) 1239 return -EBADRQC; 1240 1241 info->pipe = gen8_plane_code[v].pipe; 1242 info->plane = gen8_plane_code[v].plane; 1243 info->event = gen8_plane_code[v].event; 1244 info->stride_val = (dword1 & GENMASK(15, 6)) >> 6; 1245 info->tile_val = (dword1 & 0x1); 1246 info->surf_val = (dword2 & GENMASK(31, 12)) >> 12; 1247 info->async_flip = ((dword2 & GENMASK(1, 0)) == 0x1); 1248 1249 if (info->plane == PLANE_A) { 1250 info->ctrl_reg = DSPCNTR(info->pipe); 1251 info->stride_reg = DSPSTRIDE(info->pipe); 1252 info->surf_reg = DSPSURF(info->pipe); 1253 } else if (info->plane == PLANE_B) { 1254 info->ctrl_reg = SPRCTL(info->pipe); 1255 info->stride_reg = SPRSTRIDE(info->pipe); 1256 info->surf_reg = SPRSURF(info->pipe); 1257 } else { 1258 WARN_ON(1); 1259 return -EBADRQC; 1260 } 1261 return 0; 1262 } 1263 1264 static int skl_decode_mi_display_flip(struct parser_exec_state *s, 1265 struct mi_display_flip_command_info *info) 1266 { 1267 struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv; 1268 struct intel_vgpu *vgpu = s->vgpu; 1269 u32 dword0 = cmd_val(s, 0); 1270 u32 dword1 = cmd_val(s, 1); 1271 u32 dword2 = cmd_val(s, 2); 1272 u32 plane = (dword0 & GENMASK(12, 8)) >> 8; 1273 1274 info->plane = PRIMARY_PLANE; 1275 1276 switch (plane) { 1277 case MI_DISPLAY_FLIP_SKL_PLANE_1_A: 1278 info->pipe = PIPE_A; 1279 info->event = PRIMARY_A_FLIP_DONE; 1280 break; 1281 case MI_DISPLAY_FLIP_SKL_PLANE_1_B: 1282 info->pipe = PIPE_B; 1283 info->event = PRIMARY_B_FLIP_DONE; 1284 break; 1285 case MI_DISPLAY_FLIP_SKL_PLANE_1_C: 1286 info->pipe = PIPE_C; 1287 info->event = PRIMARY_C_FLIP_DONE; 1288 break; 1289 1290 case MI_DISPLAY_FLIP_SKL_PLANE_2_A: 1291 info->pipe = PIPE_A; 1292 info->event = SPRITE_A_FLIP_DONE; 1293 info->plane = SPRITE_PLANE; 1294 break; 1295 case MI_DISPLAY_FLIP_SKL_PLANE_2_B: 1296 info->pipe = PIPE_B; 1297 info->event = SPRITE_B_FLIP_DONE; 1298 info->plane = SPRITE_PLANE; 1299 break; 1300 case MI_DISPLAY_FLIP_SKL_PLANE_2_C: 1301 info->pipe = PIPE_C; 1302 info->event = SPRITE_C_FLIP_DONE; 1303 info->plane = SPRITE_PLANE; 1304 break; 1305 1306 default: 1307 gvt_vgpu_err("unknown plane code %d\n", plane); 1308 return -EBADRQC; 1309 } 1310 1311 info->stride_val = (dword1 & GENMASK(15, 6)) >> 6; 1312 info->tile_val = (dword1 & GENMASK(2, 0)); 1313 info->surf_val = (dword2 & GENMASK(31, 12)) >> 12; 1314 info->async_flip = ((dword2 & GENMASK(1, 0)) == 0x1); 1315 1316 info->ctrl_reg = DSPCNTR(info->pipe); 1317 info->stride_reg = DSPSTRIDE(info->pipe); 1318 info->surf_reg = DSPSURF(info->pipe); 1319 1320 return 0; 1321 } 1322 1323 static int gen8_check_mi_display_flip(struct parser_exec_state *s, 1324 struct mi_display_flip_command_info *info) 1325 { 1326 struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv; 1327 u32 stride, tile; 1328 1329 if (!info->async_flip) 1330 return 0; 1331 1332 if (INTEL_GEN(dev_priv) >= 9) { 1333 stride = vgpu_vreg_t(s->vgpu, info->stride_reg) & GENMASK(9, 0); 1334 tile = (vgpu_vreg_t(s->vgpu, info->ctrl_reg) & 1335 GENMASK(12, 10)) >> 10; 1336 } else { 1337 stride = (vgpu_vreg_t(s->vgpu, info->stride_reg) & 1338 GENMASK(15, 6)) >> 6; 1339 tile = (vgpu_vreg_t(s->vgpu, info->ctrl_reg) & (1 << 10)) >> 10; 1340 } 1341 1342 if (stride != info->stride_val) 1343 gvt_dbg_cmd("cannot change stride during async flip\n"); 1344 1345 if (tile != info->tile_val) 1346 gvt_dbg_cmd("cannot change tile during async flip\n"); 1347 1348 return 0; 1349 } 1350 1351 static int gen8_update_plane_mmio_from_mi_display_flip( 1352 struct parser_exec_state *s, 1353 struct mi_display_flip_command_info *info) 1354 { 1355 struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv; 1356 struct intel_vgpu *vgpu = s->vgpu; 1357 1358 set_mask_bits(&vgpu_vreg_t(vgpu, info->surf_reg), GENMASK(31, 12), 1359 info->surf_val << 12); 1360 if (INTEL_GEN(dev_priv) >= 9) { 1361 set_mask_bits(&vgpu_vreg_t(vgpu, info->stride_reg), GENMASK(9, 0), 1362 info->stride_val); 1363 set_mask_bits(&vgpu_vreg_t(vgpu, info->ctrl_reg), GENMASK(12, 10), 1364 info->tile_val << 10); 1365 } else { 1366 set_mask_bits(&vgpu_vreg_t(vgpu, info->stride_reg), GENMASK(15, 6), 1367 info->stride_val << 6); 1368 set_mask_bits(&vgpu_vreg_t(vgpu, info->ctrl_reg), GENMASK(10, 10), 1369 info->tile_val << 10); 1370 } 1371 1372 if (info->plane == PLANE_PRIMARY) 1373 vgpu_vreg_t(vgpu, PIPE_FLIPCOUNT_G4X(info->pipe))++; 1374 1375 if (info->async_flip) 1376 intel_vgpu_trigger_virtual_event(vgpu, info->event); 1377 else 1378 set_bit(info->event, vgpu->irq.flip_done_event[info->pipe]); 1379 1380 return 0; 1381 } 1382 1383 static int decode_mi_display_flip(struct parser_exec_state *s, 1384 struct mi_display_flip_command_info *info) 1385 { 1386 struct drm_i915_private *dev_priv = s->vgpu->gvt->dev_priv; 1387 1388 if (IS_BROADWELL(dev_priv)) 1389 return gen8_decode_mi_display_flip(s, info); 1390 if (INTEL_GEN(dev_priv) >= 9) 1391 return skl_decode_mi_display_flip(s, info); 1392 1393 return -ENODEV; 1394 } 1395 1396 static int check_mi_display_flip(struct parser_exec_state *s, 1397 struct mi_display_flip_command_info *info) 1398 { 1399 return gen8_check_mi_display_flip(s, info); 1400 } 1401 1402 static int update_plane_mmio_from_mi_display_flip( 1403 struct parser_exec_state *s, 1404 struct mi_display_flip_command_info *info) 1405 { 1406 return gen8_update_plane_mmio_from_mi_display_flip(s, info); 1407 } 1408 1409 static int cmd_handler_mi_display_flip(struct parser_exec_state *s) 1410 { 1411 struct mi_display_flip_command_info info; 1412 struct intel_vgpu *vgpu = s->vgpu; 1413 int ret; 1414 int i; 1415 int len = cmd_length(s); 1416 u32 valid_len = CMD_LEN(1); 1417 1418 /* Flip Type == Stereo 3D Flip */ 1419 if (DWORD_FIELD(2, 1, 0) == 2) 1420 valid_len++; 1421 ret = gvt_check_valid_cmd_length(cmd_length(s), 1422 valid_len); 1423 if (ret) 1424 return ret; 1425 1426 ret = decode_mi_display_flip(s, &info); 1427 if (ret) { 1428 gvt_vgpu_err("fail to decode MI display flip command\n"); 1429 return ret; 1430 } 1431 1432 ret = check_mi_display_flip(s, &info); 1433 if (ret) { 1434 gvt_vgpu_err("invalid MI display flip command\n"); 1435 return ret; 1436 } 1437 1438 ret = update_plane_mmio_from_mi_display_flip(s, &info); 1439 if (ret) { 1440 gvt_vgpu_err("fail to update plane mmio\n"); 1441 return ret; 1442 } 1443 1444 for (i = 0; i < len; i++) 1445 patch_value(s, cmd_ptr(s, i), MI_NOOP); 1446 return 0; 1447 } 1448 1449 static bool is_wait_for_flip_pending(u32 cmd) 1450 { 1451 return cmd & (MI_WAIT_FOR_PLANE_A_FLIP_PENDING | 1452 MI_WAIT_FOR_PLANE_B_FLIP_PENDING | 1453 MI_WAIT_FOR_PLANE_C_FLIP_PENDING | 1454 MI_WAIT_FOR_SPRITE_A_FLIP_PENDING | 1455 MI_WAIT_FOR_SPRITE_B_FLIP_PENDING | 1456 MI_WAIT_FOR_SPRITE_C_FLIP_PENDING); 1457 } 1458 1459 static int cmd_handler_mi_wait_for_event(struct parser_exec_state *s) 1460 { 1461 u32 cmd = cmd_val(s, 0); 1462 1463 if (!is_wait_for_flip_pending(cmd)) 1464 return 0; 1465 1466 patch_value(s, cmd_ptr(s, 0), MI_NOOP); 1467 return 0; 1468 } 1469 1470 static unsigned long get_gma_bb_from_cmd(struct parser_exec_state *s, int index) 1471 { 1472 unsigned long addr; 1473 unsigned long gma_high, gma_low; 1474 struct intel_vgpu *vgpu = s->vgpu; 1475 int gmadr_bytes = vgpu->gvt->device_info.gmadr_bytes_in_cmd; 1476 1477 if (WARN_ON(gmadr_bytes != 4 && gmadr_bytes != 8)) { 1478 gvt_vgpu_err("invalid gma bytes %d\n", gmadr_bytes); 1479 return INTEL_GVT_INVALID_ADDR; 1480 } 1481 1482 gma_low = cmd_val(s, index) & BATCH_BUFFER_ADDR_MASK; 1483 if (gmadr_bytes == 4) { 1484 addr = gma_low; 1485 } else { 1486 gma_high = cmd_val(s, index + 1) & BATCH_BUFFER_ADDR_HIGH_MASK; 1487 addr = (((unsigned long)gma_high) << 32) | gma_low; 1488 } 1489 return addr; 1490 } 1491 1492 static inline int cmd_address_audit(struct parser_exec_state *s, 1493 unsigned long guest_gma, int op_size, bool index_mode) 1494 { 1495 struct intel_vgpu *vgpu = s->vgpu; 1496 u32 max_surface_size = vgpu->gvt->device_info.max_surface_size; 1497 int i; 1498 int ret; 1499 1500 if (op_size > max_surface_size) { 1501 gvt_vgpu_err("command address audit fail name %s\n", 1502 s->info->name); 1503 return -EFAULT; 1504 } 1505 1506 if (index_mode) { 1507 if (guest_gma >= I915_GTT_PAGE_SIZE) { 1508 ret = -EFAULT; 1509 goto err; 1510 } 1511 } else if (!intel_gvt_ggtt_validate_range(vgpu, guest_gma, op_size)) { 1512 ret = -EFAULT; 1513 goto err; 1514 } 1515 1516 return 0; 1517 1518 err: 1519 gvt_vgpu_err("cmd_parser: Malicious %s detected, addr=0x%lx, len=%d!\n", 1520 s->info->name, guest_gma, op_size); 1521 1522 pr_err("cmd dump: "); 1523 for (i = 0; i < cmd_length(s); i++) { 1524 if (!(i % 4)) 1525 pr_err("\n%08x ", cmd_val(s, i)); 1526 else 1527 pr_err("%08x ", cmd_val(s, i)); 1528 } 1529 pr_err("\nvgpu%d: aperture 0x%llx - 0x%llx, hidden 0x%llx - 0x%llx\n", 1530 vgpu->id, 1531 vgpu_aperture_gmadr_base(vgpu), 1532 vgpu_aperture_gmadr_end(vgpu), 1533 vgpu_hidden_gmadr_base(vgpu), 1534 vgpu_hidden_gmadr_end(vgpu)); 1535 return ret; 1536 } 1537 1538 static int cmd_handler_mi_store_data_imm(struct parser_exec_state *s) 1539 { 1540 int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd; 1541 int op_size = (cmd_length(s) - 3) * sizeof(u32); 1542 int core_id = (cmd_val(s, 2) & (1 << 0)) ? 1 : 0; 1543 unsigned long gma, gma_low, gma_high; 1544 u32 valid_len = CMD_LEN(2); 1545 int ret = 0; 1546 1547 /* check ppggt */ 1548 if (!(cmd_val(s, 0) & (1 << 22))) 1549 return 0; 1550 1551 /* check if QWORD */ 1552 if (DWORD_FIELD(0, 21, 21)) 1553 valid_len++; 1554 ret = gvt_check_valid_cmd_length(cmd_length(s), 1555 valid_len); 1556 if (ret) 1557 return ret; 1558 1559 gma = cmd_val(s, 2) & GENMASK(31, 2); 1560 1561 if (gmadr_bytes == 8) { 1562 gma_low = cmd_val(s, 1) & GENMASK(31, 2); 1563 gma_high = cmd_val(s, 2) & GENMASK(15, 0); 1564 gma = (gma_high << 32) | gma_low; 1565 core_id = (cmd_val(s, 1) & (1 << 0)) ? 1 : 0; 1566 } 1567 ret = cmd_address_audit(s, gma + op_size * core_id, op_size, false); 1568 return ret; 1569 } 1570 1571 static inline int unexpected_cmd(struct parser_exec_state *s) 1572 { 1573 struct intel_vgpu *vgpu = s->vgpu; 1574 1575 gvt_vgpu_err("Unexpected %s in command buffer!\n", s->info->name); 1576 1577 return -EBADRQC; 1578 } 1579 1580 static int cmd_handler_mi_semaphore_wait(struct parser_exec_state *s) 1581 { 1582 return unexpected_cmd(s); 1583 } 1584 1585 static int cmd_handler_mi_report_perf_count(struct parser_exec_state *s) 1586 { 1587 return unexpected_cmd(s); 1588 } 1589 1590 static int cmd_handler_mi_op_2e(struct parser_exec_state *s) 1591 { 1592 return unexpected_cmd(s); 1593 } 1594 1595 static int cmd_handler_mi_op_2f(struct parser_exec_state *s) 1596 { 1597 int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd; 1598 int op_size = (1 << ((cmd_val(s, 0) & GENMASK(20, 19)) >> 19)) * 1599 sizeof(u32); 1600 unsigned long gma, gma_high; 1601 u32 valid_len = CMD_LEN(1); 1602 int ret = 0; 1603 1604 if (!(cmd_val(s, 0) & (1 << 22))) 1605 return ret; 1606 1607 /* check inline data */ 1608 if (cmd_val(s, 0) & BIT(18)) 1609 valid_len = CMD_LEN(9); 1610 ret = gvt_check_valid_cmd_length(cmd_length(s), 1611 valid_len); 1612 if (ret) 1613 return ret; 1614 1615 gma = cmd_val(s, 1) & GENMASK(31, 2); 1616 if (gmadr_bytes == 8) { 1617 gma_high = cmd_val(s, 2) & GENMASK(15, 0); 1618 gma = (gma_high << 32) | gma; 1619 } 1620 ret = cmd_address_audit(s, gma, op_size, false); 1621 return ret; 1622 } 1623 1624 static int cmd_handler_mi_store_data_index(struct parser_exec_state *s) 1625 { 1626 return unexpected_cmd(s); 1627 } 1628 1629 static int cmd_handler_mi_clflush(struct parser_exec_state *s) 1630 { 1631 return unexpected_cmd(s); 1632 } 1633 1634 static int cmd_handler_mi_conditional_batch_buffer_end( 1635 struct parser_exec_state *s) 1636 { 1637 return unexpected_cmd(s); 1638 } 1639 1640 static int cmd_handler_mi_update_gtt(struct parser_exec_state *s) 1641 { 1642 return unexpected_cmd(s); 1643 } 1644 1645 static int cmd_handler_mi_flush_dw(struct parser_exec_state *s) 1646 { 1647 int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd; 1648 unsigned long gma; 1649 bool index_mode = false; 1650 int ret = 0; 1651 u32 hws_pga, val; 1652 u32 valid_len = CMD_LEN(2); 1653 1654 ret = gvt_check_valid_cmd_length(cmd_length(s), 1655 valid_len); 1656 if (ret) { 1657 /* Check again for Qword */ 1658 ret = gvt_check_valid_cmd_length(cmd_length(s), 1659 ++valid_len); 1660 return ret; 1661 } 1662 1663 /* Check post-sync and ppgtt bit */ 1664 if (((cmd_val(s, 0) >> 14) & 0x3) && (cmd_val(s, 1) & (1 << 2))) { 1665 gma = cmd_val(s, 1) & GENMASK(31, 3); 1666 if (gmadr_bytes == 8) 1667 gma |= (cmd_val(s, 2) & GENMASK(15, 0)) << 32; 1668 /* Store Data Index */ 1669 if (cmd_val(s, 0) & (1 << 21)) 1670 index_mode = true; 1671 ret = cmd_address_audit(s, gma, sizeof(u64), index_mode); 1672 if (ret) 1673 return ret; 1674 if (index_mode) { 1675 hws_pga = s->vgpu->hws_pga[s->ring_id]; 1676 gma = hws_pga + gma; 1677 patch_value(s, cmd_ptr(s, 1), gma); 1678 val = cmd_val(s, 0) & (~(1 << 21)); 1679 patch_value(s, cmd_ptr(s, 0), val); 1680 } 1681 } 1682 /* Check notify bit */ 1683 if ((cmd_val(s, 0) & (1 << 8))) 1684 set_bit(cmd_interrupt_events[s->ring_id].mi_flush_dw, 1685 s->workload->pending_events); 1686 return ret; 1687 } 1688 1689 static void addr_type_update_snb(struct parser_exec_state *s) 1690 { 1691 if ((s->buf_type == RING_BUFFER_INSTRUCTION) && 1692 (BATCH_BUFFER_ADR_SPACE_BIT(cmd_val(s, 0)) == 1)) { 1693 s->buf_addr_type = PPGTT_BUFFER; 1694 } 1695 } 1696 1697 1698 static int copy_gma_to_hva(struct intel_vgpu *vgpu, struct intel_vgpu_mm *mm, 1699 unsigned long gma, unsigned long end_gma, void *va) 1700 { 1701 unsigned long copy_len, offset; 1702 unsigned long len = 0; 1703 unsigned long gpa; 1704 1705 while (gma != end_gma) { 1706 gpa = intel_vgpu_gma_to_gpa(mm, gma); 1707 if (gpa == INTEL_GVT_INVALID_ADDR) { 1708 gvt_vgpu_err("invalid gma address: %lx\n", gma); 1709 return -EFAULT; 1710 } 1711 1712 offset = gma & (I915_GTT_PAGE_SIZE - 1); 1713 1714 copy_len = (end_gma - gma) >= (I915_GTT_PAGE_SIZE - offset) ? 1715 I915_GTT_PAGE_SIZE - offset : end_gma - gma; 1716 1717 intel_gvt_hypervisor_read_gpa(vgpu, gpa, va + len, copy_len); 1718 1719 len += copy_len; 1720 gma += copy_len; 1721 } 1722 return len; 1723 } 1724 1725 1726 /* 1727 * Check whether a batch buffer needs to be scanned. Currently 1728 * the only criteria is based on privilege. 1729 */ 1730 static int batch_buffer_needs_scan(struct parser_exec_state *s) 1731 { 1732 /* Decide privilege based on address space */ 1733 if (cmd_val(s, 0) & (1 << 8) && 1734 !(s->vgpu->scan_nonprivbb & (1 << s->ring_id))) 1735 return 0; 1736 return 1; 1737 } 1738 1739 static int find_bb_size(struct parser_exec_state *s, 1740 unsigned long *bb_size, 1741 unsigned long *bb_end_cmd_offset) 1742 { 1743 unsigned long gma = 0; 1744 const struct cmd_info *info; 1745 u32 cmd_len = 0; 1746 bool bb_end = false; 1747 struct intel_vgpu *vgpu = s->vgpu; 1748 u32 cmd; 1749 struct intel_vgpu_mm *mm = (s->buf_addr_type == GTT_BUFFER) ? 1750 s->vgpu->gtt.ggtt_mm : s->workload->shadow_mm; 1751 1752 *bb_size = 0; 1753 *bb_end_cmd_offset = 0; 1754 1755 /* get the start gm address of the batch buffer */ 1756 gma = get_gma_bb_from_cmd(s, 1); 1757 if (gma == INTEL_GVT_INVALID_ADDR) 1758 return -EFAULT; 1759 1760 cmd = cmd_val(s, 0); 1761 info = get_cmd_info(s->vgpu->gvt, cmd, s->ring_id); 1762 if (info == NULL) { 1763 gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x, addr_type=%s, ring %d, workload=%p\n", 1764 cmd, get_opcode(cmd, s->ring_id), 1765 (s->buf_addr_type == PPGTT_BUFFER) ? 1766 "ppgtt" : "ggtt", s->ring_id, s->workload); 1767 return -EBADRQC; 1768 } 1769 do { 1770 if (copy_gma_to_hva(s->vgpu, mm, 1771 gma, gma + 4, &cmd) < 0) 1772 return -EFAULT; 1773 info = get_cmd_info(s->vgpu->gvt, cmd, s->ring_id); 1774 if (info == NULL) { 1775 gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x, addr_type=%s, ring %d, workload=%p\n", 1776 cmd, get_opcode(cmd, s->ring_id), 1777 (s->buf_addr_type == PPGTT_BUFFER) ? 1778 "ppgtt" : "ggtt", s->ring_id, s->workload); 1779 return -EBADRQC; 1780 } 1781 1782 if (info->opcode == OP_MI_BATCH_BUFFER_END) { 1783 bb_end = true; 1784 } else if (info->opcode == OP_MI_BATCH_BUFFER_START) { 1785 if (BATCH_BUFFER_2ND_LEVEL_BIT(cmd) == 0) 1786 /* chained batch buffer */ 1787 bb_end = true; 1788 } 1789 1790 if (bb_end) 1791 *bb_end_cmd_offset = *bb_size; 1792 1793 cmd_len = get_cmd_length(info, cmd) << 2; 1794 *bb_size += cmd_len; 1795 gma += cmd_len; 1796 } while (!bb_end); 1797 1798 return 0; 1799 } 1800 1801 static int audit_bb_end(struct parser_exec_state *s, void *va) 1802 { 1803 struct intel_vgpu *vgpu = s->vgpu; 1804 u32 cmd = *(u32 *)va; 1805 const struct cmd_info *info; 1806 1807 info = get_cmd_info(s->vgpu->gvt, cmd, s->ring_id); 1808 if (info == NULL) { 1809 gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x, addr_type=%s, ring %d, workload=%p\n", 1810 cmd, get_opcode(cmd, s->ring_id), 1811 (s->buf_addr_type == PPGTT_BUFFER) ? 1812 "ppgtt" : "ggtt", s->ring_id, s->workload); 1813 return -EBADRQC; 1814 } 1815 1816 if ((info->opcode == OP_MI_BATCH_BUFFER_END) || 1817 ((info->opcode == OP_MI_BATCH_BUFFER_START) && 1818 (BATCH_BUFFER_2ND_LEVEL_BIT(cmd) == 0))) 1819 return 0; 1820 1821 return -EBADRQC; 1822 } 1823 1824 static int perform_bb_shadow(struct parser_exec_state *s) 1825 { 1826 struct intel_vgpu *vgpu = s->vgpu; 1827 struct intel_vgpu_shadow_bb *bb; 1828 unsigned long gma = 0; 1829 unsigned long bb_size; 1830 unsigned long bb_end_cmd_offset; 1831 int ret = 0; 1832 struct intel_vgpu_mm *mm = (s->buf_addr_type == GTT_BUFFER) ? 1833 s->vgpu->gtt.ggtt_mm : s->workload->shadow_mm; 1834 unsigned long start_offset = 0; 1835 1836 /* get the start gm address of the batch buffer */ 1837 gma = get_gma_bb_from_cmd(s, 1); 1838 if (gma == INTEL_GVT_INVALID_ADDR) 1839 return -EFAULT; 1840 1841 ret = find_bb_size(s, &bb_size, &bb_end_cmd_offset); 1842 if (ret) 1843 return ret; 1844 1845 bb = kzalloc(sizeof(*bb), GFP_KERNEL); 1846 if (!bb) 1847 return -ENOMEM; 1848 1849 bb->ppgtt = (s->buf_addr_type == GTT_BUFFER) ? false : true; 1850 1851 /* the start_offset stores the batch buffer's start gma's 1852 * offset relative to page boundary. so for non-privileged batch 1853 * buffer, the shadowed gem object holds exactly the same page 1854 * layout as original gem object. This is for the convience of 1855 * replacing the whole non-privilged batch buffer page to this 1856 * shadowed one in PPGTT at the same gma address. (this replacing 1857 * action is not implemented yet now, but may be necessary in 1858 * future). 1859 * for prileged batch buffer, we just change start gma address to 1860 * that of shadowed page. 1861 */ 1862 if (bb->ppgtt) 1863 start_offset = gma & ~I915_GTT_PAGE_MASK; 1864 1865 bb->obj = i915_gem_object_create_shmem(s->vgpu->gvt->dev_priv, 1866 round_up(bb_size + start_offset, 1867 PAGE_SIZE)); 1868 if (IS_ERR(bb->obj)) { 1869 ret = PTR_ERR(bb->obj); 1870 goto err_free_bb; 1871 } 1872 1873 ret = i915_gem_object_prepare_write(bb->obj, &bb->clflush); 1874 if (ret) 1875 goto err_free_obj; 1876 1877 bb->va = i915_gem_object_pin_map(bb->obj, I915_MAP_WB); 1878 if (IS_ERR(bb->va)) { 1879 ret = PTR_ERR(bb->va); 1880 goto err_finish_shmem_access; 1881 } 1882 1883 if (bb->clflush & CLFLUSH_BEFORE) { 1884 drm_clflush_virt_range(bb->va, bb->obj->base.size); 1885 bb->clflush &= ~CLFLUSH_BEFORE; 1886 } 1887 1888 ret = copy_gma_to_hva(s->vgpu, mm, 1889 gma, gma + bb_size, 1890 bb->va + start_offset); 1891 if (ret < 0) { 1892 gvt_vgpu_err("fail to copy guest ring buffer\n"); 1893 ret = -EFAULT; 1894 goto err_unmap; 1895 } 1896 1897 ret = audit_bb_end(s, bb->va + start_offset + bb_end_cmd_offset); 1898 if (ret) 1899 goto err_unmap; 1900 1901 INIT_LIST_HEAD(&bb->list); 1902 list_add(&bb->list, &s->workload->shadow_bb); 1903 1904 bb->accessing = true; 1905 bb->bb_start_cmd_va = s->ip_va; 1906 1907 if ((s->buf_type == BATCH_BUFFER_INSTRUCTION) && (!s->is_ctx_wa)) 1908 bb->bb_offset = s->ip_va - s->rb_va; 1909 else 1910 bb->bb_offset = 0; 1911 1912 /* 1913 * ip_va saves the virtual address of the shadow batch buffer, while 1914 * ip_gma saves the graphics address of the original batch buffer. 1915 * As the shadow batch buffer is just a copy from the originial one, 1916 * it should be right to use shadow batch buffer'va and original batch 1917 * buffer's gma in pair. After all, we don't want to pin the shadow 1918 * buffer here (too early). 1919 */ 1920 s->ip_va = bb->va + start_offset; 1921 s->ip_gma = gma; 1922 return 0; 1923 err_unmap: 1924 i915_gem_object_unpin_map(bb->obj); 1925 err_finish_shmem_access: 1926 i915_gem_object_finish_access(bb->obj); 1927 err_free_obj: 1928 i915_gem_object_put(bb->obj); 1929 err_free_bb: 1930 kfree(bb); 1931 return ret; 1932 } 1933 1934 static int cmd_handler_mi_batch_buffer_start(struct parser_exec_state *s) 1935 { 1936 bool second_level; 1937 int ret = 0; 1938 struct intel_vgpu *vgpu = s->vgpu; 1939 1940 if (s->buf_type == BATCH_BUFFER_2ND_LEVEL) { 1941 gvt_vgpu_err("Found MI_BATCH_BUFFER_START in 2nd level BB\n"); 1942 return -EFAULT; 1943 } 1944 1945 second_level = BATCH_BUFFER_2ND_LEVEL_BIT(cmd_val(s, 0)) == 1; 1946 if (second_level && (s->buf_type != BATCH_BUFFER_INSTRUCTION)) { 1947 gvt_vgpu_err("Jumping to 2nd level BB from RB is not allowed\n"); 1948 return -EFAULT; 1949 } 1950 1951 s->saved_buf_addr_type = s->buf_addr_type; 1952 addr_type_update_snb(s); 1953 if (s->buf_type == RING_BUFFER_INSTRUCTION) { 1954 s->ret_ip_gma_ring = s->ip_gma + cmd_length(s) * sizeof(u32); 1955 s->buf_type = BATCH_BUFFER_INSTRUCTION; 1956 } else if (second_level) { 1957 s->buf_type = BATCH_BUFFER_2ND_LEVEL; 1958 s->ret_ip_gma_bb = s->ip_gma + cmd_length(s) * sizeof(u32); 1959 s->ret_bb_va = s->ip_va + cmd_length(s) * sizeof(u32); 1960 } 1961 1962 if (batch_buffer_needs_scan(s)) { 1963 ret = perform_bb_shadow(s); 1964 if (ret < 0) 1965 gvt_vgpu_err("invalid shadow batch buffer\n"); 1966 } else { 1967 /* emulate a batch buffer end to do return right */ 1968 ret = cmd_handler_mi_batch_buffer_end(s); 1969 if (ret < 0) 1970 return ret; 1971 } 1972 return ret; 1973 } 1974 1975 static int mi_noop_index; 1976 1977 static const struct cmd_info cmd_info[] = { 1978 {"MI_NOOP", OP_MI_NOOP, F_LEN_CONST, R_ALL, D_ALL, 0, 1, NULL}, 1979 1980 {"MI_SET_PREDICATE", OP_MI_SET_PREDICATE, F_LEN_CONST, R_ALL, D_ALL, 1981 0, 1, NULL}, 1982 1983 {"MI_USER_INTERRUPT", OP_MI_USER_INTERRUPT, F_LEN_CONST, R_ALL, D_ALL, 1984 0, 1, cmd_handler_mi_user_interrupt}, 1985 1986 {"MI_WAIT_FOR_EVENT", OP_MI_WAIT_FOR_EVENT, F_LEN_CONST, R_RCS | R_BCS, 1987 D_ALL, 0, 1, cmd_handler_mi_wait_for_event}, 1988 1989 {"MI_FLUSH", OP_MI_FLUSH, F_LEN_CONST, R_ALL, D_ALL, 0, 1, NULL}, 1990 1991 {"MI_ARB_CHECK", OP_MI_ARB_CHECK, F_LEN_CONST, R_ALL, D_ALL, 0, 1, 1992 NULL}, 1993 1994 {"MI_RS_CONTROL", OP_MI_RS_CONTROL, F_LEN_CONST, R_RCS, D_ALL, 0, 1, 1995 NULL}, 1996 1997 {"MI_REPORT_HEAD", OP_MI_REPORT_HEAD, F_LEN_CONST, R_ALL, D_ALL, 0, 1, 1998 NULL}, 1999 2000 {"MI_ARB_ON_OFF", OP_MI_ARB_ON_OFF, F_LEN_CONST, R_ALL, D_ALL, 0, 1, 2001 NULL}, 2002 2003 {"MI_URB_ATOMIC_ALLOC", OP_MI_URB_ATOMIC_ALLOC, F_LEN_CONST, R_RCS, 2004 D_ALL, 0, 1, NULL}, 2005 2006 {"MI_BATCH_BUFFER_END", OP_MI_BATCH_BUFFER_END, 2007 F_IP_ADVANCE_CUSTOM | F_LEN_CONST, R_ALL, D_ALL, 0, 1, 2008 cmd_handler_mi_batch_buffer_end}, 2009 2010 {"MI_SUSPEND_FLUSH", OP_MI_SUSPEND_FLUSH, F_LEN_CONST, R_ALL, D_ALL, 2011 0, 1, NULL}, 2012 2013 {"MI_PREDICATE", OP_MI_PREDICATE, F_LEN_CONST, R_RCS, D_ALL, 0, 1, 2014 NULL}, 2015 2016 {"MI_TOPOLOGY_FILTER", OP_MI_TOPOLOGY_FILTER, F_LEN_CONST, R_ALL, 2017 D_ALL, 0, 1, NULL}, 2018 2019 {"MI_SET_APPID", OP_MI_SET_APPID, F_LEN_CONST, R_ALL, D_ALL, 0, 1, 2020 NULL}, 2021 2022 {"MI_RS_CONTEXT", OP_MI_RS_CONTEXT, F_LEN_CONST, R_RCS, D_ALL, 0, 1, 2023 NULL}, 2024 2025 {"MI_DISPLAY_FLIP", OP_MI_DISPLAY_FLIP, F_LEN_VAR, 2026 R_RCS | R_BCS, D_ALL, 0, 8, cmd_handler_mi_display_flip}, 2027 2028 {"MI_SEMAPHORE_MBOX", OP_MI_SEMAPHORE_MBOX, F_LEN_VAR | F_LEN_VAR_FIXED, 2029 R_ALL, D_ALL, 0, 8, NULL, CMD_LEN(1)}, 2030 2031 {"MI_MATH", OP_MI_MATH, F_LEN_VAR, R_ALL, D_ALL, 0, 8, NULL}, 2032 2033 {"MI_URB_CLEAR", OP_MI_URB_CLEAR, F_LEN_VAR | F_LEN_VAR_FIXED, R_RCS, 2034 D_ALL, 0, 8, NULL, CMD_LEN(0)}, 2035 2036 {"MI_SEMAPHORE_SIGNAL", OP_MI_SEMAPHORE_SIGNAL, 2037 F_LEN_VAR | F_LEN_VAR_FIXED, R_ALL, D_BDW_PLUS, 0, 8, 2038 NULL, CMD_LEN(0)}, 2039 2040 {"MI_SEMAPHORE_WAIT", OP_MI_SEMAPHORE_WAIT, 2041 F_LEN_VAR | F_LEN_VAR_FIXED, R_ALL, D_BDW_PLUS, ADDR_FIX_1(2), 2042 8, cmd_handler_mi_semaphore_wait, CMD_LEN(2)}, 2043 2044 {"MI_STORE_DATA_IMM", OP_MI_STORE_DATA_IMM, F_LEN_VAR, R_ALL, D_BDW_PLUS, 2045 ADDR_FIX_1(1), 10, cmd_handler_mi_store_data_imm}, 2046 2047 {"MI_STORE_DATA_INDEX", OP_MI_STORE_DATA_INDEX, F_LEN_VAR, R_ALL, D_ALL, 2048 0, 8, cmd_handler_mi_store_data_index}, 2049 2050 {"MI_LOAD_REGISTER_IMM", OP_MI_LOAD_REGISTER_IMM, F_LEN_VAR, R_ALL, 2051 D_ALL, 0, 8, cmd_handler_lri}, 2052 2053 {"MI_UPDATE_GTT", OP_MI_UPDATE_GTT, F_LEN_VAR, R_ALL, D_BDW_PLUS, 0, 10, 2054 cmd_handler_mi_update_gtt}, 2055 2056 {"MI_STORE_REGISTER_MEM", OP_MI_STORE_REGISTER_MEM, 2057 F_LEN_VAR | F_LEN_VAR_FIXED, R_ALL, D_ALL, ADDR_FIX_1(2), 8, 2058 cmd_handler_srm, CMD_LEN(2)}, 2059 2060 {"MI_FLUSH_DW", OP_MI_FLUSH_DW, F_LEN_VAR, R_ALL, D_ALL, 0, 6, 2061 cmd_handler_mi_flush_dw}, 2062 2063 {"MI_CLFLUSH", OP_MI_CLFLUSH, F_LEN_VAR, R_ALL, D_ALL, ADDR_FIX_1(1), 2064 10, cmd_handler_mi_clflush}, 2065 2066 {"MI_REPORT_PERF_COUNT", OP_MI_REPORT_PERF_COUNT, 2067 F_LEN_VAR | F_LEN_VAR_FIXED, R_ALL, D_ALL, ADDR_FIX_1(1), 6, 2068 cmd_handler_mi_report_perf_count, CMD_LEN(2)}, 2069 2070 {"MI_LOAD_REGISTER_MEM", OP_MI_LOAD_REGISTER_MEM, 2071 F_LEN_VAR | F_LEN_VAR_FIXED, R_ALL, D_ALL, ADDR_FIX_1(2), 8, 2072 cmd_handler_lrm, CMD_LEN(2)}, 2073 2074 {"MI_LOAD_REGISTER_REG", OP_MI_LOAD_REGISTER_REG, 2075 F_LEN_VAR | F_LEN_VAR_FIXED, R_ALL, D_ALL, 0, 8, 2076 cmd_handler_lrr, CMD_LEN(1)}, 2077 2078 {"MI_RS_STORE_DATA_IMM", OP_MI_RS_STORE_DATA_IMM, 2079 F_LEN_VAR | F_LEN_VAR_FIXED, R_RCS, D_ALL, 0, 2080 8, NULL, CMD_LEN(2)}, 2081 2082 {"MI_LOAD_URB_MEM", OP_MI_LOAD_URB_MEM, F_LEN_VAR | F_LEN_VAR_FIXED, 2083 R_RCS, D_ALL, ADDR_FIX_1(2), 8, NULL, CMD_LEN(2)}, 2084 2085 {"MI_STORE_URM_MEM", OP_MI_STORE_URM_MEM, F_LEN_VAR, R_RCS, D_ALL, 2086 ADDR_FIX_1(2), 8, NULL}, 2087 2088 {"MI_OP_2E", OP_MI_2E, F_LEN_VAR | F_LEN_VAR_FIXED, R_ALL, D_BDW_PLUS, 2089 ADDR_FIX_2(1, 2), 8, cmd_handler_mi_op_2e, CMD_LEN(3)}, 2090 2091 {"MI_OP_2F", OP_MI_2F, F_LEN_VAR, R_ALL, D_BDW_PLUS, ADDR_FIX_1(1), 2092 8, cmd_handler_mi_op_2f}, 2093 2094 {"MI_BATCH_BUFFER_START", OP_MI_BATCH_BUFFER_START, 2095 F_IP_ADVANCE_CUSTOM, R_ALL, D_ALL, 0, 8, 2096 cmd_handler_mi_batch_buffer_start}, 2097 2098 {"MI_CONDITIONAL_BATCH_BUFFER_END", OP_MI_CONDITIONAL_BATCH_BUFFER_END, 2099 F_LEN_VAR | F_LEN_VAR_FIXED, R_ALL, D_ALL, ADDR_FIX_1(2), 8, 2100 cmd_handler_mi_conditional_batch_buffer_end, CMD_LEN(2)}, 2101 2102 {"MI_LOAD_SCAN_LINES_INCL", OP_MI_LOAD_SCAN_LINES_INCL, F_LEN_CONST, 2103 R_RCS | R_BCS, D_ALL, 0, 2, NULL}, 2104 2105 {"XY_SETUP_BLT", OP_XY_SETUP_BLT, F_LEN_VAR, R_BCS, D_ALL, 2106 ADDR_FIX_2(4, 7), 8, NULL}, 2107 2108 {"XY_SETUP_CLIP_BLT", OP_XY_SETUP_CLIP_BLT, F_LEN_VAR, R_BCS, D_ALL, 2109 0, 8, NULL}, 2110 2111 {"XY_SETUP_MONO_PATTERN_SL_BLT", OP_XY_SETUP_MONO_PATTERN_SL_BLT, 2112 F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_1(4), 8, NULL}, 2113 2114 {"XY_PIXEL_BLT", OP_XY_PIXEL_BLT, F_LEN_VAR, R_BCS, D_ALL, 0, 8, NULL}, 2115 2116 {"XY_SCANLINES_BLT", OP_XY_SCANLINES_BLT, F_LEN_VAR, R_BCS, D_ALL, 2117 0, 8, NULL}, 2118 2119 {"XY_TEXT_BLT", OP_XY_TEXT_BLT, F_LEN_VAR, R_BCS, D_ALL, 2120 ADDR_FIX_1(3), 8, NULL}, 2121 2122 {"XY_TEXT_IMMEDIATE_BLT", OP_XY_TEXT_IMMEDIATE_BLT, F_LEN_VAR, R_BCS, 2123 D_ALL, 0, 8, NULL}, 2124 2125 {"XY_COLOR_BLT", OP_XY_COLOR_BLT, F_LEN_VAR, R_BCS, D_ALL, 2126 ADDR_FIX_1(4), 8, NULL}, 2127 2128 {"XY_PAT_BLT", OP_XY_PAT_BLT, F_LEN_VAR, R_BCS, D_ALL, 2129 ADDR_FIX_2(4, 5), 8, NULL}, 2130 2131 {"XY_MONO_PAT_BLT", OP_XY_MONO_PAT_BLT, F_LEN_VAR, R_BCS, D_ALL, 2132 ADDR_FIX_1(4), 8, NULL}, 2133 2134 {"XY_SRC_COPY_BLT", OP_XY_SRC_COPY_BLT, F_LEN_VAR, R_BCS, D_ALL, 2135 ADDR_FIX_2(4, 7), 8, NULL}, 2136 2137 {"XY_MONO_SRC_COPY_BLT", OP_XY_MONO_SRC_COPY_BLT, F_LEN_VAR, R_BCS, 2138 D_ALL, ADDR_FIX_2(4, 5), 8, NULL}, 2139 2140 {"XY_FULL_BLT", OP_XY_FULL_BLT, F_LEN_VAR, R_BCS, D_ALL, 0, 8, NULL}, 2141 2142 {"XY_FULL_MONO_SRC_BLT", OP_XY_FULL_MONO_SRC_BLT, F_LEN_VAR, R_BCS, 2143 D_ALL, ADDR_FIX_3(4, 5, 8), 8, NULL}, 2144 2145 {"XY_FULL_MONO_PATTERN_BLT", OP_XY_FULL_MONO_PATTERN_BLT, F_LEN_VAR, 2146 R_BCS, D_ALL, ADDR_FIX_2(4, 7), 8, NULL}, 2147 2148 {"XY_FULL_MONO_PATTERN_MONO_SRC_BLT", 2149 OP_XY_FULL_MONO_PATTERN_MONO_SRC_BLT, 2150 F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_2(4, 5), 8, NULL}, 2151 2152 {"XY_MONO_PAT_FIXED_BLT", OP_XY_MONO_PAT_FIXED_BLT, F_LEN_VAR, R_BCS, 2153 D_ALL, ADDR_FIX_1(4), 8, NULL}, 2154 2155 {"XY_MONO_SRC_COPY_IMMEDIATE_BLT", OP_XY_MONO_SRC_COPY_IMMEDIATE_BLT, 2156 F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_1(4), 8, NULL}, 2157 2158 {"XY_PAT_BLT_IMMEDIATE", OP_XY_PAT_BLT_IMMEDIATE, F_LEN_VAR, R_BCS, 2159 D_ALL, ADDR_FIX_1(4), 8, NULL}, 2160 2161 {"XY_SRC_COPY_CHROMA_BLT", OP_XY_SRC_COPY_CHROMA_BLT, F_LEN_VAR, R_BCS, 2162 D_ALL, ADDR_FIX_2(4, 7), 8, NULL}, 2163 2164 {"XY_FULL_IMMEDIATE_PATTERN_BLT", OP_XY_FULL_IMMEDIATE_PATTERN_BLT, 2165 F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_2(4, 7), 8, NULL}, 2166 2167 {"XY_FULL_MONO_SRC_IMMEDIATE_PATTERN_BLT", 2168 OP_XY_FULL_MONO_SRC_IMMEDIATE_PATTERN_BLT, 2169 F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_2(4, 5), 8, NULL}, 2170 2171 {"XY_PAT_CHROMA_BLT", OP_XY_PAT_CHROMA_BLT, F_LEN_VAR, R_BCS, D_ALL, 2172 ADDR_FIX_2(4, 5), 8, NULL}, 2173 2174 {"XY_PAT_CHROMA_BLT_IMMEDIATE", OP_XY_PAT_CHROMA_BLT_IMMEDIATE, 2175 F_LEN_VAR, R_BCS, D_ALL, ADDR_FIX_1(4), 8, NULL}, 2176 2177 {"3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP", 2178 OP_3DSTATE_VIEWPORT_STATE_POINTERS_SF_CLIP, 2179 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2180 2181 {"3DSTATE_VIEWPORT_STATE_POINTERS_CC", 2182 OP_3DSTATE_VIEWPORT_STATE_POINTERS_CC, 2183 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2184 2185 {"3DSTATE_BLEND_STATE_POINTERS", 2186 OP_3DSTATE_BLEND_STATE_POINTERS, 2187 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2188 2189 {"3DSTATE_DEPTH_STENCIL_STATE_POINTERS", 2190 OP_3DSTATE_DEPTH_STENCIL_STATE_POINTERS, 2191 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2192 2193 {"3DSTATE_BINDING_TABLE_POINTERS_VS", 2194 OP_3DSTATE_BINDING_TABLE_POINTERS_VS, 2195 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2196 2197 {"3DSTATE_BINDING_TABLE_POINTERS_HS", 2198 OP_3DSTATE_BINDING_TABLE_POINTERS_HS, 2199 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2200 2201 {"3DSTATE_BINDING_TABLE_POINTERS_DS", 2202 OP_3DSTATE_BINDING_TABLE_POINTERS_DS, 2203 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2204 2205 {"3DSTATE_BINDING_TABLE_POINTERS_GS", 2206 OP_3DSTATE_BINDING_TABLE_POINTERS_GS, 2207 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2208 2209 {"3DSTATE_BINDING_TABLE_POINTERS_PS", 2210 OP_3DSTATE_BINDING_TABLE_POINTERS_PS, 2211 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2212 2213 {"3DSTATE_SAMPLER_STATE_POINTERS_VS", 2214 OP_3DSTATE_SAMPLER_STATE_POINTERS_VS, 2215 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2216 2217 {"3DSTATE_SAMPLER_STATE_POINTERS_HS", 2218 OP_3DSTATE_SAMPLER_STATE_POINTERS_HS, 2219 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2220 2221 {"3DSTATE_SAMPLER_STATE_POINTERS_DS", 2222 OP_3DSTATE_SAMPLER_STATE_POINTERS_DS, 2223 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2224 2225 {"3DSTATE_SAMPLER_STATE_POINTERS_GS", 2226 OP_3DSTATE_SAMPLER_STATE_POINTERS_GS, 2227 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2228 2229 {"3DSTATE_SAMPLER_STATE_POINTERS_PS", 2230 OP_3DSTATE_SAMPLER_STATE_POINTERS_PS, 2231 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2232 2233 {"3DSTATE_URB_VS", OP_3DSTATE_URB_VS, F_LEN_VAR, R_RCS, D_ALL, 2234 0, 8, NULL}, 2235 2236 {"3DSTATE_URB_HS", OP_3DSTATE_URB_HS, F_LEN_VAR, R_RCS, D_ALL, 2237 0, 8, NULL}, 2238 2239 {"3DSTATE_URB_DS", OP_3DSTATE_URB_DS, F_LEN_VAR, R_RCS, D_ALL, 2240 0, 8, NULL}, 2241 2242 {"3DSTATE_URB_GS", OP_3DSTATE_URB_GS, F_LEN_VAR, R_RCS, D_ALL, 2243 0, 8, NULL}, 2244 2245 {"3DSTATE_GATHER_CONSTANT_VS", OP_3DSTATE_GATHER_CONSTANT_VS, 2246 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2247 2248 {"3DSTATE_GATHER_CONSTANT_GS", OP_3DSTATE_GATHER_CONSTANT_GS, 2249 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2250 2251 {"3DSTATE_GATHER_CONSTANT_HS", OP_3DSTATE_GATHER_CONSTANT_HS, 2252 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2253 2254 {"3DSTATE_GATHER_CONSTANT_DS", OP_3DSTATE_GATHER_CONSTANT_DS, 2255 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2256 2257 {"3DSTATE_GATHER_CONSTANT_PS", OP_3DSTATE_GATHER_CONSTANT_PS, 2258 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2259 2260 {"3DSTATE_DX9_CONSTANTF_VS", OP_3DSTATE_DX9_CONSTANTF_VS, 2261 F_LEN_VAR, R_RCS, D_ALL, 0, 11, NULL}, 2262 2263 {"3DSTATE_DX9_CONSTANTF_PS", OP_3DSTATE_DX9_CONSTANTF_PS, 2264 F_LEN_VAR, R_RCS, D_ALL, 0, 11, NULL}, 2265 2266 {"3DSTATE_DX9_CONSTANTI_VS", OP_3DSTATE_DX9_CONSTANTI_VS, 2267 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2268 2269 {"3DSTATE_DX9_CONSTANTI_PS", OP_3DSTATE_DX9_CONSTANTI_PS, 2270 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2271 2272 {"3DSTATE_DX9_CONSTANTB_VS", OP_3DSTATE_DX9_CONSTANTB_VS, 2273 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2274 2275 {"3DSTATE_DX9_CONSTANTB_PS", OP_3DSTATE_DX9_CONSTANTB_PS, 2276 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2277 2278 {"3DSTATE_DX9_LOCAL_VALID_VS", OP_3DSTATE_DX9_LOCAL_VALID_VS, 2279 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2280 2281 {"3DSTATE_DX9_LOCAL_VALID_PS", OP_3DSTATE_DX9_LOCAL_VALID_PS, 2282 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2283 2284 {"3DSTATE_DX9_GENERATE_ACTIVE_VS", OP_3DSTATE_DX9_GENERATE_ACTIVE_VS, 2285 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2286 2287 {"3DSTATE_DX9_GENERATE_ACTIVE_PS", OP_3DSTATE_DX9_GENERATE_ACTIVE_PS, 2288 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2289 2290 {"3DSTATE_BINDING_TABLE_EDIT_VS", OP_3DSTATE_BINDING_TABLE_EDIT_VS, 2291 F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL}, 2292 2293 {"3DSTATE_BINDING_TABLE_EDIT_GS", OP_3DSTATE_BINDING_TABLE_EDIT_GS, 2294 F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL}, 2295 2296 {"3DSTATE_BINDING_TABLE_EDIT_HS", OP_3DSTATE_BINDING_TABLE_EDIT_HS, 2297 F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL}, 2298 2299 {"3DSTATE_BINDING_TABLE_EDIT_DS", OP_3DSTATE_BINDING_TABLE_EDIT_DS, 2300 F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL}, 2301 2302 {"3DSTATE_BINDING_TABLE_EDIT_PS", OP_3DSTATE_BINDING_TABLE_EDIT_PS, 2303 F_LEN_VAR, R_RCS, D_ALL, 0, 9, NULL}, 2304 2305 {"3DSTATE_VF_INSTANCING", OP_3DSTATE_VF_INSTANCING, F_LEN_VAR, R_RCS, 2306 D_BDW_PLUS, 0, 8, NULL}, 2307 2308 {"3DSTATE_VF_SGVS", OP_3DSTATE_VF_SGVS, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, 8, 2309 NULL}, 2310 2311 {"3DSTATE_VF_TOPOLOGY", OP_3DSTATE_VF_TOPOLOGY, F_LEN_VAR, R_RCS, 2312 D_BDW_PLUS, 0, 8, NULL}, 2313 2314 {"3DSTATE_WM_CHROMAKEY", OP_3DSTATE_WM_CHROMAKEY, F_LEN_VAR, R_RCS, 2315 D_BDW_PLUS, 0, 8, NULL}, 2316 2317 {"3DSTATE_PS_BLEND", OP_3DSTATE_PS_BLEND, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, 2318 8, NULL}, 2319 2320 {"3DSTATE_WM_DEPTH_STENCIL", OP_3DSTATE_WM_DEPTH_STENCIL, F_LEN_VAR, 2321 R_RCS, D_BDW_PLUS, 0, 8, NULL}, 2322 2323 {"3DSTATE_PS_EXTRA", OP_3DSTATE_PS_EXTRA, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, 2324 8, NULL}, 2325 2326 {"3DSTATE_RASTER", OP_3DSTATE_RASTER, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, 8, 2327 NULL}, 2328 2329 {"3DSTATE_SBE_SWIZ", OP_3DSTATE_SBE_SWIZ, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, 8, 2330 NULL}, 2331 2332 {"3DSTATE_WM_HZ_OP", OP_3DSTATE_WM_HZ_OP, F_LEN_VAR, R_RCS, D_BDW_PLUS, 0, 8, 2333 NULL}, 2334 2335 {"3DSTATE_VERTEX_BUFFERS", OP_3DSTATE_VERTEX_BUFFERS, F_LEN_VAR, R_RCS, 2336 D_BDW_PLUS, 0, 8, NULL}, 2337 2338 {"3DSTATE_VERTEX_ELEMENTS", OP_3DSTATE_VERTEX_ELEMENTS, F_LEN_VAR, 2339 R_RCS, D_ALL, 0, 8, NULL}, 2340 2341 {"3DSTATE_INDEX_BUFFER", OP_3DSTATE_INDEX_BUFFER, F_LEN_VAR, R_RCS, 2342 D_BDW_PLUS, ADDR_FIX_1(2), 8, NULL}, 2343 2344 {"3DSTATE_VF_STATISTICS", OP_3DSTATE_VF_STATISTICS, F_LEN_CONST, 2345 R_RCS, D_ALL, 0, 1, NULL}, 2346 2347 {"3DSTATE_VF", OP_3DSTATE_VF, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2348 2349 {"3DSTATE_CC_STATE_POINTERS", OP_3DSTATE_CC_STATE_POINTERS, F_LEN_VAR, 2350 R_RCS, D_ALL, 0, 8, NULL}, 2351 2352 {"3DSTATE_SCISSOR_STATE_POINTERS", OP_3DSTATE_SCISSOR_STATE_POINTERS, 2353 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2354 2355 {"3DSTATE_GS", OP_3DSTATE_GS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2356 2357 {"3DSTATE_CLIP", OP_3DSTATE_CLIP, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2358 2359 {"3DSTATE_WM", OP_3DSTATE_WM, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2360 2361 {"3DSTATE_CONSTANT_GS", OP_3DSTATE_CONSTANT_GS, F_LEN_VAR, R_RCS, 2362 D_BDW_PLUS, 0, 8, NULL}, 2363 2364 {"3DSTATE_CONSTANT_PS", OP_3DSTATE_CONSTANT_PS, F_LEN_VAR, R_RCS, 2365 D_BDW_PLUS, 0, 8, NULL}, 2366 2367 {"3DSTATE_SAMPLE_MASK", OP_3DSTATE_SAMPLE_MASK, F_LEN_VAR, R_RCS, 2368 D_ALL, 0, 8, NULL}, 2369 2370 {"3DSTATE_CONSTANT_HS", OP_3DSTATE_CONSTANT_HS, F_LEN_VAR, R_RCS, 2371 D_BDW_PLUS, 0, 8, NULL}, 2372 2373 {"3DSTATE_CONSTANT_DS", OP_3DSTATE_CONSTANT_DS, F_LEN_VAR, R_RCS, 2374 D_BDW_PLUS, 0, 8, NULL}, 2375 2376 {"3DSTATE_HS", OP_3DSTATE_HS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2377 2378 {"3DSTATE_TE", OP_3DSTATE_TE, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2379 2380 {"3DSTATE_DS", OP_3DSTATE_DS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2381 2382 {"3DSTATE_STREAMOUT", OP_3DSTATE_STREAMOUT, F_LEN_VAR, R_RCS, 2383 D_ALL, 0, 8, NULL}, 2384 2385 {"3DSTATE_SBE", OP_3DSTATE_SBE, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2386 2387 {"3DSTATE_PS", OP_3DSTATE_PS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2388 2389 {"3DSTATE_DRAWING_RECTANGLE", OP_3DSTATE_DRAWING_RECTANGLE, F_LEN_VAR, 2390 R_RCS, D_ALL, 0, 8, NULL}, 2391 2392 {"3DSTATE_SAMPLER_PALETTE_LOAD0", OP_3DSTATE_SAMPLER_PALETTE_LOAD0, 2393 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2394 2395 {"3DSTATE_CHROMA_KEY", OP_3DSTATE_CHROMA_KEY, F_LEN_VAR, R_RCS, D_ALL, 2396 0, 8, NULL}, 2397 2398 {"3DSTATE_DEPTH_BUFFER", OP_3DSTATE_DEPTH_BUFFER, F_LEN_VAR, R_RCS, 2399 D_ALL, ADDR_FIX_1(2), 8, NULL}, 2400 2401 {"3DSTATE_POLY_STIPPLE_OFFSET", OP_3DSTATE_POLY_STIPPLE_OFFSET, 2402 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2403 2404 {"3DSTATE_POLY_STIPPLE_PATTERN", OP_3DSTATE_POLY_STIPPLE_PATTERN, 2405 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2406 2407 {"3DSTATE_LINE_STIPPLE", OP_3DSTATE_LINE_STIPPLE, F_LEN_VAR, R_RCS, 2408 D_ALL, 0, 8, NULL}, 2409 2410 {"3DSTATE_AA_LINE_PARAMS", OP_3DSTATE_AA_LINE_PARAMS, F_LEN_VAR, R_RCS, 2411 D_ALL, 0, 8, NULL}, 2412 2413 {"3DSTATE_GS_SVB_INDEX", OP_3DSTATE_GS_SVB_INDEX, F_LEN_VAR, R_RCS, 2414 D_ALL, 0, 8, NULL}, 2415 2416 {"3DSTATE_SAMPLER_PALETTE_LOAD1", OP_3DSTATE_SAMPLER_PALETTE_LOAD1, 2417 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2418 2419 {"3DSTATE_MULTISAMPLE", OP_3DSTATE_MULTISAMPLE_BDW, F_LEN_VAR, R_RCS, 2420 D_BDW_PLUS, 0, 8, NULL}, 2421 2422 {"3DSTATE_STENCIL_BUFFER", OP_3DSTATE_STENCIL_BUFFER, F_LEN_VAR, R_RCS, 2423 D_ALL, ADDR_FIX_1(2), 8, NULL}, 2424 2425 {"3DSTATE_HIER_DEPTH_BUFFER", OP_3DSTATE_HIER_DEPTH_BUFFER, F_LEN_VAR, 2426 R_RCS, D_ALL, ADDR_FIX_1(2), 8, NULL}, 2427 2428 {"3DSTATE_CLEAR_PARAMS", OP_3DSTATE_CLEAR_PARAMS, F_LEN_VAR, 2429 R_RCS, D_ALL, 0, 8, NULL}, 2430 2431 {"3DSTATE_PUSH_CONSTANT_ALLOC_VS", OP_3DSTATE_PUSH_CONSTANT_ALLOC_VS, 2432 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2433 2434 {"3DSTATE_PUSH_CONSTANT_ALLOC_HS", OP_3DSTATE_PUSH_CONSTANT_ALLOC_HS, 2435 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2436 2437 {"3DSTATE_PUSH_CONSTANT_ALLOC_DS", OP_3DSTATE_PUSH_CONSTANT_ALLOC_DS, 2438 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2439 2440 {"3DSTATE_PUSH_CONSTANT_ALLOC_GS", OP_3DSTATE_PUSH_CONSTANT_ALLOC_GS, 2441 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2442 2443 {"3DSTATE_PUSH_CONSTANT_ALLOC_PS", OP_3DSTATE_PUSH_CONSTANT_ALLOC_PS, 2444 F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2445 2446 {"3DSTATE_MONOFILTER_SIZE", OP_3DSTATE_MONOFILTER_SIZE, F_LEN_VAR, 2447 R_RCS, D_ALL, 0, 8, NULL}, 2448 2449 {"3DSTATE_SO_DECL_LIST", OP_3DSTATE_SO_DECL_LIST, F_LEN_VAR, R_RCS, 2450 D_ALL, 0, 9, NULL}, 2451 2452 {"3DSTATE_SO_BUFFER", OP_3DSTATE_SO_BUFFER, F_LEN_VAR, R_RCS, D_BDW_PLUS, 2453 ADDR_FIX_2(2, 4), 8, NULL}, 2454 2455 {"3DSTATE_BINDING_TABLE_POOL_ALLOC", 2456 OP_3DSTATE_BINDING_TABLE_POOL_ALLOC, 2457 F_LEN_VAR, R_RCS, D_BDW_PLUS, ADDR_FIX_1(1), 8, NULL}, 2458 2459 {"3DSTATE_GATHER_POOL_ALLOC", OP_3DSTATE_GATHER_POOL_ALLOC, 2460 F_LEN_VAR, R_RCS, D_BDW_PLUS, ADDR_FIX_1(1), 8, NULL}, 2461 2462 {"3DSTATE_DX9_CONSTANT_BUFFER_POOL_ALLOC", 2463 OP_3DSTATE_DX9_CONSTANT_BUFFER_POOL_ALLOC, 2464 F_LEN_VAR, R_RCS, D_BDW_PLUS, ADDR_FIX_1(1), 8, NULL}, 2465 2466 {"3DSTATE_SAMPLE_PATTERN", OP_3DSTATE_SAMPLE_PATTERN, F_LEN_VAR, R_RCS, 2467 D_BDW_PLUS, 0, 8, NULL}, 2468 2469 {"PIPE_CONTROL", OP_PIPE_CONTROL, F_LEN_VAR, R_RCS, D_ALL, 2470 ADDR_FIX_1(2), 8, cmd_handler_pipe_control}, 2471 2472 {"3DPRIMITIVE", OP_3DPRIMITIVE, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2473 2474 {"PIPELINE_SELECT", OP_PIPELINE_SELECT, F_LEN_CONST, R_RCS, D_ALL, 0, 2475 1, NULL}, 2476 2477 {"STATE_PREFETCH", OP_STATE_PREFETCH, F_LEN_VAR, R_RCS, D_ALL, 2478 ADDR_FIX_1(1), 8, NULL}, 2479 2480 {"STATE_SIP", OP_STATE_SIP, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2481 2482 {"STATE_BASE_ADDRESS", OP_STATE_BASE_ADDRESS, F_LEN_VAR, R_RCS, D_BDW_PLUS, 2483 ADDR_FIX_5(1, 3, 4, 5, 6), 8, NULL}, 2484 2485 {"OP_3D_MEDIA_0_1_4", OP_3D_MEDIA_0_1_4, F_LEN_VAR, R_RCS, D_ALL, 2486 ADDR_FIX_1(1), 8, NULL}, 2487 2488 {"3DSTATE_VS", OP_3DSTATE_VS, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2489 2490 {"3DSTATE_SF", OP_3DSTATE_SF, F_LEN_VAR, R_RCS, D_ALL, 0, 8, NULL}, 2491 2492 {"3DSTATE_CONSTANT_VS", OP_3DSTATE_CONSTANT_VS, F_LEN_VAR, R_RCS, D_BDW_PLUS, 2493 0, 8, NULL}, 2494 2495 {"3DSTATE_COMPONENT_PACKING", OP_3DSTATE_COMPONENT_PACKING, F_LEN_VAR, R_RCS, 2496 D_SKL_PLUS, 0, 8, NULL}, 2497 2498 {"MEDIA_INTERFACE_DESCRIPTOR_LOAD", OP_MEDIA_INTERFACE_DESCRIPTOR_LOAD, 2499 F_LEN_VAR, R_RCS, D_ALL, 0, 16, NULL}, 2500 2501 {"MEDIA_GATEWAY_STATE", OP_MEDIA_GATEWAY_STATE, F_LEN_VAR, R_RCS, D_ALL, 2502 0, 16, NULL}, 2503 2504 {"MEDIA_STATE_FLUSH", OP_MEDIA_STATE_FLUSH, F_LEN_VAR, R_RCS, D_ALL, 2505 0, 16, NULL}, 2506 2507 {"MEDIA_POOL_STATE", OP_MEDIA_POOL_STATE, F_LEN_VAR, R_RCS, D_ALL, 2508 0, 16, NULL}, 2509 2510 {"MEDIA_OBJECT", OP_MEDIA_OBJECT, F_LEN_VAR, R_RCS, D_ALL, 0, 16, NULL}, 2511 2512 {"MEDIA_CURBE_LOAD", OP_MEDIA_CURBE_LOAD, F_LEN_VAR, R_RCS, D_ALL, 2513 0, 16, NULL}, 2514 2515 {"MEDIA_OBJECT_PRT", OP_MEDIA_OBJECT_PRT, F_LEN_VAR, R_RCS, D_ALL, 2516 0, 16, NULL}, 2517 2518 {"MEDIA_OBJECT_WALKER", OP_MEDIA_OBJECT_WALKER, F_LEN_VAR, R_RCS, D_ALL, 2519 0, 16, NULL}, 2520 2521 {"GPGPU_WALKER", OP_GPGPU_WALKER, F_LEN_VAR, R_RCS, D_ALL, 2522 0, 8, NULL}, 2523 2524 {"MEDIA_VFE_STATE", OP_MEDIA_VFE_STATE, F_LEN_VAR, R_RCS, D_ALL, 0, 16, 2525 NULL}, 2526 2527 {"3DSTATE_VF_STATISTICS_GM45", OP_3DSTATE_VF_STATISTICS_GM45, 2528 F_LEN_CONST, R_ALL, D_ALL, 0, 1, NULL}, 2529 2530 {"MFX_PIPE_MODE_SELECT", OP_MFX_PIPE_MODE_SELECT, F_LEN_VAR, 2531 R_VCS, D_ALL, 0, 12, NULL}, 2532 2533 {"MFX_SURFACE_STATE", OP_MFX_SURFACE_STATE, F_LEN_VAR, 2534 R_VCS, D_ALL, 0, 12, NULL}, 2535 2536 {"MFX_PIPE_BUF_ADDR_STATE", OP_MFX_PIPE_BUF_ADDR_STATE, F_LEN_VAR, 2537 R_VCS, D_BDW_PLUS, 0, 12, NULL}, 2538 2539 {"MFX_IND_OBJ_BASE_ADDR_STATE", OP_MFX_IND_OBJ_BASE_ADDR_STATE, 2540 F_LEN_VAR, R_VCS, D_BDW_PLUS, 0, 12, NULL}, 2541 2542 {"MFX_BSP_BUF_BASE_ADDR_STATE", OP_MFX_BSP_BUF_BASE_ADDR_STATE, 2543 F_LEN_VAR, R_VCS, D_BDW_PLUS, ADDR_FIX_3(1, 3, 5), 12, NULL}, 2544 2545 {"OP_2_0_0_5", OP_2_0_0_5, F_LEN_VAR, R_VCS, D_BDW_PLUS, 0, 12, NULL}, 2546 2547 {"MFX_STATE_POINTER", OP_MFX_STATE_POINTER, F_LEN_VAR, 2548 R_VCS, D_ALL, 0, 12, NULL}, 2549 2550 {"MFX_QM_STATE", OP_MFX_QM_STATE, F_LEN_VAR, 2551 R_VCS, D_ALL, 0, 12, NULL}, 2552 2553 {"MFX_FQM_STATE", OP_MFX_FQM_STATE, F_LEN_VAR, 2554 R_VCS, D_ALL, 0, 12, NULL}, 2555 2556 {"MFX_PAK_INSERT_OBJECT", OP_MFX_PAK_INSERT_OBJECT, F_LEN_VAR, 2557 R_VCS, D_ALL, 0, 12, NULL}, 2558 2559 {"MFX_STITCH_OBJECT", OP_MFX_STITCH_OBJECT, F_LEN_VAR, 2560 R_VCS, D_ALL, 0, 12, NULL}, 2561 2562 {"MFD_IT_OBJECT", OP_MFD_IT_OBJECT, F_LEN_VAR, 2563 R_VCS, D_ALL, 0, 12, NULL}, 2564 2565 {"MFX_WAIT", OP_MFX_WAIT, F_LEN_VAR, 2566 R_VCS, D_ALL, 0, 6, NULL}, 2567 2568 {"MFX_AVC_IMG_STATE", OP_MFX_AVC_IMG_STATE, F_LEN_VAR, 2569 R_VCS, D_ALL, 0, 12, NULL}, 2570 2571 {"MFX_AVC_QM_STATE", OP_MFX_AVC_QM_STATE, F_LEN_VAR, 2572 R_VCS, D_ALL, 0, 12, NULL}, 2573 2574 {"MFX_AVC_DIRECTMODE_STATE", OP_MFX_AVC_DIRECTMODE_STATE, F_LEN_VAR, 2575 R_VCS, D_ALL, 0, 12, NULL}, 2576 2577 {"MFX_AVC_SLICE_STATE", OP_MFX_AVC_SLICE_STATE, F_LEN_VAR, 2578 R_VCS, D_ALL, 0, 12, NULL}, 2579 2580 {"MFX_AVC_REF_IDX_STATE", OP_MFX_AVC_REF_IDX_STATE, F_LEN_VAR, 2581 R_VCS, D_ALL, 0, 12, NULL}, 2582 2583 {"MFX_AVC_WEIGHTOFFSET_STATE", OP_MFX_AVC_WEIGHTOFFSET_STATE, F_LEN_VAR, 2584 R_VCS, D_ALL, 0, 12, NULL}, 2585 2586 {"MFD_AVC_PICID_STATE", OP_MFD_AVC_PICID_STATE, F_LEN_VAR, 2587 R_VCS, D_ALL, 0, 12, NULL}, 2588 {"MFD_AVC_DPB_STATE", OP_MFD_AVC_DPB_STATE, F_LEN_VAR, 2589 R_VCS, D_ALL, 0, 12, NULL}, 2590 2591 {"MFD_AVC_BSD_OBJECT", OP_MFD_AVC_BSD_OBJECT, F_LEN_VAR, 2592 R_VCS, D_ALL, 0, 12, NULL}, 2593 2594 {"MFD_AVC_SLICEADDR", OP_MFD_AVC_SLICEADDR, F_LEN_VAR, 2595 R_VCS, D_ALL, ADDR_FIX_1(2), 12, NULL}, 2596 2597 {"MFC_AVC_PAK_OBJECT", OP_MFC_AVC_PAK_OBJECT, F_LEN_VAR, 2598 R_VCS, D_ALL, 0, 12, NULL}, 2599 2600 {"MFX_VC1_PRED_PIPE_STATE", OP_MFX_VC1_PRED_PIPE_STATE, F_LEN_VAR, 2601 R_VCS, D_ALL, 0, 12, NULL}, 2602 2603 {"MFX_VC1_DIRECTMODE_STATE", OP_MFX_VC1_DIRECTMODE_STATE, F_LEN_VAR, 2604 R_VCS, D_ALL, 0, 12, NULL}, 2605 2606 {"MFD_VC1_SHORT_PIC_STATE", OP_MFD_VC1_SHORT_PIC_STATE, F_LEN_VAR, 2607 R_VCS, D_ALL, 0, 12, NULL}, 2608 2609 {"MFD_VC1_LONG_PIC_STATE", OP_MFD_VC1_LONG_PIC_STATE, F_LEN_VAR, 2610 R_VCS, D_ALL, 0, 12, NULL}, 2611 2612 {"MFD_VC1_BSD_OBJECT", OP_MFD_VC1_BSD_OBJECT, F_LEN_VAR, 2613 R_VCS, D_ALL, 0, 12, NULL}, 2614 2615 {"MFC_MPEG2_SLICEGROUP_STATE", OP_MFC_MPEG2_SLICEGROUP_STATE, F_LEN_VAR, 2616 R_VCS, D_ALL, 0, 12, NULL}, 2617 2618 {"MFC_MPEG2_PAK_OBJECT", OP_MFC_MPEG2_PAK_OBJECT, F_LEN_VAR, 2619 R_VCS, D_ALL, 0, 12, NULL}, 2620 2621 {"MFX_MPEG2_PIC_STATE", OP_MFX_MPEG2_PIC_STATE, F_LEN_VAR, 2622 R_VCS, D_ALL, 0, 12, NULL}, 2623 2624 {"MFX_MPEG2_QM_STATE", OP_MFX_MPEG2_QM_STATE, F_LEN_VAR, 2625 R_VCS, D_ALL, 0, 12, NULL}, 2626 2627 {"MFD_MPEG2_BSD_OBJECT", OP_MFD_MPEG2_BSD_OBJECT, F_LEN_VAR, 2628 R_VCS, D_ALL, 0, 12, NULL}, 2629 2630 {"MFX_2_6_0_0", OP_MFX_2_6_0_0, F_LEN_VAR, R_VCS, D_ALL, 2631 0, 16, NULL}, 2632 2633 {"MFX_2_6_0_9", OP_MFX_2_6_0_9, F_LEN_VAR, R_VCS, D_ALL, 0, 16, NULL}, 2634 2635 {"MFX_2_6_0_8", OP_MFX_2_6_0_8, F_LEN_VAR, R_VCS, D_ALL, 0, 16, NULL}, 2636 2637 {"MFX_JPEG_PIC_STATE", OP_MFX_JPEG_PIC_STATE, F_LEN_VAR, 2638 R_VCS, D_ALL, 0, 12, NULL}, 2639 2640 {"MFX_JPEG_HUFF_TABLE_STATE", OP_MFX_JPEG_HUFF_TABLE_STATE, F_LEN_VAR, 2641 R_VCS, D_ALL, 0, 12, NULL}, 2642 2643 {"MFD_JPEG_BSD_OBJECT", OP_MFD_JPEG_BSD_OBJECT, F_LEN_VAR, 2644 R_VCS, D_ALL, 0, 12, NULL}, 2645 2646 {"VEBOX_STATE", OP_VEB_STATE, F_LEN_VAR, R_VECS, D_ALL, 0, 12, NULL}, 2647 2648 {"VEBOX_SURFACE_STATE", OP_VEB_SURFACE_STATE, F_LEN_VAR, R_VECS, D_ALL, 2649 0, 12, NULL}, 2650 2651 {"VEB_DI_IECP", OP_VEB_DNDI_IECP_STATE, F_LEN_VAR, R_VECS, D_BDW_PLUS, 2652 0, 12, NULL}, 2653 }; 2654 2655 static void add_cmd_entry(struct intel_gvt *gvt, struct cmd_entry *e) 2656 { 2657 hash_add(gvt->cmd_table, &e->hlist, e->info->opcode); 2658 } 2659 2660 /* call the cmd handler, and advance ip */ 2661 static int cmd_parser_exec(struct parser_exec_state *s) 2662 { 2663 struct intel_vgpu *vgpu = s->vgpu; 2664 const struct cmd_info *info; 2665 u32 cmd; 2666 int ret = 0; 2667 2668 cmd = cmd_val(s, 0); 2669 2670 /* fastpath for MI_NOOP */ 2671 if (cmd == MI_NOOP) 2672 info = &cmd_info[mi_noop_index]; 2673 else 2674 info = get_cmd_info(s->vgpu->gvt, cmd, s->ring_id); 2675 2676 if (info == NULL) { 2677 gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x, addr_type=%s, ring %d, workload=%p\n", 2678 cmd, get_opcode(cmd, s->ring_id), 2679 (s->buf_addr_type == PPGTT_BUFFER) ? 2680 "ppgtt" : "ggtt", s->ring_id, s->workload); 2681 return -EBADRQC; 2682 } 2683 2684 s->info = info; 2685 2686 trace_gvt_command(vgpu->id, s->ring_id, s->ip_gma, s->ip_va, 2687 cmd_length(s), s->buf_type, s->buf_addr_type, 2688 s->workload, info->name); 2689 2690 if ((info->flag & F_LEN_MASK) == F_LEN_VAR_FIXED) { 2691 ret = gvt_check_valid_cmd_length(cmd_length(s), 2692 info->valid_len); 2693 if (ret) 2694 return ret; 2695 } 2696 2697 if (info->handler) { 2698 ret = info->handler(s); 2699 if (ret < 0) { 2700 gvt_vgpu_err("%s handler error\n", info->name); 2701 return ret; 2702 } 2703 } 2704 2705 if (!(info->flag & F_IP_ADVANCE_CUSTOM)) { 2706 ret = cmd_advance_default(s); 2707 if (ret) { 2708 gvt_vgpu_err("%s IP advance error\n", info->name); 2709 return ret; 2710 } 2711 } 2712 return 0; 2713 } 2714 2715 static inline bool gma_out_of_range(unsigned long gma, 2716 unsigned long gma_head, unsigned int gma_tail) 2717 { 2718 if (gma_tail >= gma_head) 2719 return (gma < gma_head) || (gma > gma_tail); 2720 else 2721 return (gma > gma_tail) && (gma < gma_head); 2722 } 2723 2724 /* Keep the consistent return type, e.g EBADRQC for unknown 2725 * cmd, EFAULT for invalid address, EPERM for nonpriv. later 2726 * works as the input of VM healthy status. 2727 */ 2728 static int command_scan(struct parser_exec_state *s, 2729 unsigned long rb_head, unsigned long rb_tail, 2730 unsigned long rb_start, unsigned long rb_len) 2731 { 2732 2733 unsigned long gma_head, gma_tail, gma_bottom; 2734 int ret = 0; 2735 struct intel_vgpu *vgpu = s->vgpu; 2736 2737 gma_head = rb_start + rb_head; 2738 gma_tail = rb_start + rb_tail; 2739 gma_bottom = rb_start + rb_len; 2740 2741 while (s->ip_gma != gma_tail) { 2742 if (s->buf_type == RING_BUFFER_INSTRUCTION) { 2743 if (!(s->ip_gma >= rb_start) || 2744 !(s->ip_gma < gma_bottom)) { 2745 gvt_vgpu_err("ip_gma %lx out of ring scope." 2746 "(base:0x%lx, bottom: 0x%lx)\n", 2747 s->ip_gma, rb_start, 2748 gma_bottom); 2749 parser_exec_state_dump(s); 2750 return -EFAULT; 2751 } 2752 if (gma_out_of_range(s->ip_gma, gma_head, gma_tail)) { 2753 gvt_vgpu_err("ip_gma %lx out of range." 2754 "base 0x%lx head 0x%lx tail 0x%lx\n", 2755 s->ip_gma, rb_start, 2756 rb_head, rb_tail); 2757 parser_exec_state_dump(s); 2758 break; 2759 } 2760 } 2761 ret = cmd_parser_exec(s); 2762 if (ret) { 2763 gvt_vgpu_err("cmd parser error\n"); 2764 parser_exec_state_dump(s); 2765 break; 2766 } 2767 } 2768 2769 return ret; 2770 } 2771 2772 static int scan_workload(struct intel_vgpu_workload *workload) 2773 { 2774 unsigned long gma_head, gma_tail, gma_bottom; 2775 struct parser_exec_state s; 2776 int ret = 0; 2777 2778 /* ring base is page aligned */ 2779 if (WARN_ON(!IS_ALIGNED(workload->rb_start, I915_GTT_PAGE_SIZE))) 2780 return -EINVAL; 2781 2782 gma_head = workload->rb_start + workload->rb_head; 2783 gma_tail = workload->rb_start + workload->rb_tail; 2784 gma_bottom = workload->rb_start + _RING_CTL_BUF_SIZE(workload->rb_ctl); 2785 2786 s.buf_type = RING_BUFFER_INSTRUCTION; 2787 s.buf_addr_type = GTT_BUFFER; 2788 s.vgpu = workload->vgpu; 2789 s.ring_id = workload->ring_id; 2790 s.ring_start = workload->rb_start; 2791 s.ring_size = _RING_CTL_BUF_SIZE(workload->rb_ctl); 2792 s.ring_head = gma_head; 2793 s.ring_tail = gma_tail; 2794 s.rb_va = workload->shadow_ring_buffer_va; 2795 s.workload = workload; 2796 s.is_ctx_wa = false; 2797 2798 if ((bypass_scan_mask & (1 << workload->ring_id)) || 2799 gma_head == gma_tail) 2800 return 0; 2801 2802 ret = ip_gma_set(&s, gma_head); 2803 if (ret) 2804 goto out; 2805 2806 ret = command_scan(&s, workload->rb_head, workload->rb_tail, 2807 workload->rb_start, _RING_CTL_BUF_SIZE(workload->rb_ctl)); 2808 2809 out: 2810 return ret; 2811 } 2812 2813 static int scan_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) 2814 { 2815 2816 unsigned long gma_head, gma_tail, gma_bottom, ring_size, ring_tail; 2817 struct parser_exec_state s; 2818 int ret = 0; 2819 struct intel_vgpu_workload *workload = container_of(wa_ctx, 2820 struct intel_vgpu_workload, 2821 wa_ctx); 2822 2823 /* ring base is page aligned */ 2824 if (WARN_ON(!IS_ALIGNED(wa_ctx->indirect_ctx.guest_gma, 2825 I915_GTT_PAGE_SIZE))) 2826 return -EINVAL; 2827 2828 ring_tail = wa_ctx->indirect_ctx.size + 3 * sizeof(u32); 2829 ring_size = round_up(wa_ctx->indirect_ctx.size + CACHELINE_BYTES, 2830 PAGE_SIZE); 2831 gma_head = wa_ctx->indirect_ctx.guest_gma; 2832 gma_tail = wa_ctx->indirect_ctx.guest_gma + ring_tail; 2833 gma_bottom = wa_ctx->indirect_ctx.guest_gma + ring_size; 2834 2835 s.buf_type = RING_BUFFER_INSTRUCTION; 2836 s.buf_addr_type = GTT_BUFFER; 2837 s.vgpu = workload->vgpu; 2838 s.ring_id = workload->ring_id; 2839 s.ring_start = wa_ctx->indirect_ctx.guest_gma; 2840 s.ring_size = ring_size; 2841 s.ring_head = gma_head; 2842 s.ring_tail = gma_tail; 2843 s.rb_va = wa_ctx->indirect_ctx.shadow_va; 2844 s.workload = workload; 2845 s.is_ctx_wa = true; 2846 2847 ret = ip_gma_set(&s, gma_head); 2848 if (ret) 2849 goto out; 2850 2851 ret = command_scan(&s, 0, ring_tail, 2852 wa_ctx->indirect_ctx.guest_gma, ring_size); 2853 out: 2854 return ret; 2855 } 2856 2857 static int shadow_workload_ring_buffer(struct intel_vgpu_workload *workload) 2858 { 2859 struct intel_vgpu *vgpu = workload->vgpu; 2860 struct intel_vgpu_submission *s = &vgpu->submission; 2861 unsigned long gma_head, gma_tail, gma_top, guest_rb_size; 2862 void *shadow_ring_buffer_va; 2863 int ring_id = workload->ring_id; 2864 int ret; 2865 2866 guest_rb_size = _RING_CTL_BUF_SIZE(workload->rb_ctl); 2867 2868 /* calculate workload ring buffer size */ 2869 workload->rb_len = (workload->rb_tail + guest_rb_size - 2870 workload->rb_head) % guest_rb_size; 2871 2872 gma_head = workload->rb_start + workload->rb_head; 2873 gma_tail = workload->rb_start + workload->rb_tail; 2874 gma_top = workload->rb_start + guest_rb_size; 2875 2876 if (workload->rb_len > s->ring_scan_buffer_size[ring_id]) { 2877 void *p; 2878 2879 /* realloc the new ring buffer if needed */ 2880 p = krealloc(s->ring_scan_buffer[ring_id], workload->rb_len, 2881 GFP_KERNEL); 2882 if (!p) { 2883 gvt_vgpu_err("fail to re-alloc ring scan buffer\n"); 2884 return -ENOMEM; 2885 } 2886 s->ring_scan_buffer[ring_id] = p; 2887 s->ring_scan_buffer_size[ring_id] = workload->rb_len; 2888 } 2889 2890 shadow_ring_buffer_va = s->ring_scan_buffer[ring_id]; 2891 2892 /* get shadow ring buffer va */ 2893 workload->shadow_ring_buffer_va = shadow_ring_buffer_va; 2894 2895 /* head > tail --> copy head <-> top */ 2896 if (gma_head > gma_tail) { 2897 ret = copy_gma_to_hva(vgpu, vgpu->gtt.ggtt_mm, 2898 gma_head, gma_top, shadow_ring_buffer_va); 2899 if (ret < 0) { 2900 gvt_vgpu_err("fail to copy guest ring buffer\n"); 2901 return ret; 2902 } 2903 shadow_ring_buffer_va += ret; 2904 gma_head = workload->rb_start; 2905 } 2906 2907 /* copy head or start <-> tail */ 2908 ret = copy_gma_to_hva(vgpu, vgpu->gtt.ggtt_mm, gma_head, gma_tail, 2909 shadow_ring_buffer_va); 2910 if (ret < 0) { 2911 gvt_vgpu_err("fail to copy guest ring buffer\n"); 2912 return ret; 2913 } 2914 return 0; 2915 } 2916 2917 int intel_gvt_scan_and_shadow_ringbuffer(struct intel_vgpu_workload *workload) 2918 { 2919 int ret; 2920 struct intel_vgpu *vgpu = workload->vgpu; 2921 2922 ret = shadow_workload_ring_buffer(workload); 2923 if (ret) { 2924 gvt_vgpu_err("fail to shadow workload ring_buffer\n"); 2925 return ret; 2926 } 2927 2928 ret = scan_workload(workload); 2929 if (ret) { 2930 gvt_vgpu_err("scan workload error\n"); 2931 return ret; 2932 } 2933 return 0; 2934 } 2935 2936 static int shadow_indirect_ctx(struct intel_shadow_wa_ctx *wa_ctx) 2937 { 2938 int ctx_size = wa_ctx->indirect_ctx.size; 2939 unsigned long guest_gma = wa_ctx->indirect_ctx.guest_gma; 2940 struct intel_vgpu_workload *workload = container_of(wa_ctx, 2941 struct intel_vgpu_workload, 2942 wa_ctx); 2943 struct intel_vgpu *vgpu = workload->vgpu; 2944 struct drm_i915_gem_object *obj; 2945 int ret = 0; 2946 void *map; 2947 2948 obj = i915_gem_object_create_shmem(workload->vgpu->gvt->dev_priv, 2949 roundup(ctx_size + CACHELINE_BYTES, 2950 PAGE_SIZE)); 2951 if (IS_ERR(obj)) 2952 return PTR_ERR(obj); 2953 2954 /* get the va of the shadow batch buffer */ 2955 map = i915_gem_object_pin_map(obj, I915_MAP_WB); 2956 if (IS_ERR(map)) { 2957 gvt_vgpu_err("failed to vmap shadow indirect ctx\n"); 2958 ret = PTR_ERR(map); 2959 goto put_obj; 2960 } 2961 2962 i915_gem_object_lock(obj); 2963 ret = i915_gem_object_set_to_cpu_domain(obj, false); 2964 i915_gem_object_unlock(obj); 2965 if (ret) { 2966 gvt_vgpu_err("failed to set shadow indirect ctx to CPU\n"); 2967 goto unmap_src; 2968 } 2969 2970 ret = copy_gma_to_hva(workload->vgpu, 2971 workload->vgpu->gtt.ggtt_mm, 2972 guest_gma, guest_gma + ctx_size, 2973 map); 2974 if (ret < 0) { 2975 gvt_vgpu_err("fail to copy guest indirect ctx\n"); 2976 goto unmap_src; 2977 } 2978 2979 wa_ctx->indirect_ctx.obj = obj; 2980 wa_ctx->indirect_ctx.shadow_va = map; 2981 return 0; 2982 2983 unmap_src: 2984 i915_gem_object_unpin_map(obj); 2985 put_obj: 2986 i915_gem_object_put(obj); 2987 return ret; 2988 } 2989 2990 static int combine_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) 2991 { 2992 u32 per_ctx_start[CACHELINE_DWORDS] = {0}; 2993 unsigned char *bb_start_sva; 2994 2995 if (!wa_ctx->per_ctx.valid) 2996 return 0; 2997 2998 per_ctx_start[0] = 0x18800001; 2999 per_ctx_start[1] = wa_ctx->per_ctx.guest_gma; 3000 3001 bb_start_sva = (unsigned char *)wa_ctx->indirect_ctx.shadow_va + 3002 wa_ctx->indirect_ctx.size; 3003 3004 memcpy(bb_start_sva, per_ctx_start, CACHELINE_BYTES); 3005 3006 return 0; 3007 } 3008 3009 int intel_gvt_scan_and_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx) 3010 { 3011 int ret; 3012 struct intel_vgpu_workload *workload = container_of(wa_ctx, 3013 struct intel_vgpu_workload, 3014 wa_ctx); 3015 struct intel_vgpu *vgpu = workload->vgpu; 3016 3017 if (wa_ctx->indirect_ctx.size == 0) 3018 return 0; 3019 3020 ret = shadow_indirect_ctx(wa_ctx); 3021 if (ret) { 3022 gvt_vgpu_err("fail to shadow indirect ctx\n"); 3023 return ret; 3024 } 3025 3026 combine_wa_ctx(wa_ctx); 3027 3028 ret = scan_wa_ctx(wa_ctx); 3029 if (ret) { 3030 gvt_vgpu_err("scan wa ctx error\n"); 3031 return ret; 3032 } 3033 3034 return 0; 3035 } 3036 3037 static const struct cmd_info *find_cmd_entry_any_ring(struct intel_gvt *gvt, 3038 unsigned int opcode, unsigned long rings) 3039 { 3040 const struct cmd_info *info = NULL; 3041 unsigned int ring; 3042 3043 for_each_set_bit(ring, &rings, I915_NUM_ENGINES) { 3044 info = find_cmd_entry(gvt, opcode, ring); 3045 if (info) 3046 break; 3047 } 3048 return info; 3049 } 3050 3051 static int init_cmd_table(struct intel_gvt *gvt) 3052 { 3053 int i; 3054 struct cmd_entry *e; 3055 const struct cmd_info *info; 3056 unsigned int gen_type; 3057 3058 gen_type = intel_gvt_get_device_type(gvt); 3059 3060 for (i = 0; i < ARRAY_SIZE(cmd_info); i++) { 3061 if (!(cmd_info[i].devices & gen_type)) 3062 continue; 3063 3064 e = kzalloc(sizeof(*e), GFP_KERNEL); 3065 if (!e) 3066 return -ENOMEM; 3067 3068 e->info = &cmd_info[i]; 3069 info = find_cmd_entry_any_ring(gvt, 3070 e->info->opcode, e->info->rings); 3071 if (info) { 3072 gvt_err("%s %s duplicated\n", e->info->name, 3073 info->name); 3074 kfree(e); 3075 return -EEXIST; 3076 } 3077 if (cmd_info[i].opcode == OP_MI_NOOP) 3078 mi_noop_index = i; 3079 3080 INIT_HLIST_NODE(&e->hlist); 3081 add_cmd_entry(gvt, e); 3082 gvt_dbg_cmd("add %-30s op %04x flag %x devs %02x rings %02x\n", 3083 e->info->name, e->info->opcode, e->info->flag, 3084 e->info->devices, e->info->rings); 3085 } 3086 return 0; 3087 } 3088 3089 static void clean_cmd_table(struct intel_gvt *gvt) 3090 { 3091 struct hlist_node *tmp; 3092 struct cmd_entry *e; 3093 int i; 3094 3095 hash_for_each_safe(gvt->cmd_table, i, tmp, e, hlist) 3096 kfree(e); 3097 3098 hash_init(gvt->cmd_table); 3099 } 3100 3101 void intel_gvt_clean_cmd_parser(struct intel_gvt *gvt) 3102 { 3103 clean_cmd_table(gvt); 3104 } 3105 3106 int intel_gvt_init_cmd_parser(struct intel_gvt *gvt) 3107 { 3108 int ret; 3109 3110 ret = init_cmd_table(gvt); 3111 if (ret) { 3112 intel_gvt_clean_cmd_parser(gvt); 3113 return ret; 3114 } 3115 return 0; 3116 } 3117