1 /* $NetBSD: intel_mocs.c,v 1.2 2021/12/18 23:45:30 riastradh Exp $ */
2
3 /*
4 * Copyright (c) 2015 Intel Corporation
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions: *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24
25 #include <sys/cdefs.h>
26 __KERNEL_RCSID(0, "$NetBSD: intel_mocs.c,v 1.2 2021/12/18 23:45:30 riastradh Exp $");
27
28 #include "i915_drv.h"
29
30 #include "intel_engine.h"
31 #include "intel_gt.h"
32 #include "intel_mocs.h"
33 #include "intel_lrc.h"
34 #include "intel_ring.h"
35
36 /* structures required */
37 struct drm_i915_mocs_entry {
38 u32 control_value;
39 u16 l3cc_value;
40 u16 used;
41 };
42
43 struct drm_i915_mocs_table {
44 unsigned int size;
45 unsigned int n_entries;
46 const struct drm_i915_mocs_entry *table;
47 };
48
49 /* Defines for the tables (XXX_MOCS_0 - XXX_MOCS_63) */
50 #define _LE_CACHEABILITY(value) ((value) << 0)
51 #define _LE_TGT_CACHE(value) ((value) << 2)
52 #define LE_LRUM(value) ((value) << 4)
53 #define LE_AOM(value) ((value) << 6)
54 #define LE_RSC(value) ((value) << 7)
55 #define LE_SCC(value) ((value) << 8)
56 #define LE_PFM(value) ((value) << 11)
57 #define LE_SCF(value) ((value) << 14)
58 #define LE_COS(value) ((value) << 15)
59 #define LE_SSE(value) ((value) << 17)
60
61 /* Defines for the tables (LNCFMOCS0 - LNCFMOCS31) - two entries per word */
62 #define L3_ESC(value) ((value) << 0)
63 #define L3_SCC(value) ((value) << 1)
64 #define _L3_CACHEABILITY(value) ((value) << 4)
65
66 /* Helper defines */
67 #define GEN9_NUM_MOCS_ENTRIES 62 /* 62 out of 64 - 63 & 64 are reserved. */
68 #define GEN11_NUM_MOCS_ENTRIES 64 /* 63-64 are reserved, but configured. */
69
70 /* (e)LLC caching options */
71 /*
72 * Note: LE_0_PAGETABLE works only up to Gen11; for newer gens it means
73 * the same as LE_UC
74 */
75 #define LE_0_PAGETABLE _LE_CACHEABILITY(0)
76 #define LE_1_UC _LE_CACHEABILITY(1)
77 #define LE_2_WT _LE_CACHEABILITY(2)
78 #define LE_3_WB _LE_CACHEABILITY(3)
79
80 /* Target cache */
81 #define LE_TC_0_PAGETABLE _LE_TGT_CACHE(0)
82 #define LE_TC_1_LLC _LE_TGT_CACHE(1)
83 #define LE_TC_2_LLC_ELLC _LE_TGT_CACHE(2)
84 #define LE_TC_3_LLC_ELLC_ALT _LE_TGT_CACHE(3)
85
86 /* L3 caching options */
87 #define L3_0_DIRECT _L3_CACHEABILITY(0)
88 #define L3_1_UC _L3_CACHEABILITY(1)
89 #define L3_2_RESERVED _L3_CACHEABILITY(2)
90 #define L3_3_WB _L3_CACHEABILITY(3)
91
92 #define MOCS_ENTRY(__idx, __control_value, __l3cc_value) \
93 [__idx] = { \
94 .control_value = __control_value, \
95 .l3cc_value = __l3cc_value, \
96 .used = 1, \
97 }
98
99 /*
100 * MOCS tables
101 *
102 * These are the MOCS tables that are programmed across all the rings.
103 * The control value is programmed to all the rings that support the
104 * MOCS registers. While the l3cc_values are only programmed to the
105 * LNCFCMOCS0 - LNCFCMOCS32 registers.
106 *
107 * These tables are intended to be kept reasonably consistent across
108 * HW platforms, and for ICL+, be identical across OSes. To achieve
109 * that, for Icelake and above, list of entries is published as part
110 * of bspec.
111 *
112 * Entries not part of the following tables are undefined as far as
113 * userspace is concerned and shouldn't be relied upon. For Gen < 12
114 * they will be initialized to PTE. Gen >= 12 onwards don't have a setting for
115 * PTE and will be initialized to an invalid value.
116 *
117 * The last two entries are reserved by the hardware. For ICL+ they
118 * should be initialized according to bspec and never used, for older
119 * platforms they should never be written to.
120 *
121 * NOTE: These tables are part of bspec and defined as part of hardware
122 * interface for ICL+. For older platforms, they are part of kernel
123 * ABI. It is expected that, for specific hardware platform, existing
124 * entries will remain constant and the table will only be updated by
125 * adding new entries, filling unused positions.
126 */
127 #define GEN9_MOCS_ENTRIES \
128 MOCS_ENTRY(I915_MOCS_UNCACHED, \
129 LE_1_UC | LE_TC_2_LLC_ELLC, \
130 L3_1_UC), \
131 MOCS_ENTRY(I915_MOCS_PTE, \
132 LE_0_PAGETABLE | LE_TC_2_LLC_ELLC | LE_LRUM(3), \
133 L3_3_WB)
134
135 static const struct drm_i915_mocs_entry skl_mocs_table[] = {
136 GEN9_MOCS_ENTRIES,
137 MOCS_ENTRY(I915_MOCS_CACHED,
138 LE_3_WB | LE_TC_2_LLC_ELLC | LE_LRUM(3),
139 L3_3_WB)
140 };
141
142 /* NOTE: the LE_TGT_CACHE is not used on Broxton */
143 static const struct drm_i915_mocs_entry broxton_mocs_table[] = {
144 GEN9_MOCS_ENTRIES,
145 MOCS_ENTRY(I915_MOCS_CACHED,
146 LE_1_UC | LE_TC_2_LLC_ELLC | LE_LRUM(3),
147 L3_3_WB)
148 };
149
150 #define GEN11_MOCS_ENTRIES \
151 /* Entries 0 and 1 are defined per-platform */ \
152 /* Base - L3 + LLC */ \
153 MOCS_ENTRY(2, \
154 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \
155 L3_3_WB), \
156 /* Base - Uncached */ \
157 MOCS_ENTRY(3, \
158 LE_1_UC | LE_TC_1_LLC, \
159 L3_1_UC), \
160 /* Base - L3 */ \
161 MOCS_ENTRY(4, \
162 LE_1_UC | LE_TC_1_LLC, \
163 L3_3_WB), \
164 /* Base - LLC */ \
165 MOCS_ENTRY(5, \
166 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \
167 L3_1_UC), \
168 /* Age 0 - LLC */ \
169 MOCS_ENTRY(6, \
170 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1), \
171 L3_1_UC), \
172 /* Age 0 - L3 + LLC */ \
173 MOCS_ENTRY(7, \
174 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1), \
175 L3_3_WB), \
176 /* Age: Don't Chg. - LLC */ \
177 MOCS_ENTRY(8, \
178 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2), \
179 L3_1_UC), \
180 /* Age: Don't Chg. - L3 + LLC */ \
181 MOCS_ENTRY(9, \
182 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2), \
183 L3_3_WB), \
184 /* No AOM - LLC */ \
185 MOCS_ENTRY(10, \
186 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_AOM(1), \
187 L3_1_UC), \
188 /* No AOM - L3 + LLC */ \
189 MOCS_ENTRY(11, \
190 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_AOM(1), \
191 L3_3_WB), \
192 /* No AOM; Age 0 - LLC */ \
193 MOCS_ENTRY(12, \
194 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1) | LE_AOM(1), \
195 L3_1_UC), \
196 /* No AOM; Age 0 - L3 + LLC */ \
197 MOCS_ENTRY(13, \
198 LE_3_WB | LE_TC_1_LLC | LE_LRUM(1) | LE_AOM(1), \
199 L3_3_WB), \
200 /* No AOM; Age:DC - LLC */ \
201 MOCS_ENTRY(14, \
202 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1), \
203 L3_1_UC), \
204 /* No AOM; Age:DC - L3 + LLC */ \
205 MOCS_ENTRY(15, \
206 LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1), \
207 L3_3_WB), \
208 /* Self-Snoop - L3 + LLC */ \
209 MOCS_ENTRY(18, \
210 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SSE(3), \
211 L3_3_WB), \
212 /* Skip Caching - L3 + LLC(12.5%) */ \
213 MOCS_ENTRY(19, \
214 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(7), \
215 L3_3_WB), \
216 /* Skip Caching - L3 + LLC(25%) */ \
217 MOCS_ENTRY(20, \
218 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(3), \
219 L3_3_WB), \
220 /* Skip Caching - L3 + LLC(50%) */ \
221 MOCS_ENTRY(21, \
222 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(1), \
223 L3_3_WB), \
224 /* Skip Caching - L3 + LLC(75%) */ \
225 MOCS_ENTRY(22, \
226 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_RSC(1) | LE_SCC(3), \
227 L3_3_WB), \
228 /* Skip Caching - L3 + LLC(87.5%) */ \
229 MOCS_ENTRY(23, \
230 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_RSC(1) | LE_SCC(7), \
231 L3_3_WB), \
232 /* HW Reserved - SW program but never use */ \
233 MOCS_ENTRY(62, \
234 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \
235 L3_1_UC), \
236 /* HW Reserved - SW program but never use */ \
237 MOCS_ENTRY(63, \
238 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \
239 L3_1_UC)
240
241 static const struct drm_i915_mocs_entry tgl_mocs_table[] = {
242 /* Base - Error (Reserved for Non-Use) */
243 MOCS_ENTRY(0, 0x0, 0x0),
244 /* Base - Reserved */
245 MOCS_ENTRY(1, 0x0, 0x0),
246
247 GEN11_MOCS_ENTRIES,
248
249 /* Implicitly enable L1 - HDC:L1 + L3 + LLC */
250 MOCS_ENTRY(48,
251 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),
252 L3_3_WB),
253 /* Implicitly enable L1 - HDC:L1 + L3 */
254 MOCS_ENTRY(49,
255 LE_1_UC | LE_TC_1_LLC,
256 L3_3_WB),
257 /* Implicitly enable L1 - HDC:L1 + LLC */
258 MOCS_ENTRY(50,
259 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),
260 L3_1_UC),
261 /* Implicitly enable L1 - HDC:L1 */
262 MOCS_ENTRY(51,
263 LE_1_UC | LE_TC_1_LLC,
264 L3_1_UC),
265 /* HW Special Case (CCS) */
266 MOCS_ENTRY(60,
267 LE_3_WB | LE_TC_1_LLC | LE_LRUM(3),
268 L3_1_UC),
269 /* HW Special Case (Displayable) */
270 MOCS_ENTRY(61,
271 LE_1_UC | LE_TC_1_LLC,
272 L3_3_WB),
273 };
274
275 static const struct drm_i915_mocs_entry icl_mocs_table[] = {
276 /* Base - Uncached (Deprecated) */
277 MOCS_ENTRY(I915_MOCS_UNCACHED,
278 LE_1_UC | LE_TC_1_LLC,
279 L3_1_UC),
280 /* Base - L3 + LeCC:PAT (Deprecated) */
281 MOCS_ENTRY(I915_MOCS_PTE,
282 LE_0_PAGETABLE | LE_TC_1_LLC,
283 L3_3_WB),
284
285 GEN11_MOCS_ENTRIES
286 };
287
get_mocs_settings(const struct drm_i915_private * i915,struct drm_i915_mocs_table * table)288 static bool get_mocs_settings(const struct drm_i915_private *i915,
289 struct drm_i915_mocs_table *table)
290 {
291 if (INTEL_GEN(i915) >= 12) {
292 table->size = ARRAY_SIZE(tgl_mocs_table);
293 table->table = tgl_mocs_table;
294 table->n_entries = GEN11_NUM_MOCS_ENTRIES;
295 } else if (IS_GEN(i915, 11)) {
296 table->size = ARRAY_SIZE(icl_mocs_table);
297 table->table = icl_mocs_table;
298 table->n_entries = GEN11_NUM_MOCS_ENTRIES;
299 } else if (IS_GEN9_BC(i915) || IS_CANNONLAKE(i915)) {
300 table->size = ARRAY_SIZE(skl_mocs_table);
301 table->n_entries = GEN9_NUM_MOCS_ENTRIES;
302 table->table = skl_mocs_table;
303 } else if (IS_GEN9_LP(i915)) {
304 table->size = ARRAY_SIZE(broxton_mocs_table);
305 table->n_entries = GEN9_NUM_MOCS_ENTRIES;
306 table->table = broxton_mocs_table;
307 } else {
308 WARN_ONCE(INTEL_GEN(i915) >= 9,
309 "Platform that should have a MOCS table does not.\n");
310 return false;
311 }
312
313 if (GEM_DEBUG_WARN_ON(table->size > table->n_entries))
314 return false;
315
316 /* WaDisableSkipCaching:skl,bxt,kbl,glk */
317 if (IS_GEN(i915, 9)) {
318 int i;
319
320 for (i = 0; i < table->size; i++)
321 if (GEM_DEBUG_WARN_ON(table->table[i].l3cc_value &
322 (L3_ESC(1) | L3_SCC(0x7))))
323 return false;
324 }
325
326 return true;
327 }
328
329 /*
330 * Get control_value from MOCS entry taking into account when it's not used:
331 * I915_MOCS_PTE's value is returned in this case.
332 */
get_entry_control(const struct drm_i915_mocs_table * table,unsigned int index)333 static u32 get_entry_control(const struct drm_i915_mocs_table *table,
334 unsigned int index)
335 {
336 if (index < table->size && table->table[index].used)
337 return table->table[index].control_value;
338
339 return table->table[I915_MOCS_PTE].control_value;
340 }
341
342 #define for_each_mocs(mocs, t, i) \
343 for (i = 0; \
344 i < (t)->n_entries ? (mocs = get_entry_control((t), i)), 1 : 0;\
345 i++)
346
__init_mocs_table(struct intel_uncore * uncore,const struct drm_i915_mocs_table * table,u32 addr)347 static void __init_mocs_table(struct intel_uncore *uncore,
348 const struct drm_i915_mocs_table *table,
349 u32 addr)
350 {
351 unsigned int i;
352 u32 mocs;
353
354 for_each_mocs(mocs, table, i)
355 intel_uncore_write_fw(uncore, _MMIO(addr + i * 4), mocs);
356 }
357
mocs_offset(const struct intel_engine_cs * engine)358 static u32 mocs_offset(const struct intel_engine_cs *engine)
359 {
360 static const u32 offset[] = {
361 [RCS0] = __GEN9_RCS0_MOCS0,
362 [VCS0] = __GEN9_VCS0_MOCS0,
363 [VCS1] = __GEN9_VCS1_MOCS0,
364 [VECS0] = __GEN9_VECS0_MOCS0,
365 [BCS0] = __GEN9_BCS0_MOCS0,
366 [VCS2] = __GEN11_VCS2_MOCS0,
367 };
368
369 GEM_BUG_ON(engine->id >= ARRAY_SIZE(offset));
370 return offset[engine->id];
371 }
372
init_mocs_table(struct intel_engine_cs * engine,const struct drm_i915_mocs_table * table)373 static void init_mocs_table(struct intel_engine_cs *engine,
374 const struct drm_i915_mocs_table *table)
375 {
376 __init_mocs_table(engine->uncore, table, mocs_offset(engine));
377 }
378
379 /*
380 * Get l3cc_value from MOCS entry taking into account when it's not used:
381 * I915_MOCS_PTE's value is returned in this case.
382 */
get_entry_l3cc(const struct drm_i915_mocs_table * table,unsigned int index)383 static u16 get_entry_l3cc(const struct drm_i915_mocs_table *table,
384 unsigned int index)
385 {
386 if (index < table->size && table->table[index].used)
387 return table->table[index].l3cc_value;
388
389 return table->table[I915_MOCS_PTE].l3cc_value;
390 }
391
l3cc_combine(u16 low,u16 high)392 static inline u32 l3cc_combine(u16 low, u16 high)
393 {
394 return low | (u32)high << 16;
395 }
396
397 #define for_each_l3cc(l3cc, t, i) \
398 for (i = 0; \
399 i < ((t)->n_entries + 1) / 2 ? \
400 (l3cc = l3cc_combine(get_entry_l3cc((t), 2 * i), \
401 get_entry_l3cc((t), 2 * i + 1))), 1 : \
402 0; \
403 i++)
404
init_l3cc_table(struct intel_engine_cs * engine,const struct drm_i915_mocs_table * table)405 static void init_l3cc_table(struct intel_engine_cs *engine,
406 const struct drm_i915_mocs_table *table)
407 {
408 struct intel_uncore *uncore = engine->uncore;
409 unsigned int i;
410 u32 l3cc;
411
412 for_each_l3cc(l3cc, table, i)
413 intel_uncore_write_fw(uncore, GEN9_LNCFCMOCS(i), l3cc);
414 }
415
intel_mocs_init_engine(struct intel_engine_cs * engine)416 void intel_mocs_init_engine(struct intel_engine_cs *engine)
417 {
418 struct drm_i915_mocs_table table;
419
420 /* Called under a blanket forcewake */
421 assert_forcewakes_active(engine->uncore, FORCEWAKE_ALL);
422
423 if (!get_mocs_settings(engine->i915, &table))
424 return;
425
426 /* Platforms with global MOCS do not need per-engine initialization. */
427 if (!HAS_GLOBAL_MOCS_REGISTERS(engine->i915))
428 init_mocs_table(engine, &table);
429
430 if (engine->class == RENDER_CLASS)
431 init_l3cc_table(engine, &table);
432 }
433
global_mocs_offset(void)434 static u32 global_mocs_offset(void)
435 {
436 return i915_mmio_reg_offset(GEN12_GLOBAL_MOCS(0));
437 }
438
init_global_mocs(struct intel_gt * gt)439 static void init_global_mocs(struct intel_gt *gt)
440 {
441 struct drm_i915_mocs_table table;
442
443 /*
444 * LLC and eDRAM control values are not applicable to dgfx
445 */
446 if (IS_DGFX(gt->i915))
447 return;
448
449 if (!get_mocs_settings(gt->i915, &table))
450 return;
451
452 __init_mocs_table(gt->uncore, &table, global_mocs_offset());
453 }
454
intel_mocs_init(struct intel_gt * gt)455 void intel_mocs_init(struct intel_gt *gt)
456 {
457 if (HAS_GLOBAL_MOCS_REGISTERS(gt->i915))
458 init_global_mocs(gt);
459 }
460
461 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
462 #include "selftest_mocs.c"
463 #endif
464