1 /* $NetBSD: i915_memcpy.c,v 1.4 2021/12/19 11:33:49 riastradh Exp $ */
2
3 /*
4 * Copyright © 2016 Intel Corporation
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice (including the next
14 * paragraph) shall be included in all copies or substantial portions of the
15 * Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
23 * IN THE SOFTWARE.
24 *
25 */
26
27 #include <sys/cdefs.h>
28 __KERNEL_RCSID(0, "$NetBSD: i915_memcpy.c,v 1.4 2021/12/19 11:33:49 riastradh Exp $");
29
30 #include <linux/kernel.h>
31 #include <asm/fpu/api.h>
32
33 #include "i915_memcpy.h"
34
35 #include <linux/nbsd-namespace.h>
36
37 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG)
38 #define CI_BUG_ON(expr) BUG_ON(expr)
39 #else
40 #define CI_BUG_ON(expr) BUILD_BUG_ON_INVALID(expr)
41 #endif
42
43 static DEFINE_STATIC_KEY_FALSE(has_movntdqa);
44
45 #ifdef CONFIG_AS_MOVNTDQA
__memcpy_ntdqa(void * dst,const void * src,unsigned long len)46 static void __memcpy_ntdqa(void *dst, const void *src, unsigned long len)
47 {
48 kernel_fpu_begin();
49
50 while (len >= 4) {
51 asm("movntdqa (%0), %%xmm0\n"
52 "movntdqa 16(%0), %%xmm1\n"
53 "movntdqa 32(%0), %%xmm2\n"
54 "movntdqa 48(%0), %%xmm3\n"
55 "movaps %%xmm0, (%1)\n"
56 "movaps %%xmm1, 16(%1)\n"
57 "movaps %%xmm2, 32(%1)\n"
58 "movaps %%xmm3, 48(%1)\n"
59 :: "r" (src), "r" (dst) : "memory");
60 src += 64;
61 dst += 64;
62 len -= 4;
63 }
64 while (len--) {
65 asm("movntdqa (%0), %%xmm0\n"
66 "movaps %%xmm0, (%1)\n"
67 :: "r" (src), "r" (dst) : "memory");
68 src += 16;
69 dst += 16;
70 }
71
72 kernel_fpu_end();
73 }
74
__memcpy_ntdqu(void * dst,const void * src,unsigned long len)75 static void __memcpy_ntdqu(void *dst, const void *src, unsigned long len)
76 {
77 kernel_fpu_begin();
78
79 while (len >= 4) {
80 asm("movntdqa (%0), %%xmm0\n"
81 "movntdqa 16(%0), %%xmm1\n"
82 "movntdqa 32(%0), %%xmm2\n"
83 "movntdqa 48(%0), %%xmm3\n"
84 "movups %%xmm0, (%1)\n"
85 "movups %%xmm1, 16(%1)\n"
86 "movups %%xmm2, 32(%1)\n"
87 "movups %%xmm3, 48(%1)\n"
88 :: "r" (src), "r" (dst) : "memory");
89 src += 64;
90 dst += 64;
91 len -= 4;
92 }
93 while (len--) {
94 asm("movntdqa (%0), %%xmm0\n"
95 "movups %%xmm0, (%1)\n"
96 :: "r" (src), "r" (dst) : "memory");
97 src += 16;
98 dst += 16;
99 }
100
101 kernel_fpu_end();
102 }
103 #else
__memcpy_ntdqa(void * dst,const void * src,unsigned long len)104 static void __memcpy_ntdqa(void *dst, const void *src, unsigned long len) {}
__memcpy_ntdqu(void * dst,const void * src,unsigned long len)105 static void __memcpy_ntdqu(void *dst, const void *src, unsigned long len) {}
106 #endif
107
108 /**
109 * i915_memcpy_from_wc: perform an accelerated *aligned* read from WC
110 * @dst: destination pointer
111 * @src: source pointer
112 * @len: how many bytes to copy
113 *
114 * i915_memcpy_from_wc copies @len bytes from @src to @dst using
115 * non-temporal instructions where available. Note that all arguments
116 * (@src, @dst) must be aligned to 16 bytes and @len must be a multiple
117 * of 16.
118 *
119 * To test whether accelerated reads from WC are supported, use
120 * i915_memcpy_from_wc(NULL, NULL, 0);
121 *
122 * Returns true if the copy was successful, false if the preconditions
123 * are not met.
124 */
i915_memcpy_from_wc(void * dst,const void * src,unsigned long len)125 bool i915_memcpy_from_wc(void *dst, const void *src, unsigned long len)
126 {
127 if (unlikely(((unsigned long)dst | (unsigned long)src | len) & 15))
128 return false;
129
130 if (static_branch_likely(&has_movntdqa)) {
131 if (likely(len))
132 __memcpy_ntdqa(dst, src, len >> 4);
133 return true;
134 }
135
136 return false;
137 }
138
139 /**
140 * i915_unaligned_memcpy_from_wc: perform a mostly accelerated read from WC
141 * @dst: destination pointer
142 * @src: source pointer
143 * @len: how many bytes to copy
144 *
145 * Like i915_memcpy_from_wc(), the unaligned variant copies @len bytes from
146 * @src to @dst using * non-temporal instructions where available, but
147 * accepts that its arguments may not be aligned, but are valid for the
148 * potential 16-byte read past the end.
149 */
i915_unaligned_memcpy_from_wc(void * dst,void * src,unsigned long len)150 void i915_unaligned_memcpy_from_wc(void *dst, void *src, unsigned long len)
151 {
152 unsigned long addr;
153
154 CI_BUG_ON(!i915_has_memcpy_from_wc());
155
156 addr = (unsigned long)src;
157 if (!IS_ALIGNED(addr, 16)) {
158 unsigned long x = min(ALIGN(addr, 16) - addr, len);
159
160 memcpy(dst, src, x);
161
162 len -= x;
163 dst += x;
164 src += x;
165 }
166
167 if (likely(len))
168 __memcpy_ntdqu(dst, src, DIV_ROUND_UP(len, 16));
169 }
170
i915_memcpy_init_early(struct drm_i915_private * dev_priv)171 void i915_memcpy_init_early(struct drm_i915_private *dev_priv)
172 {
173 #ifdef CONFIG_AS_MOVNTDQA
174 /*
175 * Some hypervisors (e.g. KVM) don't support VEX-prefix instructions
176 * emulation. So don't enable movntdqa in hypervisor guest.
177 */
178 if (static_cpu_has(X86_FEATURE_XMM4_1) &&
179 !boot_cpu_has(X86_FEATURE_HYPERVISOR))
180 static_branch_enable(&has_movntdqa);
181 #endif
182 }
183