1 /*- 2 * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 * $FreeBSD$ 26 */ 27 28 #include <linux/kernel.h> 29 #include <linux/module.h> 30 #include <dev/mlx5/driver.h> 31 #include "mlx5_core.h" 32 33 struct mlx5_pages_req { 34 struct mlx5_core_dev *dev; 35 u16 func_id; 36 s32 npages; 37 struct work_struct work; 38 }; 39 40 struct fw_page { 41 struct rb_node rb_node; 42 u64 addr; 43 struct page *page; 44 u16 func_id; 45 unsigned long bitmask; 46 struct list_head list; 47 unsigned free_count; 48 }; 49 50 struct mlx5_manage_pages_inbox { 51 struct mlx5_inbox_hdr hdr; 52 __be16 rsvd; 53 __be16 func_id; 54 __be32 num_entries; 55 __be64 pas[0]; 56 }; 57 58 struct mlx5_manage_pages_outbox { 59 struct mlx5_outbox_hdr hdr; 60 __be32 num_entries; 61 u8 rsvd[4]; 62 __be64 pas[0]; 63 }; 64 65 enum { 66 MAX_RECLAIM_TIME_MSECS = 5000, 67 }; 68 69 enum { 70 MLX5_MAX_RECLAIM_TIME_MILI = 5000, 71 MLX5_NUM_4K_IN_PAGE = PAGE_SIZE / MLX5_ADAPTER_PAGE_SIZE, 72 }; 73 74 static int insert_page(struct mlx5_core_dev *dev, u64 addr, struct page *page, u16 func_id) 75 { 76 struct rb_root *root = &dev->priv.page_root; 77 struct rb_node **new = &root->rb_node; 78 struct rb_node *parent = NULL; 79 struct fw_page *nfp; 80 struct fw_page *tfp; 81 int i; 82 83 while (*new) { 84 parent = *new; 85 tfp = rb_entry(parent, struct fw_page, rb_node); 86 if (tfp->addr < addr) 87 new = &parent->rb_left; 88 else if (tfp->addr > addr) 89 new = &parent->rb_right; 90 else 91 return -EEXIST; 92 } 93 94 nfp = kzalloc(sizeof(*nfp), GFP_KERNEL); 95 96 nfp->addr = addr; 97 nfp->page = page; 98 nfp->func_id = func_id; 99 nfp->free_count = MLX5_NUM_4K_IN_PAGE; 100 for (i = 0; i < MLX5_NUM_4K_IN_PAGE; i++) 101 set_bit(i, &nfp->bitmask); 102 103 rb_link_node(&nfp->rb_node, parent, new); 104 rb_insert_color(&nfp->rb_node, root); 105 list_add(&nfp->list, &dev->priv.free_list); 106 107 return 0; 108 } 109 110 static struct fw_page *find_fw_page(struct mlx5_core_dev *dev, u64 addr) 111 { 112 struct rb_root *root = &dev->priv.page_root; 113 struct rb_node *tmp = root->rb_node; 114 struct fw_page *result = NULL; 115 struct fw_page *tfp; 116 117 while (tmp) { 118 tfp = rb_entry(tmp, struct fw_page, rb_node); 119 if (tfp->addr < addr) { 120 tmp = tmp->rb_left; 121 } else if (tfp->addr > addr) { 122 tmp = tmp->rb_right; 123 } else { 124 result = tfp; 125 break; 126 } 127 } 128 129 return result; 130 } 131 132 static int mlx5_cmd_query_pages(struct mlx5_core_dev *dev, u16 *func_id, 133 s32 *npages, int boot) 134 { 135 u32 in[MLX5_ST_SZ_DW(query_pages_in)]; 136 u32 out[MLX5_ST_SZ_DW(query_pages_out)]; 137 int err; 138 139 memset(in, 0, sizeof(in)); 140 141 MLX5_SET(query_pages_in, in, opcode, MLX5_CMD_OP_QUERY_PAGES); 142 MLX5_SET(query_pages_in, in, op_mod, 143 boot ? MLX5_BOOT_PAGES : MLX5_INIT_PAGES); 144 145 memset(out, 0, sizeof(out)); 146 err = mlx5_cmd_exec_check_status(dev, in, sizeof(in), out, sizeof(out)); 147 if (err) 148 return err; 149 150 *npages = MLX5_GET(query_pages_out, out, num_pages); 151 *func_id = MLX5_GET(query_pages_out, out, function_id); 152 153 return 0; 154 } 155 156 static int alloc_4k(struct mlx5_core_dev *dev, u64 *addr) 157 { 158 struct fw_page *fp; 159 unsigned n; 160 161 if (list_empty(&dev->priv.free_list)) 162 return -ENOMEM; 163 164 fp = list_entry(dev->priv.free_list.next, struct fw_page, list); 165 n = find_first_bit(&fp->bitmask, 8 * sizeof(fp->bitmask)); 166 if (n >= MLX5_NUM_4K_IN_PAGE) { 167 mlx5_core_warn(dev, "alloc 4k bug\n"); 168 return -ENOENT; 169 } 170 clear_bit(n, &fp->bitmask); 171 fp->free_count--; 172 if (!fp->free_count) 173 list_del(&fp->list); 174 175 *addr = fp->addr + n * MLX5_ADAPTER_PAGE_SIZE; 176 177 return 0; 178 } 179 180 static void free_4k(struct mlx5_core_dev *dev, u64 addr) 181 { 182 struct fw_page *fwp; 183 int n; 184 185 fwp = find_fw_page(dev, addr & PAGE_MASK); 186 if (!fwp) { 187 mlx5_core_warn(dev, "page not found\n"); 188 return; 189 } 190 191 n = (addr & ~PAGE_MASK) >> MLX5_ADAPTER_PAGE_SHIFT; 192 fwp->free_count++; 193 set_bit(n, &fwp->bitmask); 194 if (fwp->free_count == MLX5_NUM_4K_IN_PAGE) { 195 rb_erase(&fwp->rb_node, &dev->priv.page_root); 196 if (fwp->free_count != 1) 197 list_del(&fwp->list); 198 dma_unmap_page(&dev->pdev->dev, addr & PAGE_MASK, PAGE_SIZE, 199 DMA_BIDIRECTIONAL); 200 __free_page(fwp->page); 201 kfree(fwp); 202 } else if (fwp->free_count == 1) { 203 list_add(&fwp->list, &dev->priv.free_list); 204 } 205 } 206 207 static int alloc_system_page(struct mlx5_core_dev *dev, u16 func_id) 208 { 209 struct page *page; 210 u64 addr; 211 int err; 212 213 page = alloc_page(GFP_HIGHUSER); 214 if (!page) { 215 mlx5_core_warn(dev, "failed to allocate page\n"); 216 return -ENOMEM; 217 } 218 addr = dma_map_page(&dev->pdev->dev, page, 0, 219 PAGE_SIZE, DMA_BIDIRECTIONAL); 220 if (dma_mapping_error(&dev->pdev->dev, addr)) { 221 mlx5_core_warn(dev, "failed dma mapping page\n"); 222 err = -ENOMEM; 223 goto out_alloc; 224 } 225 err = insert_page(dev, addr, page, func_id); 226 if (err) { 227 mlx5_core_err(dev, "failed to track allocated page\n"); 228 goto out_mapping; 229 } 230 231 return 0; 232 233 out_mapping: 234 dma_unmap_page(&dev->pdev->dev, addr, PAGE_SIZE, DMA_BIDIRECTIONAL); 235 236 out_alloc: 237 __free_page(page); 238 return err; 239 } 240 static int give_pages(struct mlx5_core_dev *dev, u16 func_id, int npages, 241 int notify_fail) 242 { 243 struct mlx5_manage_pages_inbox *in; 244 struct mlx5_manage_pages_outbox out; 245 struct mlx5_manage_pages_inbox *nin; 246 int inlen; 247 u64 addr; 248 int err; 249 int i; 250 251 inlen = sizeof(*in) + npages * sizeof(in->pas[0]); 252 in = mlx5_vzalloc(inlen); 253 if (!in) { 254 mlx5_core_warn(dev, "vzalloc failed %d\n", inlen); 255 return -ENOMEM; 256 } 257 memset(&out, 0, sizeof(out)); 258 259 for (i = 0; i < npages; i++) { 260 retry: 261 err = alloc_4k(dev, &addr); 262 if (err) { 263 if (err == -ENOMEM) 264 err = alloc_system_page(dev, func_id); 265 if (err) 266 goto out_4k; 267 268 goto retry; 269 } 270 in->pas[i] = cpu_to_be64(addr); 271 } 272 273 in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES); 274 in->hdr.opmod = cpu_to_be16(MLX5_PAGES_GIVE); 275 in->func_id = cpu_to_be16(func_id); 276 in->num_entries = cpu_to_be32(npages); 277 err = mlx5_cmd_exec(dev, in, inlen, &out, sizeof(out)); 278 if (err) { 279 mlx5_core_warn(dev, "func_id 0x%x, npages %d, err %d\n", 280 func_id, npages, err); 281 goto out_alloc; 282 } 283 dev->priv.fw_pages += npages; 284 285 if (out.hdr.status) { 286 err = mlx5_cmd_status_to_err(&out.hdr); 287 if (err) { 288 mlx5_core_warn(dev, "func_id 0x%x, npages %d, status %d\n", 289 func_id, npages, out.hdr.status); 290 goto out_alloc; 291 } 292 } 293 294 mlx5_core_dbg(dev, "err %d\n", err); 295 296 goto out_free; 297 298 out_alloc: 299 if (notify_fail) { 300 nin = kzalloc(sizeof(*nin), GFP_KERNEL); 301 memset(&out, 0, sizeof(out)); 302 nin->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES); 303 nin->hdr.opmod = cpu_to_be16(MLX5_PAGES_CANT_GIVE); 304 if (mlx5_cmd_exec(dev, nin, sizeof(*nin), &out, sizeof(out))) 305 mlx5_core_warn(dev, "page notify failed\n"); 306 kfree(nin); 307 } 308 309 out_4k: 310 for (i--; i >= 0; i--) 311 free_4k(dev, be64_to_cpu(in->pas[i])); 312 out_free: 313 kvfree(in); 314 return err; 315 } 316 317 static int reclaim_pages(struct mlx5_core_dev *dev, u32 func_id, int npages, 318 int *nclaimed) 319 { 320 struct mlx5_manage_pages_inbox in; 321 struct mlx5_manage_pages_outbox *out; 322 int num_claimed; 323 int outlen; 324 u64 addr; 325 int err; 326 int i; 327 328 if (nclaimed) 329 *nclaimed = 0; 330 331 memset(&in, 0, sizeof(in)); 332 outlen = sizeof(*out) + npages * sizeof(out->pas[0]); 333 out = mlx5_vzalloc(outlen); 334 if (!out) 335 return -ENOMEM; 336 337 in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_MANAGE_PAGES); 338 in.hdr.opmod = cpu_to_be16(MLX5_PAGES_TAKE); 339 in.func_id = cpu_to_be16(func_id); 340 in.num_entries = cpu_to_be32(npages); 341 mlx5_core_dbg(dev, "npages %d, outlen %d\n", npages, outlen); 342 err = mlx5_cmd_exec(dev, &in, sizeof(in), out, outlen); 343 if (err) { 344 mlx5_core_err(dev, "failed reclaiming pages\n"); 345 goto out_free; 346 } 347 348 if (out->hdr.status) { 349 err = mlx5_cmd_status_to_err(&out->hdr); 350 goto out_free; 351 } 352 353 num_claimed = be32_to_cpu(out->num_entries); 354 if (nclaimed) 355 *nclaimed = num_claimed; 356 357 dev->priv.fw_pages -= num_claimed; 358 359 for (i = 0; i < num_claimed; i++) { 360 addr = be64_to_cpu(out->pas[i]); 361 free_4k(dev, addr); 362 } 363 364 out_free: 365 kvfree(out); 366 return err; 367 } 368 369 static void pages_work_handler(struct work_struct *work) 370 { 371 struct mlx5_pages_req *req = container_of(work, struct mlx5_pages_req, work); 372 struct mlx5_core_dev *dev = req->dev; 373 int err = 0; 374 375 if (req->npages < 0) 376 err = reclaim_pages(dev, req->func_id, -1 * req->npages, NULL); 377 else if (req->npages > 0) 378 err = give_pages(dev, req->func_id, req->npages, 1); 379 380 if (err) 381 mlx5_core_warn(dev, "%s fail %d\n", 382 req->npages < 0 ? "reclaim" : "give", err); 383 384 kfree(req); 385 } 386 387 void mlx5_core_req_pages_handler(struct mlx5_core_dev *dev, u16 func_id, 388 s32 npages) 389 { 390 struct mlx5_pages_req *req; 391 392 req = kzalloc(sizeof(*req), GFP_ATOMIC); 393 if (!req) { 394 mlx5_core_warn(dev, "failed to allocate pages request\n"); 395 return; 396 } 397 398 req->dev = dev; 399 req->func_id = func_id; 400 req->npages = npages; 401 INIT_WORK(&req->work, pages_work_handler); 402 if (!queue_work(dev->priv.pg_wq, &req->work)) 403 mlx5_core_warn(dev, "failed to queue pages handler work\n"); 404 } 405 406 int mlx5_satisfy_startup_pages(struct mlx5_core_dev *dev, int boot) 407 { 408 u16 uninitialized_var(func_id); 409 s32 uninitialized_var(npages); 410 int err; 411 412 err = mlx5_cmd_query_pages(dev, &func_id, &npages, boot); 413 if (err) 414 return err; 415 416 mlx5_core_dbg(dev, "requested %d %s pages for func_id 0x%x\n", 417 npages, boot ? "boot" : "init", func_id); 418 419 return give_pages(dev, func_id, npages, 0); 420 } 421 422 enum { 423 MLX5_BLKS_FOR_RECLAIM_PAGES = 12 424 }; 425 426 static int optimal_reclaimed_pages(void) 427 { 428 struct mlx5_cmd_prot_block *block; 429 struct mlx5_cmd_layout *lay; 430 int ret; 431 432 ret = (sizeof(lay->out) + MLX5_BLKS_FOR_RECLAIM_PAGES * sizeof(block->data) - 433 sizeof(struct mlx5_manage_pages_outbox)) / 434 FIELD_SIZEOF(struct mlx5_manage_pages_outbox, pas[0]); 435 436 return ret; 437 } 438 439 int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev) 440 { 441 int end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); 442 struct fw_page *fwp; 443 struct rb_node *p; 444 int nclaimed = 0; 445 int err; 446 447 do { 448 p = rb_first(&dev->priv.page_root); 449 if (p) { 450 fwp = rb_entry(p, struct fw_page, rb_node); 451 err = reclaim_pages(dev, fwp->func_id, 452 optimal_reclaimed_pages(), 453 &nclaimed); 454 if (err) { 455 mlx5_core_warn(dev, "failed reclaiming pages (%d)\n", 456 err); 457 return err; 458 } 459 if (nclaimed) 460 end = jiffies + msecs_to_jiffies(MAX_RECLAIM_TIME_MSECS); 461 } 462 if (time_after(jiffies, end)) { 463 mlx5_core_warn(dev, "FW did not return all pages. giving up...\n"); 464 break; 465 } 466 } while (p); 467 468 return 0; 469 } 470 471 void mlx5_pagealloc_init(struct mlx5_core_dev *dev) 472 { 473 dev->priv.page_root = RB_ROOT; 474 INIT_LIST_HEAD(&dev->priv.free_list); 475 } 476 477 void mlx5_pagealloc_cleanup(struct mlx5_core_dev *dev) 478 { 479 /* nothing */ 480 } 481 482 int mlx5_pagealloc_start(struct mlx5_core_dev *dev) 483 { 484 dev->priv.pg_wq = create_singlethread_workqueue("mlx5_page_allocator"); 485 if (!dev->priv.pg_wq) 486 return -ENOMEM; 487 488 return 0; 489 } 490 491 void mlx5_pagealloc_stop(struct mlx5_core_dev *dev) 492 { 493 destroy_workqueue(dev->priv.pg_wq); 494 } 495