1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2015 Intel Corporation. 3 * Copyright (c) 2017, IBM Corporation. 4 * All rights reserved. 5 */ 6 7 /** \file 8 * Memory barriers 9 */ 10 11 #ifndef SPDK_BARRIER_H 12 #define SPDK_BARRIER_H 13 14 #include "spdk/stdinc.h" 15 16 #ifdef __cplusplus 17 extern "C" { 18 #endif 19 20 /** Compiler memory barrier */ 21 #define spdk_compiler_barrier() __asm volatile("" ::: "memory") 22 23 /** Read memory barrier */ 24 #define spdk_rmb() _spdk_rmb() 25 26 /** Write memory barrier */ 27 #define spdk_wmb() _spdk_wmb() 28 29 /** Full read/write memory barrier */ 30 #define spdk_mb() _spdk_mb() 31 32 /** SMP read memory barrier. */ 33 #define spdk_smp_rmb() _spdk_smp_rmb() 34 35 /** SMP write memory barrier. */ 36 #define spdk_smp_wmb() _spdk_smp_wmb() 37 38 /** SMP read/write memory barrier. */ 39 #define spdk_smp_mb() _spdk_smp_mb() 40 41 /** Invalidate data cache, input is data pointer */ 42 #define spdk_ivdt_dcache(pdata) _spdk_ivdt_dcache(pdata) 43 44 #ifdef __PPC64__ 45 46 #define _spdk_rmb() __asm volatile("sync" ::: "memory") 47 #define _spdk_wmb() __asm volatile("sync" ::: "memory") 48 #define _spdk_mb() __asm volatile("sync" ::: "memory") 49 #define _spdk_smp_rmb() __asm volatile("lwsync" ::: "memory") 50 #define _spdk_smp_wmb() __asm volatile("lwsync" ::: "memory") 51 #define _spdk_smp_mb() spdk_mb() 52 #define _spdk_ivdt_dcache(pdata) 53 54 #elif defined(__aarch64__) 55 56 #define _spdk_rmb() __asm volatile("dsb ld" ::: "memory") 57 #define _spdk_wmb() __asm volatile("dsb st" ::: "memory") 58 #define _spdk_mb() __asm volatile("dsb sy" ::: "memory") 59 #define _spdk_smp_rmb() __asm volatile("dmb ishld" ::: "memory") 60 #define _spdk_smp_wmb() __asm volatile("dmb ishst" ::: "memory") 61 #define _spdk_smp_mb() __asm volatile("dmb ish" ::: "memory") 62 #define _spdk_ivdt_dcache(pdata) asm volatile("dc civac, %0" : : "r"(pdata) : "memory"); 63 64 #elif defined(__i386__) || defined(__x86_64__) 65 66 #define _spdk_rmb() __asm volatile("lfence" ::: "memory") 67 #define _spdk_wmb() __asm volatile("sfence" ::: "memory") 68 #define _spdk_mb() __asm volatile("mfence" ::: "memory") 69 #define _spdk_smp_rmb() spdk_compiler_barrier() 70 #define _spdk_smp_wmb() spdk_compiler_barrier() 71 #if defined(__x86_64__) 72 #define _spdk_smp_mb() __asm volatile("lock addl $0, -128(%%rsp); " ::: "memory"); 73 #elif defined(__i386__) 74 #define _spdk_smp_mb() __asm volatile("lock addl $0, -128(%%esp); " ::: "memory"); 75 #endif 76 #define _spdk_ivdt_dcache(pdata) 77 78 #elif defined(__riscv) 79 80 #define _spdk_rmb() __asm__ __volatile__("fence ir, ir" ::: "memory") 81 #define _spdk_wmb() __asm__ __volatile__("fence ow, ow" ::: "memory") 82 #define _spdk_mb() __asm__ __volatile__("fence iorw, iorw" ::: "memory") 83 #define _spdk_smp_rmb() __asm__ __volatile__("fence r, r" ::: "memory") 84 #define _spdk_smp_wmb() __asm__ __volatile__("fence w, w" ::: "memory") 85 #define _spdk_smp_mb() __asm__ __volatile__("fence rw, rw" ::: "memory") 86 #define _spdk_ivdt_dcache(pdata) 87 88 #elif defined(__loongarch__) 89 90 #define _spdk_rmb() __asm volatile("dbar 0" ::: "memory") 91 #define _spdk_wmb() __asm volatile("dbar 0" ::: "memory") 92 #define _spdk_mb() __asm volatile("dbar 0" ::: "memory") 93 #define _spdk_smp_rmb() __asm volatile("dbar 0" ::: "memory") 94 #define _spdk_smp_wmb() __asm volatile("dbar 0" ::: "memory") 95 #define _spdk_smp_mb() __asm volatile("dbar 0" ::: "memory") 96 #define _spdk_ivdt_dcache(pdata) 97 98 #else 99 100 #define _spdk_rmb() 101 #define _spdk_wmb() 102 #define _spdk_mb() 103 #define _spdk_smp_rmb() 104 #define _spdk_smp_wmb() 105 #define _spdk_smp_mb() 106 #define _spdk_ivdt_dcache(pdata) 107 #error Unknown architecture 108 109 #endif 110 111 #ifdef __cplusplus 112 } 113 #endif 114 115 #endif 116