1 // RUN: %libomp-compile && env LIBOMP_NUM_HIDDEN_HELPER_THREADS=0 OMP_PROC_BIND=close OMP_PLACES=cores KMP_AFFINITY=verbose %libomp-run 8 1 4
2 // REQUIRES: linux
3 //
4 // This test pthread_creates 8 root threads before any OpenMP
5 // runtime entry is ever called. We have all the root threads
6 // register with the runtime by calling omp_set_num_threads(),
7 // but this does not initialize their affinity. The fourth root thread
8 // then calls a parallel region and we make sure its affinity
9 // is correct. We also make sure all the other root threads are
10 // free-floating since they have not called into a parallel region.
11
12 #define _GNU_SOURCE
13 #include <stdio.h>
14 #include <stdlib.h>
15 #include <omp.h>
16 #include <pthread.h>
17 #include <unistd.h>
18 #include <assert.h>
19 #include <sys/types.h>
20 #include <sys/syscall.h>
21 #include "libomp_test_affinity.h"
22
23 volatile int entry_flag = 0;
24 volatile int flag = 0;
25 volatile int num_roots_arrived = 0;
26 int num_roots;
27 int spawner = 0;
28 pthread_mutex_t lock;
29 int register_workers = 0; // boolean
30 affinity_mask_t *full_mask;
31
32 int __kmpc_global_thread_num(void*);
33
get_os_thread_id()34 int get_os_thread_id() {
35 return (int)syscall(SYS_gettid);
36 }
37
place_and_affinity_match()38 int place_and_affinity_match() {
39 int i, max_cpu;
40 char buf[512];
41 affinity_mask_t *mask = affinity_mask_alloc();
42 int place = omp_get_place_num();
43 int num_procs = omp_get_place_num_procs(place);
44 int *ids = (int*)malloc(sizeof(int) * num_procs);
45 omp_get_place_proc_ids(place, ids);
46 get_thread_affinity(mask);
47 affinity_mask_snprintf(buf, sizeof(buf), mask);
48 printf("Primary Thread Place: %d\n", place);
49 printf("Primary Thread mask: %s\n", buf);
50
51 for (i = 0; i < num_procs; ++i) {
52 int cpu = ids[i];
53 if (!affinity_mask_isset(mask, cpu))
54 return 0;
55 }
56
57 max_cpu = AFFINITY_MAX_CPUS;
58 for (i = 0; i < max_cpu; ++i) {
59 int cpu = i;
60 if (affinity_mask_isset(mask, cpu)) {
61 int j, found = 0;
62 for (j = 0; j < num_procs; ++j) {
63 if (ids[j] == cpu) {
64 found = 1;
65 break;
66 }
67 }
68 if (!found)
69 return 0;
70 }
71 }
72
73 affinity_mask_free(mask);
74 free(ids);
75 return 1;
76 }
77
thread_func(void * arg)78 void* thread_func(void *arg) {
79 int place, nplaces;
80 int root_id = *((int*)arg);
81 int pid = getpid();
82 int tid = get_os_thread_id();
83
84 // Order how the root threads are assigned a gtid in the runtime
85 // i.e., root_id = gtid
86 while (1) {
87 int v = entry_flag;
88 if (v == root_id)
89 break;
90 }
91
92 // If main root thread
93 if (root_id == spawner) {
94 printf("Initial application thread (pid=%d, tid=%d, spawner=%d) reached thread_func (will call OpenMP)\n", pid, tid, spawner);
95 omp_set_num_threads(4);
96 #pragma omp atomic
97 entry_flag++;
98 // Wait for the workers to signal their arrival before #pragma omp parallel
99 while (num_roots_arrived < num_roots - 1) {}
100 // This will trigger the output for KMP_AFFINITY in this case
101 #pragma omp parallel
102 {
103 int gtid = __kmpc_global_thread_num(NULL);
104 #pragma omp single
105 {
106 printf("Exactly %d threads in the #pragma omp parallel\n",
107 omp_get_num_threads());
108 }
109 #pragma omp critical
110 {
111 printf("OpenMP thread %d: gtid=%d\n", omp_get_thread_num(), gtid);
112 }
113 }
114 flag = 1;
115 if (!place_and_affinity_match()) {
116 fprintf(stderr, "error: place and affinity mask do not match for primary thread\n");
117 exit (EXIT_FAILURE);
118 }
119
120 } else { // If worker root thread
121 // Worker root threads, register with OpenMP through omp_set_num_threads()
122 // if designated to, signal their arrival and then wait for the main root
123 // thread to signal them to exit.
124 printf("New root pthread (pid=%d, tid=%d) reached thread_func\n", pid, tid);
125 if (register_workers)
126 omp_set_num_threads(4);
127 #pragma omp atomic
128 entry_flag++;
129
130 pthread_mutex_lock(&lock);
131 num_roots_arrived++;
132 pthread_mutex_unlock(&lock);
133 while (flag == 0) {}
134
135 // Main check whether root threads' mask is equal to the
136 // initial affinity mask
137 affinity_mask_t *mask = affinity_mask_alloc();
138 get_thread_affinity(mask);
139 if (!affinity_mask_equal(mask, full_mask)) {
140 char buf[1024];
141 printf("root thread %d mask: ", root_id);
142 affinity_mask_snprintf(buf, sizeof(buf), mask);
143 printf("initial affinity mask: %s\n", buf);
144 fprintf(stderr, "error: root thread %d affinity mask not equal"
145 " to initial full mask\n", root_id);
146 affinity_mask_free(mask);
147 exit(EXIT_FAILURE);
148 }
149 affinity_mask_free(mask);
150 }
151 return NULL;
152 }
153
main(int argc,char ** argv)154 int main(int argc, char** argv) {
155 int i;
156 if (argc != 3 && argc != 4) {
157 fprintf(stderr, "usage: %s <num_roots> <register_workers_bool> [<spawn_root_number>]\n", argv[0]);
158 exit(EXIT_FAILURE);
159 }
160
161 // Initialize pthread mutex
162 pthread_mutex_init(&lock, NULL);
163
164 // Get initial full mask
165 full_mask = affinity_mask_alloc();
166 get_thread_affinity(full_mask);
167
168 // Get the number of root pthreads to create and allocate resources for them
169 num_roots = atoi(argv[1]);
170 pthread_t *roots = (pthread_t*)malloc(sizeof(pthread_t) * num_roots);
171 int *root_ids = (int*)malloc(sizeof(int) * num_roots);
172
173 // Get the flag indicating whether to have root pthreads call omp_set_num_threads() or not
174 register_workers = atoi(argv[2]);
175
176 if (argc == 4)
177 spawner = atoi(argv[3]);
178
179 // Spawn worker root threads
180 for (i = 1; i < num_roots; ++i) {
181 *(root_ids + i) = i;
182 pthread_create(roots + i, NULL, thread_func, root_ids + i);
183 }
184 // Have main root thread (root 0) go into thread_func
185 *root_ids = 0;
186 thread_func(root_ids);
187
188 // Cleanup all resources
189 for (i = 1; i < num_roots; ++i) {
190 void *status;
191 pthread_join(roots[i], &status);
192 }
193 free(roots);
194 free(root_ids);
195 pthread_mutex_destroy(&lock);
196 return EXIT_SUCCESS;
197 }
198