xref: /spdk/test/nvme/aer/aer.c (revision 8dd1cd2104ea4001e4a0da2a4851ccd62c82f8e8)
1 /*   SPDX-License-Identifier: BSD-3-Clause
2  *   Copyright (c) Intel Corporation.
3  *   All rights reserved.
4  */
5 
6 #include "spdk/stdinc.h"
7 
8 #include "spdk/log.h"
9 #include "spdk/nvme.h"
10 #include "spdk/env.h"
11 #include "spdk/string.h"
12 
13 #define MAX_DEVS 64
14 
15 struct dev {
16 	struct spdk_nvme_ctrlr				*ctrlr;
17 	/* Expected changed NS ID state before AER */
18 	bool						ns_test_active;
19 	struct spdk_nvme_health_information_page	*health_page;
20 	uint32_t					orig_temp_threshold;
21 	char						name[SPDK_NVMF_TRADDR_MAX_LEN + 1];
22 };
23 
24 static void get_feature_test(struct dev *dev);
25 
26 static struct dev g_devs[MAX_DEVS];
27 static int g_num_devs = 0;
28 
29 #define foreach_dev(iter) \
30 	for (iter = g_devs; iter - g_devs < g_num_devs; iter++)
31 #define AER_PRINTF(format, ...) printf("%s" format, g_parent_process ? "" : "[Child] ", \
32 	##__VA_ARGS__)
33 #define AER_FPRINTF(f, format, ...) fprintf(f, "%s" format, g_parent_process ? \
34 	"" : "[Child] ", ##__VA_ARGS__)
35 
36 static int g_outstanding_commands = 0;
37 static int g_aer_done = 0;
38 static int g_temperature_done = 0;
39 static int g_failed = 0;
40 static struct spdk_nvme_transport_id g_trid;
41 static char *g_touch_file;
42 
43 /* Enable AER temperature test */
44 static int g_enable_temp_test = 0;
45 /* Expected changed NS ID */
46 static uint32_t g_expected_ns_test = 0;
47 /* For multi-process test */
48 static int g_multi_process_test = 0;
49 static bool g_parent_process = true;
50 static const char *g_sem_init_name = "/init";
51 static const char *g_sem_child_name = "/child";
52 static sem_t *g_sem_init_id;
53 static sem_t *g_sem_child_id;
54 
55 static void
56 set_temp_completion(void *cb_arg, const struct spdk_nvme_cpl *cpl)
57 {
58 	struct dev *dev = cb_arg;
59 
60 	g_outstanding_commands--;
61 
62 	if (spdk_nvme_cpl_is_error(cpl)) {
63 		AER_PRINTF("%s: set feature (temp threshold) failed\n", dev->name);
64 		g_failed = 1;
65 		return;
66 	}
67 
68 	/* Admin command completions are synchronized by the NVMe driver,
69 	 * so we don't need to do any special locking here. */
70 	g_temperature_done++;
71 }
72 
73 static int
74 set_temp_threshold(struct dev *dev, uint32_t temp)
75 {
76 	struct spdk_nvme_cmd cmd = {};
77 	int rc;
78 
79 	cmd.opc = SPDK_NVME_OPC_SET_FEATURES;
80 	cmd.cdw10_bits.set_features.fid = SPDK_NVME_FEAT_TEMPERATURE_THRESHOLD;
81 	cmd.cdw11_bits.feat_temp_threshold.bits.tmpth = temp;
82 
83 	rc = spdk_nvme_ctrlr_cmd_admin_raw(dev->ctrlr, &cmd, NULL, 0, set_temp_completion, dev);
84 	if (rc == 0) {
85 		g_outstanding_commands++;
86 	} else {
87 		AER_FPRINTF(stderr, "Submitting Admin cmd failed with rc: %d\n", rc);
88 	}
89 
90 	return rc;
91 }
92 
93 static void
94 get_temp_completion(void *cb_arg, const struct spdk_nvme_cpl *cpl)
95 {
96 	struct dev *dev = cb_arg;
97 
98 	g_outstanding_commands--;
99 
100 	if (spdk_nvme_cpl_is_error(cpl)) {
101 		AER_PRINTF("%s: get feature (temp threshold) failed\n", dev->name);
102 		g_failed = 1;
103 		return;
104 	}
105 
106 	dev->orig_temp_threshold = cpl->cdw0;
107 	AER_PRINTF("%s: original temperature threshold: %u Kelvin (%d Celsius)\n",
108 		   dev->name, dev->orig_temp_threshold, dev->orig_temp_threshold - 273);
109 
110 	g_temperature_done++;
111 }
112 
113 static int
114 get_temp_threshold(struct dev *dev)
115 {
116 	struct spdk_nvme_cmd cmd = {};
117 	int rc;
118 
119 	cmd.opc = SPDK_NVME_OPC_GET_FEATURES;
120 	cmd.cdw10_bits.get_features.fid = SPDK_NVME_FEAT_TEMPERATURE_THRESHOLD;
121 
122 	rc = spdk_nvme_ctrlr_cmd_admin_raw(dev->ctrlr, &cmd, NULL, 0, get_temp_completion, dev);
123 	if (rc == 0) {
124 		g_outstanding_commands++;
125 	}
126 
127 	return rc;
128 }
129 
130 static void
131 print_health_page(struct dev *dev, struct spdk_nvme_health_information_page *hip)
132 {
133 	AER_PRINTF("%s: Current Temperature:         %u Kelvin (%d Celsius)\n",
134 		   dev->name, hip->temperature, hip->temperature - 273);
135 }
136 
137 static void
138 get_health_log_page_completion(void *cb_arg, const struct spdk_nvme_cpl *cpl)
139 {
140 	struct dev *dev = cb_arg;
141 
142 	g_outstanding_commands--;
143 
144 	if (spdk_nvme_cpl_is_error(cpl)) {
145 		AER_PRINTF("%s: get log page failed\n", dev->name);
146 		g_failed = 1;
147 		return;
148 	}
149 
150 	print_health_page(dev, dev->health_page);
151 	g_aer_done++;
152 }
153 
154 static int
155 get_health_log_page(struct dev *dev)
156 {
157 	int rc;
158 
159 	rc = spdk_nvme_ctrlr_cmd_get_log_page(dev->ctrlr, SPDK_NVME_LOG_HEALTH_INFORMATION,
160 					      SPDK_NVME_GLOBAL_NS_TAG, dev->health_page,
161 					      sizeof(*dev->health_page), 0,
162 					      get_health_log_page_completion, dev);
163 
164 	if (rc == 0) {
165 		g_outstanding_commands++;
166 	}
167 
168 	return rc;
169 }
170 
171 static void
172 get_ns_state_test(struct dev *dev, uint32_t nsid)
173 {
174 	bool new_ns_state;
175 
176 	new_ns_state = spdk_nvme_ctrlr_is_active_ns(dev->ctrlr, nsid);
177 	if (new_ns_state == dev->ns_test_active) {
178 		g_failed = 1;
179 	}
180 }
181 
182 static void
183 cleanup(void)
184 {
185 	struct dev *dev;
186 
187 	foreach_dev(dev) {
188 		if (dev->health_page) {
189 			spdk_free(dev->health_page);
190 		}
191 	}
192 }
193 
194 static void
195 aer_cb(void *arg, const struct spdk_nvme_cpl *cpl)
196 {
197 	struct dev				*dev = arg;
198 	uint32_t				log_page_id;
199 	uint32_t				aen_event_info;
200 	uint32_t				aen_event_type;
201 	union spdk_nvme_async_event_completion	aen_cpl;
202 
203 	aen_cpl.raw = cpl->cdw0;
204 	aen_event_info = aen_cpl.bits.async_event_info;
205 	aen_event_type = aen_cpl.bits.async_event_type;
206 	log_page_id = aen_cpl.bits.log_page_identifier;
207 
208 	if (spdk_nvme_cpl_is_error(cpl)) {
209 		AER_FPRINTF(stderr, "%s: AER failed\n", dev->name);
210 		g_failed = 1;
211 		return;
212 	}
213 
214 	AER_PRINTF("%s: aer_cb for log page %d, aen_event_type: 0x%02x, aen_event_info: 0x%02x\n",
215 		   dev->name, log_page_id, aen_event_type, aen_event_info);
216 	/* Temp Test: Verify proper EventType, Event Info and Log Page.
217 	 * NOTE: QEMU NVMe controllers return Spare Below Threshold Status event info
218 	 * instead of Temperate Threshold even info which is why it's used in the check
219 	 * below.
220 	 */
221 	if ((log_page_id == SPDK_NVME_LOG_HEALTH_INFORMATION) && \
222 	    (aen_event_type == SPDK_NVME_ASYNC_EVENT_TYPE_SMART) && \
223 	    ((aen_event_info == SPDK_NVME_ASYNC_EVENT_TEMPERATURE_THRESHOLD) || \
224 	     (aen_event_info == SPDK_NVME_ASYNC_EVENT_SPARE_BELOW_THRESHOLD))) {
225 		/* Set the temperature threshold back to the original value to stop triggering  */
226 		AER_PRINTF("aer_cb - Resetting Temp Threshold for device: %s\n", dev->name);
227 		set_temp_threshold(dev, dev->orig_temp_threshold);
228 		get_health_log_page(dev);
229 	} else if (log_page_id == SPDK_NVME_LOG_CHANGED_NS_LIST) {
230 		AER_PRINTF("aer_cb - Changed Namespace\n");
231 		get_ns_state_test(dev, g_expected_ns_test);
232 		g_aer_done++;
233 	} else {
234 		AER_PRINTF("aer_cb - Unknown Log Page\n");
235 	}
236 }
237 
238 static void
239 usage(const char *program_name)
240 {
241 	AER_PRINTF("%s [options]", program_name);
242 	AER_PRINTF("\n");
243 	AER_PRINTF("options:\n");
244 	AER_PRINTF(" -g         use single file descriptor for DPDK memory segments]\n");
245 	AER_PRINTF(" -T         enable temperature tests\n");
246 	AER_PRINTF(" -n         expected Namespace attribute notice ID\n");
247 	AER_PRINTF(" -t <file>  touch specified file when ready to receive AER\n");
248 	AER_PRINTF(" -r trid    remote NVMe over Fabrics target address\n");
249 	AER_PRINTF("    Format: 'key:value [key:value] ...'\n");
250 	AER_PRINTF("    Keys:\n");
251 	AER_PRINTF("     trtype      Transport type (e.g. RDMA)\n");
252 	AER_PRINTF("     adrfam      Address family (e.g. IPv4, IPv6)\n");
253 	AER_PRINTF("     traddr      Transport address (e.g. 192.168.100.8)\n");
254 	AER_PRINTF("     trsvcid     Transport service identifier (e.g. 4420)\n");
255 	AER_PRINTF("     subnqn      Subsystem NQN (default: %s)\n", SPDK_NVMF_DISCOVERY_NQN);
256 	AER_PRINTF("    Example: -r 'trtype:RDMA adrfam:IPv4 traddr:192.168.100.8 trsvcid:4420'\n");
257 
258 	spdk_log_usage(stdout, "-L");
259 
260 	AER_PRINTF(" -i <id>    shared memory group ID\n");
261 	AER_PRINTF(" -m         Multi-Process AER Test (only with Temp Test)\n");
262 	AER_PRINTF(" -H         show this usage\n");
263 }
264 
265 static int
266 parse_args(int argc, char **argv, struct spdk_env_opts *env_opts)
267 {
268 	int op, rc;
269 	long int val;
270 
271 	spdk_nvme_trid_populate_transport(&g_trid, SPDK_NVME_TRANSPORT_PCIE);
272 	snprintf(g_trid.subnqn, sizeof(g_trid.subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN);
273 
274 	while ((op = getopt(argc, argv, "gi:mn:r:t:HL:T")) != -1) {
275 		switch (op) {
276 		case 'n':
277 			val = spdk_strtol(optarg, 10);
278 			if (val < 0) {
279 				AER_FPRINTF(stderr, "Invalid NS attribute notice ID\n");
280 				return val;
281 			}
282 			g_expected_ns_test = (uint32_t)val;
283 			break;
284 		case 'g':
285 			env_opts->hugepage_single_segments = true;
286 			break;
287 		case 'r':
288 			if (spdk_nvme_transport_id_parse(&g_trid, optarg) != 0) {
289 				AER_FPRINTF(stderr, "Error parsing transport address\n");
290 				return 1;
291 			}
292 			break;
293 		case 't':
294 			g_touch_file = optarg;
295 			break;
296 		case 'L':
297 			rc = spdk_log_set_flag(optarg);
298 			if (rc < 0) {
299 				AER_FPRINTF(stderr, "unknown flag\n");
300 				usage(argv[0]);
301 				exit(EXIT_FAILURE);
302 			}
303 #ifdef DEBUG
304 			spdk_log_set_print_level(SPDK_LOG_DEBUG);
305 #endif
306 			break;
307 		case 'T':
308 			g_enable_temp_test = 1;
309 			break;
310 		case 'H':
311 			usage(argv[0]);
312 			exit(EXIT_SUCCESS);
313 		case 'i':
314 			env_opts->shm_id = spdk_strtol(optarg, 10);
315 			if (env_opts->shm_id < 0) {
316 				AER_FPRINTF(stderr, "Invalid shared memory ID\n");
317 				return env_opts->shm_id;
318 			}
319 			break;
320 		case 'm':
321 			g_multi_process_test = 1;
322 			break;
323 		default:
324 			usage(argv[0]);
325 			return 1;
326 		}
327 	}
328 
329 	return 0;
330 }
331 
332 static bool
333 probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
334 	 struct spdk_nvme_ctrlr_opts *opts)
335 {
336 	AER_PRINTF("Attaching to %s\n", trid->traddr);
337 
338 	return true;
339 }
340 
341 static void
342 attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid,
343 	  struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts)
344 {
345 	struct dev *dev;
346 
347 	/* add to dev list */
348 	dev = &g_devs[g_num_devs++];
349 
350 	dev->ctrlr = ctrlr;
351 
352 	snprintf(dev->name, sizeof(dev->name), "%s",
353 		 trid->traddr);
354 
355 	AER_PRINTF("Attached to %s\n", dev->name);
356 
357 	dev->health_page = spdk_zmalloc(sizeof(*dev->health_page), 4096, NULL, SPDK_ENV_LCORE_ID_ANY,
358 					SPDK_MALLOC_DMA);
359 	if (dev->health_page == NULL) {
360 		AER_PRINTF("Allocation error (health page)\n");
361 		g_failed = 1;
362 	}
363 }
364 
365 static void
366 get_feature_test_cb(void *cb_arg, const struct spdk_nvme_cpl *cpl)
367 {
368 	struct dev *dev = cb_arg;
369 
370 	g_outstanding_commands--;
371 
372 	if (spdk_nvme_cpl_is_error(cpl)) {
373 		AER_PRINTF("%s: get number of queues failed\n", dev->name);
374 		g_failed = 1;
375 		return;
376 	}
377 
378 	if (g_aer_done < g_num_devs) {
379 		/*
380 		 * Resubmit Get Features command to continue filling admin queue
381 		 * while the test is running.
382 		 */
383 		get_feature_test(dev);
384 	}
385 }
386 
387 static void
388 get_feature_test(struct dev *dev)
389 {
390 	struct spdk_nvme_cmd cmd;
391 
392 	memset(&cmd, 0, sizeof(cmd));
393 	cmd.opc = SPDK_NVME_OPC_GET_FEATURES;
394 	cmd.cdw10_bits.get_features.fid = SPDK_NVME_FEAT_NUMBER_OF_QUEUES;
395 	if (spdk_nvme_ctrlr_cmd_admin_raw(dev->ctrlr, &cmd, NULL, 0,
396 					  get_feature_test_cb, dev) != 0) {
397 		AER_PRINTF("Failed to send Get Features command for dev=%p\n", dev);
398 		g_failed = 1;
399 		return;
400 	}
401 
402 	g_outstanding_commands++;
403 }
404 
405 static int
406 spdk_aer_temperature_test(void)
407 {
408 	struct dev *dev;
409 
410 	AER_PRINTF("Getting orig temperature thresholds of all controllers\n");
411 	foreach_dev(dev) {
412 		/* Get the original temperature threshold */
413 		get_temp_threshold(dev);
414 	}
415 
416 	while (!g_failed && g_temperature_done < g_num_devs) {
417 		foreach_dev(dev) {
418 			spdk_nvme_ctrlr_process_admin_completions(dev->ctrlr);
419 		}
420 	}
421 
422 	if (g_failed) {
423 		return g_failed;
424 	}
425 	g_temperature_done = 0;
426 	g_aer_done = 0;
427 
428 	/* Send admin commands to test admin queue wraparound while waiting for the AER */
429 	foreach_dev(dev) {
430 		get_feature_test(dev);
431 	}
432 
433 	if (g_failed) {
434 		return g_failed;
435 	}
436 
437 	/* Only single process needs to set and verify lower threshold */
438 	if (g_parent_process) {
439 		/* Wait until child has init'd and ready for test to continue */
440 		if (g_multi_process_test) {
441 			sem_wait(g_sem_child_id);
442 		}
443 		AER_PRINTF("Setting all controllers temperature threshold low to trigger AER\n");
444 		foreach_dev(dev) {
445 			/* Set the temperature threshold to a low value */
446 			set_temp_threshold(dev, 200);
447 		}
448 
449 		AER_PRINTF("Waiting for all controllers temperature threshold to be set lower\n");
450 		while (!g_failed && (g_temperature_done < g_num_devs)) {
451 			foreach_dev(dev) {
452 				spdk_nvme_ctrlr_process_admin_completions(dev->ctrlr);
453 			}
454 		}
455 		g_temperature_done = 0;
456 
457 		if (g_failed) {
458 			return g_failed;
459 		}
460 	}
461 
462 	AER_PRINTF("Waiting for all controllers to trigger AER and reset threshold\n");
463 	/* Let parent know init is done and it's okay to continue */
464 	if (!g_parent_process) {
465 		sem_post(g_sem_child_id);
466 	}
467 	/* Waiting for AEN to be occur here */
468 	while (!g_failed && (g_aer_done < g_num_devs || g_temperature_done < g_num_devs)) {
469 		foreach_dev(dev) {
470 			spdk_nvme_ctrlr_process_admin_completions(dev->ctrlr);
471 		}
472 	}
473 
474 	if (g_failed) {
475 		return g_failed;
476 	}
477 
478 	return 0;
479 }
480 
481 static int
482 spdk_aer_changed_ns_test(void)
483 {
484 	struct dev *dev;
485 
486 	g_aer_done = 0;
487 
488 	AER_PRINTF("Starting namespace attribute notice tests for all controllers...\n");
489 
490 	foreach_dev(dev) {
491 		get_feature_test(dev);
492 		dev->ns_test_active = spdk_nvme_ctrlr_is_active_ns(dev->ctrlr, g_expected_ns_test);
493 	}
494 
495 	if (g_failed) {
496 		return g_failed;
497 	}
498 
499 	while (!g_failed && (g_aer_done < g_num_devs)) {
500 		foreach_dev(dev) {
501 			spdk_nvme_ctrlr_process_admin_completions(dev->ctrlr);
502 		}
503 	}
504 
505 	if (g_failed) {
506 		return g_failed;
507 	}
508 
509 	return 0;
510 }
511 
512 static int
513 setup_multi_process(void)
514 {
515 	pid_t pid;
516 	int rc = 0;
517 
518 	/* If AEN test was killed, remove named semaphore to start again */
519 	rc = sem_unlink(g_sem_init_name);
520 	if (rc < 0 && errno != ENOENT) {
521 		AER_FPRINTF(stderr, "Init semaphore removal failure: %s", spdk_strerror(errno));
522 		return rc;
523 	}
524 	rc = sem_unlink(g_sem_child_name);
525 	if (rc < 0 && errno != ENOENT) {
526 		AER_FPRINTF(stderr, "Child semaphore removal failure: %s", spdk_strerror(errno));
527 		return rc;
528 	}
529 	pid = fork();
530 	if (pid == -1) {
531 		perror("Failed to fork\n");
532 		return -1;
533 	} else if (pid == 0) {
534 		AER_PRINTF("Child process pid: %d\n", getpid());
535 		g_parent_process = false;
536 		g_sem_init_id = sem_open(g_sem_init_name, O_CREAT, 0600, 0);
537 		g_sem_child_id = sem_open(g_sem_child_name, O_CREAT, 0600, 0);
538 		if ((g_sem_init_id == SEM_FAILED) || (g_sem_child_id == SEM_FAILED)) {
539 			AER_FPRINTF(stderr, "Sem Open failed for child: %s\n",
540 				    spdk_strerror(errno));
541 			return -1;
542 		}
543 	}
544 	/* Parent process */
545 	else {
546 		g_parent_process = true;
547 		g_sem_init_id = sem_open(g_sem_init_name, O_CREAT, 0600, 0);
548 		g_sem_child_id = sem_open(g_sem_child_name, O_CREAT, 0600, 0);
549 		if ((g_sem_init_id == SEM_FAILED) || (g_sem_child_id == SEM_FAILED)) {
550 			AER_FPRINTF(stderr, "Sem Open failed for parent: %s\n",
551 				    spdk_strerror(errno));
552 			return -1;
553 		}
554 	}
555 	return 0;
556 }
557 
558 int
559 main(int argc, char **argv)
560 {
561 	struct dev		*dev;
562 	struct spdk_env_opts	opts;
563 	int			rc;
564 	struct spdk_nvme_detach_ctx *detach_ctx = NULL;
565 
566 	spdk_env_opts_init(&opts);
567 	rc = parse_args(argc, argv, &opts);
568 	if (rc != 0) {
569 		return rc;
570 	}
571 
572 	if (g_multi_process_test)  {
573 		/* Multi-Process test only available with Temp Test */
574 		if (!g_enable_temp_test) {
575 			AER_FPRINTF(stderr, "Multi Process only available with Temp Test (-T)\n");
576 			return 1;
577 		}
578 		if (opts.shm_id < 0) {
579 			AER_FPRINTF(stderr, "Multi Process requires shared memory id (-i <id>)\n");
580 			return 1;
581 		}
582 		rc = setup_multi_process();
583 		if (rc != 0) {
584 			AER_FPRINTF(stderr, "Multi Process test failed to setup\n");
585 			return rc;
586 		}
587 	} else {
588 		/* Only one process in test, set it to the parent process */
589 		g_parent_process = true;
590 	}
591 	opts.name = "aer";
592 	if (g_parent_process) {
593 		opts.core_mask = "0x1";
594 	} else {
595 		opts.core_mask = "0x2";
596 	}
597 
598 	/*
599 	 * For multi-process test, parent (primary) and child (secondary) processes
600 	 * will execute all following code but DPDK setup is serialized
601 	 */
602 	if (!g_parent_process) {
603 		if (sem_wait(g_sem_init_id) < 0) {
604 			AER_FPRINTF(stderr, "sem_wait failed for child process\n");
605 			return (-1);
606 		}
607 	}
608 	if (spdk_env_init(&opts) < 0) {
609 		AER_FPRINTF(stderr, "Unable to initialize SPDK env\n");
610 		return 1;
611 	}
612 
613 	AER_PRINTF("Asynchronous Event Request test\n");
614 
615 	if (spdk_nvme_probe(&g_trid, NULL, probe_cb, attach_cb, NULL) != 0) {
616 		AER_FPRINTF(stderr, "spdk_nvme_probe() failed\n");
617 		return 1;
618 	}
619 
620 	if (g_num_devs == 0) {
621 		AER_FPRINTF(stderr, "No controllers found - exiting\n");
622 		g_failed = 1;
623 	}
624 	if (g_failed) {
625 		goto done;
626 	}
627 
628 	if (g_parent_process && g_enable_temp_test) {
629 		AER_PRINTF("Reset controller to setup AER completions for this process\n");
630 		foreach_dev(dev) {
631 			if (spdk_nvme_ctrlr_reset(dev->ctrlr) < 0) {
632 				AER_FPRINTF(stderr, "nvme reset failed.\n");
633 				return -1;
634 			}
635 		}
636 	}
637 	if (g_parent_process && g_multi_process_test) {
638 		/* Primary can release child/secondary for init now */
639 		sem_post(g_sem_init_id);
640 	}
641 
642 	AER_PRINTF("Registering asynchronous event callbacks...\n");
643 	foreach_dev(dev) {
644 		spdk_nvme_ctrlr_register_aer_callback(dev->ctrlr, aer_cb, dev);
645 	}
646 
647 	if (g_touch_file) {
648 		int fd;
649 
650 		fd = open(g_touch_file, O_CREAT | O_EXCL | O_RDWR, S_IFREG);
651 		if (fd == -1) {
652 			AER_FPRINTF(stderr, "Could not touch %s (%s).\n", g_touch_file,
653 				    strerror(errno));
654 			g_failed = true;
655 			goto done;
656 		}
657 		close(fd);
658 	}
659 
660 	/* AER temperature test */
661 	if (g_enable_temp_test) {
662 		if (spdk_aer_temperature_test()) {
663 			goto done;
664 		}
665 	}
666 
667 	/* AER changed namespace list test */
668 	if (g_expected_ns_test) {
669 		if (spdk_aer_changed_ns_test()) {
670 			goto done;
671 		}
672 	}
673 
674 	AER_PRINTF("Cleaning up...\n");
675 
676 	while (g_outstanding_commands) {
677 		foreach_dev(dev) {
678 			spdk_nvme_ctrlr_process_admin_completions(dev->ctrlr);
679 		}
680 	}
681 
682 	/* Only one process cleans up at a time - let child go first */
683 	if (g_multi_process_test && g_parent_process) {
684 		/* Parent waits for child to clean up before executing clean up process */
685 		sem_wait(g_sem_child_id);
686 	}
687 	/* unregister AER callback so we don't fail on aborted AERs when we close out qpairs. */
688 	foreach_dev(dev) {
689 		spdk_nvme_ctrlr_register_aer_callback(dev->ctrlr, NULL, NULL);
690 	}
691 
692 	foreach_dev(dev) {
693 		spdk_nvme_detach_async(dev->ctrlr, &detach_ctx);
694 	}
695 
696 	if (detach_ctx) {
697 		spdk_nvme_detach_poll(detach_ctx);
698 	}
699 
700 	/* Release semaphore to allow parent to cleanup */
701 	if (!g_parent_process) {
702 		sem_post(g_sem_child_id);
703 		sem_wait(g_sem_init_id);
704 	}
705 done:
706 	cleanup();
707 
708 	/* Wait for child process to finish and verify it finished correctly before detaching
709 	 * resources */
710 	if (g_multi_process_test && g_parent_process) {
711 		int status;
712 		sem_post(g_sem_init_id);
713 		wait(&status);
714 		if (WIFEXITED(status)) {
715 			/* Child ended normally */
716 			if (WEXITSTATUS(status) != 0) {
717 				AER_FPRINTF(stderr, "Child Failed with status: %d.\n",
718 					    (int8_t)(WEXITSTATUS(status)));
719 				g_failed = true;
720 			}
721 		}
722 		if (sem_close(g_sem_init_id) != 0) {
723 			perror("sem_close Failed for init\n");
724 			g_failed = true;
725 		}
726 		if (sem_close(g_sem_child_id) != 0) {
727 			perror("sem_close Failed for child\n");
728 			g_failed = true;
729 		}
730 
731 		if (sem_unlink(g_sem_init_name) != 0) {
732 			perror("sem_unlink Failed for init\n");
733 			g_failed = true;
734 		}
735 		if (sem_unlink(g_sem_child_name) != 0) {
736 			perror("sem_unlink Failed for child\n");
737 			g_failed = true;
738 		}
739 	}
740 	return g_failed;
741 }
742