1.\" 2.\" Automated Testing Framework (atf) 3.\" 4.\" Copyright (c) 2008, 2010 The NetBSD Foundation, Inc. 5.\" All rights reserved. 6.\" 7.\" Redistribution and use in source and binary forms, with or without 8.\" modification, are permitted provided that the following conditions 9.\" are met: 10.\" 1. Redistributions of source code must retain the above copyright 11.\" notice, this list of conditions and the following disclaimer. 12.\" 2. Redistributions in binary form must reproduce the above copyright 13.\" notice, this list of conditions and the following disclaimer in the 14.\" documentation and/or other materials provided with the distribution. 15.\" 16.\" THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND 17.\" CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, 18.\" INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 19.\" MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20.\" IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS BE LIABLE FOR ANY 21.\" DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 23.\" GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24.\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 25.\" IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 26.\" OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 27.\" IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28.\" 29.Dd December 10, 2010 30.Dt ATF-C++-API 3 31.Os 32.Sh NAME 33.Nm ATF_ADD_TEST_CASE , 34.Nm ATF_CHECK_ERRNO , 35.Nm ATF_FAIL , 36.Nm ATF_INIT_TEST_CASES , 37.Nm ATF_PASS , 38.Nm ATF_REQUIRE , 39.Nm ATF_REQUIRE_EQ , 40.Nm ATF_REQUIRE_ERRNO , 41.Nm ATF_REQUIRE_IN , 42.Nm ATF_REQUIRE_MATCH , 43.Nm ATF_REQUIRE_NOT_IN , 44.Nm ATF_REQUIRE_THROW , 45.Nm ATF_REQUIRE_THROW_RE , 46.Nm ATF_SKIP , 47.Nm ATF_TEST_CASE , 48.Nm ATF_TEST_CASE_BODY , 49.Nm ATF_TEST_CASE_CLEANUP , 50.Nm ATF_TEST_CASE_HEAD , 51.Nm ATF_TEST_CASE_WITH_CLEANUP , 52.Nm ATF_TEST_CASE_WITHOUT_HEAD , 53.Nd C++ API to write ATF-based test programs 54.Sh SYNOPSIS 55.In atf-c++.hpp 56.Fn ATF_ADD_TEST_CASE "tcs" "name" 57.Fn ATF_CHECK_ERRNO "exp_errno" "bool_expression" 58.Fn ATF_FAIL "reason" 59.Fn ATF_INIT_TEST_CASES "tcs" 60.Fn ATF_PASS 61.Fn ATF_REQUIRE "expression" 62.Fn ATF_REQUIRE_EQ "expression_1" "expression_2" 63.Fn ATF_REQUIRE_ERRNO "exp_errno" "bool_expression" 64.Fn ATF_REQUIRE_IN "element" "collection" 65.Fn ATF_REQUIRE_MATCH "regexp" "string_expression" 66.Fn ATF_REQUIRE_NOT_IN "element" "collection" 67.Fn ATF_REQUIRE_THROW "expected_exception" "statement" 68.Fn ATF_REQUIRE_THROW_RE "expected_exception" "regexp" "statement" 69.Fn ATF_SKIP "reason" 70.Fn ATF_TEST_CASE "name" 71.Fn ATF_TEST_CASE_BODY "name" 72.Fn ATF_TEST_CASE_CLEANUP "name" 73.Fn ATF_TEST_CASE_HEAD "name" 74.Fn ATF_TEST_CASE_WITH_CLEANUP "name" 75.Fn ATF_TEST_CASE_WITHOUT_HEAD "name" 76.Sh DESCRIPTION 77ATF provides a mostly-macro-based programming interface to implement test 78programs in C or C++. 79This interface is backed by a C++ implementation, but this fact is 80hidden from the developer as much as possible through the use of 81macros to simplify programming. 82However, the use of C++ is not hidden everywhere and while you can 83implement test cases without knowing anything at all about the object model 84underneath the provided calls, you might need some minimum notions of the 85language in very specific circumstances. 86.Pp 87C++-based test programs always follow this template: 88.Bd -literal -offset indent 89extern "C" { 90.Ns ... C-specific includes go here ... 91} 92 93.Ns ... C++-specific includes go here ... 94 95#include <atf-c++.hpp> 96 97ATF_TEST_CASE(tc1); 98ATF_TEST_CASE_HEAD(tc1) 99{ 100 ... first test case's header ... 101} 102ATF_TEST_CASE_BODY(tc1) 103{ 104 ... first test case's body ... 105} 106 107ATF_TEST_CASE_WITH_CLEANUP(tc2); 108ATF_TEST_CASE_HEAD(tc2) 109{ 110 ... second test case's header ... 111} 112ATF_TEST_CASE_BODY(tc2) 113{ 114 ... second test case's body ... 115} 116ATF_TEST_CASE_CLEANUP(tc2) 117{ 118 ... second test case's cleanup ... 119} 120 121ATF_TEST_CASE(tc3); 122ATF_TEST_CASE_BODY(tc3) 123{ 124 ... third test case's body ... 125} 126 127.Ns ... additional test cases ... 128 129ATF_INIT_TEST_CASES(tcs) 130{ 131 ATF_ADD_TEST_CASE(tcs, tc1); 132 ATF_ADD_TEST_CASE(tcs, tc2); 133 ATF_ADD_TEST_CASE(tcs, tc3); 134 ... add additional test cases ... 135} 136.Ed 137.Ss Definition of test cases 138Test cases have an identifier and are composed of three different parts: 139the header, the body and an optional cleanup routine, all of which are 140described in 141.Xr atf-test-case 4 . 142To define test cases, one can use the 143.Fn ATF_TEST_CASE , 144.Fn ATF_TEST_CASE_WITH_CLEANUP 145or the 146.Fn ATF_TEST_CASE_WITHOUT_HEAD 147macros, which take a single parameter specifiying the test case's 148name. 149.Fn ATF_TEST_CASE , 150requires to define a head and a body for the test case, 151.Fn ATF_TEST_CASE_WITH_CLEANUP 152requires to define a head, a body and a cleanup for the test case and 153.Fn ATF_TEST_CASE_WITHOUT_HEAD 154requires only a body for the test case. 155It is important to note that these 156.Em do not 157set the test case up for execution when the program is run. 158In order to do so, a later registration is needed through the 159.Fn ATF_ADD_TEST_CASE 160macro detailed in 161.Sx Program initialization . 162.Pp 163Later on, one must define the three parts of the body by means of three 164functions. 165Their headers are given by the 166.Fn ATF_TEST_CASE_HEAD , 167.Fn ATF_TEST_CASE_BODY 168and 169.Fn ATF_TEST_CASE_CLEANUP 170macros, all of which take the test case's name. 171Following each of these, a block of code is expected, surrounded by the 172opening and closing brackets. 173.Ss Program initialization 174The library provides a way to easily define the test program's 175.Fn main 176function. 177You should never define one on your own, but rely on the 178library to do it for you. 179This is done by using the 180.Fn ATF_INIT_TEST_CASES 181macro, which is passed the name of the list that will hold the test cases. 182This name can be whatever you want as long as it is a valid variable value. 183.Pp 184After the macro, you are supposed to provide the body of a function, which 185should only use the 186.Fn ATF_ADD_TEST_CASE 187macro to register the test cases the test program will execute. 188The first parameter of this macro matches the name you provided in the 189former call. 190.Ss Header definitions 191The test case's header can define the meta-data by using the 192.Fn set 193method, which takes two parameters: the first one specifies the 194meta-data variable to be set and the second one specifies its value. 195Both of them are strings. 196.Ss Configuration variables 197The test case has read-only access to the current configuration variables 198by means of the 199.Ft bool 200.Fn has_config_var 201and the 202.Ft std::string 203.Fn get_config_var 204methods, which can be called in any of the three parts of a test case. 205.Ss Access to the source directory 206It is possible to get the path to the test case's source directory from any 207of its three components by querying the 208.Sq srcdir 209configuration variable. 210.Ss Requiring programs 211Aside from the 212.Va require.progs 213meta-data variable available in the header only, one can also check for 214additional programs in the test case's body by using the 215.Fn require_prog 216function, which takes the base name or full path of a single binary. 217Relative paths are forbidden. 218If it is not found, the test case will be automatically skipped. 219.Ss Test case finalization 220The test case finalizes either when the body reaches its end, at which 221point the test is assumed to have 222.Em passed , 223or at any explicit call to 224.Fn ATF_PASS , 225.Fn ATF_FAIL 226or 227.Fn ATF_SKIP . 228These three macros terminate the execution of the test case immediately. 229The cleanup routine will be processed afterwards in a completely automated 230way, regardless of the test case's termination reason. 231.Pp 232.Fn ATF_PASS 233does not take any parameters. 234.Fn ATF_FAIL 235and 236.Fn ATF_SKIP 237take a single string that describes why the test case failed or 238was skipped, respectively. 239It is very important to provide a clear error message in both cases so that 240the user can quickly know why the test did not pass. 241.Ss Expectations 242Everything explained in the previous section changes when the test case 243expectations are redefined by the programmer. 244.Pp 245Each test case has an internal state called 246.Sq expect 247that describes what the test case expectations are at any point in time. 248The value of this property can change during execution by any of: 249.Bl -tag -width indent 250.It Fn expect_death "reason" 251Expects the test case to exit prematurely regardless of the nature of the 252exit. 253.It Fn expect_exit "exitcode" "reason" 254Expects the test case to exit cleanly. 255If 256.Va exitcode 257is not 258.Sq -1 , 259.Xr atf-run 1 260will validate that the exit code of the test case matches the one provided 261in this call. 262Otherwise, the exact value will be ignored. 263.It Fn expect_fail "reason" 264Any failure (be it fatal or non-fatal) raised in this mode is recorded. 265However, such failures do not report the test case as failed; instead, the 266test case finalizes cleanly and is reported as 267.Sq expected failure ; 268this report includes the provided 269.Fa reason 270as part of it. 271If no error is raised while running in this mode, then the test case is 272reported as 273.Sq failed . 274.Pp 275This mode is useful to reproduce actual known bugs in tests. 276Whenever the developer fixes the bug later on, the test case will start 277reporting a failure, signaling the developer that the test case must be 278adjusted to the new conditions. 279In this situation, it is useful, for example, to set 280.Fa reason 281as the bug number for tracking purposes. 282.It Fn expect_pass 283This is the normal mode of execution. 284In this mode, any failure is reported as such to the user and the test case 285is marked as 286.Sq failed . 287.It Fn expect_race "reason" 288Any failure or timeout during the execution of the test case will be 289considered as if a race condition has been triggered and reported as such. 290If no problems arise, the test will continue execution as usual. 291.It Fn expect_signal "signo" "reason" 292Expects the test case to terminate due to the reception of a signal. 293If 294.Va signo 295is not 296.Sq -1 , 297.Xr atf-run 1 298will validate that the signal that terminated the test case matches the one 299provided in this call. 300Otherwise, the exact value will be ignored. 301.It Fn expect_timeout "reason" 302Expects the test case to execute for longer than its timeout. 303.El 304.Ss Helper macros for common checks 305The library provides several macros that are very handy in multiple 306situations. 307These basically check some condition after executing a given statement or 308processing a given expression and, if the condition is not met, they 309automatically call 310.Fn ATF_FAIL 311with an appropriate error message. 312.Pp 313.Fn ATF_REQUIRE 314takes an expression and raises a failure if it evaluates to false. 315.Pp 316.Fn ATF_REQUIRE_EQ 317takes two expressions and raises a failure if the two do not evaluate to 318the same exact value. 319.Pp 320.Fn ATF_REQUIRE_IN 321takes an element and a collection and validates that the element is present in 322the collection. 323.Pp 324.Fn ATF_REQUIRE_MATCH 325takes a regular expression and a string and raises a failure if the regular 326expression does not match the string. 327.Pp 328.Fn ATF_REQUIRE_NOT_IN 329takes an element and a collection and validates that the element is not present 330in the collection. 331.Pp 332.Fn ATF_REQUIRE_THROW 333takes the name of an exception and a statement and raises a failure if 334the statement does not throw the specified exception. 335.Fn ATF_REQUIRE_THROW_EQ 336takes the name of an exception, a regular expresion and a statement and raises a 337failure if the statement does not throw the specified exception and if the 338message of the exception does not match the regular expression. 339.Pp 340.Fn ATF_CHECK_ERRNO 341and 342.Fn ATF_REQUIRE_ERRNO 343take, first, the error code that the check is expecting to find in the 344.Va errno 345variable and, second, a boolean expression that, if evaluates to true, 346means that a call failed and 347.Va errno 348has to be checked against the first value. 349.Sh EXAMPLES 350The following shows a complete test program with a single test case that 351validates the addition operator: 352.Bd -literal -offset indent 353#include <atf-c++.hpp> 354 355ATF_TEST_CASE(addition); 356ATF_TEST_CASE_HEAD(addition) 357{ 358 set("descr", "Sample tests for the addition operator"); 359} 360ATF_TEST_CASE_BODY(addition) 361{ 362 ATF_REQUIRE_EQ(0 + 0, 0); 363 ATF_REQUIRE_EQ(0 + 1, 1); 364 ATF_REQUIRE_EQ(1 + 0, 1); 365 366 ATF_REQUIRE_EQ(1 + 1, 2); 367 368 ATF_REQUIRE_EQ(100 + 200, 300); 369} 370 371ATF_TEST_CASE(open_failure); 372ATF_TEST_CASE_HEAD(open_failure) 373{ 374 set("descr", "Sample tests for the open function"); 375} 376ATF_TEST_CASE_BODY(open_failure) 377{ 378 ATF_REQUIRE_ERRNO(ENOENT, open("non-existent", O_RDONLY) == -1); 379} 380 381ATF_TEST_CASE(known_bug); 382ATF_TEST_CASE_HEAD(known_bug) 383{ 384 set("descr", "Reproduces a known bug"); 385} 386ATF_TEST_CASE_BODY(known_bug) 387{ 388 expect_fail("See bug number foo/bar"); 389 ATF_REQUIRE_EQ(3, 1 + 1); 390 expect_pass(); 391 ATF_REQUIRE_EQ(3, 1 + 2); 392} 393 394ATF_INIT_TEST_CASES(tcs) 395{ 396 ATF_ADD_TEST_CASE(tcs, addition); 397 ATF_ADD_TEST_CASE(tcs, open_failure); 398 ATF_ADD_TEST_CASE(tcs, known_bug); 399} 400.Ed 401.Sh SEE ALSO 402.Xr atf-test-program 1 , 403.Xr atf-test-case 4 , 404.Xr atf 7 405