1 //===- unittests/Basic/SarifTest.cpp - Test writing SARIF documents -------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "clang/Basic/Sarif.h" 10 #include "clang/Basic/DiagnosticOptions.h" 11 #include "clang/Basic/FileManager.h" 12 #include "clang/Basic/FileSystemOptions.h" 13 #include "clang/Basic/LangOptions.h" 14 #include "clang/Basic/SourceLocation.h" 15 #include "clang/Basic/SourceManager.h" 16 #include "llvm/ADT/StringRef.h" 17 #include "llvm/Support/FormatVariadic.h" 18 #include "llvm/Support/JSON.h" 19 #include "llvm/Support/MemoryBuffer.h" 20 #include "llvm/Support/VirtualFileSystem.h" 21 #include "llvm/Support/raw_ostream.h" 22 #include "gmock/gmock.h" 23 #include "gtest/gtest.h" 24 25 #include <algorithm> 26 27 using namespace clang; 28 29 namespace { 30 31 using LineCol = std::pair<unsigned int, unsigned int>; 32 33 static std::string serializeSarifDocument(llvm::json::Object &&Doc) { 34 std::string Output; 35 llvm::json::Value Value(std::move(Doc)); 36 llvm::raw_string_ostream OS{Output}; 37 OS << llvm::formatv("{0}", Value); 38 return Output; 39 } 40 41 class SarifDocumentWriterTest : public ::testing::Test { 42 protected: 43 SarifDocumentWriterTest() 44 : InMemoryFileSystem(new llvm::vfs::InMemoryFileSystem), 45 FileMgr(FileSystemOptions(), InMemoryFileSystem), 46 DiagID(new DiagnosticIDs()), DiagOpts(new DiagnosticOptions()), 47 Diags(DiagID, DiagOpts.get(), new IgnoringDiagConsumer()), 48 SourceMgr(Diags, FileMgr) {} 49 50 IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> InMemoryFileSystem; 51 FileManager FileMgr; 52 IntrusiveRefCntPtr<DiagnosticIDs> DiagID; 53 IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts; 54 DiagnosticsEngine Diags; 55 SourceManager SourceMgr; 56 LangOptions LangOpts; 57 58 FileID registerSource(llvm::StringRef Name, const char *SourceText, 59 bool IsMainFile = false) { 60 std::unique_ptr<llvm::MemoryBuffer> SourceBuf = 61 llvm::MemoryBuffer::getMemBuffer(SourceText); 62 FileEntryRef SourceFile = 63 FileMgr.getVirtualFileRef(Name, SourceBuf->getBufferSize(), 0); 64 SourceMgr.overrideFileContents(SourceFile, std::move(SourceBuf)); 65 FileID FID = SourceMgr.getOrCreateFileID(SourceFile, SrcMgr::C_User); 66 if (IsMainFile) 67 SourceMgr.setMainFileID(FID); 68 return FID; 69 } 70 71 CharSourceRange getFakeCharSourceRange(FileID FID, LineCol Begin, 72 LineCol End) { 73 auto BeginLoc = SourceMgr.translateLineCol(FID, Begin.first, Begin.second); 74 auto EndLoc = SourceMgr.translateLineCol(FID, End.first, End.second); 75 return CharSourceRange{SourceRange{BeginLoc, EndLoc}, /* ITR = */ false}; 76 } 77 }; 78 79 TEST_F(SarifDocumentWriterTest, canCreateEmptyDocument) { 80 // GIVEN: 81 SarifDocumentWriter Writer{SourceMgr}; 82 83 // WHEN: 84 const llvm::json::Object &EmptyDoc = Writer.createDocument(); 85 std::vector<StringRef> Keys(EmptyDoc.size()); 86 std::transform(EmptyDoc.begin(), EmptyDoc.end(), Keys.begin(), 87 [](auto Item) { return Item.getFirst(); }); 88 89 // THEN: 90 ASSERT_THAT(Keys, testing::UnorderedElementsAre("$schema", "version")); 91 } 92 93 // Test that a newly inserted run will associate correct tool names 94 TEST_F(SarifDocumentWriterTest, canCreateDocumentWithOneRun) { 95 // GIVEN: 96 SarifDocumentWriter Writer{SourceMgr}; 97 const char *ShortName = "sariftest"; 98 const char *LongName = "sarif writer test"; 99 100 // WHEN: 101 Writer.createRun(ShortName, LongName); 102 Writer.endRun(); 103 const llvm::json::Object &Doc = Writer.createDocument(); 104 const llvm::json::Array *Runs = Doc.getArray("runs"); 105 106 // THEN: 107 // A run was created 108 ASSERT_THAT(Runs, testing::NotNull()); 109 110 // It is the only run 111 ASSERT_EQ(Runs->size(), 1UL); 112 113 // The tool associated with the run was the tool 114 const llvm::json::Object *Driver = 115 Runs->begin()->getAsObject()->getObject("tool")->getObject("driver"); 116 ASSERT_THAT(Driver, testing::NotNull()); 117 118 ASSERT_TRUE(Driver->getString("name").has_value()); 119 ASSERT_TRUE(Driver->getString("fullName").has_value()); 120 ASSERT_TRUE(Driver->getString("language").has_value()); 121 122 EXPECT_EQ(*Driver->getString("name"), ShortName); 123 EXPECT_EQ(*Driver->getString("fullName"), LongName); 124 EXPECT_EQ(*Driver->getString("language"), "en-US"); 125 } 126 127 TEST_F(SarifDocumentWriterTest, addingResultsWillCrashIfThereIsNoRun) { 128 #if defined(NDEBUG) || !GTEST_HAS_DEATH_TEST 129 GTEST_SKIP() << "This death test is only available for debug builds."; 130 #endif 131 // GIVEN: 132 SarifDocumentWriter Writer{SourceMgr}; 133 134 // WHEN: 135 // A SarifDocumentWriter::createRun(...) was not called prior to 136 // SarifDocumentWriter::appendResult(...) 137 // But a rule exists 138 auto RuleIdx = Writer.createRule(SarifRule::create()); 139 const SarifResult &EmptyResult = SarifResult::create(RuleIdx); 140 141 // THEN: 142 auto Matcher = ::testing::AnyOf( 143 ::testing::HasSubstr("create a run first"), 144 ::testing::HasSubstr("no runs associated with the document")); 145 ASSERT_DEATH(Writer.appendResult(EmptyResult), Matcher); 146 } 147 148 TEST_F(SarifDocumentWriterTest, settingInvalidRankWillCrash) { 149 #if defined(NDEBUG) || !GTEST_HAS_DEATH_TEST 150 GTEST_SKIP() << "This death test is only available for debug builds."; 151 #endif 152 // GIVEN: 153 SarifDocumentWriter Writer{SourceMgr}; 154 155 // WHEN: 156 // A SarifReportingConfiguration is created with an invalid "rank" 157 // * Ranks below 0.0 are invalid 158 // * Ranks above 100.0 are invalid 159 160 // THEN: The builder will crash in either case 161 EXPECT_DEATH(SarifReportingConfiguration::create().setRank(-1.0), 162 ::testing::HasSubstr("Rule rank cannot be smaller than 0.0")); 163 EXPECT_DEATH(SarifReportingConfiguration::create().setRank(101.0), 164 ::testing::HasSubstr("Rule rank cannot be larger than 100.0")); 165 } 166 167 TEST_F(SarifDocumentWriterTest, creatingResultWithDisabledRuleWillCrash) { 168 #if defined(NDEBUG) || !GTEST_HAS_DEATH_TEST 169 GTEST_SKIP() << "This death test is only available for debug builds."; 170 #endif 171 172 // GIVEN: 173 SarifDocumentWriter Writer{SourceMgr}; 174 175 // WHEN: 176 // A disabled Rule is created, and a result is create referencing this rule 177 const auto &Config = SarifReportingConfiguration::create().disable(); 178 auto RuleIdx = 179 Writer.createRule(SarifRule::create().setDefaultConfiguration(Config)); 180 const SarifResult &Result = SarifResult::create(RuleIdx); 181 182 // THEN: 183 // SarifResult::create(...) will produce a crash 184 ASSERT_DEATH( 185 Writer.appendResult(Result), 186 ::testing::HasSubstr("Cannot add a result referencing a disabled Rule")); 187 } 188 189 // Test adding rule and result shows up in the final document 190 TEST_F(SarifDocumentWriterTest, addingResultWithValidRuleAndRunIsOk) { 191 // GIVEN: 192 SarifDocumentWriter Writer{SourceMgr}; 193 const SarifRule &Rule = 194 SarifRule::create() 195 .setRuleId("clang.unittest") 196 .setDescription("Example rule created during unit tests") 197 .setName("clang unit test"); 198 199 // WHEN: 200 Writer.createRun("sarif test", "sarif test runner"); 201 unsigned RuleIdx = Writer.createRule(Rule); 202 const SarifResult &Result = SarifResult::create(RuleIdx); 203 204 Writer.appendResult(Result); 205 const llvm::json::Object &Doc = Writer.createDocument(); 206 207 // THEN: 208 // A document with a valid schema and version exists 209 ASSERT_THAT(Doc.get("$schema"), ::testing::NotNull()); 210 ASSERT_THAT(Doc.get("version"), ::testing::NotNull()); 211 const llvm::json::Array *Runs = Doc.getArray("runs"); 212 213 // A run exists on this document 214 ASSERT_THAT(Runs, ::testing::NotNull()); 215 ASSERT_EQ(Runs->size(), 1UL); 216 const llvm::json::Object *TheRun = Runs->back().getAsObject(); 217 218 // The run has slots for tools, results, rules and artifacts 219 ASSERT_THAT(TheRun->get("tool"), ::testing::NotNull()); 220 ASSERT_THAT(TheRun->get("results"), ::testing::NotNull()); 221 ASSERT_THAT(TheRun->get("artifacts"), ::testing::NotNull()); 222 const llvm::json::Object *Driver = 223 TheRun->getObject("tool")->getObject("driver"); 224 const llvm::json::Array *Results = TheRun->getArray("results"); 225 const llvm::json::Array *Artifacts = TheRun->getArray("artifacts"); 226 227 // The tool is as expected 228 ASSERT_TRUE(Driver->getString("name").has_value()); 229 ASSERT_TRUE(Driver->getString("fullName").has_value()); 230 231 EXPECT_EQ(*Driver->getString("name"), "sarif test"); 232 EXPECT_EQ(*Driver->getString("fullName"), "sarif test runner"); 233 234 // The results are as expected 235 EXPECT_EQ(Results->size(), 1UL); 236 237 // The artifacts are as expected 238 EXPECT_TRUE(Artifacts->empty()); 239 } 240 241 TEST_F(SarifDocumentWriterTest, checkSerializingResultsWithDefaultRuleConfig) { 242 // GIVEN: 243 const std::string ExpectedOutput = 244 R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[],"columnKind":"unicodeCodePoints","results":[{"level":"warning","message":{"text":""},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"defaultConfiguration":{"enabled":true,"level":"warning","rank":-1},"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})"; 245 246 SarifDocumentWriter Writer{SourceMgr}; 247 const SarifRule &Rule = 248 SarifRule::create() 249 .setRuleId("clang.unittest") 250 .setDescription("Example rule created during unit tests") 251 .setName("clang unit test"); 252 253 // WHEN: A run contains a result 254 Writer.createRun("sarif test", "sarif test runner", "1.0.0"); 255 unsigned RuleIdx = Writer.createRule(Rule); 256 const SarifResult &Result = SarifResult::create(RuleIdx); 257 Writer.appendResult(Result); 258 std::string Output = serializeSarifDocument(Writer.createDocument()); 259 260 // THEN: 261 ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput)); 262 } 263 264 TEST_F(SarifDocumentWriterTest, checkSerializingResultsWithCustomRuleConfig) { 265 // GIVEN: 266 const std::string ExpectedOutput = 267 R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[],"columnKind":"unicodeCodePoints","results":[{"level":"error","message":{"text":""},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"defaultConfiguration":{"enabled":true,"level":"error","rank":35.5},"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})"; 268 269 SarifDocumentWriter Writer{SourceMgr}; 270 const SarifRule &Rule = 271 SarifRule::create() 272 .setRuleId("clang.unittest") 273 .setDescription("Example rule created during unit tests") 274 .setName("clang unit test") 275 .setDefaultConfiguration(SarifReportingConfiguration::create() 276 .setLevel(SarifResultLevel::Error) 277 .setRank(35.5)); 278 279 // WHEN: A run contains a result 280 Writer.createRun("sarif test", "sarif test runner", "1.0.0"); 281 unsigned RuleIdx = Writer.createRule(Rule); 282 const SarifResult &Result = SarifResult::create(RuleIdx); 283 Writer.appendResult(Result); 284 std::string Output = serializeSarifDocument(Writer.createDocument()); 285 286 // THEN: 287 ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput)); 288 } 289 290 // Check that serializing artifacts from results produces valid SARIF 291 TEST_F(SarifDocumentWriterTest, checkSerializingArtifacts) { 292 // GIVEN: 293 const std::string ExpectedOutput = 294 R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[{"length":40,"location":{"index":0,"uri":"file:///main.cpp"},"mimeType":"text/plain","roles":["resultFile"]}],"columnKind":"unicodeCodePoints","results":[{"level":"error","locations":[{"physicalLocation":{"artifactLocation":{"index":0,"uri":"file:///main.cpp"},"region":{"endColumn":14,"startColumn":14,"startLine":3}}}],"message":{"text":"expected ';' after top level declarator"},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"defaultConfiguration":{"enabled":true,"level":"warning","rank":-1},"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})"; 295 296 SarifDocumentWriter Writer{SourceMgr}; 297 const SarifRule &Rule = 298 SarifRule::create() 299 .setRuleId("clang.unittest") 300 .setDescription("Example rule created during unit tests") 301 .setName("clang unit test"); 302 303 // WHEN: A result is added with valid source locations for its diagnostics 304 Writer.createRun("sarif test", "sarif test runner", "1.0.0"); 305 unsigned RuleIdx = Writer.createRule(Rule); 306 307 llvm::SmallVector<CharSourceRange, 1> DiagLocs; 308 const char *SourceText = "int foo = 0;\n" 309 "int bar = 1;\n" 310 "float x = 0.0\n"; 311 312 FileID MainFileID = 313 registerSource("/main.cpp", SourceText, /* IsMainFile = */ true); 314 CharSourceRange SourceCSR = 315 getFakeCharSourceRange(MainFileID, {3, 14}, {3, 14}); 316 317 DiagLocs.push_back(SourceCSR); 318 319 const SarifResult &Result = 320 SarifResult::create(RuleIdx) 321 .setLocations(DiagLocs) 322 .setDiagnosticMessage("expected ';' after top level declarator") 323 .setDiagnosticLevel(SarifResultLevel::Error); 324 Writer.appendResult(Result); 325 std::string Output = serializeSarifDocument(Writer.createDocument()); 326 327 // THEN: Assert that the serialized SARIF is as expected 328 ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput)); 329 } 330 331 TEST_F(SarifDocumentWriterTest, checkSerializingCodeflows) { 332 // GIVEN: 333 const std::string ExpectedOutput = 334 R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[{"length":41,"location":{"index":0,"uri":"file:///main.cpp"},"mimeType":"text/plain","roles":["resultFile"]},{"length":27,"location":{"index":1,"uri":"file:///test-header-1.h"},"mimeType":"text/plain","roles":["resultFile"]},{"length":30,"location":{"index":2,"uri":"file:///test-header-2.h"},"mimeType":"text/plain","roles":["resultFile"]},{"length":28,"location":{"index":3,"uri":"file:///test-header-3.h"},"mimeType":"text/plain","roles":["resultFile"]}],"columnKind":"unicodeCodePoints","results":[{"codeFlows":[{"threadFlows":[{"locations":[{"importance":"essential","location":{"message":{"text":"Message #1"},"physicalLocation":{"artifactLocation":{"index":1,"uri":"file:///test-header-1.h"},"region":{"endColumn":8,"endLine":2,"startColumn":1,"startLine":1}}}},{"importance":"important","location":{"message":{"text":"Message #2"},"physicalLocation":{"artifactLocation":{"index":2,"uri":"file:///test-header-2.h"},"region":{"endColumn":8,"endLine":2,"startColumn":1,"startLine":1}}}},{"importance":"unimportant","location":{"message":{"text":"Message #3"},"physicalLocation":{"artifactLocation":{"index":3,"uri":"file:///test-header-3.h"},"region":{"endColumn":8,"endLine":2,"startColumn":1,"startLine":1}}}}]}]}],"level":"warning","locations":[{"physicalLocation":{"artifactLocation":{"index":0,"uri":"file:///main.cpp"},"region":{"endColumn":8,"endLine":2,"startColumn":5,"startLine":2}}}],"message":{"text":"Redefinition of 'foo'"},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"defaultConfiguration":{"enabled":true,"level":"warning","rank":-1},"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})"; 335 336 const char *SourceText = "int foo = 0;\n" 337 "int foo = 1;\n" 338 "float x = 0.0;\n"; 339 FileID MainFileID = 340 registerSource("/main.cpp", SourceText, /* IsMainFile = */ true); 341 CharSourceRange DiagLoc{getFakeCharSourceRange(MainFileID, {2, 5}, {2, 8})}; 342 343 SarifDocumentWriter Writer{SourceMgr}; 344 const SarifRule &Rule = 345 SarifRule::create() 346 .setRuleId("clang.unittest") 347 .setDescription("Example rule created during unit tests") 348 .setName("clang unit test"); 349 350 constexpr unsigned int NumCases = 3; 351 llvm::SmallVector<ThreadFlow, NumCases> Threadflows; 352 const char *HeaderTexts[NumCases]{("#pragma once\n" 353 "#include <foo>"), 354 ("#ifndef FOO\n" 355 "#define FOO\n" 356 "#endif"), 357 ("#ifdef FOO\n" 358 "#undef FOO\n" 359 "#endif")}; 360 const char *HeaderNames[NumCases]{"/test-header-1.h", "/test-header-2.h", 361 "/test-header-3.h"}; 362 ThreadFlowImportance Importances[NumCases]{ThreadFlowImportance::Essential, 363 ThreadFlowImportance::Important, 364 ThreadFlowImportance::Unimportant}; 365 for (size_t Idx = 0; Idx != NumCases; ++Idx) { 366 FileID FID = registerSource(HeaderNames[Idx], HeaderTexts[Idx]); 367 CharSourceRange &&CSR = getFakeCharSourceRange(FID, {1, 1}, {2, 8}); 368 std::string Message = llvm::formatv("Message #{0}", Idx + 1); 369 ThreadFlow Item = ThreadFlow::create() 370 .setRange(CSR) 371 .setImportance(Importances[Idx]) 372 .setMessage(Message); 373 Threadflows.push_back(Item); 374 } 375 376 // WHEN: A result containing code flows and diagnostic locations is added 377 Writer.createRun("sarif test", "sarif test runner", "1.0.0"); 378 unsigned RuleIdx = Writer.createRule(Rule); 379 const SarifResult &Result = 380 SarifResult::create(RuleIdx) 381 .setLocations({DiagLoc}) 382 .setDiagnosticMessage("Redefinition of 'foo'") 383 .setThreadFlows(Threadflows) 384 .setDiagnosticLevel(SarifResultLevel::Warning); 385 Writer.appendResult(Result); 386 std::string Output = serializeSarifDocument(Writer.createDocument()); 387 388 // THEN: Assert that the serialized SARIF is as expected 389 ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput)); 390 } 391 392 } // namespace 393