1 //===- unittests/Basic/SarifTest.cpp - Test writing SARIF documents -------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "clang/Basic/Sarif.h" 10 #include "clang/Basic/DiagnosticIDs.h" 11 #include "clang/Basic/DiagnosticOptions.h" 12 #include "clang/Basic/FileManager.h" 13 #include "clang/Basic/FileSystemOptions.h" 14 #include "clang/Basic/LangOptions.h" 15 #include "clang/Basic/SourceLocation.h" 16 #include "clang/Basic/SourceManager.h" 17 #include "llvm/ADT/StringRef.h" 18 #include "llvm/Support/FormatVariadic.h" 19 #include "llvm/Support/JSON.h" 20 #include "llvm/Support/MemoryBuffer.h" 21 #include "llvm/Support/VirtualFileSystem.h" 22 #include "llvm/Support/raw_ostream.h" 23 #include "gmock/gmock.h" 24 #include "gtest/gtest.h" 25 26 #include <algorithm> 27 28 using namespace clang; 29 30 namespace { 31 32 using LineCol = std::pair<unsigned int, unsigned int>; 33 34 static std::string serializeSarifDocument(llvm::json::Object &&Doc) { 35 std::string Output; 36 llvm::json::Value value(std::move(Doc)); 37 llvm::raw_string_ostream OS{Output}; 38 OS << llvm::formatv("{0}", value); 39 OS.flush(); 40 return Output; 41 } 42 43 class SarifDocumentWriterTest : public ::testing::Test { 44 protected: 45 SarifDocumentWriterTest() 46 : InMemoryFileSystem(new llvm::vfs::InMemoryFileSystem), 47 FileMgr(FileSystemOptions(), InMemoryFileSystem), 48 DiagID(new DiagnosticIDs()), DiagOpts(new DiagnosticOptions()), 49 Diags(DiagID, DiagOpts.get(), new IgnoringDiagConsumer()), 50 SourceMgr(Diags, FileMgr) {} 51 52 IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> InMemoryFileSystem; 53 FileManager FileMgr; 54 IntrusiveRefCntPtr<DiagnosticIDs> DiagID; 55 IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts; 56 DiagnosticsEngine Diags; 57 SourceManager SourceMgr; 58 LangOptions LangOpts; 59 60 FileID registerSource(llvm::StringRef Name, const char *SourceText, 61 bool IsMainFile = false) { 62 std::unique_ptr<llvm::MemoryBuffer> SourceBuf = 63 llvm::MemoryBuffer::getMemBuffer(SourceText); 64 const FileEntry *SourceFile = 65 FileMgr.getVirtualFile(Name, SourceBuf->getBufferSize(), 0); 66 SourceMgr.overrideFileContents(SourceFile, std::move(SourceBuf)); 67 FileID FID = SourceMgr.getOrCreateFileID(SourceFile, SrcMgr::C_User); 68 if (IsMainFile) 69 SourceMgr.setMainFileID(FID); 70 return FID; 71 } 72 73 CharSourceRange getFakeCharSourceRange(FileID FID, LineCol Begin, 74 LineCol End) { 75 auto BeginLoc = SourceMgr.translateLineCol(FID, Begin.first, Begin.second); 76 auto EndLoc = SourceMgr.translateLineCol(FID, End.first, End.second); 77 return CharSourceRange{SourceRange{BeginLoc, EndLoc}, /* ITR = */ false}; 78 } 79 }; 80 81 TEST_F(SarifDocumentWriterTest, canCreateEmptyDocument) { 82 // GIVEN: 83 SarifDocumentWriter Writer{SourceMgr}; 84 85 // WHEN: 86 const llvm::json::Object &EmptyDoc = Writer.createDocument(); 87 std::vector<StringRef> Keys(EmptyDoc.size()); 88 std::transform(EmptyDoc.begin(), EmptyDoc.end(), Keys.begin(), 89 [](auto item) { return item.getFirst(); }); 90 91 // THEN: 92 ASSERT_THAT(Keys, testing::UnorderedElementsAre("$schema", "version")); 93 } 94 95 // Test that a newly inserted run will associate correct tool names 96 TEST_F(SarifDocumentWriterTest, canCreateDocumentWithOneRun) { 97 // GIVEN: 98 SarifDocumentWriter Writer{SourceMgr}; 99 const char *ShortName = "sariftest"; 100 const char *LongName = "sarif writer test"; 101 102 // WHEN: 103 Writer.createRun(ShortName, LongName); 104 Writer.endRun(); 105 const llvm::json::Object &Doc = Writer.createDocument(); 106 const llvm::json::Array *Runs = Doc.getArray("runs"); 107 108 // THEN: 109 // A run was created 110 ASSERT_THAT(Runs, testing::NotNull()); 111 112 // It is the only run 113 ASSERT_EQ(Runs->size(), 1UL); 114 115 // The tool associated with the run was the tool 116 const llvm::json::Object *driver = 117 Runs->begin()->getAsObject()->getObject("tool")->getObject("driver"); 118 ASSERT_THAT(driver, testing::NotNull()); 119 120 ASSERT_TRUE(driver->getString("name").has_value()); 121 ASSERT_TRUE(driver->getString("fullName").has_value()); 122 ASSERT_TRUE(driver->getString("language").has_value()); 123 124 EXPECT_EQ(driver->getString("name").getValue(), ShortName); 125 EXPECT_EQ(driver->getString("fullName").getValue(), LongName); 126 EXPECT_EQ(driver->getString("language").getValue(), "en-US"); 127 } 128 129 TEST_F(SarifDocumentWriterTest, addingResultsWillCrashIfThereIsNoRun) { 130 #if defined(NDEBUG) || !GTEST_HAS_DEATH_TEST 131 GTEST_SKIP() << "This death test is only available for debug builds."; 132 #endif 133 // GIVEN: 134 SarifDocumentWriter Writer{SourceMgr}; 135 136 // WHEN: 137 // A SarifDocumentWriter::createRun(...) was not called prior to 138 // SarifDocumentWriter::appendResult(...) 139 // But a rule exists 140 auto RuleIdx = Writer.createRule(SarifRule::create()); 141 const SarifResult &EmptyResult = SarifResult::create(RuleIdx); 142 143 // THEN: 144 auto Matcher = ::testing::AnyOf( 145 ::testing::HasSubstr("create a run first"), 146 ::testing::HasSubstr("no runs associated with the document")); 147 ASSERT_DEATH(Writer.appendResult(EmptyResult), Matcher); 148 } 149 150 // Test adding rule and result shows up in the final document 151 TEST_F(SarifDocumentWriterTest, addingResultWithValidRuleAndRunIsOk) { 152 // GIVEN: 153 SarifDocumentWriter Writer{SourceMgr}; 154 const SarifRule &Rule = 155 SarifRule::create() 156 .setRuleId("clang.unittest") 157 .setDescription("Example rule created during unit tests") 158 .setName("clang unit test"); 159 160 // WHEN: 161 Writer.createRun("sarif test", "sarif test runner"); 162 unsigned RuleIdx = Writer.createRule(Rule); 163 const SarifResult &result = SarifResult::create(RuleIdx); 164 165 Writer.appendResult(result); 166 const llvm::json::Object &Doc = Writer.createDocument(); 167 168 // THEN: 169 // A document with a valid schema and version exists 170 ASSERT_THAT(Doc.get("$schema"), ::testing::NotNull()); 171 ASSERT_THAT(Doc.get("version"), ::testing::NotNull()); 172 const llvm::json::Array *Runs = Doc.getArray("runs"); 173 174 // A run exists on this document 175 ASSERT_THAT(Runs, ::testing::NotNull()); 176 ASSERT_EQ(Runs->size(), 1UL); 177 const llvm::json::Object *TheRun = Runs->back().getAsObject(); 178 179 // The run has slots for tools, results, rules and artifacts 180 ASSERT_THAT(TheRun->get("tool"), ::testing::NotNull()); 181 ASSERT_THAT(TheRun->get("results"), ::testing::NotNull()); 182 ASSERT_THAT(TheRun->get("artifacts"), ::testing::NotNull()); 183 const llvm::json::Object *Driver = 184 TheRun->getObject("tool")->getObject("driver"); 185 const llvm::json::Array *Results = TheRun->getArray("results"); 186 const llvm::json::Array *Artifacts = TheRun->getArray("artifacts"); 187 188 // The tool is as expected 189 ASSERT_TRUE(Driver->getString("name").has_value()); 190 ASSERT_TRUE(Driver->getString("fullName").has_value()); 191 192 EXPECT_EQ(Driver->getString("name").getValue(), "sarif test"); 193 EXPECT_EQ(Driver->getString("fullName").getValue(), "sarif test runner"); 194 195 // The results are as expected 196 EXPECT_EQ(Results->size(), 1UL); 197 198 // The artifacts are as expected 199 EXPECT_TRUE(Artifacts->empty()); 200 } 201 202 TEST_F(SarifDocumentWriterTest, checkSerializingResults) { 203 // GIVEN: 204 const std::string ExpectedOutput = 205 R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[],"columnKind":"unicodeCodePoints","results":[{"message":{"text":""},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})"; 206 207 SarifDocumentWriter Writer{SourceMgr}; 208 const SarifRule &Rule = 209 SarifRule::create() 210 .setRuleId("clang.unittest") 211 .setDescription("Example rule created during unit tests") 212 .setName("clang unit test"); 213 214 // WHEN: A run contains a result 215 Writer.createRun("sarif test", "sarif test runner", "1.0.0"); 216 unsigned ruleIdx = Writer.createRule(Rule); 217 const SarifResult &Result = SarifResult::create(ruleIdx); 218 Writer.appendResult(Result); 219 std::string Output = serializeSarifDocument(Writer.createDocument()); 220 221 // THEN: 222 ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput)); 223 } 224 225 // Check that serializing artifacts from results produces valid SARIF 226 TEST_F(SarifDocumentWriterTest, checkSerializingArtifacts) { 227 // GIVEN: 228 const std::string ExpectedOutput = 229 R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[{"length":40,"location":{"index":0,"uri":"file:///main.cpp"},"mimeType":"text/plain","roles":["resultFile"]}],"columnKind":"unicodeCodePoints","results":[{"locations":[{"physicalLocation":{"artifactLocation":{"index":0},"region":{"endColumn":14,"startColumn":14,"startLine":3}}}],"message":{"text":"expected ';' after top level declarator"},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})"; 230 231 SarifDocumentWriter Writer{SourceMgr}; 232 const SarifRule &Rule = 233 SarifRule::create() 234 .setRuleId("clang.unittest") 235 .setDescription("Example rule created during unit tests") 236 .setName("clang unit test"); 237 238 // WHEN: A result is added with valid source locations for its diagnostics 239 Writer.createRun("sarif test", "sarif test runner", "1.0.0"); 240 unsigned RuleIdx = Writer.createRule(Rule); 241 242 llvm::SmallVector<CharSourceRange, 1> DiagLocs; 243 const char *SourceText = "int foo = 0;\n" 244 "int bar = 1;\n" 245 "float x = 0.0\n"; 246 247 FileID MainFileID = 248 registerSource("/main.cpp", SourceText, /* IsMainFile = */ true); 249 CharSourceRange SourceCSR = 250 getFakeCharSourceRange(MainFileID, {3, 14}, {3, 14}); 251 252 DiagLocs.push_back(SourceCSR); 253 254 const SarifResult &Result = 255 SarifResult::create(RuleIdx).setLocations(DiagLocs).setDiagnosticMessage( 256 "expected ';' after top level declarator"); 257 Writer.appendResult(Result); 258 std::string Output = serializeSarifDocument(Writer.createDocument()); 259 260 // THEN: Assert that the serialized SARIF is as expected 261 ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput)); 262 } 263 264 TEST_F(SarifDocumentWriterTest, checkSerializingCodeflows) { 265 // GIVEN: 266 const std::string ExpectedOutput = 267 R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[{"length":27,"location":{"index":1,"uri":"file:///test-header-1.h"},"mimeType":"text/plain","roles":["resultFile"]},{"length":30,"location":{"index":2,"uri":"file:///test-header-2.h"},"mimeType":"text/plain","roles":["resultFile"]},{"length":28,"location":{"index":3,"uri":"file:///test-header-3.h"},"mimeType":"text/plain","roles":["resultFile"]},{"length":41,"location":{"index":0,"uri":"file:///main.cpp"},"mimeType":"text/plain","roles":["resultFile"]}],"columnKind":"unicodeCodePoints","results":[{"codeFlows":[{"threadFlows":[{"locations":[{"importance":"essential","location":{"message":{"text":"Message #1"},"physicalLocation":{"artifactLocation":{"index":1},"region":{"endColumn":8,"endLine":2,"startColumn":1,"startLine":1}}}},{"importance":"important","location":{"message":{"text":"Message #2"},"physicalLocation":{"artifactLocation":{"index":2},"region":{"endColumn":8,"endLine":2,"startColumn":1,"startLine":1}}}},{"importance":"unimportant","location":{"message":{"text":"Message #3"},"physicalLocation":{"artifactLocation":{"index":3},"region":{"endColumn":8,"endLine":2,"startColumn":1,"startLine":1}}}}]}]}],"locations":[{"physicalLocation":{"artifactLocation":{"index":0},"region":{"endColumn":8,"endLine":2,"startColumn":5,"startLine":2}}}],"message":{"text":"Redefinition of 'foo'"},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})"; 268 269 const char *SourceText = "int foo = 0;\n" 270 "int foo = 1;\n" 271 "float x = 0.0;\n"; 272 FileID MainFileID = 273 registerSource("/main.cpp", SourceText, /* IsMainFile = */ true); 274 CharSourceRange DiagLoc{getFakeCharSourceRange(MainFileID, {2, 5}, {2, 8})}; 275 276 SarifDocumentWriter Writer{SourceMgr}; 277 const SarifRule &Rule = 278 SarifRule::create() 279 .setRuleId("clang.unittest") 280 .setDescription("Example rule created during unit tests") 281 .setName("clang unit test"); 282 283 constexpr unsigned int NUM_CASES = 3; 284 llvm::SmallVector<ThreadFlow, NUM_CASES> Threadflows; 285 const char *HeaderTexts[NUM_CASES]{("#pragma once\n" 286 "#include <foo>"), 287 ("#ifndef FOO\n" 288 "#define FOO\n" 289 "#endif"), 290 ("#ifdef FOO\n" 291 "#undef FOO\n" 292 "#endif")}; 293 const char *HeaderNames[NUM_CASES]{"/test-header-1.h", "/test-header-2.h", 294 "/test-header-3.h"}; 295 ThreadFlowImportance Importances[NUM_CASES]{ 296 ThreadFlowImportance::Essential, ThreadFlowImportance::Important, 297 ThreadFlowImportance::Unimportant}; 298 for (size_t Idx = 0; Idx != NUM_CASES; ++Idx) { 299 FileID FID = registerSource(HeaderNames[Idx], HeaderTexts[Idx]); 300 CharSourceRange &&CSR = getFakeCharSourceRange(FID, {1, 1}, {2, 8}); 301 std::string Message = llvm::formatv("Message #{0}", Idx + 1); 302 ThreadFlow Item = ThreadFlow::create() 303 .setRange(CSR) 304 .setImportance(Importances[Idx]) 305 .setMessage(Message); 306 Threadflows.push_back(Item); 307 } 308 309 // WHEN: A result containing code flows and diagnostic locations is added 310 Writer.createRun("sarif test", "sarif test runner", "1.0.0"); 311 unsigned RuleIdx = Writer.createRule(Rule); 312 const SarifResult &Result = SarifResult::create(RuleIdx) 313 .setLocations({DiagLoc}) 314 .setDiagnosticMessage("Redefinition of 'foo'") 315 .setThreadFlows(Threadflows); 316 Writer.appendResult(Result); 317 std::string Output = serializeSarifDocument(Writer.createDocument()); 318 319 // THEN: Assert that the serialized SARIF is as expected 320 ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput)); 321 } 322 323 } // namespace 324