1 //===- unittests/Basic/SarifTest.cpp - Test writing SARIF documents -------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "clang/Basic/Sarif.h" 10 #include "clang/Basic/DiagnosticIDs.h" 11 #include "clang/Basic/DiagnosticOptions.h" 12 #include "clang/Basic/FileManager.h" 13 #include "clang/Basic/FileSystemOptions.h" 14 #include "clang/Basic/LangOptions.h" 15 #include "clang/Basic/SourceLocation.h" 16 #include "clang/Basic/SourceManager.h" 17 #include "llvm/ADT/StringRef.h" 18 #include "llvm/Support/FormatVariadic.h" 19 #include "llvm/Support/JSON.h" 20 #include "llvm/Support/MemoryBuffer.h" 21 #include "llvm/Support/VirtualFileSystem.h" 22 #include "llvm/Support/raw_ostream.h" 23 #include "gmock/gmock.h" 24 #include "gtest/gtest-death-test.h" 25 #include "gtest/gtest-matchers.h" 26 #include "gtest/gtest.h" 27 28 #include <algorithm> 29 30 using namespace clang; 31 32 namespace { 33 34 using LineCol = std::pair<unsigned int, unsigned int>; 35 36 static std::string serializeSarifDocument(llvm::json::Object &&Doc) { 37 std::string Output; 38 llvm::json::Value value(std::move(Doc)); 39 llvm::raw_string_ostream OS{Output}; 40 OS << llvm::formatv("{0}", value); 41 OS.flush(); 42 return Output; 43 } 44 45 class SarifDocumentWriterTest : public ::testing::Test { 46 protected: 47 SarifDocumentWriterTest() 48 : InMemoryFileSystem(new llvm::vfs::InMemoryFileSystem), 49 FileMgr(FileSystemOptions(), InMemoryFileSystem), 50 DiagID(new DiagnosticIDs()), DiagOpts(new DiagnosticOptions()), 51 Diags(DiagID, DiagOpts.get(), new IgnoringDiagConsumer()), 52 SourceMgr(Diags, FileMgr) {} 53 54 IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> InMemoryFileSystem; 55 FileManager FileMgr; 56 IntrusiveRefCntPtr<DiagnosticIDs> DiagID; 57 IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts; 58 DiagnosticsEngine Diags; 59 SourceManager SourceMgr; 60 LangOptions LangOpts; 61 62 FileID registerSource(llvm::StringRef Name, const char *SourceText, 63 bool IsMainFile = false) { 64 std::unique_ptr<llvm::MemoryBuffer> SourceBuf = 65 llvm::MemoryBuffer::getMemBuffer(SourceText); 66 const FileEntry *SourceFile = 67 FileMgr.getVirtualFile(Name, SourceBuf->getBufferSize(), 0); 68 SourceMgr.overrideFileContents(SourceFile, std::move(SourceBuf)); 69 FileID FID = SourceMgr.getOrCreateFileID(SourceFile, SrcMgr::C_User); 70 if (IsMainFile) 71 SourceMgr.setMainFileID(FID); 72 return FID; 73 } 74 75 CharSourceRange getFakeCharSourceRange(FileID FID, LineCol Begin, 76 LineCol End) { 77 auto BeginLoc = SourceMgr.translateLineCol(FID, Begin.first, Begin.second); 78 auto EndLoc = SourceMgr.translateLineCol(FID, End.first, End.second); 79 return CharSourceRange{SourceRange{BeginLoc, EndLoc}, /* ITR = */ false}; 80 } 81 }; 82 83 TEST_F(SarifDocumentWriterTest, createEmptyDocument) { 84 // GIVEN: 85 SarifDocumentWriter Writer{SourceMgr}; 86 87 // WHEN: 88 const llvm::json::Object &EmptyDoc = Writer.createDocument(); 89 std::vector<StringRef> Keys(EmptyDoc.size()); 90 std::transform(EmptyDoc.begin(), EmptyDoc.end(), Keys.begin(), 91 [](auto item) { return item.getFirst(); }); 92 93 // THEN: 94 ASSERT_THAT(Keys, testing::UnorderedElementsAre("$schema", "version")); 95 } 96 97 // Test that a newly inserted run will associate correct tool names 98 TEST_F(SarifDocumentWriterTest, documentWithARun) { 99 // GIVEN: 100 SarifDocumentWriter Writer{SourceMgr}; 101 const char *ShortName = "sariftest"; 102 const char *LongName = "sarif writer test"; 103 104 // WHEN: 105 Writer.createRun(ShortName, LongName); 106 Writer.endRun(); 107 const llvm::json::Object &Doc = Writer.createDocument(); 108 const llvm::json::Array *Runs = Doc.getArray("runs"); 109 110 // THEN: 111 // A run was created 112 ASSERT_THAT(Runs, testing::NotNull()); 113 114 // It is the only run 115 ASSERT_EQ(Runs->size(), 1UL); 116 117 // The tool associated with the run was the tool 118 const llvm::json::Object *driver = 119 Runs->begin()->getAsObject()->getObject("tool")->getObject("driver"); 120 ASSERT_THAT(driver, testing::NotNull()); 121 122 ASSERT_TRUE(driver->getString("name").hasValue()); 123 ASSERT_TRUE(driver->getString("fullName").hasValue()); 124 ASSERT_TRUE(driver->getString("language").hasValue()); 125 126 EXPECT_EQ(driver->getString("name").getValue(), ShortName); 127 EXPECT_EQ(driver->getString("fullName").getValue(), LongName); 128 EXPECT_EQ(driver->getString("language").getValue(), "en-US"); 129 } 130 131 // Test adding result without a run causes a crash 132 TEST_F(SarifDocumentWriterTest, addingResultsWillCrashIfThereIsNoRun) { 133 // GIVEN: 134 SarifDocumentWriter Writer{SourceMgr}; 135 136 // WHEN: 137 // A SarifDocumentWriter::createRun(...) was not called prior to 138 // SarifDocumentWriter::appendResult(...) 139 // But a rule exists 140 auto RuleIdx = Writer.createRule(SarifRule::create()); 141 SarifResult &&EmptyResult = SarifResult::create(RuleIdx); 142 143 // THEN: 144 ASSERT_DEATH({ Writer.appendResult(EmptyResult); }, ".*create a run first.*"); 145 } 146 147 // Test adding rule and result shows up in the final document 148 TEST_F(SarifDocumentWriterTest, addResultWithValidRuleIsOk) { 149 // GIVEN: 150 SarifDocumentWriter Writer{SourceMgr}; 151 const SarifRule &Rule = 152 SarifRule::create() 153 .setRuleId("clang.unittest") 154 .setDescription("Example rule created during unit tests") 155 .setName("clang unit test"); 156 157 // WHEN: 158 Writer.createRun("sarif test", "sarif test runner"); 159 unsigned RuleIdx = Writer.createRule(Rule); 160 const SarifResult &result = SarifResult::create(RuleIdx); 161 162 Writer.appendResult(result); 163 const llvm::json::Object &Doc = Writer.createDocument(); 164 165 // THEN: 166 // A document with a valid schema and version exists 167 ASSERT_THAT(Doc.get("$schema"), ::testing::NotNull()); 168 ASSERT_THAT(Doc.get("version"), ::testing::NotNull()); 169 const llvm::json::Array *Runs = Doc.getArray("runs"); 170 171 // A run exists on this document 172 ASSERT_THAT(Runs, ::testing::NotNull()); 173 ASSERT_EQ(Runs->size(), 1UL); 174 const llvm::json::Object *TheRun = Runs->back().getAsObject(); 175 176 // The run has slots for tools, results, rules and artifacts 177 ASSERT_THAT(TheRun->get("tool"), ::testing::NotNull()); 178 ASSERT_THAT(TheRun->get("results"), ::testing::NotNull()); 179 ASSERT_THAT(TheRun->get("artifacts"), ::testing::NotNull()); 180 const llvm::json::Object *Driver = 181 TheRun->getObject("tool")->getObject("driver"); 182 const llvm::json::Array *Results = TheRun->getArray("results"); 183 const llvm::json::Array *Artifacts = TheRun->getArray("artifacts"); 184 185 // The tool is as expected 186 ASSERT_TRUE(Driver->getString("name").hasValue()); 187 ASSERT_TRUE(Driver->getString("fullName").hasValue()); 188 189 EXPECT_EQ(Driver->getString("name").getValue(), "sarif test"); 190 EXPECT_EQ(Driver->getString("fullName").getValue(), "sarif test runner"); 191 192 // The results are as expected 193 EXPECT_EQ(Results->size(), 1UL); 194 195 // The artifacts are as expected 196 EXPECT_TRUE(Artifacts->empty()); 197 } 198 199 TEST_F(SarifDocumentWriterTest, checkSerializingResults) { 200 // GIVEN: 201 const std::string ExpectedOutput = 202 R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[],"columnKind":"unicodeCodePoints","results":[{"message":{"text":""},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})"; 203 204 SarifDocumentWriter Writer{SourceMgr}; 205 const SarifRule &Rule = 206 SarifRule::create() 207 .setRuleId("clang.unittest") 208 .setDescription("Example rule created during unit tests") 209 .setName("clang unit test"); 210 211 // WHEN: A run contains a result 212 Writer.createRun("sarif test", "sarif test runner", "1.0.0"); 213 unsigned ruleIdx = Writer.createRule(Rule); 214 const SarifResult &Result = SarifResult::create(ruleIdx); 215 Writer.appendResult(Result); 216 std::string Output = serializeSarifDocument(Writer.createDocument()); 217 218 // THEN: 219 ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput)); 220 } 221 222 // Check that serializing artifacts from results produces valid SARIF 223 TEST_F(SarifDocumentWriterTest, checkSerializingArtifacts) { 224 // GIVEN: 225 const std::string ExpectedOutput = 226 R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[{"length":40,"location":{"index":0,"uri":"file:///main.cpp"},"mimeType":"text/plain","roles":["resultFile"]}],"columnKind":"unicodeCodePoints","results":[{"locations":[{"physicalLocation":{"artifactLocation":{"index":0},"region":{"endColumn":14,"startColumn":14,"startLine":3}}}],"message":{"text":"expected ';' after top level declarator"},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})"; 227 228 SarifDocumentWriter Writer{SourceMgr}; 229 const SarifRule &Rule = 230 SarifRule::create() 231 .setRuleId("clang.unittest") 232 .setDescription("Example rule created during unit tests") 233 .setName("clang unit test"); 234 235 // WHEN: A result is added with valid source locations for its diagnostics 236 Writer.createRun("sarif test", "sarif test runner", "1.0.0"); 237 unsigned RuleIdx = Writer.createRule(Rule); 238 239 llvm::SmallVector<CharSourceRange, 1> DiagLocs; 240 const char *SourceText = "int foo = 0;\n" 241 "int bar = 1;\n" 242 "float x = 0.0\n"; 243 244 FileID MainFileID = 245 registerSource("/main.cpp", SourceText, /* IsMainFile = */ true); 246 CharSourceRange SourceCSR = 247 getFakeCharSourceRange(MainFileID, {3, 14}, {3, 14}); 248 249 DiagLocs.push_back(SourceCSR); 250 251 const SarifResult &Result = 252 SarifResult::create(RuleIdx).setLocations(DiagLocs).setDiagnosticMessage( 253 "expected ';' after top level declarator"); 254 Writer.appendResult(Result); 255 std::string Output = serializeSarifDocument(Writer.createDocument()); 256 257 // THEN: Assert that the serialized SARIF is as expected 258 ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput)); 259 } 260 261 TEST_F(SarifDocumentWriterTest, checkSerializingCodeflows) { 262 // GIVEN: 263 const std::string ExpectedOutput = 264 R"({"$schema":"https://docs.oasis-open.org/sarif/sarif/v2.1.0/cos02/schemas/sarif-schema-2.1.0.json","runs":[{"artifacts":[{"length":27,"location":{"index":1,"uri":"file:///test-header-1.h"},"mimeType":"text/plain","roles":["resultFile"]},{"length":30,"location":{"index":2,"uri":"file:///test-header-2.h"},"mimeType":"text/plain","roles":["resultFile"]},{"length":28,"location":{"index":3,"uri":"file:///test-header-3.h"},"mimeType":"text/plain","roles":["resultFile"]},{"length":41,"location":{"index":0,"uri":"file:///main.cpp"},"mimeType":"text/plain","roles":["resultFile"]}],"columnKind":"unicodeCodePoints","results":[{"codeFlows":[{"threadFlows":[{"locations":[{"importance":"essential","location":{"message":{"text":"Message #1"},"physicalLocation":{"artifactLocation":{"index":1},"region":{"endColumn":8,"endLine":2,"startColumn":1,"startLine":1}}}},{"importance":"important","location":{"message":{"text":"Message #2"},"physicalLocation":{"artifactLocation":{"index":2},"region":{"endColumn":8,"endLine":2,"startColumn":1,"startLine":1}}}},{"importance":"unimportant","location":{"message":{"text":"Message #3"},"physicalLocation":{"artifactLocation":{"index":3},"region":{"endColumn":8,"endLine":2,"startColumn":1,"startLine":1}}}}]}]}],"locations":[{"physicalLocation":{"artifactLocation":{"index":0},"region":{"endColumn":8,"endLine":2,"startColumn":5,"startLine":2}}}],"message":{"text":"Redefinition of 'foo'"},"ruleId":"clang.unittest","ruleIndex":0}],"tool":{"driver":{"fullName":"sarif test runner","informationUri":"https://clang.llvm.org/docs/UsersManual.html","language":"en-US","name":"sarif test","rules":[{"fullDescription":{"text":"Example rule created during unit tests"},"id":"clang.unittest","name":"clang unit test"}],"version":"1.0.0"}}}],"version":"2.1.0"})"; 265 266 const char *SourceText = "int foo = 0;\n" 267 "int foo = 1;\n" 268 "float x = 0.0;\n"; 269 FileID MainFileID = 270 registerSource("/main.cpp", SourceText, /* IsMainFile = */ true); 271 CharSourceRange DiagLoc{getFakeCharSourceRange(MainFileID, {2, 5}, {2, 8})}; 272 273 SarifDocumentWriter Writer{SourceMgr}; 274 const SarifRule &Rule = 275 SarifRule::create() 276 .setRuleId("clang.unittest") 277 .setDescription("Example rule created during unit tests") 278 .setName("clang unit test"); 279 280 constexpr unsigned int NUM_CASES = 3; 281 llvm::SmallVector<ThreadFlow, NUM_CASES> Threadflows; 282 const char *HeaderTexts[NUM_CASES]{("#pragma once\n" 283 "#include <foo>"), 284 ("#ifndef FOO\n" 285 "#define FOO\n" 286 "#endif"), 287 ("#ifdef FOO\n" 288 "#undef FOO\n" 289 "#endif")}; 290 const char *HeaderNames[NUM_CASES]{"/test-header-1.h", "/test-header-2.h", 291 "/test-header-3.h"}; 292 ThreadFlowImportance Importances[NUM_CASES]{ 293 ThreadFlowImportance::Essential, ThreadFlowImportance::Important, 294 ThreadFlowImportance::Unimportant}; 295 for (size_t Idx = 0; Idx != NUM_CASES; ++Idx) { 296 FileID FID = registerSource(HeaderNames[Idx], HeaderTexts[Idx]); 297 CharSourceRange &&CSR = getFakeCharSourceRange(FID, {1, 1}, {2, 8}); 298 std::string Message = llvm::formatv("Message #{0}", Idx + 1); 299 ThreadFlow Item = ThreadFlow::create() 300 .setRange(CSR) 301 .setImportance(Importances[Idx]) 302 .setMessage(Message); 303 Threadflows.push_back(Item); 304 } 305 306 // WHEN: A result containing code flows and diagnostic locations is added 307 Writer.createRun("sarif test", "sarif test runner", "1.0.0"); 308 unsigned RuleIdx = Writer.createRule(Rule); 309 const SarifResult &Result = SarifResult::create(RuleIdx) 310 .setLocations({DiagLoc}) 311 .setDiagnosticMessage("Redefinition of 'foo'") 312 .setThreadFlows(Threadflows); 313 Writer.appendResult(Result); 314 std::string Output = serializeSarifDocument(Writer.createDocument()); 315 316 // THEN: Assert that the serialized SARIF is as expected 317 ASSERT_THAT(Output, ::testing::StrEq(ExpectedOutput)); 318 } 319 320 } // namespace 321