1 //===-- QualityTests.cpp ----------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Evaluating scoring functions isn't a great fit for assert-based tests.
10 // For interesting cases, both exact scores and "X beats Y" are too brittle to
11 // make good hard assertions.
12 //
13 // Here we test the signal extraction and sanity-check that signals point in
14 // the right direction. This should be supplemented by quality metrics which
15 // we can compute from a corpus of queries and preferred rankings.
16 //
17 //===----------------------------------------------------------------------===//
18
19 #include "FileDistance.h"
20 #include "Quality.h"
21 #include "TestFS.h"
22 #include "TestTU.h"
23 #include "index/FileIndex.h"
24 #include "clang/AST/Decl.h"
25 #include "clang/AST/DeclCXX.h"
26 #include "clang/Sema/CodeCompleteConsumer.h"
27 #include "llvm/Support/Casting.h"
28 #include "gmock/gmock.h"
29 #include "gtest/gtest.h"
30 #include <vector>
31
32 namespace clang {
33 namespace clangd {
34
35 // Force the unittest URI scheme to be linked,
36 static int LLVM_ATTRIBUTE_UNUSED UnittestSchemeAnchorDest =
37 UnittestSchemeAnchorSource;
38
39 namespace {
40
TEST(QualityTests,SymbolQualitySignalExtraction)41 TEST(QualityTests, SymbolQualitySignalExtraction) {
42 auto Header = TestTU::withHeaderCode(R"cpp(
43 int _X;
44
45 [[deprecated]]
46 int _f() { return _X; }
47
48 #define DECL_NAME(x, y) x##_##y##_Decl
49 #define DECL(x, y) class DECL_NAME(x, y) {};
50 DECL(X, Y); // X_Y_Decl
51 )cpp");
52
53 auto Symbols = Header.headerSymbols();
54 auto AST = Header.build();
55
56 SymbolQualitySignals Quality;
57 Quality.merge(findSymbol(Symbols, "X_Y_Decl"));
58 EXPECT_TRUE(Quality.ImplementationDetail);
59
60 Symbol F = findSymbol(Symbols, "_f");
61 F.References = 24; // TestTU doesn't count references, so fake it.
62 Quality = {};
63 Quality.merge(F);
64 EXPECT_TRUE(Quality.Deprecated);
65 EXPECT_FALSE(Quality.ReservedName);
66 EXPECT_EQ(Quality.References, 24u);
67 EXPECT_EQ(Quality.Category, SymbolQualitySignals::Function);
68
69 Quality = {};
70 Quality.merge(CodeCompletionResult(&findDecl(AST, "_f"), /*Priority=*/42));
71 EXPECT_TRUE(Quality.Deprecated);
72 EXPECT_FALSE(Quality.ReservedName);
73 EXPECT_EQ(Quality.References, SymbolQualitySignals().References);
74 EXPECT_EQ(Quality.Category, SymbolQualitySignals::Function);
75
76 Quality = {};
77 Quality.merge(CodeCompletionResult("if"));
78 EXPECT_EQ(Quality.Category, SymbolQualitySignals::Keyword);
79
80 // Testing ReservedName in main file, we don't index those symbols in headers.
81 auto MainAST = TestTU::withCode("int _X;").build();
82 SymbolSlab MainSymbols = std::get<0>(indexMainDecls(MainAST));
83
84 Quality = {};
85 Quality.merge(findSymbol(MainSymbols, "_X"));
86 EXPECT_FALSE(Quality.Deprecated);
87 EXPECT_FALSE(Quality.ImplementationDetail);
88 EXPECT_TRUE(Quality.ReservedName);
89 }
90
TEST(QualityTests,SymbolRelevanceSignalExtraction)91 TEST(QualityTests, SymbolRelevanceSignalExtraction) {
92 TestTU Test;
93 Test.HeaderCode = R"cpp(
94 int header();
95 int header_main();
96
97 namespace hdr { class Bar {}; } // namespace hdr
98
99 #define DEFINE_FLAG(X) \
100 namespace flags { \
101 int FLAGS_##X; \
102 } \
103
104 DEFINE_FLAG(FOO)
105 )cpp";
106 Test.Code = R"cpp(
107 using hdr::Bar;
108
109 using flags::FLAGS_FOO;
110
111 int ::header_main() {}
112 int main();
113
114 [[deprecated]]
115 int deprecated() { return 0; }
116
117 namespace { struct X { void y() { int z; } }; }
118 struct S{};
119 )cpp";
120 auto AST = Test.build();
121
122 SymbolRelevanceSignals Relevance;
123 Relevance.merge(CodeCompletionResult(&findDecl(AST, "deprecated"),
124 /*Priority=*/42, nullptr, false,
125 /*Accessible=*/false));
126 EXPECT_EQ(Relevance.NameMatch, SymbolRelevanceSignals().NameMatch);
127 EXPECT_TRUE(Relevance.Forbidden);
128 EXPECT_EQ(Relevance.Scope, SymbolRelevanceSignals::GlobalScope);
129
130 Relevance = {};
131 Relevance.merge(CodeCompletionResult(&findDecl(AST, "main"), 42));
132 EXPECT_FLOAT_EQ(Relevance.SemaFileProximityScore, 1.0f)
133 << "Decl in current file";
134 Relevance = {};
135 Relevance.merge(CodeCompletionResult(&findDecl(AST, "header"), 42));
136 EXPECT_FLOAT_EQ(Relevance.SemaFileProximityScore, 0.6f) << "Decl from header";
137 Relevance = {};
138 Relevance.merge(CodeCompletionResult(&findDecl(AST, "header_main"), 42));
139 EXPECT_FLOAT_EQ(Relevance.SemaFileProximityScore, 1.0f)
140 << "Current file and header";
141
142 auto ConstructShadowDeclCompletionResult = [&](const std::string DeclName) {
143 auto *Shadow =
144 *dyn_cast<UsingDecl>(&findDecl(AST, [&](const NamedDecl &ND) {
145 if (const UsingDecl *Using = dyn_cast<UsingDecl>(&ND))
146 if (Using->shadow_size() &&
147 Using->getQualifiedNameAsString() == DeclName)
148 return true;
149 return false;
150 }))->shadow_begin();
151 CodeCompletionResult Result(Shadow->getTargetDecl(), 42);
152 Result.ShadowDecl = Shadow;
153 return Result;
154 };
155
156 Relevance = {};
157 Relevance.merge(ConstructShadowDeclCompletionResult("Bar"));
158 EXPECT_FLOAT_EQ(Relevance.SemaFileProximityScore, 1.0f)
159 << "Using declaration in main file";
160 Relevance.merge(ConstructShadowDeclCompletionResult("FLAGS_FOO"));
161 EXPECT_FLOAT_EQ(Relevance.SemaFileProximityScore, 1.0f)
162 << "Using declaration in main file";
163
164 Relevance = {};
165 Relevance.merge(CodeCompletionResult(&findUnqualifiedDecl(AST, "X"), 42));
166 EXPECT_EQ(Relevance.Scope, SymbolRelevanceSignals::FileScope);
167 Relevance = {};
168 Relevance.merge(CodeCompletionResult(&findUnqualifiedDecl(AST, "y"), 42));
169 EXPECT_EQ(Relevance.Scope, SymbolRelevanceSignals::ClassScope);
170 Relevance = {};
171 Relevance.merge(CodeCompletionResult(&findUnqualifiedDecl(AST, "z"), 42));
172 EXPECT_EQ(Relevance.Scope, SymbolRelevanceSignals::FunctionScope);
173 // The injected class name is treated as the outer class name.
174 Relevance = {};
175 Relevance.merge(CodeCompletionResult(&findDecl(AST, "S::S"), 42));
176 EXPECT_EQ(Relevance.Scope, SymbolRelevanceSignals::GlobalScope);
177
178 Relevance = {};
179 EXPECT_FALSE(Relevance.InBaseClass);
180 auto BaseMember = CodeCompletionResult(&findUnqualifiedDecl(AST, "y"), 42);
181 BaseMember.InBaseClass = true;
182 Relevance.merge(BaseMember);
183 EXPECT_TRUE(Relevance.InBaseClass);
184
185 auto Index = Test.index();
186 FuzzyFindRequest Req;
187 Req.Query = "X";
188 Req.AnyScope = true;
189 bool Matched = false;
190 Index->fuzzyFind(Req, [&](const Symbol &S) {
191 Matched = true;
192 Relevance = {};
193 Relevance.merge(S);
194 EXPECT_EQ(Relevance.Scope, SymbolRelevanceSignals::FileScope);
195 });
196 EXPECT_TRUE(Matched);
197 }
198
199 // Do the signals move the scores in the direction we expect?
TEST(QualityTests,SymbolQualitySignalsSanity)200 TEST(QualityTests, SymbolQualitySignalsSanity) {
201 SymbolQualitySignals Default;
202 EXPECT_EQ(Default.evaluateHeuristics(), 1);
203
204 SymbolQualitySignals Deprecated;
205 Deprecated.Deprecated = true;
206 EXPECT_LT(Deprecated.evaluateHeuristics(), Default.evaluateHeuristics());
207
208 SymbolQualitySignals ReservedName;
209 ReservedName.ReservedName = true;
210 EXPECT_LT(ReservedName.evaluateHeuristics(), Default.evaluateHeuristics());
211
212 SymbolQualitySignals ImplementationDetail;
213 ImplementationDetail.ImplementationDetail = true;
214 EXPECT_LT(ImplementationDetail.evaluateHeuristics(),
215 Default.evaluateHeuristics());
216
217 SymbolQualitySignals WithReferences, ManyReferences;
218 WithReferences.References = 20;
219 ManyReferences.References = 1000;
220 EXPECT_GT(WithReferences.evaluateHeuristics(), Default.evaluateHeuristics());
221 EXPECT_GT(ManyReferences.evaluateHeuristics(),
222 WithReferences.evaluateHeuristics());
223
224 SymbolQualitySignals Keyword, Variable, Macro, Constructor, Function,
225 Destructor, Operator;
226 Keyword.Category = SymbolQualitySignals::Keyword;
227 Variable.Category = SymbolQualitySignals::Variable;
228 Macro.Category = SymbolQualitySignals::Macro;
229 Constructor.Category = SymbolQualitySignals::Constructor;
230 Destructor.Category = SymbolQualitySignals::Destructor;
231 Destructor.Category = SymbolQualitySignals::Destructor;
232 Operator.Category = SymbolQualitySignals::Operator;
233 Function.Category = SymbolQualitySignals::Function;
234 EXPECT_GT(Variable.evaluateHeuristics(), Default.evaluateHeuristics());
235 EXPECT_GT(Keyword.evaluateHeuristics(), Variable.evaluateHeuristics());
236 EXPECT_LT(Macro.evaluateHeuristics(), Default.evaluateHeuristics());
237 EXPECT_LT(Operator.evaluateHeuristics(), Default.evaluateHeuristics());
238 EXPECT_LT(Constructor.evaluateHeuristics(), Function.evaluateHeuristics());
239 EXPECT_LT(Destructor.evaluateHeuristics(), Constructor.evaluateHeuristics());
240 }
241
TEST(QualityTests,SymbolRelevanceSignalsSanity)242 TEST(QualityTests, SymbolRelevanceSignalsSanity) {
243 SymbolRelevanceSignals Default;
244 EXPECT_EQ(Default.evaluateHeuristics(), 1);
245
246 SymbolRelevanceSignals Forbidden;
247 Forbidden.Forbidden = true;
248 EXPECT_LT(Forbidden.evaluateHeuristics(), Default.evaluateHeuristics());
249
250 SymbolRelevanceSignals PoorNameMatch;
251 PoorNameMatch.NameMatch = 0.2f;
252 EXPECT_LT(PoorNameMatch.evaluateHeuristics(), Default.evaluateHeuristics());
253
254 SymbolRelevanceSignals WithSemaFileProximity;
255 WithSemaFileProximity.SemaFileProximityScore = 0.2f;
256 EXPECT_GT(WithSemaFileProximity.evaluateHeuristics(),
257 Default.evaluateHeuristics());
258
259 ScopeDistance ScopeProximity({"x::y::"});
260
261 SymbolRelevanceSignals WithSemaScopeProximity;
262 WithSemaScopeProximity.ScopeProximityMatch = &ScopeProximity;
263 WithSemaScopeProximity.SemaSaysInScope = true;
264 EXPECT_GT(WithSemaScopeProximity.evaluateHeuristics(),
265 Default.evaluateHeuristics());
266
267 SymbolRelevanceSignals WithIndexScopeProximity;
268 WithIndexScopeProximity.ScopeProximityMatch = &ScopeProximity;
269 WithIndexScopeProximity.SymbolScope = "x::";
270 EXPECT_GT(WithSemaScopeProximity.evaluateHeuristics(),
271 Default.evaluateHeuristics());
272
273 SymbolRelevanceSignals IndexProximate;
274 IndexProximate.SymbolURI = "unittest:/foo/bar.h";
275 llvm::StringMap<SourceParams> ProxSources;
276 ProxSources.try_emplace(testPath("foo/baz.h"));
277 URIDistance Distance(ProxSources);
278 IndexProximate.FileProximityMatch = &Distance;
279 EXPECT_GT(IndexProximate.evaluateHeuristics(), Default.evaluateHeuristics());
280 SymbolRelevanceSignals IndexDistant = IndexProximate;
281 IndexDistant.SymbolURI = "unittest:/elsewhere/path.h";
282 EXPECT_GT(IndexProximate.evaluateHeuristics(),
283 IndexDistant.evaluateHeuristics())
284 << IndexProximate << IndexDistant;
285 EXPECT_GT(IndexDistant.evaluateHeuristics(), Default.evaluateHeuristics());
286
287 SymbolRelevanceSignals Scoped;
288 Scoped.Scope = SymbolRelevanceSignals::FileScope;
289 EXPECT_LT(Scoped.evaluateHeuristics(), Default.evaluateHeuristics());
290 Scoped.Query = SymbolRelevanceSignals::CodeComplete;
291 EXPECT_GT(Scoped.evaluateHeuristics(), Default.evaluateHeuristics());
292
293 SymbolRelevanceSignals Instance;
294 Instance.IsInstanceMember = false;
295 EXPECT_EQ(Instance.evaluateHeuristics(), Default.evaluateHeuristics());
296 Instance.Context = CodeCompletionContext::CCC_DotMemberAccess;
297 EXPECT_LT(Instance.evaluateHeuristics(), Default.evaluateHeuristics());
298 Instance.IsInstanceMember = true;
299 EXPECT_EQ(Instance.evaluateHeuristics(), Default.evaluateHeuristics());
300
301 SymbolRelevanceSignals InBaseClass;
302 InBaseClass.InBaseClass = true;
303 EXPECT_LT(InBaseClass.evaluateHeuristics(), Default.evaluateHeuristics());
304
305 llvm::StringSet<> Words = {"one", "two", "three"};
306 SymbolRelevanceSignals WithoutMatchingWord;
307 WithoutMatchingWord.ContextWords = &Words;
308 WithoutMatchingWord.Name = "four";
309 EXPECT_EQ(WithoutMatchingWord.evaluateHeuristics(),
310 Default.evaluateHeuristics());
311 SymbolRelevanceSignals WithMatchingWord;
312 WithMatchingWord.ContextWords = &Words;
313 WithMatchingWord.Name = "TheTwoTowers";
314 EXPECT_GT(WithMatchingWord.evaluateHeuristics(),
315 Default.evaluateHeuristics());
316 }
317
TEST(QualityTests,ScopeProximity)318 TEST(QualityTests, ScopeProximity) {
319 SymbolRelevanceSignals Relevance;
320 ScopeDistance ScopeProximity({"x::y::z::", "x::", "llvm::", ""});
321 Relevance.ScopeProximityMatch = &ScopeProximity;
322
323 Relevance.SymbolScope = "other::";
324 float NotMatched = Relevance.evaluateHeuristics();
325
326 Relevance.SymbolScope = "";
327 float Global = Relevance.evaluateHeuristics();
328 EXPECT_GT(Global, NotMatched);
329
330 Relevance.SymbolScope = "llvm::";
331 float NonParent = Relevance.evaluateHeuristics();
332 EXPECT_GT(NonParent, Global);
333
334 Relevance.SymbolScope = "x::";
335 float GrandParent = Relevance.evaluateHeuristics();
336 EXPECT_GT(GrandParent, Global);
337
338 Relevance.SymbolScope = "x::y::";
339 float Parent = Relevance.evaluateHeuristics();
340 EXPECT_GT(Parent, GrandParent);
341
342 Relevance.SymbolScope = "x::y::z::";
343 float Enclosing = Relevance.evaluateHeuristics();
344 EXPECT_GT(Enclosing, Parent);
345 }
346
TEST(QualityTests,SortText)347 TEST(QualityTests, SortText) {
348 EXPECT_LT(sortText(std::numeric_limits<float>::infinity()),
349 sortText(1000.2f));
350 EXPECT_LT(sortText(1000.2f), sortText(1));
351 EXPECT_LT(sortText(1), sortText(0.3f));
352 EXPECT_LT(sortText(0.3f), sortText(0));
353 EXPECT_LT(sortText(0), sortText(-10));
354 EXPECT_LT(sortText(-10), sortText(-std::numeric_limits<float>::infinity()));
355
356 EXPECT_LT(sortText(1, "z"), sortText(0, "a"));
357 EXPECT_LT(sortText(0, "a"), sortText(0, "z"));
358 }
359
TEST(QualityTests,NoBoostForClassConstructor)360 TEST(QualityTests, NoBoostForClassConstructor) {
361 auto Header = TestTU::withHeaderCode(R"cpp(
362 class Foo {
363 public:
364 Foo(int);
365 };
366 )cpp");
367 auto Symbols = Header.headerSymbols();
368 auto AST = Header.build();
369
370 const NamedDecl *Foo = &findDecl(AST, "Foo");
371 SymbolRelevanceSignals Cls;
372 Cls.merge(CodeCompletionResult(Foo, /*Priority=*/0));
373
374 const NamedDecl *CtorDecl = &findDecl(AST, [](const NamedDecl &ND) {
375 return (ND.getQualifiedNameAsString() == "Foo::Foo") &&
376 isa<CXXConstructorDecl>(&ND);
377 });
378 SymbolRelevanceSignals Ctor;
379 Ctor.merge(CodeCompletionResult(CtorDecl, /*Priority=*/0));
380
381 EXPECT_EQ(Cls.Scope, SymbolRelevanceSignals::GlobalScope);
382 EXPECT_EQ(Ctor.Scope, SymbolRelevanceSignals::GlobalScope);
383 }
384
TEST(QualityTests,IsInstanceMember)385 TEST(QualityTests, IsInstanceMember) {
386 auto Header = TestTU::withHeaderCode(R"cpp(
387 class Foo {
388 public:
389 static void foo() {}
390
391 template <typename T> void tpl(T *t) {}
392
393 void bar() {}
394 };
395 )cpp");
396 auto Symbols = Header.headerSymbols();
397
398 SymbolRelevanceSignals Rel;
399 const Symbol &FooSym = findSymbol(Symbols, "Foo::foo");
400 Rel.merge(FooSym);
401 EXPECT_FALSE(Rel.IsInstanceMember);
402 const Symbol &BarSym = findSymbol(Symbols, "Foo::bar");
403 Rel.merge(BarSym);
404 EXPECT_TRUE(Rel.IsInstanceMember);
405
406 Rel.IsInstanceMember = false;
407 const Symbol &TplSym = findSymbol(Symbols, "Foo::tpl");
408 Rel.merge(TplSym);
409 EXPECT_TRUE(Rel.IsInstanceMember);
410
411 auto AST = Header.build();
412 const NamedDecl *Foo = &findDecl(AST, "Foo::foo");
413 const NamedDecl *Bar = &findDecl(AST, "Foo::bar");
414 const NamedDecl *Tpl = &findDecl(AST, "Foo::tpl");
415
416 Rel.IsInstanceMember = false;
417 Rel.merge(CodeCompletionResult(Foo, /*Priority=*/0));
418 EXPECT_FALSE(Rel.IsInstanceMember);
419 Rel.merge(CodeCompletionResult(Bar, /*Priority=*/0));
420 EXPECT_TRUE(Rel.IsInstanceMember);
421 Rel.IsInstanceMember = false;
422 Rel.merge(CodeCompletionResult(Tpl, /*Priority=*/0));
423 EXPECT_TRUE(Rel.IsInstanceMember);
424 }
425
TEST(QualityTests,ConstructorDestructor)426 TEST(QualityTests, ConstructorDestructor) {
427 auto Header = TestTU::withHeaderCode(R"cpp(
428 class Foo {
429 public:
430 Foo(int);
431 ~Foo();
432 };
433 )cpp");
434 auto Symbols = Header.headerSymbols();
435 auto AST = Header.build();
436
437 const NamedDecl *CtorDecl = &findDecl(AST, [](const NamedDecl &ND) {
438 return (ND.getQualifiedNameAsString() == "Foo::Foo") &&
439 isa<CXXConstructorDecl>(&ND);
440 });
441 const NamedDecl *DtorDecl = &findDecl(AST, [](const NamedDecl &ND) {
442 return (ND.getQualifiedNameAsString() == "Foo::~Foo") &&
443 isa<CXXDestructorDecl>(&ND);
444 });
445
446 SymbolQualitySignals CtorQ;
447 CtorQ.merge(CodeCompletionResult(CtorDecl, /*Priority=*/0));
448 EXPECT_EQ(CtorQ.Category, SymbolQualitySignals::Constructor);
449
450 CtorQ.Category = SymbolQualitySignals::Unknown;
451 const Symbol &CtorSym = findSymbol(Symbols, "Foo::Foo");
452 CtorQ.merge(CtorSym);
453 EXPECT_EQ(CtorQ.Category, SymbolQualitySignals::Constructor);
454
455 SymbolQualitySignals DtorQ;
456 DtorQ.merge(CodeCompletionResult(DtorDecl, /*Priority=*/0));
457 EXPECT_EQ(DtorQ.Category, SymbolQualitySignals::Destructor);
458 }
459
TEST(QualityTests,Operator)460 TEST(QualityTests, Operator) {
461 auto Header = TestTU::withHeaderCode(R"cpp(
462 class Foo {
463 public:
464 bool operator<(const Foo& f1);
465 };
466 )cpp");
467 auto AST = Header.build();
468
469 const NamedDecl *Operator = &findDecl(AST, [](const NamedDecl &ND) {
470 if (const auto *OD = dyn_cast<FunctionDecl>(&ND))
471 if (OD->isOverloadedOperator())
472 return true;
473 return false;
474 });
475 SymbolQualitySignals Q;
476 Q.merge(CodeCompletionResult(Operator, /*Priority=*/0));
477 EXPECT_EQ(Q.Category, SymbolQualitySignals::Operator);
478 }
479
TEST(QualityTests,ItemWithFixItsRankedDown)480 TEST(QualityTests, ItemWithFixItsRankedDown) {
481 CodeCompleteOptions Opts;
482 Opts.IncludeFixIts = true;
483
484 auto Header = TestTU::withHeaderCode(R"cpp(
485 int x;
486 )cpp");
487 auto AST = Header.build();
488
489 SymbolRelevanceSignals RelevanceWithFixIt;
490 RelevanceWithFixIt.merge(CodeCompletionResult(&findDecl(AST, "x"), 0, nullptr,
491 false, true, {FixItHint{}}));
492 EXPECT_TRUE(RelevanceWithFixIt.NeedsFixIts);
493
494 SymbolRelevanceSignals RelevanceWithoutFixIt;
495 RelevanceWithoutFixIt.merge(
496 CodeCompletionResult(&findDecl(AST, "x"), 0, nullptr, false, true, {}));
497 EXPECT_FALSE(RelevanceWithoutFixIt.NeedsFixIts);
498
499 EXPECT_LT(RelevanceWithFixIt.evaluateHeuristics(),
500 RelevanceWithoutFixIt.evaluateHeuristics());
501 }
502
503 } // namespace
504 } // namespace clangd
505 } // namespace clang
506