1 //===- unittests/DirectoryWatcher/DirectoryWatcherTest.cpp ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "clang/DirectoryWatcher/DirectoryWatcher.h"
10 #include "llvm/Support/FileSystem.h"
11 #include "llvm/Support/Path.h"
12 #include "llvm/Support/raw_ostream.h"
13 #include "llvm/Testing/Support/Error.h"
14 #include "gtest/gtest.h"
15 #include <condition_variable>
16 #include <future>
17 #include <mutex>
18 #include <optional>
19 #include <thread>
20
21 using namespace llvm;
22 using namespace llvm::sys;
23 using namespace llvm::sys::fs;
24 using namespace clang;
25
26 namespace clang {
operator ==(const DirectoryWatcher::Event & lhs,const DirectoryWatcher::Event & rhs)27 static bool operator==(const DirectoryWatcher::Event &lhs,
28 const DirectoryWatcher::Event &rhs) {
29 return lhs.Filename == rhs.Filename &&
30 static_cast<int>(lhs.Kind) == static_cast<int>(rhs.Kind);
31 }
32 } // namespace clang
33
34 namespace {
35
36 typedef DirectoryWatcher::Event::EventKind EventKind;
37
38 // We've observed this test being significantly flaky when running on a heavily
39 // loaded machine (e.g. when it's being run as part of the full check-clang
40 // suite). Set a high timeout value to avoid this flakiness. The 60s timeout
41 // value was determined empirically. It's a timeout value, not a sleep value,
42 // and the test should require much less time in practice the vast majority of
43 // instances. The cases where we do come close to (or still end up hitting) the
44 // longer timeout are most likely to occur when other tests are also running at
45 // the same time (e.g. as part of the full check-clang suite), in which case the
46 // latency of the timeout will be masked by the latency of the other tests.
47 constexpr std::chrono::seconds EventualResultTimeout(60);
48
49 struct DirectoryWatcherTestFixture {
50 std::string TestRootDir;
51 std::string TestWatchedDir;
52
DirectoryWatcherTestFixture__anon421a5f3e0111::DirectoryWatcherTestFixture53 DirectoryWatcherTestFixture() {
54 SmallString<128> pathBuf;
55 #ifndef NDEBUG
56 std::error_code UniqDirRes =
57 #endif
58 createUniqueDirectory("dirwatcher", pathBuf);
59 assert(!UniqDirRes);
60 TestRootDir = std::string(pathBuf.str());
61 path::append(pathBuf, "watch");
62 TestWatchedDir = std::string(pathBuf.str());
63 #ifndef NDEBUG
64 std::error_code CreateDirRes =
65 #endif
66 create_directory(TestWatchedDir, false);
67 assert(!CreateDirRes);
68 }
69
~DirectoryWatcherTestFixture__anon421a5f3e0111::DirectoryWatcherTestFixture70 ~DirectoryWatcherTestFixture() { remove_directories(TestRootDir); }
71
getPathInWatched__anon421a5f3e0111::DirectoryWatcherTestFixture72 SmallString<128> getPathInWatched(const std::string &testFile) {
73 SmallString<128> pathBuf;
74 pathBuf = TestWatchedDir;
75 path::append(pathBuf, testFile);
76 return pathBuf;
77 }
78
addFile__anon421a5f3e0111::DirectoryWatcherTestFixture79 void addFile(const std::string &testFile) {
80 Expected<file_t> ft = openNativeFileForWrite(getPathInWatched(testFile),
81 CD_CreateNew, OF_None);
82 if (ft) {
83 closeFile(*ft);
84 } else {
85 llvm::errs() << llvm::toString(ft.takeError()) << "\n";
86 llvm::errs() << getPathInWatched(testFile) << "\n";
87 llvm_unreachable("Couldn't create test file.");
88 }
89 }
90
deleteFile__anon421a5f3e0111::DirectoryWatcherTestFixture91 void deleteFile(const std::string &testFile) {
92 std::error_code EC =
93 remove(getPathInWatched(testFile), /*IgnoreNonExisting=*/false);
94 ASSERT_FALSE(EC);
95 }
96 };
97
eventKindToString(const EventKind K)98 std::string eventKindToString(const EventKind K) {
99 switch (K) {
100 case EventKind::Removed:
101 return "Removed";
102 case EventKind::Modified:
103 return "Modified";
104 case EventKind::WatchedDirRemoved:
105 return "WatchedDirRemoved";
106 case EventKind::WatcherGotInvalidated:
107 return "WatcherGotInvalidated";
108 }
109 llvm_unreachable("unknown event kind");
110 }
111
112 struct VerifyingConsumer {
113 std::vector<DirectoryWatcher::Event> ExpectedInitial;
114 const std::vector<DirectoryWatcher::Event> ExpectedInitialCopy;
115 std::vector<DirectoryWatcher::Event> ExpectedNonInitial;
116 const std::vector<DirectoryWatcher::Event> ExpectedNonInitialCopy;
117 std::vector<DirectoryWatcher::Event> OptionalNonInitial;
118 std::vector<DirectoryWatcher::Event> UnexpectedInitial;
119 std::vector<DirectoryWatcher::Event> UnexpectedNonInitial;
120 std::mutex Mtx;
121 std::condition_variable ResultIsReady;
122
VerifyingConsumer__anon421a5f3e0111::VerifyingConsumer123 VerifyingConsumer(
124 const std::vector<DirectoryWatcher::Event> &ExpectedInitial,
125 const std::vector<DirectoryWatcher::Event> &ExpectedNonInitial,
126 const std::vector<DirectoryWatcher::Event> &OptionalNonInitial = {})
127 : ExpectedInitial(ExpectedInitial), ExpectedInitialCopy(ExpectedInitial),
128 ExpectedNonInitial(ExpectedNonInitial), ExpectedNonInitialCopy(ExpectedNonInitial),
129 OptionalNonInitial(OptionalNonInitial) {}
130
131 // This method is used by DirectoryWatcher.
consume__anon421a5f3e0111::VerifyingConsumer132 void consume(DirectoryWatcher::Event E, bool IsInitial) {
133 if (IsInitial)
134 consumeInitial(E);
135 else
136 consumeNonInitial(E);
137 }
138
consumeInitial__anon421a5f3e0111::VerifyingConsumer139 void consumeInitial(DirectoryWatcher::Event E) {
140 std::unique_lock<std::mutex> L(Mtx);
141 auto It = std::find(ExpectedInitial.begin(), ExpectedInitial.end(), E);
142 if (It == ExpectedInitial.end()) {
143 UnexpectedInitial.push_back(E);
144 } else {
145 ExpectedInitial.erase(It);
146 }
147 if (result()) {
148 L.unlock();
149 ResultIsReady.notify_one();
150 }
151 }
152
consumeNonInitial__anon421a5f3e0111::VerifyingConsumer153 void consumeNonInitial(DirectoryWatcher::Event E) {
154 std::unique_lock<std::mutex> L(Mtx);
155 auto It =
156 std::find(ExpectedNonInitial.begin(), ExpectedNonInitial.end(), E);
157 if (It == ExpectedNonInitial.end()) {
158 auto OptIt =
159 std::find(OptionalNonInitial.begin(), OptionalNonInitial.end(), E);
160 if (OptIt != OptionalNonInitial.end()) {
161 OptionalNonInitial.erase(OptIt);
162 } else {
163 UnexpectedNonInitial.push_back(E);
164 }
165 } else {
166 ExpectedNonInitial.erase(It);
167 }
168 if (result()) {
169 L.unlock();
170 ResultIsReady.notify_one();
171 }
172 }
173
174 // This method is used by DirectoryWatcher.
consume__anon421a5f3e0111::VerifyingConsumer175 void consume(llvm::ArrayRef<DirectoryWatcher::Event> Es, bool IsInitial) {
176 for (const auto &E : Es)
177 consume(E, IsInitial);
178 }
179
180 // Not locking - caller has to lock Mtx.
result__anon421a5f3e0111::VerifyingConsumer181 std::optional<bool> result() const {
182 if (ExpectedInitial.empty() && ExpectedNonInitial.empty() &&
183 UnexpectedInitial.empty() && UnexpectedNonInitial.empty())
184 return true;
185 if (!UnexpectedInitial.empty() || !UnexpectedNonInitial.empty())
186 return false;
187 return std::nullopt;
188 }
189
190 // This method is used by tests.
191 // \returns true on success
blockUntilResult__anon421a5f3e0111::VerifyingConsumer192 bool blockUntilResult() {
193 std::unique_lock<std::mutex> L(Mtx);
194 while (true) {
195 if (result())
196 return *result();
197
198 ResultIsReady.wait(L, [this]() { return result().has_value(); });
199 }
200 return false; // Just to make compiler happy.
201 }
202
printUnmetExpectations__anon421a5f3e0111::VerifyingConsumer203 void printUnmetExpectations(llvm::raw_ostream &OS) {
204 // If there was any issue, print the expected state
205 if (
206 !ExpectedInitial.empty()
207 ||
208 !ExpectedNonInitial.empty()
209 ||
210 !UnexpectedInitial.empty()
211 ||
212 !UnexpectedNonInitial.empty()
213 ) {
214 OS << "Expected initial events: \n";
215 for (const auto &E : ExpectedInitialCopy) {
216 OS << eventKindToString(E.Kind) << " " << E.Filename << "\n";
217 }
218 OS << "Expected non-initial events: \n";
219 for (const auto &E : ExpectedNonInitialCopy) {
220 OS << eventKindToString(E.Kind) << " " << E.Filename << "\n";
221 }
222 }
223
224 if (!ExpectedInitial.empty()) {
225 OS << "Expected but not seen initial events: \n";
226 for (const auto &E : ExpectedInitial) {
227 OS << eventKindToString(E.Kind) << " " << E.Filename << "\n";
228 }
229 }
230 if (!ExpectedNonInitial.empty()) {
231 OS << "Expected but not seen non-initial events: \n";
232 for (const auto &E : ExpectedNonInitial) {
233 OS << eventKindToString(E.Kind) << " " << E.Filename << "\n";
234 }
235 }
236 if (!UnexpectedInitial.empty()) {
237 OS << "Unexpected initial events seen: \n";
238 for (const auto &E : UnexpectedInitial) {
239 OS << eventKindToString(E.Kind) << " " << E.Filename << "\n";
240 }
241 }
242 if (!UnexpectedNonInitial.empty()) {
243 OS << "Unexpected non-initial events seen: \n";
244 for (const auto &E : UnexpectedNonInitial) {
245 OS << eventKindToString(E.Kind) << " " << E.Filename << "\n";
246 }
247 }
248 }
249 };
250
checkEventualResultWithTimeout(VerifyingConsumer & TestConsumer)251 void checkEventualResultWithTimeout(VerifyingConsumer &TestConsumer) {
252 std::packaged_task<int(void)> task(
253 [&TestConsumer]() { return TestConsumer.blockUntilResult(); });
254 std::future<int> WaitForExpectedStateResult = task.get_future();
255 std::thread worker(std::move(task));
256 worker.detach();
257
258 EXPECT_TRUE(WaitForExpectedStateResult.wait_for(EventualResultTimeout) ==
259 std::future_status::ready)
260 << "The expected result state wasn't reached before the time-out.";
261 std::unique_lock<std::mutex> L(TestConsumer.Mtx);
262 EXPECT_TRUE(TestConsumer.result().has_value());
263 if (TestConsumer.result()) {
264 EXPECT_TRUE(*TestConsumer.result());
265 }
266 if ((TestConsumer.result() && !*TestConsumer.result()) ||
267 !TestConsumer.result())
268 TestConsumer.printUnmetExpectations(llvm::outs());
269 }
270 } // namespace
271
TEST(DirectoryWatcherTest,InitialScanSync)272 TEST(DirectoryWatcherTest, InitialScanSync) {
273 DirectoryWatcherTestFixture fixture;
274
275 fixture.addFile("a");
276 fixture.addFile("b");
277 fixture.addFile("c");
278
279 VerifyingConsumer TestConsumer{
280 {{EventKind::Modified, "a"},
281 {EventKind::Modified, "b"},
282 {EventKind::Modified, "c"}},
283 {},
284 // We have to ignore these as it's a race between the test process
285 // which is scanning the directory and kernel which is sending
286 // notification.
287 {{EventKind::Modified, "a"},
288 {EventKind::Modified, "b"},
289 {EventKind::Modified, "c"}}
290 };
291
292 llvm::Expected<std::unique_ptr<DirectoryWatcher>> DW =
293 DirectoryWatcher::create(
294 fixture.TestWatchedDir,
295 [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events,
296 bool IsInitial) {
297 TestConsumer.consume(Events, IsInitial);
298 },
299 /*waitForInitialSync=*/true);
300 ASSERT_THAT_ERROR(DW.takeError(), Succeeded());
301
302 checkEventualResultWithTimeout(TestConsumer);
303 }
304
TEST(DirectoryWatcherTest,InitialScanAsync)305 TEST(DirectoryWatcherTest, InitialScanAsync) {
306 DirectoryWatcherTestFixture fixture;
307
308 fixture.addFile("a");
309 fixture.addFile("b");
310 fixture.addFile("c");
311
312 VerifyingConsumer TestConsumer{
313 {{EventKind::Modified, "a"},
314 {EventKind::Modified, "b"},
315 {EventKind::Modified, "c"}},
316 {},
317 // We have to ignore these as it's a race between the test process
318 // which is scanning the directory and kernel which is sending
319 // notification.
320 {{EventKind::Modified, "a"},
321 {EventKind::Modified, "b"},
322 {EventKind::Modified, "c"}}
323 };
324
325 llvm::Expected<std::unique_ptr<DirectoryWatcher>> DW =
326 DirectoryWatcher::create(
327 fixture.TestWatchedDir,
328 [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events,
329 bool IsInitial) {
330 TestConsumer.consume(Events, IsInitial);
331 },
332 /*waitForInitialSync=*/false);
333 ASSERT_THAT_ERROR(DW.takeError(), Succeeded());
334
335 checkEventualResultWithTimeout(TestConsumer);
336 }
337
TEST(DirectoryWatcherTest,AddFiles)338 TEST(DirectoryWatcherTest, AddFiles) {
339 DirectoryWatcherTestFixture fixture;
340
341 VerifyingConsumer TestConsumer{
342 {},
343 {{EventKind::Modified, "a"},
344 {EventKind::Modified, "b"},
345 {EventKind::Modified, "c"}}};
346
347 llvm::Expected<std::unique_ptr<DirectoryWatcher>> DW =
348 DirectoryWatcher::create(
349 fixture.TestWatchedDir,
350 [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events,
351 bool IsInitial) {
352 TestConsumer.consume(Events, IsInitial);
353 },
354 /*waitForInitialSync=*/true);
355 ASSERT_THAT_ERROR(DW.takeError(), Succeeded());
356
357 fixture.addFile("a");
358 fixture.addFile("b");
359 fixture.addFile("c");
360
361 checkEventualResultWithTimeout(TestConsumer);
362 }
363
TEST(DirectoryWatcherTest,ModifyFile)364 TEST(DirectoryWatcherTest, ModifyFile) {
365 DirectoryWatcherTestFixture fixture;
366
367 fixture.addFile("a");
368
369 VerifyingConsumer TestConsumer{
370 {{EventKind::Modified, "a"}},
371 {{EventKind::Modified, "a"}},
372 {{EventKind::Modified, "a"}}};
373
374 llvm::Expected<std::unique_ptr<DirectoryWatcher>> DW =
375 DirectoryWatcher::create(
376 fixture.TestWatchedDir,
377 [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events,
378 bool IsInitial) {
379 TestConsumer.consume(Events, IsInitial);
380 },
381 /*waitForInitialSync=*/true);
382 ASSERT_THAT_ERROR(DW.takeError(), Succeeded());
383
384 // modify the file
385 {
386 std::error_code error;
387 llvm::raw_fd_ostream bStream(fixture.getPathInWatched("a"), error,
388 CD_OpenExisting);
389 assert(!error);
390 bStream << "foo";
391 }
392
393 checkEventualResultWithTimeout(TestConsumer);
394 }
395
TEST(DirectoryWatcherTest,DeleteFile)396 TEST(DirectoryWatcherTest, DeleteFile) {
397 DirectoryWatcherTestFixture fixture;
398
399 fixture.addFile("a");
400
401 VerifyingConsumer TestConsumer{
402 {{EventKind::Modified, "a"}},
403 {{EventKind::Removed, "a"}},
404 {{EventKind::Modified, "a"}, {EventKind::Removed, "a"}}};
405
406 llvm::Expected<std::unique_ptr<DirectoryWatcher>> DW =
407 DirectoryWatcher::create(
408 fixture.TestWatchedDir,
409 [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events,
410 bool IsInitial) {
411 TestConsumer.consume(Events, IsInitial);
412 },
413 /*waitForInitialSync=*/true);
414 ASSERT_THAT_ERROR(DW.takeError(), Succeeded());
415
416 fixture.deleteFile("a");
417
418 checkEventualResultWithTimeout(TestConsumer);
419 }
420
TEST(DirectoryWatcherTest,DeleteWatchedDir)421 TEST(DirectoryWatcherTest, DeleteWatchedDir) {
422 DirectoryWatcherTestFixture fixture;
423
424 VerifyingConsumer TestConsumer{
425 {},
426 {{EventKind::WatchedDirRemoved, ""},
427 {EventKind::WatcherGotInvalidated, ""}}};
428
429 llvm::Expected<std::unique_ptr<DirectoryWatcher>> DW =
430 DirectoryWatcher::create(
431 fixture.TestWatchedDir,
432 [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events,
433 bool IsInitial) {
434 TestConsumer.consume(Events, IsInitial);
435 },
436 /*waitForInitialSync=*/true);
437 ASSERT_THAT_ERROR(DW.takeError(), Succeeded());
438
439 remove_directories(fixture.TestWatchedDir);
440
441 checkEventualResultWithTimeout(TestConsumer);
442 }
443
TEST(DirectoryWatcherTest,InvalidatedWatcher)444 TEST(DirectoryWatcherTest, InvalidatedWatcher) {
445 DirectoryWatcherTestFixture fixture;
446
447 VerifyingConsumer TestConsumer{
448 {}, {{EventKind::WatcherGotInvalidated, ""}}};
449
450 {
451 llvm::Expected<std::unique_ptr<DirectoryWatcher>> DW =
452 DirectoryWatcher::create(
453 fixture.TestWatchedDir,
454 [&TestConsumer](llvm::ArrayRef<DirectoryWatcher::Event> Events,
455 bool IsInitial) {
456 TestConsumer.consume(Events, IsInitial);
457 },
458 /*waitForInitialSync=*/true);
459 ASSERT_THAT_ERROR(DW.takeError(), Succeeded());
460 } // DW is destructed here.
461
462 checkEventualResultWithTimeout(TestConsumer);
463 }
464
TEST(DirectoryWatcherTest,InvalidatedWatcherAsync)465 TEST(DirectoryWatcherTest, InvalidatedWatcherAsync) {
466 DirectoryWatcherTestFixture fixture;
467 fixture.addFile("a");
468
469 // This test is checking that we get the initial notification for 'a' before
470 // the final 'invalidated'. Strictly speaking, we do not care whether 'a' is
471 // processed or not, only that it is neither racing with, nor after
472 // 'invalidated'. In practice, it is always processed in our implementations.
473 VerifyingConsumer TestConsumer{
474 {{EventKind::Modified, "a"}},
475 {{EventKind::WatcherGotInvalidated, ""}},
476 // We have to ignore these as it's a race between the test process
477 // which is scanning the directory and kernel which is sending
478 // notification.
479 {{EventKind::Modified, "a"}},
480 };
481
482 // A counter that can help detect data races on the event receiver,
483 // particularly if used with TSan. Expected count will be 2 or 3 depending on
484 // whether we get the kernel event or not (see comment above).
485 unsigned Count = 0;
486 {
487 llvm::Expected<std::unique_ptr<DirectoryWatcher>> DW =
488 DirectoryWatcher::create(
489 fixture.TestWatchedDir,
490 [&TestConsumer,
491 &Count](llvm::ArrayRef<DirectoryWatcher::Event> Events,
492 bool IsInitial) {
493 Count += 1;
494 TestConsumer.consume(Events, IsInitial);
495 },
496 /*waitForInitialSync=*/false);
497 ASSERT_THAT_ERROR(DW.takeError(), Succeeded());
498 } // DW is destructed here.
499
500 checkEventualResultWithTimeout(TestConsumer);
501 ASSERT_TRUE(Count == 2u || Count == 3u);
502 }
503