1 /**
2 * File utilities.
3 *
4 * Functions and objects dedicated to file I/O and management. TODO: Move here artifacts
5 * from places such as root/ so both the frontend and the backend have access to them.
6 *
7 * Copyright: Copyright (C) 1999-2022 by The D Language Foundation, All Rights Reserved
8 * Authors: Walter Bright, https://www.digitalmars.com
9 * License: $(LINK2 https://www.boost.org/LICENSE_1_0.txt, Boost License 1.0)
10 * Source: $(LINK2 https://github.com/dlang/dmd/blob/master/src/dmd/common/file.d, common/_file.d)
11 * Documentation: https://dlang.org/phobos/dmd_common_file.html
12 * Coverage: https://codecov.io/gh/dlang/dmd/src/master/src/dmd/common/file.d
13 */
14
15 module dmd.common.file;
16
17 import core.stdc.errno : errno;
18 import core.stdc.stdio : fprintf, remove, rename, stderr;
19 import core.stdc.stdlib : exit;
20 import core.stdc.string : strerror;
21 import core.sys.windows.winbase;
22 import core.sys.windows.winnt;
23 import core.sys.posix.fcntl;
24 import core.sys.posix.unistd;
25
26 import dmd.common.string;
27
28 nothrow:
29
30 /**
31 Encapsulated management of a memory-mapped file.
32
33 Params:
34 Datum = the mapped data type: Use a POD of size 1 for read/write mapping
35 and a `const` version thereof for read-only mapping. Other primitive types
36 should work, but have not been yet tested.
37 */
FileMapping(Datum)38 struct FileMapping(Datum)
39 {
40 static assert(__traits(isPOD, Datum) && Datum.sizeof == 1,
41 "Not tested with other data types yet. Add new types with care.");
42
43 version(Posix) enum invalidHandle = -1;
44 else version(Windows) enum invalidHandle = INVALID_HANDLE_VALUE;
45
46 // state {
47 /// Handle of underlying file
48 private auto handle = invalidHandle;
49 /// File mapping object needed on Windows
50 version(Windows) private HANDLE fileMappingObject = invalidHandle;
51 /// Memory-mapped array
52 private Datum[] data;
53 /// Name of underlying file, zero-terminated
54 private const(char)* name;
55 // state }
56
57 nothrow:
58
59 /**
60 Open `filename` and map it in memory. If `Datum` is `const`, opens for
61 read-only and maps the content in memory; no error is issued if the file
62 does not exist. This makes it easy to treat a non-existing file as empty.
63
64 If `Datum` is mutable, opens for read/write (creates file if it does not
65 exist) and fails fatally on any error.
66
67 Due to quirks in `mmap`, if the file is empty, `handle` is valid but `data`
68 is `null`. This state is valid and accounted for.
69
70 Params:
71 filename = the name of the file to be mapped in memory
72 */
73 this(const char* filename)
74 {
75 version (Posix)
76 {
77 import core.sys.posix.sys.mman;
78 import core.sys.posix.fcntl : open, O_CREAT, O_RDONLY, O_RDWR, S_IRGRP, S_IROTH, S_IRUSR, S_IWUSR;
79
80 handle = open(filename, is(Datum == const) ? O_RDONLY : (O_CREAT | O_RDWR),
81 S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
82
83 if (handle == invalidHandle)
84 {
85 static if (is(Datum == const))
86 {
87 // No error, nonexisting file in read mode behaves like an empty file.
88 return;
89 }
90 else
91 {
92 fprintf(stderr, "open(\"%s\") failed: %s\n", filename, strerror(errno));
93 exit(1);
94 }
95 }
96
97 const size = fileSize(handle);
98
99 if (size > 0 && size != ulong.max && size <= size_t.max)
100 {
101 auto p = mmap(null, cast(size_t) size, is(Datum == const) ? PROT_READ : PROT_WRITE, MAP_SHARED, handle, 0);
102 if (p == MAP_FAILED)
103 {
104 fprintf(stderr, "mmap(null, %zu) for \"%s\" failed: %s\n", cast(size_t) size, filename, strerror(errno));
105 exit(1);
106 }
107 // The cast below will always work because it's gated by the `size <= size_t.max` condition.
108 data = cast(Datum[]) p[0 .. cast(size_t) size];
109 }
110 }
111 else version(Windows)
112 {
113 static if (is(Datum == const))
114 {
115 enum createFileMode = GENERIC_READ;
116 enum openFlags = OPEN_EXISTING;
117 }
118 else
119 {
120 enum createFileMode = GENERIC_READ | GENERIC_WRITE;
121 enum openFlags = CREATE_ALWAYS;
122 }
123
124 handle = filename.asDString.extendedPathThen!(p => CreateFileW(p.ptr, createFileMode, 0, null, openFlags, FILE_ATTRIBUTE_NORMAL, null));
125 if (handle == invalidHandle)
126 {
127 static if (is(Datum == const))
128 {
129 return;
130 }
131 else
132 {
133 fprintf(stderr, "CreateFileW() failed for \"%s\": %d\n", filename, GetLastError());
134 exit(1);
135 }
136 }
137 createMapping(filename, fileSize(handle));
138 }
139 else static assert(0);
140
141 // Save the name for later. Technically there's no need: on Linux one can use readlink on /proc/self/fd/NNN.
142 // On BSD and OSX one can use fcntl with F_GETPATH. On Windows one can use GetFileInformationByHandleEx.
143 // But just saving the name is simplest, fastest, and most portable...
144 import core.stdc.string : strlen;
145 import core.stdc.stdlib : malloc;
146 import core.stdc.string : memcpy;
147 auto totalNameLength = filename.strlen() + 1;
148 name = cast(char*) memcpy(malloc(totalNameLength), filename, totalNameLength);
149 name || assert(0, "FileMapping: Out of memory.");
150 }
151
152 /**
153 Common code factored opportunistically. Windows only. Assumes `handle` is
154 already pointing to an opened file. Initializes the `fileMappingObject`
155 and `data` members.
156
157 Params:
158 filename = the file to be mapped
159 size = the size of the file in bytes
160 */
161 version(Windows) private void createMapping(const char* filename, ulong size)
162 {
163 assert(size <= size_t.max || size == ulong.max);
164 assert(handle != invalidHandle);
165 assert(data is null);
166 assert(fileMappingObject == invalidHandle);
167
168 if (size == 0 || size == ulong.max)
169 return;
170
171 static if (is(Datum == const))
172 {
173 enum fileMappingFlags = PAGE_READONLY;
174 enum mapViewFlags = FILE_MAP_READ;
175 }
176 else
177 {
178 enum fileMappingFlags = PAGE_READWRITE;
179 enum mapViewFlags = FILE_MAP_WRITE;
180 }
181
182 fileMappingObject = CreateFileMappingW(handle, null, fileMappingFlags, 0, 0, null);
183 if (!fileMappingObject)
184 {
185 fprintf(stderr, "CreateFileMappingW(%p) failed for %llu bytes of \"%s\": %d\n",
186 handle, size, filename, GetLastError());
187 fileMappingObject = invalidHandle; // by convention always use invalidHandle, not null
188 exit(1);
189 }
190 auto p = MapViewOfFile(fileMappingObject, mapViewFlags, 0, 0, 0);
191 if (!p)
192 {
193 fprintf(stderr, "MapViewOfFile() failed for \"%s\": %d\n", filename, GetLastError());
194 exit(1);
195 }
196 data = cast(Datum[]) p[0 .. cast(size_t) size];
197 }
198
199 // Not copyable or assignable (for now).
200 @disable this(const FileMapping!Datum rhs);
201 @disable void opAssign(const ref FileMapping!Datum rhs);
202
203 /**
204 Frees resources associated with this mapping. However, it does not deallocate the name.
205 */
206 ~this() pure nothrow
207 {
208 if (!active)
209 return;
210 fakePure({
211 version (Posix)
212 {
213 import core.sys.posix.sys.mman : munmap;
214 import core.sys.posix.unistd : close;
215
216 // Cannot call fprintf from inside a destructor, so exiting silently.
217
218 if (data.ptr && munmap(cast(void*) data.ptr, data.length) != 0)
219 {
220 exit(1);
221 }
222 data = null;
223 if (handle != invalidHandle && close(handle) != 0)
224 {
225 exit(1);
226 }
227 handle = invalidHandle;
228 }
229 else version(Windows)
230 {
231 if (data.ptr !is null && UnmapViewOfFile(cast(void*) data.ptr) == 0)
232 {
233 exit(1);
234 }
235 data = null;
236 if (fileMappingObject != invalidHandle && CloseHandle(fileMappingObject) == 0)
237 {
238 exit(1);
239 }
240 fileMappingObject = invalidHandle;
241 if (handle != invalidHandle && CloseHandle(handle) == 0)
242 {
243 exit(1);
244 }
245 handle = invalidHandle;
246 }
247 else static assert(0);
248 });
249 }
250
251 /**
252 Returns the zero-terminated file name associated with the mapping. Can NOT
253 be saved beyond the lifetime of `this`.
254 */
255 private const(char)* filename() const pure @nogc @safe nothrow { return name; }
256
257 /**
258 Frees resources associated with this mapping. However, it does not deallocate the name.
259 Reinitializes `this` as a fresh object that can be reused.
260 */
261 void close()
262 {
263 __dtor();
264 handle = invalidHandle;
265 version(Windows) fileMappingObject = invalidHandle;
266 data = null;
267 name = null;
268 }
269
270 /**
271 Deletes the underlying file and frees all resources associated.
272 Reinitializes `this` as a fresh object that can be reused.
273
274 This function does not abort if the file cannot be deleted, but does print
275 a message on `stderr` and returns `false` to the caller. The underlying
276 rationale is to give the caller the option to continue execution if
277 deleting the file is not important.
278
279 Returns: `true` iff the file was successfully deleted. If the file was not
280 deleted, prints a message to `stderr` and returns `false`.
281 */
282 static if (!is(Datum == const))
283 bool discard()
284 {
285 // Truncate file to zero so unflushed buffers are not flushed unnecessarily.
286 resize(0);
287 auto deleteme = name;
288 close();
289 // In-memory resource freed, now get rid of the underlying temp file.
290 version(Posix)
291 {
292 import core.sys.posix.unistd : unlink;
293 if (unlink(deleteme) != 0)
294 {
295 fprintf(stderr, "unlink(\"%s\") failed: %s\n", filename, strerror(errno));
296 return false;
297 }
298 }
299 else version(Windows)
300 {
301 import core.sys.windows.winbase;
302 if (deleteme.asDString.extendedPathThen!(p => DeleteFileW(p.ptr)) == 0)
303 {
304 fprintf(stderr, "DeleteFileW error %d\n", GetLastError());
305 return false;
306 }
307 }
308 else static assert(0);
309 return true;
310 }
311
312 /**
313 Queries whether `this` is currently associated with a file.
314
315 Returns: `true` iff there is an active mapping.
316 */
317 bool active() const pure @nogc nothrow
318 {
319 return handle !is invalidHandle;
320 }
321
322 /**
323 Queries the length of the file associated with this mapping. If not
324 active, returns 0.
325
326 Returns: the length of the file, or 0 if no file associated.
327 */
328 size_t length() const pure @nogc @safe nothrow { return data.length; }
329
330 /**
331 Get a slice to the contents of the entire file.
332
333 Returns: the contents of the file. If not active, returns the `null` slice.
334 */
335 auto opSlice() pure @nogc @safe nothrow { return data; }
336
337 /**
338 Resizes the file and mapping to the specified `size`.
339
340 Params:
341 size = new length requested
342 */
343 static if (!is(Datum == const))
344 void resize(size_t size) pure
345 {
346 assert(handle != invalidHandle);
347 fakePure({
348 version(Posix)
349 {
350 import core.sys.posix.unistd : ftruncate;
351 import core.sys.posix.sys.mman;
352
353 if (data.length)
354 {
355 assert(data.ptr, "Corrupt memory mapping");
356 // assert(0) here because it would indicate an internal error
357 munmap(cast(void*) data.ptr, data.length) == 0 || assert(0);
358 data = null;
359 }
360 if (ftruncate(handle, size) != 0)
361 {
362 fprintf(stderr, "ftruncate() failed for \"%s\": %s\n", filename, strerror(errno));
363 exit(1);
364 }
365 if (size > 0)
366 {
367 auto p = mmap(null, size, PROT_WRITE, MAP_SHARED, handle, 0);
368 if (cast(ssize_t) p == -1)
369 {
370 fprintf(stderr, "mmap() failed for \"%s\": %s\n", filename, strerror(errno));
371 exit(1);
372 }
373 data = cast(Datum[]) p[0 .. size];
374 }
375 }
376 else version(Windows)
377 {
378 // Per documentation, must unmap first.
379 if (data.length > 0 && UnmapViewOfFile(cast(void*) data.ptr) == 0)
380 {
381 fprintf(stderr, "UnmapViewOfFile(%p) failed for memory mapping of \"%s\": %d\n",
382 data.ptr, filename, GetLastError());
383 exit(1);
384 }
385 data = null;
386 if (fileMappingObject != invalidHandle && CloseHandle(fileMappingObject) == 0)
387 {
388 fprintf(stderr, "CloseHandle() failed for memory mapping of \"%s\": %d\n", filename, GetLastError());
389 exit(1);
390 }
391 fileMappingObject = invalidHandle;
392 LARGE_INTEGER biggie;
393 biggie.QuadPart = size;
394 if (SetFilePointerEx(handle, biggie, null, FILE_BEGIN) == 0 || SetEndOfFile(handle) == 0)
395 {
396 fprintf(stderr, "SetFilePointer() failed for \"%s\": %d\n", filename, GetLastError());
397 exit(1);
398 }
399 createMapping(name, size);
400 }
401 else static assert(0);
402 });
403 }
404
405 /**
406 Unconditionally and destructively moves the underlying file to `filename`.
407 If the operation succeeds, returns true. Upon failure, prints a message to
408 `stderr` and returns `false`. In all cases it closes the underlying file.
409
410 Params: filename = zero-terminated name of the file to move to.
411
412 Returns: `true` iff the operation was successful.
413 */
414 bool moveToFile(const char* filename)
415 {
416 assert(name !is null);
417
418 // Fetch the name and then set it to `null` so it doesn't get deallocated
419 auto oldname = name;
420 import core.stdc.stdlib;
421 scope(exit) free(cast(void*) oldname);
422 name = null;
423 close();
424
425 // Rename the underlying file to the target, no copy necessary.
426 version(Posix)
427 {
428 if (.rename(oldname, filename) != 0)
429 {
430 fprintf(stderr, "rename(\"%s\", \"%s\") failed: %s\n", oldname, filename, strerror(errno));
431 return false;
432 }
433 }
434 else version(Windows)
435 {
436 import core.sys.windows.winbase;
437 auto r = oldname.asDString.extendedPathThen!(
438 p1 => filename.asDString.extendedPathThen!(p2 => MoveFileExW(p1.ptr, p2.ptr, MOVEFILE_REPLACE_EXISTING))
439 );
440 if (r == 0)
441 {
442 fprintf(stderr, "MoveFileExW(\"%s\", \"%s\") failed: %d\n", oldname, filename, GetLastError());
443 return false;
444 }
445 }
446 else static assert(0);
447 return true;
448 }
449 }
450
451 /// Write a file, returning `true` on success.
writeFile(const (char)* name,const void[]data)452 extern(D) static bool writeFile(const(char)* name, const void[] data) nothrow
453 {
454 version (Posix)
455 {
456 int fd = open(name, O_CREAT | O_WRONLY | O_TRUNC, (6 << 6) | (4 << 3) | 4);
457 if (fd == -1)
458 goto err;
459 if (.write(fd, data.ptr, data.length) != data.length)
460 goto err2;
461 if (close(fd) == -1)
462 goto err;
463 return true;
464 err2:
465 close(fd);
466 .remove(name);
467 err:
468 return false;
469 }
470 else version (Windows)
471 {
472 DWORD numwritten; // here because of the gotos
473 const nameStr = name.asDString;
474 // work around Windows file path length limitation
475 // (see documentation for extendedPathThen).
476 HANDLE h = nameStr.extendedPathThen!
477 (p => CreateFileW(p.ptr,
478 GENERIC_WRITE,
479 0,
480 null,
481 CREATE_ALWAYS,
482 FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN,
483 null));
484 if (h == INVALID_HANDLE_VALUE)
485 goto err;
486
487 if (WriteFile(h, data.ptr, cast(DWORD)data.length, &numwritten, null) != TRUE)
488 goto err2;
489 if (numwritten != data.length)
490 goto err2;
491 if (!CloseHandle(h))
492 goto err;
493 return true;
494 err2:
495 CloseHandle(h);
496 nameStr.extendedPathThen!(p => DeleteFileW(p.ptr));
497 err:
498 return false;
499 }
500 else
501 {
502 static assert(0);
503 }
504 }
505
506 /// Touch a file to current date
touchFile(const char * namez)507 bool touchFile(const char* namez)
508 {
509 version (Windows)
510 {
511 FILETIME ft = void;
512 SYSTEMTIME st = void;
513 GetSystemTime(&st);
514 SystemTimeToFileTime(&st, &ft);
515
516 import core.stdc.string : strlen;
517
518 // get handle to file
519 HANDLE h = namez[0 .. namez.strlen()].extendedPathThen!(p => CreateFile(p.ptr,
520 FILE_WRITE_ATTRIBUTES, FILE_SHARE_READ | FILE_SHARE_WRITE,
521 null, OPEN_EXISTING,
522 FILE_ATTRIBUTE_NORMAL, null));
523 if (h == INVALID_HANDLE_VALUE)
524 return false;
525
526 const f = SetFileTime(h, null, null, &ft); // set last write time
527
528 if (!CloseHandle(h))
529 return false;
530
531 return f != 0;
532 }
533 else version (Posix)
534 {
535 import core.sys.posix.utime;
536 return utime(namez, null) == 0;
537 }
538 else
539 static assert(0);
540 }
541
542 // Feel free to make these public if used elsewhere.
543 /**
544 Size of a file in bytes.
545 Params: fd = file handle
546 Returns: file size in bytes, or `ulong.max` on any error.
547 */
version(Posix)548 version (Posix)
549 private ulong fileSize(int fd)
550 {
551 import core.sys.posix.sys.stat;
552 stat_t buf;
553 if (fstat(fd, &buf) == 0)
554 return buf.st_size;
555 return ulong.max;
556 }
557
558 /// Ditto
version(Windows)559 version (Windows)
560 private ulong fileSize(HANDLE fd)
561 {
562 ulong result;
563 if (GetFileSizeEx(fd, cast(LARGE_INTEGER*) &result) == 0)
564 return result;
565 return ulong.max;
566 }
567
568 /**
569 Runs a non-pure function or delegate as pure code. Use with caution.
570
571 Params:
572 fun = the delegate to run, usually inlined: `fakePure({ ... });`
573
574 Returns: whatever `fun` returns.
575 */
fakePure(F)576 private auto ref fakePure(F)(scope F fun) pure
577 {
578 mixin("alias PureFun = " ~ F.stringof ~ " pure;");
579 return (cast(PureFun) fun)();
580 }
581