[pkg-d-commits] [ldc] 04/211: Add ir2obj cache pruning support.
Matthias Klumpp
mak at moszumanska.debian.org
Sun Apr 23 22:36:04 UTC 2017
This is an automated email from the git hooks/post-receive script.
mak pushed a commit to annotated tag v1.1.0
in repository ldc.
commit df4cd3fc20a916d449fe5737cfb6d19733d906d9
Author: Johan Engelen <jbc.engelen at gmail.com>
Date: Fri Sep 9 16:19:42 2016 +0200
Add ir2obj cache pruning support.
---
CMakeLists.txt | 1 +
driver/ir2obj_cache.cpp | 83 ++++++++++++++
driver/ir2obj_cache.h | 11 +-
driver/ir2obj_cache_pruning.d | 208 ++++++++++++++++++++++++++++++++++
driver/ir2obj_cache_pruning.h | 23 ++++
driver/main.cpp | 3 +
tests/linking/ir2obj_cache_pruning.d | 13 +++
tests/linking/ir2obj_cache_pruning2.d | 38 +++++++
8 files changed, 378 insertions(+), 2 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index e6457e5..daeab2e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -346,6 +346,7 @@ set(DRV_HDR
driver/configfile.h
driver/exe_path.h
driver/ir2obj_cache.h
+ driver/ir2obj_cache_pruning.h
driver/ldc-version.h
driver/targetmachine.h
driver/toobj.h
diff --git a/driver/ir2obj_cache.cpp b/driver/ir2obj_cache.cpp
index 9891041..0e2cd7a 100644
--- a/driver/ir2obj_cache.cpp
+++ b/driver/ir2obj_cache.cpp
@@ -32,6 +32,7 @@
#include "ddmd/errors.h"
#include "driver/cl_options.h"
#include "driver/ldc-version.h"
+#include "driver/ir2obj_cache_pruning.h"
#include "gen/logger.h"
#include "gen/optimizer.h"
@@ -41,8 +42,57 @@
#include "llvm/Support/Path.h"
#include "llvm/Support/raw_ostream.h"
+// Include close() declaration.
+#if !defined(_MSC_VER) && !defined(__MINGW32__)
+#include <unistd.h>
+#else
+#include <io.h>
+#endif
+
namespace {
+// Options for the cache pruning algorithm
+llvm::cl::opt<bool> pruneEnabled("ir2obj-cache-prune",
+ llvm::cl::desc("Enable cache pruning."),
+ llvm::cl::ZeroOrMore);
+llvm::cl::opt<unsigned long long> pruneSizeLimitInBytes(
+ "ir2obj-cache-prune-maxbytes",
+ llvm::cl::desc("Sets the maximum cache size to <size> bytes. Implies "
+ "-ir2obj-cache-prune."),
+ llvm::cl::value_desc("size"), llvm::cl::init(0));
+llvm::cl::opt<unsigned> pruneInterval(
+ "ir2obj-cache-prune-interval",
+ llvm::cl::desc("Sets the cache pruning interval to <dur> seconds "
+ "(default: 20 min). Set to 0 to force pruning. Implies "
+ "-ir2obj-cache-prune."),
+ llvm::cl::value_desc("dur"), llvm::cl::init(20 * 60));
+llvm::cl::opt<unsigned> pruneExpiration(
+ "ir2obj-cache-prune-expiration",
+ llvm::cl::desc(
+ "Sets the pruning expiration time of cache files to "
+ "<dur> seconds (default: 1 week). Implies -ir2obj-cache-prune."),
+ llvm::cl::value_desc("dur"), llvm::cl::init(7 * 24 * 3600));
+llvm::cl::opt<unsigned> pruneSizeLimitPercentage(
+ "ir2obj-cache-prune-maxpercentage",
+ llvm::cl::desc(
+ "Sets the cache size limit to <perc> percent of the available "
+ "space (default: 75%). Implies -ir2obj-cache-prune."),
+ llvm::cl::value_desc("perc"), llvm::cl::init(75));
+
+bool isPruningEnabled() {
+ if (pruneEnabled)
+ return true;
+
+ // Specifying cache pruning parameters implies enabling pruning.
+ if ((pruneSizeLimitInBytes.getNumOccurrences() > 0) ||
+ (pruneInterval.getNumOccurrences() > 0) ||
+ (pruneExpiration.getNumOccurrences() > 0) ||
+ (pruneSizeLimitPercentage.getNumOccurrences() > 0))
+ return true;
+
+ return false;
+}
+
/// A raw_ostream that creates a hash of what is written to it.
/// This class does not encounter output errors.
/// There is no buffering and the hasher can be used at any time.
@@ -154,6 +204,7 @@ void recoverObjectFile(llvm::StringRef cacheObjectHash,
llvm::SmallString<128> cacheFile;
storeCacheFileName(cacheObjectHash, cacheFile);
+ // Remove the potentially pre-existing output file.
llvm::sys::fs::remove(objectFile);
IF_LOG Logger::println("SymLink output to cached object file: %s -> %s",
@@ -163,5 +214,37 @@ void recoverObjectFile(llvm::StringRef cacheObjectHash,
cacheFile.c_str(), objectFile.str().c_str());
fatal();
}
+
+ // We reset the modification time to "now" such that the pruning algorithm
+ // sees that the file should be kept over older files.
+ // On some systems the last accessed time is not automatically updated so set
+ // it explicitly here. Because the file will really only be accessed later
+ // during linking, it's not perfect but it's the best we can do.
+ {
+ int FD;
+ if (llvm::sys::fs::openFileForWrite(cacheFile.c_str(), FD,
+ llvm::sys::fs::F_Append)) {
+ error(Loc(), "Failed to open the cached file for writing: %s",
+ cacheFile.c_str());
+ fatal();
+ }
+
+ if (llvm::sys::fs::setLastModificationAndAccessTime(
+ FD, llvm::sys::TimeValue::now())) {
+ error(Loc(), "Failed to set the cached file modification time: %s",
+ cacheFile.c_str());
+ fatal();
+ }
+
+ close(FD);
+ }
+}
+
+void pruneCache() {
+ if (!opts::ir2objCacheDir.empty() && isPruningEnabled()) {
+ ::pruneCache(opts::ir2objCacheDir.data(), opts::ir2objCacheDir.size(),
+ pruneInterval, pruneExpiration, pruneSizeLimitInBytes,
+ pruneSizeLimitPercentage);
+ }
}
}
\ No newline at end of file
diff --git a/driver/ir2obj_cache.h b/driver/ir2obj_cache.h
index 72a807c..ebfece0 100644
--- a/driver/ir2obj_cache.h
+++ b/driver/ir2obj_cache.h
@@ -22,8 +22,15 @@ namespace ir2obj {
void calculateModuleHash(llvm::Module *m, llvm::SmallString<32> &str);
std::string cacheLookup(llvm::StringRef cacheObjectHash);
-void cacheObjectFile(llvm::StringRef objectFile, llvm::StringRef cacheObjectHash);
-void recoverObjectFile(llvm::StringRef cacheObjectHash, llvm::StringRef objectFile);
+void cacheObjectFile(llvm::StringRef objectFile,
+ llvm::StringRef cacheObjectHash);
+void recoverObjectFile(llvm::StringRef cacheObjectHash,
+ llvm::StringRef objectFile);
+
+/// Prune the cache to avoid filling up disk space.
+///
+/// Note: Does nothing for LLVM < 3.7.
+void pruneCache();
}
#endif
diff --git a/driver/ir2obj_cache_pruning.d b/driver/ir2obj_cache_pruning.d
new file mode 100644
index 0000000..36e0658
--- /dev/null
+++ b/driver/ir2obj_cache_pruning.d
@@ -0,0 +1,208 @@
+//===-- driver/ir2obj_cache_pruning.d -----------------------------*- D -*-===//
+//
+// LDC – the LLVM D compiler
+//
+// This file is distributed under the BSD-style LDC license. See the LICENSE
+// file for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Implements cache pruning scheme.
+// 0. Check that the cache exists.
+// 1. Check that minimum pruning interval has passed.
+// 2. Prune files that have passed the expiry duration.
+// 3. Prune files to reduce total cache size to below a set limit.
+//
+// This file is imported by the ldc-prune-cache tool and should therefore depend
+// on as little LDC code as possible (currently none).
+//
+//===----------------------------------------------------------------------===//
+
+module driver.ir2obj_cache_pruning;
+
+import std.file;
+import std.datetime: Clock, dur, Duration, SysTime;
+
+// Creates a CachePruner and performs the pruning.
+// This function is meant to take care of all C++ interfacing.
+extern (C++) void pruneCache(const(char)* cacheDirectoryPtr,
+ size_t cacheDirectoryLen, uint pruneIntervalSeconds,
+ uint expireIntervalSeconds, ulong sizeLimitBytes, uint sizeLimitPercentage)
+{
+ import std.conv: to;
+
+ auto pruner = CachePruner(to!(string)(cacheDirectoryPtr[0 .. cacheDirectoryLen]),
+ pruneIntervalSeconds, expireIntervalSeconds, sizeLimitBytes, sizeLimitPercentage);
+
+ pruner.doPrune();
+}
+
+void writeEmptyFile(string filename)
+{
+ import std.stdio: File;
+ auto f = File(filename, "w");
+ f.close();
+}
+
+// Returns ulong.max when the available disk space could not be determined.
+ulong getAvailableDiskSpace(string path)
+{
+ import std.string: toStringz;
+ version (Windows)
+ {
+ import std.path;
+ import core.sys.windows.winbase;
+ import core.sys.windows.winnt;
+ import std.internal.cstring;
+
+ ULARGE_INTEGER freeBytesAvailable;
+ path ~= dirSeparator;
+ auto success = GetDiskFreeSpaceExW(path.tempCStringW(), &freeBytesAvailable, null, null);
+ return success ? freeBytesAvailable.QuadPart : ulong.max;
+ }
+ else
+ {
+ import core.sys.posix.sys.statvfs;
+
+ statvfs_t stats;
+ int err = statvfs(path.toStringz(), &stats);
+ return !err ? stats.f_bavail * stats.f_frsize : ulong.max;
+ }
+}
+
+struct CachePruner
+{
+ enum timestampFilename = "ircache_prune_timestamp";
+
+ string cachePath; // absolute path
+ Duration pruneInterval; // minimum time between pruning
+ Duration expireDuration; // cache file expiration
+ ulong sizeLimit; // in bytes
+ uint sizeLimitPercentage; // Percentage limit of available space
+ bool willPruneForSize;
+
+ this(string cachePath, uint pruneIntervalSeconds, uint expireIntervalSeconds,
+ ulong sizeLimit, uint sizeLimitPercentage)
+ {
+ import std.path;
+ if (cachePath.isRooted())
+ this.cachePath = cachePath.dup;
+ else
+ this.cachePath = absolutePath(expandTilde(cachePath));
+ this.pruneInterval = dur!"seconds"(pruneIntervalSeconds);
+ this.expireDuration = dur!"seconds"(expireIntervalSeconds);
+ this.sizeLimit = sizeLimit;
+ this.sizeLimitPercentage = sizeLimitPercentage < 100 ? sizeLimitPercentage : 100;
+ this.willPruneForSize = sizeLimit > 0;
+ }
+
+ void doPrune()
+ {
+ if (!exists(cachePath))
+ return;
+
+ if (!hasPruneIntervalPassed())
+ return;
+
+ // Only delete files that match ir2obj cache file naming.
+ // E.g. "ircache_00a13b6f918d18f9f9de499fc661ec0d.o"
+ auto filePattern = "ircache_????????????????????????????????.{o,obj}";
+ auto cacheFiles = dirEntries(cachePath, filePattern, SpanMode.shallow, /+ followSymlink +/ false);
+
+ // Files that have not yet expired, may still be removed during pruning for size later.
+ // This array holds the prune candidates after pruning for expiry.
+ DirEntry[] pruneForSizeCandidates;
+ ulong cacheSize;
+ pruneForExpiry(cacheFiles, pruneForSizeCandidates, cacheSize);
+ if (!willPruneForSize || !pruneForSizeCandidates.length)
+ return;
+
+ pruneForSize(pruneForSizeCandidates, cacheSize);
+ }
+
+private:
+ void pruneForExpiry(T)(T cacheFiles, out DirEntry[] remainingPruneCandidates, out ulong cacheSize)
+ {
+ foreach (DirEntry f; cacheFiles)
+ {
+ if (!f.isFile())
+ continue;
+
+ if (f.timeLastAccessed < (Clock.currTime - expireDuration))
+ {
+ try
+ {
+ remove(f.name);
+ }
+ catch (FileException)
+ {
+ // Simply skip the file when an error occurs.
+ continue;
+ }
+ }
+ else if (willPruneForSize)
+ {
+ cacheSize += f.size;
+ remainingPruneCandidates ~= f;
+ }
+ }
+ }
+
+ void pruneForSize(DirEntry[] candidates, ulong cacheSize)
+ {
+ ulong availableSpace = getAvailableDiskSpace(cachePath);
+ if (!isSizeAboveMaximum(cacheSize, availableSpace))
+ return;
+
+ // Create heap ordered with most recently accessed files last.
+ import std.container.binaryheap : heapify;
+ auto candidateHeap = heapify!("a.timeLastAccessed > b.timeLastAccessed")(candidates);
+ while (!candidateHeap.empty())
+ {
+ auto candidate = candidateHeap.front();
+ candidateHeap.popFront();
+
+ try
+ {
+ remove(candidate.name);
+ // Update cache size
+ cacheSize -= candidate.size;
+
+ if (!isSizeAboveMaximum(cacheSize, availableSpace))
+ break;
+ }
+ catch (FileException)
+ {
+ // Simply skip the file when an error occurs.
+ }
+ }
+ }
+
+ // Checks if the prune interval has passed, and if so, creates/updates the pruning timestamp.
+ bool hasPruneIntervalPassed()
+ {
+ import std.path: buildPath;
+ auto fname = buildPath(cachePath, timestampFilename);
+ if (pruneInterval == dur!"seconds"(0) || timeLastModified(fname,
+ SysTime.min) < (Clock.currTime - pruneInterval))
+ {
+ writeEmptyFile(fname);
+ return true;
+ }
+ return false;
+ }
+
+ bool isSizeAboveMaximum(ulong cacheSize, ulong availableSpace)
+ {
+ if (availableSpace == 0)
+ return true;
+
+ bool tooLarge = false;
+ if (sizeLimit > 0)
+ tooLarge = cacheSize > sizeLimit;
+
+ tooLarge = tooLarge || ((100 * cacheSize) / availableSpace) > sizeLimitPercentage;
+
+ return tooLarge;
+ }
+}
diff --git a/driver/ir2obj_cache_pruning.h b/driver/ir2obj_cache_pruning.h
new file mode 100644
index 0000000..42dfe82
--- /dev/null
+++ b/driver/ir2obj_cache_pruning.h
@@ -0,0 +1,23 @@
+//===-- driver/ir2obj_cache_pruning.h ---------------------------*- C++ -*-===//
+//
+// LDC – the LLVM D compiler
+//
+// This file is distributed under the BSD-style LDC license. See the LICENSE
+// file for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LDC_DRIVER_IR2OBJ_CACHE_PRUNING_H
+#define LDC_DRIVER_IR2OBJ_CACHE_PRUNING_H
+
+#if __LP64__
+using d_ulong = unsigned long;
+#else
+using d_ulong = unsigned long long;
+#endif
+
+void pruneCache(const char *cacheDirectoryPtr, size_t cacheDirectoryLen,
+ uint32_t pruneIntervalSeconds, uint32_t expireIntervalSeconds,
+ d_ulong sizeLimitBytes, uint32_t sizeLimitPercentage);
+
+#endif
diff --git a/driver/main.cpp b/driver/main.cpp
index 639acf1..73e5300 100644
--- a/driver/main.cpp
+++ b/driver/main.cpp
@@ -23,6 +23,7 @@
#include "driver/codegenerator.h"
#include "driver/configfile.h"
#include "driver/exe_path.h"
+#include "driver/ir2obj_cache.h"
#include "driver/ldc-version.h"
#include "driver/linker.h"
#include "driver/targetmachine.h"
@@ -1171,6 +1172,8 @@ void codegenModules(Modules &modules) {
}
}
+ ir2obj::pruneCache();
+
freeRuntime();
llvm::llvm_shutdown();
}
diff --git a/tests/linking/ir2obj_cache_pruning.d b/tests/linking/ir2obj_cache_pruning.d
new file mode 100644
index 0000000..3f9edad
--- /dev/null
+++ b/tests/linking/ir2obj_cache_pruning.d
@@ -0,0 +1,13 @@
+// Test recognition of -ir2obj-cache-prune-* commandline flags
+
+// RUN: %ldc %s -ir2obj-cache=%T/prunecache1 -ir2obj-cache-prune
+// RUN: %ldc %s -ir2obj-cache=%T/prunecache1 -ir2obj-cache-prune-interval=10 -ir2obj-cache-prune-maxbytes=10000
+// RUN: %ldc %s -ir2obj-cache=%T/prunecache1 -ir2obj-cache-prune -ir2obj-cache-prune-interval=0
+// RUN: %ldc %s -ir2obj-cache=%T/prunecache1 -ir2obj-cache-prune -ir2obj-cache-prune-maxbytes=10000
+// RUN: %ldc %s -ir2obj-cache=%T/prunecache1 -ir2obj-cache-prune -ir2obj-cache-prune-expiration=10000
+// RUN: %ldc %s -ir2obj-cache=%T/prunecache1 -ir2obj-cache-prune-maxpercentage=50
+// RUN: %ldc %s -ir2obj-cache=%T/prunecache1 -ir2obj-cache-prune -ir2obj-cache-prune-maxpercentage=150
+
+void main()
+{
+}
diff --git a/tests/linking/ir2obj_cache_pruning2.d b/tests/linking/ir2obj_cache_pruning2.d
new file mode 100644
index 0000000..1c03436
--- /dev/null
+++ b/tests/linking/ir2obj_cache_pruning2.d
@@ -0,0 +1,38 @@
+// Test ir2obj-cache pruning for size
+
+// This test assumes that the `void main(){}` object file size is below 200_000 bytes and above 200_000/2,
+// such that rebuilding with version(NEW_OBJ_FILE) will clear the cache of all but the latest object file.
+
+// RUN: %ldc %s -ir2obj-cache=%T/prunecache2 \
+// RUN: && %ldc %s -ir2obj-cache=%T/prunecache2 -ir2obj-cache-prune -ir2obj-cache-prune-interval=0 -d-version=SLEEP \
+// RUN: && %ldc %s -ir2obj-cache=%T/prunecache2 -ir2obj-cache-prune -ir2obj-cache-prune-interval=0 -vv | FileCheck --check-prefix=MUST_HIT %s \
+// RUN: && %ldc %s -ir2obj-cache=%T/prunecache2 -ir2obj-cache-prune -ir2obj-cache-prune-interval=0 -vv -d-version=NEW_OBJ_FILE | FileCheck --check-prefix=NO_HIT %s \
+// RUN: && %ldc %s -ir2obj-cache=%T/prunecache2 -ir2obj-cache-prune -ir2obj-cache-prune-interval=0 -vv | FileCheck --check-prefix=MUST_HIT %s \
+// RUN: && %ldc -d-version=SLEEP -run %s \
+// RUN: && %ldc %s -c -of=%t%obj -ir2obj-cache=%T/prunecache2 -ir2obj-cache-prune-interval=0 -ir2obj-cache-prune-maxbytes=200000 -vv | FileCheck --check-prefix=MUST_HIT %s \
+// RUN: && %ldc %t%obj \
+// RUN: && %ldc %s -ir2obj-cache=%T/prunecache2 -d-version=SLEEP -vv | FileCheck --check-prefix=NO_HIT %s \
+// RUN: && %ldc -d-version=SLEEP -run %s \
+// RUN: && %ldc %s -ir2obj-cache=%T/prunecache2 -ir2obj-cache-prune-interval=1 -ir2obj-cache-prune-maxbytes=200000 -d-version=NEW_OBJ_FILE \
+// RUN: && %ldc %s -ir2obj-cache=%T/prunecache2 -ir2obj-cache-prune -ir2obj-cache-prune-interval=0 -vv | FileCheck --check-prefix=NO_HIT %s
+
+// MUST_HIT: Cache object found!
+// NO_HIT-NOT: Cache object found!
+
+void main()
+{
+ // Add non-zero static data to guarantee a binary size larger than 200_000/2.
+ static byte[120_000] dummy = 1;
+
+ version (NEW_OBJ_FILE)
+ {
+ auto a = __TIME__;
+ }
+
+ version (SLEEP)
+ {
+ // Sleep for 2 seconds, so we are sure that the cache object file timestamps are "aging".
+ import core.thread;
+ Thread.sleep( dur!"seconds"(2) );
+ }
+}
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-d/ldc.git
More information about the pkg-d-commits
mailing list