package org.graalvm.compiler.hotspot;
import static org.graalvm.compiler.core.GraalCompilerOptions.ExitVMOnException;
import static org.graalvm.compiler.core.GraalCompilerOptions.PrintBailout;
import static org.graalvm.compiler.core.GraalCompilerOptions.PrintStackTraceOnException;
import static org.graalvm.compiler.core.common.util.Util.Java8OrEarlier;
import static org.graalvm.compiler.hotspot.CompileTheWorldOptions.CompileTheWorldClasspath;
import static org.graalvm.compiler.hotspot.CompileTheWorldOptions.CompileTheWorldConfig;
import static org.graalvm.compiler.hotspot.CompileTheWorldOptions.CompileTheWorldExcludeMethodFilter;
import static org.graalvm.compiler.hotspot.CompileTheWorldOptions.CompileTheWorldMethodFilter;
import static org.graalvm.compiler.hotspot.CompileTheWorldOptions.CompileTheWorldStartAt;
import static org.graalvm.compiler.hotspot.CompileTheWorldOptions.CompileTheWorldStopAt;
import static org.graalvm.compiler.hotspot.CompileTheWorldOptions.CompileTheWorldVerbose;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.lang.annotation.Annotation;
import java.lang.reflect.Constructor;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.channels.FileChannel;
import java.nio.file.FileSystems;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.ServiceLoader;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import java.util.stream.Collectors;
import org.graalvm.compiler.api.replacements.Snippet;
import org.graalvm.compiler.bytecode.Bytecodes;
import org.graalvm.compiler.core.CompilerThreadFactory;
import org.graalvm.compiler.core.CompilerThreadFactory.DebugConfigAccess;
import org.graalvm.compiler.core.common.util.Util;
import org.graalvm.compiler.debug.Debug;
import org.graalvm.compiler.debug.DebugEnvironment;
import org.graalvm.compiler.debug.GraalDebugConfig;
import org.graalvm.compiler.debug.MethodFilter;
import org.graalvm.compiler.debug.TTY;
import org.graalvm.compiler.debug.internal.DebugScope;
import org.graalvm.compiler.debug.internal.MemUseTrackerImpl;
import org.graalvm.compiler.options.OptionDescriptor;
import org.graalvm.compiler.options.OptionDescriptors;
import org.graalvm.compiler.options.OptionValue;
import org.graalvm.compiler.options.OptionValue.OverrideScope;
import org.graalvm.compiler.options.OptionsParser;
import org.graalvm.compiler.options.OptionsParser.OptionConsumer;
import jdk.vm.ci.hotspot.HotSpotCompilationRequest;
import jdk.vm.ci.hotspot.HotSpotInstalledCode;
import jdk.vm.ci.hotspot.HotSpotJVMCIRuntime;
import jdk.vm.ci.hotspot.HotSpotJVMCIRuntimeProvider;
import jdk.vm.ci.hotspot.HotSpotResolvedJavaMethod;
import jdk.vm.ci.hotspot.HotSpotResolvedObjectType;
import jdk.vm.ci.meta.ConstantPool;
import jdk.vm.ci.meta.MetaAccessProvider;
import jdk.vm.ci.runtime.JVMCI;
import jdk.vm.ci.runtime.JVMCICompiler;
public final class CompileTheWorld {
public static final String SUN_BOOT_CLASS_PATH = "sun.boot.class.path";
@SuppressWarnings("serial")
public static class Config extends HashMap<OptionValue<?>, Object> implements OptionConsumer {
public Config(String options) {
if (options != null) {
Map<String, String> optionSettings = new HashMap<>();
for (String optionSetting : options.split("\\s+|#")) {
OptionsParser.parseOptionSettingTo(optionSetting, optionSettings);
}
OptionsParser.parseOptions(optionSettings, this, ServiceLoader.load(OptionDescriptors.class, OptionDescriptors.class.getClassLoader()));
}
}
OverrideScope apply() {
return OptionValue.override(this);
}
@Override
public void set(OptionDescriptor desc, Object value) {
put(desc.getOptionValue(), value);
}
}
private final HotSpotJVMCIRuntimeProvider jvmciRuntime;
private final HotSpotGraalCompiler compiler;
private final String inputClassPath;
private final int startAt;
private final int stopAt;
private final MethodFilter[] methodFilters;
private final MethodFilter[] excludeMethodFilters;
private int classFileCounter = 0;
private AtomicLong compiledMethodsCounter = new AtomicLong();
private AtomicLong compileTime = new AtomicLong();
private AtomicLong memoryUsed = new AtomicLong();
private boolean verbose;
private final Config config;
private boolean running;
private ThreadPoolExecutor threadPool;
public CompileTheWorld(HotSpotJVMCIRuntimeProvider jvmciRuntime, HotSpotGraalCompiler compiler, String files, Config config, int startAt, int stopAt, String methodFilters,
String excludeMethodFilters, boolean verbose) {
this.jvmciRuntime = jvmciRuntime;
this.compiler = compiler;
this.inputClassPath = files;
this.startAt = startAt;
this.stopAt = stopAt;
this.methodFilters = methodFilters == null || methodFilters.isEmpty() ? null : MethodFilter.parse(methodFilters);
this.excludeMethodFilters = excludeMethodFilters == null || excludeMethodFilters.isEmpty() ? null : MethodFilter.parse(excludeMethodFilters);
this.verbose = verbose;
this.config = config;
config.putIfAbsent(ExitVMOnException, false);
config.putIfAbsent(PrintBailout, true);
config.putIfAbsent(PrintStackTraceOnException, true);
}
public CompileTheWorld(HotSpotJVMCIRuntimeProvider jvmciRuntime, HotSpotGraalCompiler compiler) {
this(jvmciRuntime, compiler, CompileTheWorldClasspath.getValue(), new Config(CompileTheWorldConfig.getValue()), CompileTheWorldStartAt.getValue(), CompileTheWorldStopAt.getValue(),
CompileTheWorldMethodFilter.getValue(), CompileTheWorldExcludeMethodFilter.getValue(), CompileTheWorldVerbose.getValue());
}
public void compile() throws Throwable {
if (!GraalDebugConfig.Options.DebugValueThreadFilter.hasBeenSet()) {
GraalDebugConfig.Options.DebugValueThreadFilter.setValue("^CompileTheWorld");
}
if (SUN_BOOT_CLASS_PATH.equals(inputClassPath)) {
String bcpEntry = null;
if (Java8OrEarlier) {
final String[] entries = System.getProperty(SUN_BOOT_CLASS_PATH).split(File.pathSeparator);
for (int i = 0; i < entries.length && bcpEntry == null; i++) {
String entry = entries[i];
File entryFile = new File(entry);
if (entryFile.getName().endsWith("rt.jar") && entryFile.isFile()) {
bcpEntry = entry;
}
}
} else {
bcpEntry = System.getProperty("java.home") + "/lib/modules".replace('/', File.separatorChar);
}
compile(bcpEntry);
} else {
compile(inputClassPath);
}
}
public void println() {
println("");
}
public void println(String format, Object... args) {
println(String.format(format, args));
}
public void println(String s) {
println(verbose, s);
}
public static void println(boolean cond, String s) {
if (cond) {
TTY.println(s);
}
}
public void printStackTrace(Throwable t) {
if (verbose) {
t.printStackTrace(TTY.out);
}
}
@SuppressWarnings("unused")
private static void dummy() {
}
abstract static class ClassPathEntry implements Closeable {
final String name;
ClassPathEntry(String name) {
this.name = name;
}
public abstract ClassLoader createClassLoader() throws IOException;
public abstract List<String> getClassNames() throws IOException;
@Override
public String toString() {
return name;
}
@Override
public void close() throws IOException {
}
}
static class DirClassPathEntry extends ClassPathEntry {
private final File dir;
DirClassPathEntry(String name) {
super(name);
dir = new File(name);
assert dir.isDirectory();
}
@Override
public ClassLoader createClassLoader() throws IOException {
URL url = dir.toURI().toURL();
return new URLClassLoader(new URL[]{url});
}
@Override
public List<String> getClassNames() throws IOException {
List<String> classNames = new ArrayList<>();
String root = dir.getPath();
SimpleFileVisitor<Path> visitor = new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
if (attrs.isRegularFile()) {
File path = file.toFile();
if (path.getName().endsWith(".class")) {
String pathString = path.getPath();
assert pathString.startsWith(root);
String classFile = pathString.substring(root.length() + 1);
String className = classFile.replace(File.separatorChar, '.');
classNames.add(className.replace('/', '.').substring(0, className.length() - ".class".length()));
}
}
return super.visitFile(file, attrs);
}
};
Files.walkFileTree(dir.toPath(), visitor);
return classNames;
}
}
static class JarClassPathEntry extends ClassPathEntry {
private final JarFile jarFile;
JarClassPathEntry(String name) throws IOException {
super(name);
jarFile = new JarFile(name);
}
@Override
public ClassLoader createClassLoader() throws IOException {
URL url = new URL("jar", "", "file:" + name + "!/");
return new URLClassLoader(new URL[]{url});
}
@Override
public List<String> getClassNames() throws IOException {
Enumeration<JarEntry> e = jarFile.entries();
List<String> classNames = new ArrayList<>(jarFile.size());
while (e.hasMoreElements()) {
JarEntry je = e.nextElement();
if (je.isDirectory() || !je.getName().endsWith(".class")) {
continue;
}
String className = je.getName().substring(0, je.getName().length() - ".class".length());
classNames.add(className.replace('/', '.'));
}
return classNames;
}
@Override
public void close() throws IOException {
jarFile.close();
}
}
public static final String LIMITMODS_PROPERTY_NAME = "CompileTheWorld.limitmods";
static class ImageClassPathEntry extends ClassPathEntry {
private final File jimage;
ImageClassPathEntry(String name) {
super(name);
jimage = new File(name);
assert jimage.isFile();
}
@Override
public ClassLoader createClassLoader() throws IOException {
URL url = jimage.toURI().toURL();
return new URLClassLoader(new URL[]{url});
}
@Override
public List<String> getClassNames() throws IOException {
String prop = System.getProperty(LIMITMODS_PROPERTY_NAME);
Set<String> limitmods = prop == null ? null : new HashSet<>(Arrays.asList(prop.split(",")));
List<String> classNames = new ArrayList<>();
String[] entries = readJimageEntries();
for (String e : entries) {
if (e.endsWith(".class") && !e.endsWith("module-info.class")) {
assert e.charAt(0) == '/' : e;
int endModule = e.indexOf('/', 1);
assert endModule != -1 : e;
if (limitmods != null) {
String module = e.substring(1, endModule);
if (!limitmods.contains(module)) {
continue;
}
}
String className = e.substring(endModule + 1).replace('/', '.');
className = className.replace('/', '.').substring(0, className.length() - ".class".length());
classNames.add(className);
}
}
return classNames;
}
private String[] readJimageEntries() {
try {
Path path = FileSystems.getDefault().getPath(name);
Method open = Class.forName("jdk.internal.jimage.BasicImageReader").getDeclaredMethod("open", Path.class);
Object reader = open.invoke(null, path);
Method getEntryNames = reader.getClass().getDeclaredMethod("getEntryNames");
getEntryNames.setAccessible(true);
String[] entries = (String[]) getEntryNames.invoke(reader);
return entries;
} catch (Exception e) {
TTY.println("Error reading entries from " + name + ": " + e);
return new String[0];
}
}
}
static boolean isJImage(String path) {
try {
FileChannel channel = FileChannel.open(Paths.get(path), StandardOpenOption.READ);
ByteBuffer map = channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size());
map.order(ByteOrder.nativeOrder()).asIntBuffer().get(0);
int magic = map.asIntBuffer().get(0);
if (magic == 0xCAFEDADA) {
return true;
}
} catch (IOException e) {
}
return false;
}
@SuppressWarnings("try")
private void compile(String classPath) throws IOException {
final String[] entries = classPath.split(File.pathSeparator);
long start = System.currentTimeMillis();
CompilerThreadFactory factory = new CompilerThreadFactory("CompileTheWorld", new DebugConfigAccess() {
@Override
public GraalDebugConfig getDebugConfig() {
if (Debug.isEnabled() && DebugScope.getConfig() == null) {
return DebugEnvironment.initialize(System.out, compiler.getGraalRuntime().getHostProviders().getSnippetReflection());
}
return null;
}
});
try {
HotSpotResolvedJavaMethod dummyMethod = (HotSpotResolvedJavaMethod) JVMCI.getRuntime().getHostJVMCIBackend().getMetaAccess().lookupJavaMethod(
CompileTheWorld.class.getDeclaredMethod("dummy"));
int entryBCI = JVMCICompiler.INVOCATION_ENTRY_BCI;
boolean useProfilingInfo = false;
boolean installAsDefault = false;
CompilationTask task = new CompilationTask(jvmciRuntime, compiler, new HotSpotCompilationRequest(dummyMethod, entryBCI, 0L), useProfilingInfo, installAsDefault);
task.runCompilation();
} catch (NoSuchMethodException | SecurityException e1) {
printStackTrace(e1);
}
int threadCount = 1;
if (CompileTheWorldOptions.CompileTheWorldMultiThreaded.getValue()) {
threadCount = CompileTheWorldOptions.CompileTheWorldThreads.getValue();
if (threadCount == 0) {
threadCount = Runtime.getRuntime().availableProcessors();
}
} else {
running = true;
}
threadPool = new ThreadPoolExecutor(threadCount, threadCount, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>(), factory);
try (OverrideScope s = config.apply()) {
for (int i = 0; i < entries.length; i++) {
final String entry = entries[i];
ClassPathEntry cpe;
if (entry.endsWith(".zip") || entry.endsWith(".jar")) {
cpe = new JarClassPathEntry(entry);
} else if (isJImage(entry)) {
assert !Java8OrEarlier;
cpe = new ImageClassPathEntry(entry);
} else {
if (!new File(entry).isDirectory()) {
println("CompileTheWorld : Skipped classes in " + entry);
println();
continue;
}
cpe = new DirClassPathEntry(entry);
}
if (methodFilters == null || methodFilters.length == 0) {
println("CompileTheWorld : Compiling all classes in " + entry);
} else {
String include = Arrays.asList(methodFilters).stream().map(MethodFilter::toString).collect(Collectors.joining(", "));
println("CompileTheWorld : Compiling all methods in " + entry + " matching one of the following filters: " + include);
}
if (excludeMethodFilters != null && excludeMethodFilters.length > 0) {
String exclude = Arrays.asList(excludeMethodFilters).stream().map(MethodFilter::toString).collect(Collectors.joining(", "));
println("CompileTheWorld : Excluding all methods matching one of the following filters: " + exclude);
}
println();
ClassLoader loader = cpe.createClassLoader();
for (String className : cpe.getClassNames()) {
if (classFileCounter >= stopAt) {
break;
}
classFileCounter++;
if (className.startsWith("jdk.management.") || className.startsWith("jdk.internal.cmm.*")) {
continue;
}
try {
Class<?> javaClass = Class.forName(className, true, loader);
try {
HotSpotResolvedObjectType objectType = HotSpotResolvedObjectType.fromObjectClass(javaClass);
ConstantPool constantPool = objectType.getConstantPool();
for (int cpi = 1; cpi < constantPool.length(); cpi++) {
constantPool.loadReferencedType(cpi, Bytecodes.LDC);
}
} catch (Throwable t) {
println("Preloading failed for (%d) %s: %s", classFileCounter, className, t);
}
if (methodFilters != null && !MethodFilter.matchesClassName(methodFilters, className)) {
continue;
}
if (excludeMethodFilters != null && MethodFilter.matchesClassName(excludeMethodFilters, className)) {
continue;
}
MetaAccessProvider metaAccess = JVMCI.getRuntime().getHostJVMCIBackend().getMetaAccess();
if (classFileCounter >= startAt) {
println("CompileTheWorld (%d) : %s", classFileCounter, className);
for (Constructor<?> constructor : javaClass.getDeclaredConstructors()) {
HotSpotResolvedJavaMethod javaMethod = (HotSpotResolvedJavaMethod) metaAccess.lookupJavaMethod(constructor);
if (canBeCompiled(javaMethod, constructor.getModifiers())) {
compileMethod(javaMethod);
}
}
for (Method method : javaClass.getDeclaredMethods()) {
HotSpotResolvedJavaMethod javaMethod = (HotSpotResolvedJavaMethod) metaAccess.lookupJavaMethod(method);
if (canBeCompiled(javaMethod, method.getModifiers())) {
compileMethod(javaMethod);
}
}
HotSpotResolvedJavaMethod clinit = (HotSpotResolvedJavaMethod) metaAccess.lookupJavaType(javaClass).getClassInitializer();
if (clinit != null && canBeCompiled(clinit, clinit.getModifiers())) {
compileMethod(clinit);
}
}
} catch (Throwable t) {
println("CompileTheWorld (%d) : Skipping %s %s", classFileCounter, className, t.toString());
printStackTrace(t);
}
}
cpe.close();
}
}
if (!running) {
startThreads();
}
int wakeups = 0;
while (threadPool.getCompletedTaskCount() != threadPool.getTaskCount()) {
if (wakeups % 15 == 0) {
TTY.println("CompileTheWorld : Waiting for " + (threadPool.getTaskCount() - threadPool.getCompletedTaskCount()) + " compiles");
}
try {
threadPool.awaitTermination(1, TimeUnit.SECONDS);
wakeups++;
} catch (InterruptedException e) {
}
}
threadPool = null;
long elapsedTime = System.currentTimeMillis() - start;
println();
if (CompileTheWorldOptions.CompileTheWorldMultiThreaded.getValue()) {
TTY.println("CompileTheWorld : Done (%d classes, %d methods, %d ms elapsed, %d ms compile time, %d bytes of memory used)", classFileCounter, compiledMethodsCounter.get(), elapsedTime,
compileTime.get(), memoryUsed.get());
} else {
TTY.println("CompileTheWorld : Done (%d classes, %d methods, %d ms, %d bytes of memory used)", classFileCounter, compiledMethodsCounter.get(), compileTime.get(), memoryUsed.get());
}
}
private synchronized void startThreads() {
running = true;
notifyAll();
}
private synchronized void waitToRun() {
while (!running) {
try {
wait();
} catch (InterruptedException e) {
}
}
}
@SuppressWarnings("try")
private void compileMethod(HotSpotResolvedJavaMethod method) throws InterruptedException, ExecutionException {
if (methodFilters != null && !MethodFilter.matches(methodFilters, method)) {
return;
}
if (excludeMethodFilters != null && MethodFilter.matches(excludeMethodFilters, method)) {
return;
}
Future<?> task = threadPool.submit(new Runnable() {
@Override
public void run() {
waitToRun();
try (OverrideScope s = config.apply()) {
compileMethod(method, classFileCounter);
}
}
});
if (threadPool.getCorePoolSize() == 1) {
task.get();
}
}
private void compileMethod(HotSpotResolvedJavaMethod method, int counter) {
try {
long start = System.currentTimeMillis();
long allocatedAtStart = MemUseTrackerImpl.getCurrentThreadAllocatedBytes();
int entryBCI = JVMCICompiler.INVOCATION_ENTRY_BCI;
HotSpotCompilationRequest request = new HotSpotCompilationRequest(method, entryBCI, 0L);
boolean useProfilingInfo = false;
boolean installAsDefault = false;
CompilationTask task = new CompilationTask(jvmciRuntime, compiler, request, useProfilingInfo, installAsDefault);
task.runCompilation();
HotSpotInstalledCode installedCode = task.getInstalledCode();
if (installedCode != null) {
installedCode.invalidate();
}
memoryUsed.getAndAdd(MemUseTrackerImpl.getCurrentThreadAllocatedBytes() - allocatedAtStart);
compileTime.getAndAdd(System.currentTimeMillis() - start);
compiledMethodsCounter.incrementAndGet();
} catch (Throwable t) {
println("CompileTheWorld (%d) : Error compiling method: %s", counter, method.format("%H.%n(%p):%r"));
printStackTrace(t);
}
}
private boolean canBeCompiled(HotSpotResolvedJavaMethod javaMethod, int modifiers) {
if (Modifier.isAbstract(modifiers) || Modifier.isNative(modifiers)) {
return false;
}
GraalHotSpotVMConfig c = compiler.getGraalRuntime().getVMConfig();
if (c.dontCompileHugeMethods && javaMethod.getCodeSize() > c.hugeMethodLimit) {
println(verbose || methodFilters != null,
String.format("CompileTheWorld (%d) : Skipping huge method %s (use -XX:-DontCompileHugeMethods or -XX:HugeMethodLimit=%d to include it)", classFileCounter,
javaMethod.format("%H.%n(%p):%r"),
javaMethod.getCodeSize()));
return false;
}
if (!javaMethod.canBeInlined()) {
return false;
}
for (Annotation annotation : javaMethod.getAnnotations()) {
if (annotation.annotationType().equals(Snippet.class)) {
return false;
}
}
return true;
}
public static void main(String[] args) throws Throwable {
HotSpotGraalCompiler compiler = (HotSpotGraalCompiler) HotSpotJVMCIRuntime.runtime().getCompiler();
compiler.compileTheWorld();
}
}