package com.oracle.svm.hosted.image;
import static com.oracle.svm.core.util.VMError.shouldNotReachHere;
import java.lang.reflect.Array;
import java.lang.reflect.Modifier;
import java.util.ArrayDeque;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Deque;
import java.util.HashMap;
import java.util.IdentityHashMap;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import org.graalvm.compiler.api.replacements.Fold;
import org.graalvm.compiler.core.common.CompressEncoding;
import org.graalvm.compiler.core.common.NumUtil;
import org.graalvm.compiler.core.common.SuppressFBWarnings;
import org.graalvm.nativeimage.ImageSingletons;
import org.graalvm.nativeimage.c.function.RelocatedPointer;
import org.graalvm.word.UnsignedWord;
import org.graalvm.word.WordBase;
import com.oracle.graal.pointsto.meta.AnalysisUniverse;
import com.oracle.graal.pointsto.util.AnalysisError;
import com.oracle.svm.core.StaticFieldsSupport;
import com.oracle.svm.core.SubstrateOptions;
import com.oracle.svm.core.config.ConfigurationValues;
import com.oracle.svm.core.config.ObjectLayout;
import com.oracle.svm.core.heap.FillerObject;
import com.oracle.svm.core.heap.Heap;
import com.oracle.svm.core.hub.DynamicHub;
import com.oracle.svm.core.hub.LayoutEncoding;
import com.oracle.svm.core.image.ImageHeap;
import com.oracle.svm.core.image.ImageHeapLayouter;
import com.oracle.svm.core.image.ImageHeapObject;
import com.oracle.svm.core.image.ImageHeapPartition;
import com.oracle.svm.core.jdk.StringInternSupport;
import com.oracle.svm.core.meta.SubstrateObjectConstant;
import com.oracle.svm.core.util.HostedStringDeduplication;
import com.oracle.svm.core.util.UserError;
import com.oracle.svm.core.util.VMError;
import com.oracle.svm.hosted.config.HybridLayout;
import com.oracle.svm.hosted.meta.HostedArrayClass;
import com.oracle.svm.hosted.meta.HostedClass;
import com.oracle.svm.hosted.meta.HostedField;
import com.oracle.svm.hosted.meta.HostedInstanceClass;
import com.oracle.svm.hosted.meta.HostedMetaAccess;
import com.oracle.svm.hosted.meta.HostedType;
import com.oracle.svm.hosted.meta.HostedUniverse;
import com.oracle.svm.hosted.meta.MaterializedConstantFields;
import com.oracle.svm.hosted.meta.UniverseBuilder;
import jdk.vm.ci.meta.JavaConstant;
import jdk.vm.ci.meta.JavaKind;
import jdk.vm.ci.meta.ResolvedJavaType;
public final class NativeImageHeap implements ImageHeap {
private final HostedUniverse universe;
private final AnalysisUniverse aUniverse;
private final HostedMetaAccess metaAccess;
private final ObjectLayout objectLayout;
private final ImageHeapLayouter heapLayouter;
private final int minInstanceSize;
private final int minArraySize;
protected final IdentityHashMap<Object, ObjectInfo> objects = new IdentityHashMap<>();
private final Set<Object> blacklist = Collections.newSetFromMap(new IdentityHashMap<>());
private final Map<HostedClass, HybridLayout<?>> hybridLayouts = new HashMap<>();
private final Map<String, String> internedStrings = new HashMap<>();
private final Phase addObjectsPhase = Phase.factory();
private final Phase internStringsPhase = Phase.factory();
private final Deque<AddObjectData> addObjectWorklist = new ArrayDeque<>();
private final Set<Object> knownImmutableObjects = Collections.newSetFromMap(new IdentityHashMap<>());
public NativeImageHeap(AnalysisUniverse aUniverse, HostedUniverse universe, HostedMetaAccess metaAccess, ImageHeapLayouter heapLayouter) {
this.aUniverse = aUniverse;
this.universe = universe;
this.metaAccess = metaAccess;
this.objectLayout = ConfigurationValues.getObjectLayout();
this.heapLayouter = heapLayouter;
this.minInstanceSize = objectLayout.getMinimumInstanceObjectSize();
this.minArraySize = objectLayout.getMinimumArraySize();
assert assertFillerObjectSizes();
}
@Override
public Collection<ObjectInfo> getObjects() {
return objects.values();
}
public int getObjectCount() {
return objects.size();
}
public ObjectInfo getObjectInfo(Object obj) {
return objects.get(obj);
}
protected HostedUniverse getUniverse() {
return universe;
}
protected HostedMetaAccess getMetaAccess() {
return metaAccess;
}
protected AnalysisUniverse getAnalysisUniverse() {
return aUniverse;
}
protected HybridLayout<?> getHybridLayout(HostedClass clazz) {
return hybridLayouts.get(clazz);
}
protected boolean isBlacklisted(Object obj) {
return blacklist.contains(obj);
}
protected ObjectLayout getObjectLayout() {
return objectLayout;
}
public ImageHeapLayouter getLayouter() {
return heapLayouter;
}
@Fold
static boolean useHeapBase() {
return SubstrateOptions.SpawnIsolates.getValue() && ImageSingletons.lookup(CompressEncoding.class).hasBase();
}
@Fold
static boolean spawnIsolates() {
return SubstrateOptions.SpawnIsolates.getValue() && useHeapBase();
}
@SuppressWarnings("try")
public void addInitialObjects() {
addObjectsPhase.allow();
internStringsPhase.allow();
addStaticFields();
}
public void addTrailingObjects() {
processAddObjectWorklist();
HostedField internedStringsField = (HostedField) StringInternFeature.getInternedStringsField(metaAccess);
boolean usesInternedStrings = internedStringsField.isAccessed();
if (usesInternedStrings) {
addObject(getMetaAccess().lookupJavaType(String[].class).getHub(), false, "internedStrings table");
internStringsPhase.disallow();
String[] imageInternedStrings = internedStrings.keySet().toArray(new String[0]);
Arrays.sort(imageInternedStrings);
ImageSingletons.lookup(StringInternSupport.class).setImageInternedStrings(imageInternedStrings);
addObject(imageInternedStrings, true, "internedStrings table");
processAddObjectWorklist();
} else {
internStringsPhase.disallow();
}
addObjectsPhase.disallow();
assert addObjectWorklist.isEmpty();
}
private static Object readObjectField(HostedField field, JavaConstant receiver) {
return SubstrateObjectConstant.asObject(field.readStorageValue(receiver));
}
private void addStaticFields() {
addObject(StaticFieldsSupport.getStaticObjectFields(), false, "staticObjectFields");
addObject(StaticFieldsSupport.getStaticPrimitiveFields(), false, "staticPrimitiveFields");
for (HostedField field : getUniverse().getFields()) {
if (Modifier.isStatic(field.getModifiers()) && field.hasLocation() && field.getType().getStorageKind() == JavaKind.Object) {
assert field.isWritten() || MaterializedConstantFields.singleton().contains(field.wrapped);
addObject(readObjectField(field, null), false, field);
}
}
}
public void registerAsImmutable(Object object) {
assert addObjectsPhase.isBefore() : "Registering immutable object too late: phase: " + addObjectsPhase.toString();
knownImmutableObjects.add(object);
}
public void addObject(final Object original, boolean immutableFromParent, final Object reason) {
assert addObjectsPhase.isAllowed() : "Objects cannot be added at phase: " + addObjectsPhase.toString() + " with reason: " + reason;
if (original == null || original instanceof WordBase) {
return;
}
if (original instanceof Class) {
throw VMError.shouldNotReachHere("Must not have Class in native image heap: " + original);
}
if (original instanceof DynamicHub && ((DynamicHub) original).getClassInitializationInfo() == null) {
throw reportIllegalType(original, reason);
}
int identityHashCode = SubstrateObjectConstant.computeIdentityHashCode(original);
VMError.guarantee(identityHashCode != 0, "0 is used as a marker value for 'hash code not yet computed'");
if (original instanceof String) {
handleImageString((String) original);
}
final ObjectInfo existing = objects.get(original);
if (existing == null) {
addObjectToBootImageHeap(original, immutableFromParent, identityHashCode, reason);
}
}
@Override
public int countDynamicHubs() {
int count = 0;
for (ObjectInfo o : getObjects()) {
if (o.getObject() instanceof DynamicHub) {
count++;
}
}
return count;
}
@Override
public ObjectInfo addFillerObject(int size) {
if (size >= minArraySize) {
int elementSize = objectLayout.getArrayIndexScale(JavaKind.Int);
int arrayLength = (size - minArraySize) / elementSize;
assert objectLayout.getArraySize(JavaKind.Int, arrayLength) == size;
return addLateToImageHeap(new int[arrayLength], "Filler object");
} else if (size >= minInstanceSize) {
return addLateToImageHeap(new FillerObject(), "Filler object");
} else {
return null;
}
}
private boolean assertFillerObjectSizes() {
assert minArraySize == objectLayout.getArraySize(JavaKind.Int, 0);
HostedType filler = metaAccess.lookupJavaType(FillerObject.class);
UnsignedWord fillerSize = LayoutEncoding.getInstanceSize(filler.getHub().getLayoutEncoding());
assert fillerSize.equal(minInstanceSize);
assert minInstanceSize * 2 >= minArraySize : "otherwise, we might need more than one non-array object";
return true;
}
private void handleImageString(final String str) {
forceHashCodeComputation(str);
if (HostedStringDeduplication.isInternedString(str)) {
assert internedStrings.containsKey(str) || internStringsPhase.isAllowed() : "Should not intern string during phase " + internStringsPhase.toString();
internedStrings.put(str, str);
}
}
@SuppressFBWarnings(value = "RV_RETURN_VALUE_IGNORED", justification = "eager hash field computation")
private static void forceHashCodeComputation(final String str) {
str.hashCode();
}
private void addObjectToBootImageHeap(final Object object, boolean immutableFromParent, final int identityHashCode, final Object reason) {
final Optional<HostedType> optionalType = getMetaAccess().optionalLookupJavaType(object.getClass());
final HostedType type = requireType(optionalType, object, reason);
final DynamicHub hub = type.getHub();
final ObjectInfo info;
boolean immutable = immutableFromParent || isKnownImmutable(object);
boolean written = false;
boolean references = false;
boolean relocatable = false;
if (type.isInstanceClass()) {
final HostedInstanceClass clazz = (HostedInstanceClass) type;
if (clazz.getMonitorFieldOffset() != 0) {
written = true;
references = true;
}
final JavaConstant con = SubstrateObjectConstant.forObject(object);
HostedField hybridTypeIDSlotsField = null;
HostedField hybridArrayField = null;
Object hybridArray = null;
final long size;
if (HybridLayout.isHybrid(clazz)) {
HybridLayout<?> hybridLayout = hybridLayouts.get(clazz);
if (hybridLayout == null) {
hybridLayout = new HybridLayout<>(clazz, objectLayout);
hybridLayouts.put(clazz, hybridLayout);
}
hybridTypeIDSlotsField = hybridLayout.getTypeIDSlotsField();
if (hybridTypeIDSlotsField != null) {
Object typeIDSlots = readObjectField(hybridTypeIDSlotsField, con);
if (typeIDSlots != null) {
blacklist.add(typeIDSlots);
}
}
hybridArrayField = hybridLayout.getArrayField();
hybridArray = readObjectField(hybridArrayField, con);
if (hybridArray != null) {
blacklist.add(hybridArray);
written = true;
}
size = hybridLayout.getTotalSize(Array.getLength(hybridArray));
} else {
size = LayoutEncoding.getInstanceSize(hub.getLayoutEncoding()).rawValue();
}
info = addToImageHeap(object, clazz, size, identityHashCode, reason);
try {
recursiveAddObject(hub, false, info);
final boolean fieldsAreImmutable = object instanceof String;
for (HostedField field : clazz.getInstanceFields(true)) {
if (field.isInImageHeap() &&
!field.equals(hybridArrayField) &&
!field.equals(hybridTypeIDSlotsField)) {
boolean fieldRelocatable = false;
if (field.getJavaKind() == JavaKind.Object) {
assert field.hasLocation();
JavaConstant fieldValueConstant = field.readValue(con);
if (fieldValueConstant.getJavaKind() == JavaKind.Object) {
Object fieldValue = SubstrateObjectConstant.asObject(fieldValueConstant);
if (spawnIsolates()) {
fieldRelocatable = fieldValue instanceof RelocatedPointer;
}
recursiveAddObject(fieldValue, fieldsAreImmutable, info);
references = true;
}
}
relocatable = relocatable || fieldRelocatable;
written = written || (field.isWritten() && !field.isFinal() && !fieldRelocatable);
}
}
if (hybridArray instanceof Object[]) {
relocatable = addArrayElements((Object[]) hybridArray, relocatable, info);
references = true;
}
} catch (AnalysisError.TypeNotFoundError ex) {
throw reportIllegalType(ex.getType(), info);
}
} else if (type.isArray()) {
HostedArrayClass clazz = (HostedArrayClass) type;
final long size = objectLayout.getArraySize(type.getComponentType().getStorageKind(), Array.getLength(object));
info = addToImageHeap(object, clazz, size, identityHashCode, reason);
try {
recursiveAddObject(hub, false, info);
if (object instanceof Object[]) {
relocatable = addArrayElements((Object[]) object, false, info);
references = true;
}
written = true;
} catch (AnalysisError.TypeNotFoundError ex) {
throw reportIllegalType(ex.getType(), info);
}
} else {
throw shouldNotReachHere();
}
if (relocatable && !isKnownImmutable(object)) {
VMError.shouldNotReachHere("Object with relocatable pointers must be explicitly immutable: " + object);
}
heapLayouter.assignObjectToPartition(info, !written || immutable, references, relocatable);
}
private static HostedType requireType(Optional<HostedType> optionalType, Object object, Object reason) {
if (!optionalType.isPresent() || !optionalType.get().isInstantiated()) {
throw reportIllegalType(object, reason);
}
return optionalType.get();
}
static RuntimeException reportIllegalType(Object object, Object reason) {
StringBuilder msg = new StringBuilder();
msg.append("Image heap writing found a class not seen during static analysis. ");
msg.append("Did a static field or an object referenced from a static field change during native image generation? ");
msg.append("For example, a lazily initialized cache could have been initialized during image generation, in which case ");
msg.append("you need to force eager initialization of the cache before static analysis or reset the cache using a field ");
msg.append("value recomputation.").append(System.lineSeparator()).append(" ");
if (object instanceof DynamicHub) {
msg.append("class: ").append(((DynamicHub) object).getName());
} else if (object instanceof ResolvedJavaType) {
msg.append("class: ").append(((ResolvedJavaType) object).toJavaName(true));
} else {
msg.append("object: ").append(object).append(" of class: ").append(object.getClass().getTypeName());
}
msg.append(System.lineSeparator()).append(" reachable through:").append(System.lineSeparator());
fillReasonStack(msg, reason);
throw UserError.abort("%s", msg);
}
private static StringBuilder fillReasonStack(StringBuilder msg, Object reason) {
if (reason instanceof ObjectInfo) {
ObjectInfo info = (ObjectInfo) reason;
msg.append(" object: ").append(info.getObject()).append(" of class: ").append(info.getObject().getClass().getTypeName()).append(System.lineSeparator());
return fillReasonStack(msg, info.reason);
}
return msg.append(" root: ").append(reason).append(System.lineSeparator());
}
private boolean isKnownImmutable(final Object obj) {
if (obj instanceof String) {
return obj.hashCode() != 0;
}
return UniverseBuilder.isKnownImmutableType(obj.getClass()) || knownImmutableObjects.contains(obj);
}
private ObjectInfo addToImageHeap(Object object, HostedClass clazz, long size, int identityHashCode, Object reason) {
ObjectInfo info = new ObjectInfo(object, size, clazz, identityHashCode, reason);
assert !objects.containsKey(object);
objects.put(object, info);
return info;
}
@Override
public ObjectInfo addLateToImageHeap(Object object, String reason) {
assert !(object instanceof DynamicHub) : "needs a different identity hashcode";
assert !(object instanceof String) : "needs String interning";
final Optional<HostedType> optionalType = getMetaAccess().optionalLookupJavaType(object.getClass());
HostedType type = requireType(optionalType, object, reason);
return addToImageHeap(object, (HostedClass) type, getSize(object, type), System.identityHashCode(object), reason);
}
private long getSize(Object object, HostedType type) {
if (type.isInstanceClass()) {
HostedInstanceClass clazz = (HostedInstanceClass) type;
assert !HybridLayout.isHybrid(clazz);
return LayoutEncoding.getInstanceSize(clazz.getHub().getLayoutEncoding()).rawValue();
} else if (type.isArray()) {
return objectLayout.getArraySize(type.getComponentType().getStorageKind(), Array.getLength(object));
} else {
throw shouldNotReachHere();
}
}
private boolean addArrayElements(Object[] array, boolean otherFieldsRelocatable, Object reason) {
boolean relocatable = otherFieldsRelocatable;
for (Object element : array) {
Object value = aUniverse.replaceObject(element);
if (spawnIsolates()) {
relocatable = relocatable || value instanceof RelocatedPointer;
}
recursiveAddObject(value, false, reason);
}
return relocatable;
}
private void recursiveAddObject(Object original, boolean immutableFromParent, Object reason) {
if (original != null) {
addObjectWorklist.push(new AddObjectData(original, immutableFromParent, reason));
}
}
private void processAddObjectWorklist() {
while (!addObjectWorklist.isEmpty()) {
AddObjectData data = addObjectWorklist.pop();
addObject(data.original, data.immutableFromParent, data.reason);
}
}
static class AddObjectData {
AddObjectData(Object original, boolean immutableFromParent, Object reason) {
this.original = original;
this.immutableFromParent = immutableFromParent;
this.reason = reason;
}
final Object original;
final boolean immutableFromParent;
final Object reason;
}
public static final class ObjectInfo implements ImageHeapObject {
private final Object object;
private final HostedClass clazz;
private final long size;
private final int identityHashCode;
private ImageHeapPartition partition;
private long offsetInPartition;
final Object reason;
ObjectInfo(Object object, long size, HostedClass clazz, int identityHashCode, Object reason) {
this.object = object;
this.clazz = clazz;
this.partition = null;
this.offsetInPartition = -1L;
this.size = size;
this.identityHashCode = identityHashCode;
this.reason = reason;
}
@Override
public Object getObject() {
return object;
}
public HostedClass getClazz() {
return clazz;
}
@Override
public long getOffset() {
assert offsetInPartition >= 0;
assert partition != null;
return partition.getStartOffset() + offsetInPartition;
}
@Override
public void setOffsetInPartition(long value) {
assert this.offsetInPartition == -1L && value >= 0;
this.offsetInPartition = value;
}
@Override
public ImageHeapPartition getPartition() {
return partition;
}
@Override
public void setHeapPartition(ImageHeapPartition value) {
assert this.partition == null;
this.partition = value;
}
public int getIndexInBuffer(long index) {
long result = getOffset() + index;
return NumUtil.safeToInt(result);
}
public long getAddress() {
return Heap.getHeap().getImageHeapOffsetInAddressSpace() + getOffset();
}
public long getAddress(long delta) {
assert delta >= 0 && delta < getSize() : "Index: " + delta + " out of bounds: [0 .. " + getSize() + ").";
return getAddress() + delta;
}
@Override
public long getSize() {
return size;
}
int getIdentityHashCode() {
return identityHashCode;
}
@Override
public String toString() {
StringBuilder result = new StringBuilder(getObject().getClass().getName()).append(" -> ");
Object cur = reason;
Object prev = null;
boolean skipped = false;
while (cur instanceof ObjectInfo) {
skipped = prev != null;
prev = cur;
cur = ((ObjectInfo) cur).reason;
}
if (skipped) {
result.append("... -> ");
}
if (prev != null) {
result.append(prev);
} else {
result.append(cur);
}
return result.toString();
}
}
protected static final class Phase {
public static Phase factory() {
return new Phase();
}
public boolean isBefore() {
return (value == PhaseValue.BEFORE);
}
public void allow() {
assert (value == PhaseValue.BEFORE) : "Can not allow while in phase " + value.toString();
value = PhaseValue.ALLOWED;
}
void disallow() {
assert (value == PhaseValue.ALLOWED) : "Can not disallow while in phase " + value.toString();
value = PhaseValue.AFTER;
}
public boolean isAllowed() {
return (value == PhaseValue.ALLOWED);
}
@Override
public String toString() {
return value.toString();
}
protected Phase() {
value = PhaseValue.BEFORE;
}
private PhaseValue value;
private enum PhaseValue {
BEFORE,
ALLOWED,
AFTER
}
}
}