Skip to content

Commit 974bb00

Browse files
kittyoraclekittyoracle
committed
[GR-63729] SVM: Constant fold the object header when initializing an object
PullRequest: graal/20470
2 parents 6df2123 + 614235e commit 974bb00

File tree

19 files changed

+569
-152
lines changed

19 files changed

+569
-152
lines changed

‎compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/api/replacements/SnippetReflectionProvider.java

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2014, 2018, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2014, 2025, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* This code is free software; you can redistribute it and/or modify it
@@ -28,6 +28,8 @@
2828
import java.lang.reflect.Field;
2929
import java.util.Objects;
3030

31+
import jdk.graal.compiler.nodes.StructuredGraph;
32+
import jdk.graal.compiler.nodes.ValueNode;
3133
import jdk.vm.ci.meta.JavaConstant;
3234
import jdk.vm.ci.meta.JavaKind;
3335
import jdk.vm.ci.meta.ResolvedJavaField;
@@ -51,6 +53,17 @@ public interface SnippetReflectionProvider {
5153
*/
5254
JavaConstant forObject(Object object);
5355

56+
/**
57+
* Tries to create a {@link ValueNode} representing the content of the header of a newly created
58+
* object. If there is no such constant, for example because the object header depends on the
59+
* value of the heap base, which may be not available at build time, this method returns
60+
* {@code null} and the header has to be calculated at runtime from the address of the type
61+
* object.
62+
*/
63+
default ValueNode forTLABObjectHeader(@SuppressWarnings("unused") Object hub, @SuppressWarnings("unused") StructuredGraph graph) {
64+
return null;
65+
}
66+
5467
/**
5568
* Gets the object reference a given constant represents if it is of a given type. The constant
5669
* must have kind {@link JavaKind#Object}.

‎compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/asm/aarch64/AArch64MacroAssembler.java

Lines changed: 19 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -447,35 +447,33 @@ public void mov(int size, Register dst, Register src) {
447447
* @param needsImmAnnotation Flag denoting if annotation should be added.
448448
*/
449449
private void mov32(Register dst, int imm, boolean needsImmAnnotation) {
450-
MovAction[] includeSet = {MovAction.SKIPPED, MovAction.SKIPPED};
451450
int pos = position();
452451

453452
// Split 32-bit imm into low16 and high16 parts.
454453
int low16 = imm & 0xFFFF;
455454
int high16 = (imm >>> 16) & 0xFFFF;
456455

456+
if (needsImmAnnotation) {
457+
movz(32, dst, low16, 0);
458+
movk(32, dst, high16, 16);
459+
MovAction[] includeSet = {MovAction.USED, MovAction.USED};
460+
annotateImmediateMovSequence(pos, includeSet);
461+
return;
462+
}
463+
457464
// Generate code sequence with a combination of MOVZ or MOVN with MOVK.
458465
if (high16 == 0) {
459466
movz(32, dst, low16, 0);
460-
includeSet[0] = MovAction.USED;
461467
} else if (high16 == 0xFFFF) {
462468
movn(32, dst, low16 ^ 0xFFFF, 0);
463-
includeSet[0] = MovAction.NEGATED;
464469
} else if (low16 == 0) {
465470
movz(32, dst, high16, 16);
466-
includeSet[1] = MovAction.USED;
467471
} else if (low16 == 0xFFFF) {
468472
movn(32, dst, high16 ^ 0xFFFF, 16);
469-
includeSet[1] = MovAction.NEGATED;
470473
} else {
471474
// Neither of the 2 parts is all-0s or all-1s. Generate 2 instructions.
472475
movz(32, dst, low16, 0);
473476
movk(32, dst, high16, 16);
474-
includeSet[0] = MovAction.USED;
475-
includeSet[1] = MovAction.USED;
476-
}
477-
if (needsImmAnnotation) {
478-
annotateImmediateMovSequence(pos, includeSet);
479477
}
480478
}
481479

@@ -494,7 +492,6 @@ public void movzPatchable(int size, Register dst, int uimm16) {
494492
* @param needsImmAnnotation Flag denoting if annotation should be added.
495493
*/
496494
private void mov64(Register dst, long imm, boolean needsImmAnnotation) {
497-
MovAction[] includeSet = {MovAction.SKIPPED, MovAction.SKIPPED, MovAction.SKIPPED, MovAction.SKIPPED};
498495
int pos = position();
499496
int[] chunks = new int[4];
500497
int zeroCount = 0;
@@ -511,21 +508,29 @@ private void mov64(Register dst, long imm, boolean needsImmAnnotation) {
511508
chunks[i] = chunk;
512509
}
513510

511+
if (needsImmAnnotation) {
512+
// Generate one MOVZ and three MOVKs
513+
movz(64, dst, chunks[0], 0);
514+
movk(64, dst, chunks[1], 16);
515+
movk(64, dst, chunks[2], 32);
516+
movk(64, dst, chunks[3], 48);
517+
MovAction[] includeSet = {MovAction.USED, MovAction.USED, MovAction.USED, MovAction.USED};
518+
annotateImmediateMovSequence(pos, includeSet);
519+
return;
520+
}
521+
514522
// Generate code sequence with a combination of MOVZ or MOVN with MOVK.
515523
if (zeroCount == 4) {
516524
// Generate only one MOVZ.
517525
movz(64, dst, 0, 0);
518-
includeSet[0] = MovAction.USED;
519526
} else if (negCount == 4) {
520527
// Generate only one MOVN.
521528
movn(64, dst, 0, 0);
522-
includeSet[0] = MovAction.NEGATED;
523529
} else if (zeroCount == 3) {
524530
// Generate only one MOVZ.
525531
for (int i = 0; i < 4; i++) {
526532
if (chunks[i] != 0) {
527533
movz(64, dst, chunks[i], i * 16);
528-
includeSet[i] = MovAction.USED;
529534
break;
530535
}
531536
}
@@ -534,7 +539,6 @@ private void mov64(Register dst, long imm, boolean needsImmAnnotation) {
534539
for (int i = 0; i < 4; i++) {
535540
if (chunks[i] != 0xFFFF) {
536541
movn(64, dst, chunks[i] ^ 0xFFFF, i * 16);
537-
includeSet[i] = MovAction.NEGATED;
538542
break;
539543
}
540544
}
@@ -544,14 +548,12 @@ private void mov64(Register dst, long imm, boolean needsImmAnnotation) {
544548
for (i = 0; i < 4; i++) {
545549
if (chunks[i] != 0) {
546550
movz(64, dst, chunks[i], i * 16);
547-
includeSet[i] = MovAction.USED;
548551
break;
549552
}
550553
}
551554
for (int k = i + 1; k < 4; k++) {
552555
if (chunks[k] != 0) {
553556
movk(64, dst, chunks[k], k * 16);
554-
includeSet[k] = MovAction.USED;
555557
break;
556558
}
557559
}
@@ -561,14 +563,12 @@ private void mov64(Register dst, long imm, boolean needsImmAnnotation) {
561563
for (i = 0; i < 4; i++) {
562564
if (chunks[i] != 0xFFFF) {
563565
movn(64, dst, chunks[i] ^ 0xFFFF, i * 16);
564-
includeSet[i] = MovAction.NEGATED;
565566
break;
566567
}
567568
}
568569
for (int k = i + 1; k < 4; k++) {
569570
if (chunks[k] != 0xFFFF) {
570571
movk(64, dst, chunks[k], k * 16);
571-
includeSet[k] = MovAction.USED;
572572
break;
573573
}
574574
}
@@ -578,15 +578,13 @@ private void mov64(Register dst, long imm, boolean needsImmAnnotation) {
578578
for (i = 0; i < 4; i++) {
579579
if (chunks[i] != 0) {
580580
movz(64, dst, chunks[i], i * 16);
581-
includeSet[i] = MovAction.USED;
582581
break;
583582
}
584583
}
585584
int numMovks = 0;
586585
for (int k = i + 1; k < 4; k++) {
587586
if (chunks[k] != 0) {
588587
movk(64, dst, chunks[k], k * 16);
589-
includeSet[k] = MovAction.USED;
590588
numMovks++;
591589
}
592590
}
@@ -597,15 +595,13 @@ private void mov64(Register dst, long imm, boolean needsImmAnnotation) {
597595
for (i = 0; i < 4; i++) {
598596
if (chunks[i] != 0xFFFF) {
599597
movn(64, dst, chunks[i] ^ 0xFFFF, i * 16);
600-
includeSet[i] = MovAction.NEGATED;
601598
break;
602599
}
603600
}
604601
int numMovks = 0;
605602
for (int k = i + 1; k < 4; k++) {
606603
if (chunks[k] != 0xFFFF) {
607604
movk(64, dst, chunks[k], k * 16);
608-
includeSet[k] = MovAction.USED;
609605
numMovks++;
610606
}
611607
}
@@ -616,13 +612,6 @@ private void mov64(Register dst, long imm, boolean needsImmAnnotation) {
616612
movk(64, dst, chunks[1], 16);
617613
movk(64, dst, chunks[2], 32);
618614
movk(64, dst, chunks[3], 48);
619-
includeSet[0] = MovAction.USED;
620-
includeSet[1] = MovAction.USED;
621-
includeSet[2] = MovAction.USED;
622-
includeSet[3] = MovAction.USED;
623-
}
624-
if (needsImmAnnotation) {
625-
annotateImmediateMovSequence(pos, includeSet);
626615
}
627616
}
628617

‎compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/asm/amd64/AMD64MacroAssembler.java

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -297,7 +297,8 @@ public final void movdbl(AMD64Address dst, Register src) {
297297
* Non-atomic write of a 64-bit constant to memory. Do not use if the address might be a
298298
* volatile field!
299299
*/
300-
public final void movlong(AMD64Address dst, long src) {
300+
public final void movlong(AMD64Address dst, long src, boolean annotateImm) {
301+
GraalError.guarantee(!annotateImm, "patching not implemented for 8-byte stores");
301302
if (NumUtil.isInt(src)) {
302303
emitAMD64MIOp(AMD64MIOp.MOV, OperandSize.QWORD, dst, (int) src, false);
303304
} else {
@@ -1485,6 +1486,14 @@ public final void moveInt(Register dst, int imm) {
14851486
movl(dst, imm);
14861487
}
14871488

1489+
public final void moveInt(Register dst, int imm, boolean annotateImm) {
1490+
if (!annotateImm) {
1491+
moveInt(dst, imm);
1492+
} else {
1493+
movl(dst, imm, true);
1494+
}
1495+
}
1496+
14881497
public final void moveInt(AMD64Address dst, int imm) {
14891498
if (imm == 0) {
14901499
Register zeroValueRegister = getZeroValueRegister();
@@ -1496,6 +1505,14 @@ public final void moveInt(AMD64Address dst, int imm) {
14961505
movl(dst, imm);
14971506
}
14981507

1508+
public final void moveInt(AMD64Address dst, int imm, boolean annotateImm) {
1509+
if (!annotateImm) {
1510+
moveInt(dst, imm);
1511+
} else {
1512+
AMD64MIOp.MOV.emit(this, OperandSize.DWORD, dst, imm, true);
1513+
}
1514+
}
1515+
14991516
public final void moveIntSignExtend(Register result, int imm) {
15001517
if (imm == 0) {
15011518
Register zeroValueRegister = getZeroValueRegister();

‎compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/aarch64/AArch64MoveFactory.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2015, 2025, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* This code is free software; you can redistribute it and/or modify it
@@ -43,6 +43,7 @@
4343
import jdk.vm.ci.meta.AllocatableValue;
4444
import jdk.vm.ci.meta.Constant;
4545
import jdk.vm.ci.meta.JavaConstant;
46+
import jdk.vm.ci.meta.PrimitiveConstant;
4647
import jdk.vm.ci.meta.Value;
4748

4849
public class AArch64MoveFactory extends MoveFactory {
@@ -113,7 +114,8 @@ public boolean canInlineConstant(Constant con) {
113114
* immediates are encoded as bitmasks (see
114115
* AArch64Assembler.LogicalBitmaskImmediateEncoding).
115116
*/
116-
return NumUtil.isSignedNbit(17, c.asLong());
117+
int optimisticMaxInlineWidth = 17;
118+
return c instanceof PrimitiveConstant && NumUtil.isSignedNbit(optimisticMaxInlineWidth, c.asLong());
117119
case Object:
118120
return c.isNull();
119121
case Float:

‎compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/amd64/AMD64ArithmeticLIRGenerator.java

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1368,22 +1368,26 @@ public Variable emitLoad(LIRKind kind, Value address, LIRFrameState state, Memor
13681368
return result;
13691369
}
13701370

1371+
// Object constants can only be inlined if getLIRGen().target().inlineObjects
1372+
private boolean canInlineStoreVMConstant(AMD64Kind kind, VMConstant c) {
1373+
if (kind.getSizeInBytes() >= Long.BYTES) {
1374+
// Store instructions cannot have 8-byte immediate
1375+
return false;
1376+
}
1377+
return !(c instanceof JavaConstant jc) || jc.getJavaKind() != JavaKind.Object || getLIRGen().target().inlineObjects;
1378+
}
1379+
13711380
protected void emitStoreConst(AMD64Kind kind, AMD64AddressValue address, ConstantValue value, LIRFrameState state) {
13721381
Constant c = value.getConstant();
13731382
if (JavaConstant.isNull(c)) {
13741383
assert kind == AMD64Kind.DWORD || kind == AMD64Kind.QWORD : Assertions.errorMessage(kind, address, value, state);
13751384
OperandSize size = kind == AMD64Kind.DWORD ? DWORD : QWORD;
13761385
getLIRGen().append(new AMD64BinaryConsumer.MemoryConstOp(AMD64MIOp.MOV, size, address, 0, state));
13771386
return;
1378-
} else if (c instanceof VMConstant) {
1379-
// only 32-bit constants can be patched
1380-
if (kind == AMD64Kind.DWORD) {
1381-
if (getLIRGen().target().inlineObjects || !(c instanceof JavaConstant)) {
1382-
// if c is a JavaConstant, it's an oop, otherwise it's a metaspace constant
1383-
assert !(c instanceof JavaConstant) || ((JavaConstant) c).getJavaKind() == JavaKind.Object : c;
1384-
getLIRGen().append(new AMD64BinaryConsumer.MemoryVMConstOp(AMD64MIOp.MOV, address, (VMConstant) c, state));
1385-
return;
1386-
}
1387+
} else if (c instanceof VMConstant vmc) {
1388+
if (canInlineStoreVMConstant(kind, vmc)) {
1389+
getLIRGen().append(new AMD64BinaryConsumer.MemoryVMConstOp(AMD64MIOp.MOV, address, (VMConstant) c, state));
1390+
return;
13871391
}
13881392
} else {
13891393
JavaConstant jc = (JavaConstant) c;

‎compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/amd64/AMD64MoveFactory.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2015, 2025, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* This code is free software; you can redistribute it and/or modify it
@@ -65,7 +65,7 @@ public boolean canInlineConstant(Constant con) {
6565
JavaConstant c = (JavaConstant) con;
6666
switch (c.getJavaKind()) {
6767
case Long:
68-
return NumUtil.isInt(c.asLong());
68+
return c instanceof PrimitiveConstant && NumUtil.isInt(c.asLong());
6969
case Float:
7070
case Double:
7171
return false;

‎compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/aarch64/AArch64Move.java

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@
7373
import jdk.vm.ci.meta.Constant;
7474
import jdk.vm.ci.meta.JavaConstant;
7575
import jdk.vm.ci.meta.JavaKind;
76+
import jdk.vm.ci.meta.VMConstant;
7677
import jdk.vm.ci.meta.Value;
7778

7879
public class AArch64Move {
@@ -673,12 +674,20 @@ static void stack2reg(AArch64Kind moveKind, CompilationResultBuilder crb, AArch6
673674
static void const2reg(AArch64Kind moveKind, CompilationResultBuilder crb, AArch64MacroAssembler masm, Register result, JavaConstant input) {
674675
JavaKind stackKind = input.getJavaKind().getStackKind();
675676
assert stackKind.isObject() || moveKind.getSizeInBytes() <= stackKind.getByteCount() : Assertions.errorMessageContext("stackKind", stackKind, "moveKind", moveKind);
677+
// VMConstant is the marker interface for patched constants
678+
boolean needsPatching = input instanceof VMConstant;
676679
switch (stackKind) {
677680
case Int:
678-
masm.mov(result, input.asInt());
681+
if (needsPatching) {
682+
crb.recordInlineDataInCode(input);
683+
}
684+
masm.mov(result, input.asInt(), needsPatching);
679685
break;
680686
case Long:
681-
masm.mov(result, input.asLong());
687+
if (needsPatching) {
688+
crb.recordInlineDataInCode(input);
689+
}
690+
masm.mov(result, input.asLong(), needsPatching);
682691
break;
683692
case Float:
684693
if (AArch64MacroAssembler.isFloatImmediate(input.asFloat()) && result.getRegisterCategory().equals(SIMD)) {

0 commit comments

Comments
 (0)