Skip to content

Commit d992d9a

Browse files
mur47x111mur47x111
committed
[GR-48679] Implement DualPivotQuicksort intrinsics.
PullRequest: graal/20811
2 parents 486370a + 6c8ad47 commit d992d9a

File tree

8 files changed

+212
-8
lines changed

8 files changed

+212
-8
lines changed

‎compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/hotspot/test/ArraySortIntrinsicficationTest.java

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
/*
2+
* Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
3+
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4+
*
5+
* This code is free software; you can redistribute it and/or modify it
6+
* under the terms of the GNU General Public License version 2 only, as
7+
* published by the Free Software Foundation. Oracle designates this
8+
* particular file as subject to the "Classpath" exception as provided
9+
* by Oracle in the LICENSE file that accompanied this code.
10+
*
11+
* This code is distributed in the hope that it will be useful, but WITHOUT
12+
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13+
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14+
* version 2 for more details (a copy is included in the LICENSE file that
15+
* accompanied this code).
16+
*
17+
* You should have received a copy of the GNU General Public License version
18+
* 2 along with this work; if not, write to the Free Software Foundation,
19+
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20+
*
21+
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22+
* or visit www.oracle.com if you need additional information or have any
23+
* questions.
24+
*/
25+
26+
package jdk.graal.compiler.hotspot.test;
27+
28+
import java.lang.reflect.InvocationTargetException;
29+
import java.lang.reflect.Method;
30+
import java.util.Arrays;
31+
import java.util.HashMap;
32+
import java.util.Random;
33+
34+
import org.graalvm.collections.Pair;
35+
import org.junit.Test;
36+
37+
import jdk.graal.compiler.api.test.ModuleSupport;
38+
import jdk.graal.compiler.core.test.GraalCompilerTest;
39+
import jdk.vm.ci.code.InstalledCode;
40+
41+
public final class ArraySortIntrinsicficationTest extends HotSpotGraalCompilerTest {
42+
43+
private static final int[] LONG_RUN_LENGTHS = {
44+
1, 3, 8, 21, 55, 100, 1_000, 10_000, 100_000};
45+
private static final int[] PARALLELISMS = {
46+
0, 87, 64 * (3 << 1)};
47+
48+
@Test
49+
public void testIntSort() throws ClassNotFoundException, InvocationTargetException, IllegalAccessException {
50+
Class<?> klass = Class.forName("java.util.DualPivotQuicksort");
51+
Class<?> sorterKlass = Class.forName("java.util.DualPivotQuicksort$Sorter");
52+
53+
Random rng = getRandomInstance();
54+
55+
ModuleSupport.exportAndOpenAllPackagesToUnnamed("java.base");
56+
Method method = getMethod(klass, "sort", int[].class, int.class, int.class, int.class);
57+
method.setAccessible(true);
58+
59+
HashMap<Pair<Integer, Integer>, int[]> goldensFullArray = new HashMap<>();
60+
HashMap<Pair<Integer, Integer>, int[]> goldensHalfArray = new HashMap<>();
61+
62+
int[][] arrays = Arrays.stream(LONG_RUN_LENGTHS).mapToObj(len -> rng.ints(len).toArray()).toArray(int[][]::new);
63+
64+
for (int[] array : arrays) {
65+
for (int parallelism : PARALLELISMS) {
66+
int[] golden = array.clone();
67+
method.invoke(null, golden, parallelism, 0, array.length);
68+
goldensFullArray.put(Pair.create(array.length, parallelism), golden);
69+
70+
golden = array.clone();
71+
method.invoke(null, golden, parallelism, array.length / 2, array.length);
72+
goldensHalfArray.put(Pair.create(array.length, parallelism), golden);
73+
}
74+
}
75+
76+
InstalledCode intrinsic = getCode(getResolvedJavaMethod(klass, "sort", sorterKlass, int[].class, int.class, int.class, int.class),
77+
null, true, true, GraalCompilerTest.getInitialOptions());
78+
79+
for (int[] array : arrays) {
80+
for (int parallelism : PARALLELISMS) {
81+
int[] golden = goldensFullArray.get(Pair.create(array.length, parallelism));
82+
int[] actual = array.clone();
83+
method.invoke(null, actual, parallelism, 0, array.length);
84+
assertDeepEquals(golden, actual);
85+
86+
golden = goldensHalfArray.get(Pair.create(array.length, parallelism));
87+
actual = array.clone();
88+
method.invoke(null, actual, parallelism, array.length / 2, array.length);
89+
assertDeepEquals(golden, actual);
90+
}
91+
}
92+
93+
intrinsic.invalidate();
94+
}
95+
}

‎compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/hotspot/test/VectorizedHashCodeTest.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,9 +56,12 @@ private static int getField(String name) {
5656
public void testJDKConstantValue() {
5757
Assert.assertEquals(getField("T_BOOLEAN"), VectorizedHashCodeInvocationPlugin.T_BOOLEAN);
5858
Assert.assertEquals(getField("T_CHAR"), VectorizedHashCodeInvocationPlugin.T_CHAR);
59+
Assert.assertEquals(getField("T_FLOAT"), VectorizedHashCodeInvocationPlugin.T_FLOAT);
60+
Assert.assertEquals(getField("T_DOUBLE"), VectorizedHashCodeInvocationPlugin.T_DOUBLE);
5961
Assert.assertEquals(getField("T_BYTE"), VectorizedHashCodeInvocationPlugin.T_BYTE);
6062
Assert.assertEquals(getField("T_SHORT"), VectorizedHashCodeInvocationPlugin.T_SHORT);
6163
Assert.assertEquals(getField("T_INT"), VectorizedHashCodeInvocationPlugin.T_INT);
64+
Assert.assertEquals(getField("T_LONG"), VectorizedHashCodeInvocationPlugin.T_LONG);
6265
}
6366

6467
// @formatter:off

‎compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -548,6 +548,9 @@ public int threadTlabTopOffset() {
548548
public final long stubKyber12To16 = getFieldValue("StubRoutines::_kyber12To16", Long.class, "address");
549549
public final long stubKyberBarrettReduce = getFieldValue("StubRoutines::_kyberBarrettReduce", Long.class, "address");
550550

551+
public final long stubArraySort = getFieldValue("StubRoutines::_array_sort", Long.class, "address");
552+
public final long stubArrayPartition = getFieldValue("StubRoutines::_array_partition", Long.class, "address");
553+
551554
// Allocation stubs that return null when allocation fails
552555
public final long newInstanceOrNullAddress = getAddress("JVMCIRuntime::new_instance_or_null");
553556
public final long newArrayOrNullAddress = getAddress("JVMCIRuntime::new_array_or_null");
@@ -702,6 +705,7 @@ public boolean supportJVMTIVThreadNotification() {
702705
getConstant("Deoptimization::_support_large_access_byte_array_virtualization", Boolean.class);
703706

704707
public final int l1LineSize = getFieldValue("CompilerToVM::Data::L1_line_size", Integer.class, "int", 0, "amd64".equals(osArch));
708+
public final boolean supportsAvx512SimdSort = getFieldValue("CompilerToVM::Data::supports_avx512_simd_sort", Boolean.class, "bool", false, "amd64".equals(osArch));
705709

706710
// Checkstyle: stop
707711
public final int VMINTRINSIC_FIRST_MH_SIG_POLY = getConstant("vmIntrinsics::FIRST_MH_SIG_POLY", Integer.class);

‎compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotBackend.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -285,6 +285,10 @@ public static void unsafeArraycopy(Word srcAddr, Word dstAddr, Word size) {
285285
WordBase.class, int.class, WordBase.class, int.class);
286286
public static final HotSpotForeignCallDescriptor KYBER_BARRETT_REDUCE = new HotSpotForeignCallDescriptor(LEAF, HAS_SIDE_EFFECT, any(), "_kyberBarrettReduce", int.class,
287287
WordBase.class);
288+
public static final HotSpotForeignCallDescriptor ARRAY_SORT = new HotSpotForeignCallDescriptor(LEAF_NO_VZERO, HAS_SIDE_EFFECT, any(), "_array_sort", void.class,
289+
WordBase.class, int.class, int.class, int.class);
290+
public static final HotSpotForeignCallDescriptor ARRAY_PARTITION = new HotSpotForeignCallDescriptor(LEAF_NO_VZERO, HAS_SIDE_EFFECT, any(), "_array_partition", void.class,
291+
WordBase.class, int.class, int.class, int.class, WordBase.class, int.class, int.class);
288292

289293
public static final HotSpotForeignCallDescriptor SHAREDRUNTIME_NOTIFY_JVMTI_VTHREAD_START = new HotSpotForeignCallDescriptor(SAFEPOINT, HAS_SIDE_EFFECT, any(),
290294
"notify_jvmti_vthread_start", void.class,

‎compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/HotSpotGraphBuilderPlugins.java

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424
*/
2525
package jdk.graal.compiler.hotspot.meta;
2626

27+
import static jdk.graal.compiler.hotspot.HotSpotBackend.ARRAY_PARTITION;
28+
import static jdk.graal.compiler.hotspot.HotSpotBackend.ARRAY_SORT;
2729
import static jdk.graal.compiler.hotspot.HotSpotBackend.BASE64_DECODE_BLOCK;
2830
import static jdk.graal.compiler.hotspot.HotSpotBackend.BASE64_ENCODE_BLOCK;
2931
import static jdk.graal.compiler.hotspot.HotSpotBackend.CHACHA20Block;
@@ -64,6 +66,10 @@
6466
import static jdk.graal.compiler.nodes.ConstantNode.forBoolean;
6567
import static jdk.graal.compiler.nodes.ProfileData.BranchProbabilityData.injected;
6668
import static jdk.graal.compiler.nodes.extended.BranchProbabilityNode.NOT_FREQUENT_PROBABILITY;
69+
import static jdk.graal.compiler.replacements.StandardGraphBuilderPlugins.VectorizedHashCodeInvocationPlugin.T_DOUBLE;
70+
import static jdk.graal.compiler.replacements.StandardGraphBuilderPlugins.VectorizedHashCodeInvocationPlugin.T_FLOAT;
71+
import static jdk.graal.compiler.replacements.StandardGraphBuilderPlugins.VectorizedHashCodeInvocationPlugin.T_INT;
72+
import static jdk.graal.compiler.replacements.StandardGraphBuilderPlugins.VectorizedHashCodeInvocationPlugin.T_LONG;
6773
import static jdk.vm.ci.meta.DeoptimizationReason.TypeCheckedInliningViolated;
6874

6975
import java.lang.annotation.Annotation;
@@ -203,6 +209,7 @@
203209
import jdk.graal.compiler.vector.replacements.vectorapi.VectorAPIIntrinsics;
204210
import jdk.graal.compiler.word.WordTypes;
205211
import jdk.vm.ci.aarch64.AArch64;
212+
import jdk.vm.ci.amd64.AMD64;
206213
import jdk.vm.ci.code.Architecture;
207214
import jdk.vm.ci.code.TargetDescription;
208215
import jdk.vm.ci.meta.ConstantReflectionProvider;
@@ -300,6 +307,7 @@ public void run() {
300307
registerPoly1305Plugins(invocationPlugins, config, replacements);
301308
registerChaCha20Plugins(invocationPlugins, config, replacements);
302309
registerP256Plugins(invocationPlugins, config, replacements);
310+
registerDualPivotQuicksortPlugins(invocationPlugins, config, replacements, target.arch);
303311
if (VectorAPIIntrinsics.intrinsificationSupported(options)) {
304312
VectorAPIIntrinsics.registerPlugins(plugins.getInvocationPlugins(), replacements);
305313
}
@@ -1706,4 +1714,76 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec
17061714
}
17071715
});
17081716
}
1717+
1718+
private static boolean isSimdSortSupported(Architecture arch, GraalHotSpotVMConfig config, int jvmType) {
1719+
if (config.stubArraySort == 0L || config.stubArrayPartition == 0L) {
1720+
return false;
1721+
}
1722+
if (arch instanceof AMD64 && !config.supportsAvx512SimdSort) {
1723+
return switch (jvmType) {
1724+
case T_INT, T_FLOAT -> true;
1725+
default -> false;
1726+
};
1727+
}
1728+
return true;
1729+
}
1730+
1731+
private static void registerDualPivotQuicksortPlugins(InvocationPlugins plugins, GraalHotSpotVMConfig config, Replacements replacements, Architecture arch) {
1732+
Registration r = new Registration(plugins, "java.util.DualPivotQuicksort", replacements);
1733+
r.registerConditional(config.stubArraySort != 0L, new InlineOnlyInvocationPlugin("sort", Class.class, Object.class, long.class, int.class, int.class,
1734+
new InvocationPlugins.OptionalLazySymbol("java.util.DualPivotQuicksort$SortOperation")) {
1735+
@Override
1736+
public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode elemType, ValueNode array,
1737+
ValueNode offset, ValueNode low, ValueNode high, ValueNode so) {
1738+
if (elemType.isConstant()) {
1739+
ResolvedJavaType type = b.getConstantReflection().asJavaType(elemType.asJavaConstant());
1740+
int jvmType = switch (type.getJavaKind()) {
1741+
case Float -> T_FLOAT;
1742+
case Double -> T_DOUBLE;
1743+
case Int -> T_INT;
1744+
case Long -> T_LONG;
1745+
default -> throw GraalError.shouldNotReachHere("Unexpected element type " + type.toJavaName());
1746+
};
1747+
if (isSimdSortSupported(arch, config, jvmType)) {
1748+
ValueNode arrayNonNull = b.nullCheckedValue(array);
1749+
ValueNode arrayStart = b.add(new ComputeObjectAddressNode(arrayNonNull, offset));
1750+
b.add(new ForeignCallNode(ARRAY_SORT, arrayStart, ConstantNode.forInt(jvmType), low, high));
1751+
return true;
1752+
}
1753+
}
1754+
return false;
1755+
}
1756+
});
1757+
r.registerConditional(config.stubArrayPartition != 0L, new InlineOnlyInvocationPlugin("partition", Class.class, Object.class, long.class, int.class, int.class, int.class, int.class,
1758+
new InvocationPlugins.OptionalLazySymbol("java.util.DualPivotQuicksort$PartitionOperation")) {
1759+
@Override
1760+
public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode elemType, ValueNode array,
1761+
ValueNode offset, ValueNode low, ValueNode high, ValueNode pivotIndex1, ValueNode pivotIndex2, ValueNode so) {
1762+
try (HotSpotInvocationPluginHelper helper = new HotSpotInvocationPluginHelper(b, targetMethod, config)) {
1763+
if (elemType.isConstant()) {
1764+
ResolvedJavaType type = b.getConstantReflection().asJavaType(elemType.asJavaConstant());
1765+
int jvmType = switch (type.getJavaKind()) {
1766+
case Float -> T_FLOAT;
1767+
case Double -> T_DOUBLE;
1768+
case Int -> T_INT;
1769+
case Long -> T_LONG;
1770+
default -> throw GraalError.shouldNotReachHere("Unexpected element type " + type.toJavaName());
1771+
};
1772+
if (isSimdSortSupported(arch, config, jvmType)) {
1773+
ValueNode arrayNonNull = b.nullCheckedValue(array);
1774+
ValueNode arrayStart = b.add(new ComputeObjectAddressNode(arrayNonNull, offset));
1775+
ValueNode pivotIndices = b.append(new NewArrayNode(b.getMetaAccess().lookupJavaType(Integer.TYPE), ConstantNode.forInt(2), false));
1776+
ValueNode pivotIndicesStart = helper.arrayStart(pivotIndices, JavaKind.Int);
1777+
1778+
ForeignCallNode call = b.append(new ForeignCallNode(ARRAY_PARTITION, arrayStart, ConstantNode.forInt(jvmType), low, high, pivotIndicesStart, pivotIndex1, pivotIndex2));
1779+
b.push(JavaKind.Object, pivotIndices);
1780+
b.setStateAfter(call);
1781+
return true;
1782+
}
1783+
}
1784+
return false;
1785+
}
1786+
}
1787+
});
1788+
}
17091789
}

‎compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/HotSpotHostForeignCallsProvider.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@
2828
import static jdk.graal.compiler.core.common.spi.ForeignCallDescriptor.CallSideEffect.NO_SIDE_EFFECT;
2929
import static jdk.graal.compiler.core.target.Backend.ARITHMETIC_DREM;
3030
import static jdk.graal.compiler.core.target.Backend.ARITHMETIC_FREM;
31+
import static jdk.graal.compiler.hotspot.HotSpotBackend.ARRAY_PARTITION;
32+
import static jdk.graal.compiler.hotspot.HotSpotBackend.ARRAY_SORT;
3133
import static jdk.graal.compiler.hotspot.HotSpotBackend.BASE64_DECODE_BLOCK;
3234
import static jdk.graal.compiler.hotspot.HotSpotBackend.BASE64_ENCODE_BLOCK;
3335
import static jdk.graal.compiler.hotspot.HotSpotBackend.BIGINTEGER_LEFT_SHIFT_WORKER;
@@ -681,6 +683,12 @@ public void initialize(HotSpotProviders providers, OptionValues options) {
681683
if (c.stubKyberBarrettReduce != 0L) {
682684
registerForeignCall(KYBER_BARRETT_REDUCE, c.stubKyberBarrettReduce, NativeCall);
683685
}
686+
if (c.stubArrayPartition != 0L) {
687+
registerForeignCall(ARRAY_PARTITION, c.stubArrayPartition, NativeCall);
688+
}
689+
if (c.stubArraySort != 0L) {
690+
registerForeignCall(ARRAY_SORT, c.stubArraySort, NativeCall);
691+
}
684692

685693
registerSnippetStubs(providers, options);
686694
registerStubCallFunctions(options, providers, runtime.getVMConfig());

‎compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/UnimplementedGraalIntrinsics.java

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232

3333
import jdk.graal.compiler.debug.GraalError;
3434
import jdk.graal.compiler.nodes.graphbuilderconf.InvocationPlugin;
35+
import jdk.graal.compiler.serviceprovider.GraalServices;
3536
import jdk.vm.ci.aarch64.AArch64;
3637
import jdk.vm.ci.amd64.AMD64;
3738
import jdk.vm.ci.code.Architecture;
@@ -85,11 +86,6 @@ private static void add(Collection<String> c, String... elements) {
8586
}
8687

8788
public UnimplementedGraalIntrinsics(Architecture arch) {
88-
add(toBeInvestigated,
89-
// JDK-8309130: x86_64 AVX512 intrinsics for Arrays.sort methods (GR-48679)
90-
"java/util/DualPivotQuicksort.partition(Ljava/lang/Class;Ljava/lang/Object;JIIIILjava/util/DualPivotQuicksort$PartitionOperation;)[I",
91-
"java/util/DualPivotQuicksort.sort(Ljava/lang/Class;Ljava/lang/Object;JIILjava/util/DualPivotQuicksort$SortOperation;)V");
92-
9389
add(toBeInvestigated, // @formatter:off
9490
// JDK-8342103: C2 compiler support for Float16 type and associated
9591
// scalar operations
@@ -148,10 +144,21 @@ public UnimplementedGraalIntrinsics(Architecture arch) {
148144
add(ignore,
149145
"sun/security/provider/SHA2.implCompress0([BI)V");
150146
}
147+
148+
if (!GraalServices.getSavedProperty("os.name").contains("Linux")) {
149+
add(ignore,
150+
"java/util/DualPivotQuicksort.partition(Ljava/lang/Class;Ljava/lang/Object;JIIIILjava/util/DualPivotQuicksort$PartitionOperation;)[I",
151+
"java/util/DualPivotQuicksort.sort(Ljava/lang/Class;Ljava/lang/Object;JIILjava/util/DualPivotQuicksort$SortOperation;)V");
152+
}
151153
} else if (arch instanceof AArch64) {
152-
// Newly added by JDK-8338694. HotSpot runtime does not implement
153-
// C2Compiler::is_intrinsic_supported for this intrinsic properly
154-
add(ignore, "java/lang/Math.tanh(D)D");
154+
// HotSpot runtime does not implement C2Compiler::is_intrinsic_supported for the
155+
// following intrinsics properly
156+
add(ignore,
157+
// JDK-8338694
158+
"java/lang/Math.tanh(D)D",
159+
// JDK-8309130
160+
"java/util/DualPivotQuicksort.partition(Ljava/lang/Class;Ljava/lang/Object;JIIIILjava/util/DualPivotQuicksort$PartitionOperation;)[I",
161+
"java/util/DualPivotQuicksort.sort(Ljava/lang/Class;Ljava/lang/Object;JIILjava/util/DualPivotQuicksort$SortOperation;)V");
155162
}
156163

157164
// These are known to be implemented down stream

‎compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/StandardGraphBuilderPlugins.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2719,9 +2719,12 @@ public static class VectorizedHashCodeInvocationPlugin extends InlineOnlyInvocat
27192719
// Sync with ArraysSupport.java
27202720
public static final int T_BOOLEAN = 4;
27212721
public static final int T_CHAR = 5;
2722+
public static final int T_FLOAT = 6;
2723+
public static final int T_DOUBLE = 7;
27222724
public static final int T_BYTE = 8;
27232725
public static final int T_SHORT = 9;
27242726
public static final int T_INT = 10;
2727+
public static final int T_LONG = 11;
27252728

27262729
public VectorizedHashCodeInvocationPlugin(String name) {
27272730
super(name, Object.class, int.class, int.class, int.class, int.class);

0 commit comments

Comments
 (0)