diff --git a/buildSrc/scriptDepVersions.gradle b/buildSrc/scriptDepVersions.gradle index 8751da632492..a6eae860b1c0 100644 --- a/buildSrc/scriptDepVersions.gradle +++ b/buildSrc/scriptDepVersions.gradle @@ -22,7 +22,7 @@ ext { scriptDepVersions = [ "apache-rat": "0.14", - "asm": "9.4", + "asm": "9.5", "commons-codec": "1.13", "ecj": "3.30.0", "flexmark": "0.61.24", diff --git a/gradle/generation/extract-jdk-apis.gradle b/gradle/generation/extract-jdk-apis.gradle index 834a490cc0d9..6a5fb5ad8fd0 100644 --- a/gradle/generation/extract-jdk-apis.gradle +++ b/gradle/generation/extract-jdk-apis.gradle @@ -27,7 +27,7 @@ configure(rootProject) { configure(project(":lucene:core")) { ext { apijars = file('src/generated/jdk'); - mrjarJavaVersions = [ 19, 20 ] + mrjarJavaVersions = [ 19, 20, 21 ] } configurations { diff --git a/gradle/generation/extract-jdk-apis/ExtractJdkApis.java b/gradle/generation/extract-jdk-apis/ExtractJdkApis.java index 8d185f9b4298..7121374850bd 100644 --- a/gradle/generation/extract-jdk-apis/ExtractJdkApis.java +++ b/gradle/generation/extract-jdk-apis/ExtractJdkApis.java @@ -55,7 +55,8 @@ public final class ExtractJdkApis { static final Map> CLASSFILE_PATTERNS = Map.of( 19, List.of(PATTERN_PANAMA_FOREIGN), - 20, List.of(PATTERN_PANAMA_FOREIGN, PATTERN_VECTOR_VM_INTERNALS, PATTERN_VECTOR_INCUBATOR) + 20, List.of(PATTERN_PANAMA_FOREIGN, PATTERN_VECTOR_VM_INTERNALS, PATTERN_VECTOR_INCUBATOR), + 21, List.of(PATTERN_PANAMA_FOREIGN) ); public static void main(String... args) throws IOException { diff --git a/gradle/template.gradle.properties b/gradle/template.gradle.properties index a626d39f3bb5..9ac8c42e9dd4 100644 --- a/gradle/template.gradle.properties +++ b/gradle/template.gradle.properties @@ -102,5 +102,5 @@ tests.jvms=@TEST_JVMS@ org.gradle.java.installations.auto-download=true # Set these to enable automatic JVM location discovery. -org.gradle.java.installations.fromEnv=JAVA17_HOME,JAVA19_HOME,JAVA20_HOME,JAVA21_HOME,RUNTIME_JAVA_HOME +org.gradle.java.installations.fromEnv=JAVA17_HOME,JAVA19_HOME,JAVA20_HOME,JAVA21_HOME,JAVA22_HOME,RUNTIME_JAVA_HOME #org.gradle.java.installations.paths=(custom paths) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 48626a8349c9..1f478e126cc5 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -147,6 +147,12 @@ New Features thoroughly and report bugs/slowness to Lucene's mailing list. (Chris Hegarty, Robert Muir, Uwe Schindler) +* GITHUB#12294: Add support for Java 21 foreign memory API. If Java 19 up to 21 is used, + MMapDirectory will mmap Lucene indexes in chunks of 16 GiB (instead of 1 GiB) and indexes + closed while queries are running can no longer crash the JVM. To disable this feature, + pass the following sysprop on Java command line: + "-Dorg.apache.lucene.store.MMapDirectory.enableMemorySegments=false" (Uwe Schindler) + Improvements --------------------- diff --git a/lucene/core/src/generated/jdk/jdk21.apijar b/lucene/core/src/generated/jdk/jdk21.apijar new file mode 100644 index 000000000000..69906ccf4b06 Binary files /dev/null and b/lucene/core/src/generated/jdk/jdk21.apijar differ diff --git a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java index 9ca636b4cd71..c9cddb4a308a 100644 --- a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java @@ -76,9 +76,9 @@ *
  • {@code permission java.lang.RuntimePermission "accessClassInPackage.sun.misc";} * * - *

    On exactly Java 19 and Java 20 this class will use the modern {@code - * MemorySegment} API which allows to safely unmap (if you discover any problems with this preview - * API, you can disable it by using system property {@link #ENABLE_MEMORY_SEGMENTS_SYSPROP}). + *

    On exactly Java 19 / 20 / 21 this class will use the modern {@code MemorySegment} API + * which allows to safely unmap (if you discover any problems with this preview API, you can disable + * it by using system property {@link #ENABLE_MEMORY_SEGMENTS_SYSPROP}). * *

    NOTE: Accessing this class either directly or indirectly from a thread while it's * interrupted can close the underlying channel immediately if at the same time the thread is @@ -123,7 +123,7 @@ public class MMapDirectory extends FSDirectory { * Default max chunk size: * *

    @@ -346,7 +346,7 @@ private static MMapIndexInputProvider lookupProvider() { } final var lookup = MethodHandles.lookup(); final int runtimeVersion = Runtime.version().feature(); - if (runtimeVersion == 19 || runtimeVersion == 20) { + if (runtimeVersion >= 19 && runtimeVersion <= 21) { try { final var cls = lookup.findClass("org.apache.lucene.store.MemorySegmentIndexInputProvider"); // we use method handles, so we do not need to deal with setAccessible as we have private @@ -366,9 +366,9 @@ private static MMapIndexInputProvider lookupProvider() { throw new LinkageError( "MemorySegmentIndexInputProvider is missing in Lucene JAR file", cnfe); } - } else if (runtimeVersion >= 21) { + } else if (runtimeVersion >= 22) { LOG.warning( - "You are running with Java 21 or later. To make full use of MMapDirectory, please update Apache Lucene."); + "You are running with Java 22 or later. To make full use of MMapDirectory, please update Apache Lucene."); } return new MappedByteBufferIndexInputProvider(); } diff --git a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java new file mode 100644 index 000000000000..7b2216add785 --- /dev/null +++ b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java @@ -0,0 +1,588 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.store; + +import java.io.EOFException; +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.ByteOrder; +import java.util.Arrays; +import java.util.Objects; +import org.apache.lucene.util.ArrayUtil; + +/** + * Base IndexInput implementation that uses an array of MemorySegments to represent a file. + * + *

    For efficiency, this class requires that the segment size are a power-of-two ( + * chunkSizePower). + */ +@SuppressWarnings("preview") +abstract class MemorySegmentIndexInput extends IndexInput implements RandomAccessInput { + static final ValueLayout.OfByte LAYOUT_BYTE = ValueLayout.JAVA_BYTE; + static final ValueLayout.OfShort LAYOUT_LE_SHORT = + ValueLayout.JAVA_SHORT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN); + static final ValueLayout.OfInt LAYOUT_LE_INT = + ValueLayout.JAVA_INT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN); + static final ValueLayout.OfLong LAYOUT_LE_LONG = + ValueLayout.JAVA_LONG_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN); + static final ValueLayout.OfFloat LAYOUT_LE_FLOAT = + ValueLayout.JAVA_FLOAT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN); + + final long length; + final long chunkSizeMask; + final int chunkSizePower; + final Arena arena; + final MemorySegment[] segments; + + int curSegmentIndex = -1; + MemorySegment + curSegment; // redundant for speed: segments[curSegmentIndex], also marker if closed! + long curPosition; // relative to curSegment, not globally + + public static MemorySegmentIndexInput newInstance( + String resourceDescription, + Arena arena, + MemorySegment[] segments, + long length, + int chunkSizePower) { + assert Arrays.stream(segments).map(MemorySegment::scope).allMatch(arena.scope()::equals); + if (segments.length == 1) { + return new SingleSegmentImpl(resourceDescription, arena, segments[0], length, chunkSizePower); + } else { + return new MultiSegmentImpl(resourceDescription, arena, segments, 0, length, chunkSizePower); + } + } + + private MemorySegmentIndexInput( + String resourceDescription, + Arena arena, + MemorySegment[] segments, + long length, + int chunkSizePower) { + super(resourceDescription); + this.arena = arena; + this.segments = segments; + this.length = length; + this.chunkSizePower = chunkSizePower; + this.chunkSizeMask = (1L << chunkSizePower) - 1L; + this.curSegment = segments[0]; + } + + void ensureOpen() { + if (curSegment == null) { + throw alreadyClosed(null); + } + } + + // the unused parameter is just to silence javac about unused variables + RuntimeException handlePositionalIOOBE(RuntimeException unused, String action, long pos) + throws IOException { + if (pos < 0L) { + return new IllegalArgumentException(action + " negative position (pos=" + pos + "): " + this); + } else { + throw new EOFException(action + " past EOF (pos=" + pos + "): " + this); + } + } + + // the unused parameter is just to silence javac about unused variables + AlreadyClosedException alreadyClosed(RuntimeException unused) { + return new AlreadyClosedException("Already closed: " + this); + } + + @Override + public final byte readByte() throws IOException { + try { + final byte v = curSegment.get(LAYOUT_BYTE, curPosition); + curPosition++; + return v; + } catch ( + @SuppressWarnings("unused") + IndexOutOfBoundsException e) { + do { + curSegmentIndex++; + if (curSegmentIndex >= segments.length) { + throw new EOFException("read past EOF: " + this); + } + curSegment = segments[curSegmentIndex]; + curPosition = 0L; + } while (curSegment.byteSize() == 0L); + final byte v = curSegment.get(LAYOUT_BYTE, curPosition); + curPosition++; + return v; + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + public final void readBytes(byte[] b, int offset, int len) throws IOException { + try { + MemorySegment.copy(curSegment, LAYOUT_BYTE, curPosition, b, offset, len); + curPosition += len; + } catch ( + @SuppressWarnings("unused") + IndexOutOfBoundsException e) { + readBytesBoundary(b, offset, len); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + private void readBytesBoundary(byte[] b, int offset, int len) throws IOException { + try { + long curAvail = curSegment.byteSize() - curPosition; + while (len > curAvail) { + MemorySegment.copy(curSegment, LAYOUT_BYTE, curPosition, b, offset, (int) curAvail); + len -= curAvail; + offset += curAvail; + curSegmentIndex++; + if (curSegmentIndex >= segments.length) { + throw new EOFException("read past EOF: " + this); + } + curSegment = segments[curSegmentIndex]; + curPosition = 0L; + curAvail = curSegment.byteSize(); + } + MemorySegment.copy(curSegment, LAYOUT_BYTE, curPosition, b, offset, len); + curPosition += len; + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + public void readInts(int[] dst, int offset, int length) throws IOException { + try { + MemorySegment.copy(curSegment, LAYOUT_LE_INT, curPosition, dst, offset, length); + curPosition += Integer.BYTES * (long) length; + } catch ( + @SuppressWarnings("unused") + IndexOutOfBoundsException iobe) { + super.readInts(dst, offset, length); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + public void readLongs(long[] dst, int offset, int length) throws IOException { + try { + MemorySegment.copy(curSegment, LAYOUT_LE_LONG, curPosition, dst, offset, length); + curPosition += Long.BYTES * (long) length; + } catch ( + @SuppressWarnings("unused") + IndexOutOfBoundsException iobe) { + super.readLongs(dst, offset, length); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + public void readFloats(float[] dst, int offset, int length) throws IOException { + try { + MemorySegment.copy(curSegment, LAYOUT_LE_FLOAT, curPosition, dst, offset, length); + curPosition += Float.BYTES * (long) length; + } catch ( + @SuppressWarnings("unused") + IndexOutOfBoundsException iobe) { + super.readFloats(dst, offset, length); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + public final short readShort() throws IOException { + try { + final short v = curSegment.get(LAYOUT_LE_SHORT, curPosition); + curPosition += Short.BYTES; + return v; + } catch ( + @SuppressWarnings("unused") + IndexOutOfBoundsException e) { + return super.readShort(); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + public final int readInt() throws IOException { + try { + final int v = curSegment.get(LAYOUT_LE_INT, curPosition); + curPosition += Integer.BYTES; + return v; + } catch ( + @SuppressWarnings("unused") + IndexOutOfBoundsException e) { + return super.readInt(); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + public final long readLong() throws IOException { + try { + final long v = curSegment.get(LAYOUT_LE_LONG, curPosition); + curPosition += Long.BYTES; + return v; + } catch ( + @SuppressWarnings("unused") + IndexOutOfBoundsException e) { + return super.readLong(); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + public long getFilePointer() { + ensureOpen(); + return (((long) curSegmentIndex) << chunkSizePower) + curPosition; + } + + @Override + public void seek(long pos) throws IOException { + ensureOpen(); + // we use >> here to preserve negative, so we will catch AIOOBE, + // in case pos + offset overflows. + final int si = (int) (pos >> chunkSizePower); + try { + if (si != curSegmentIndex) { + final MemorySegment seg = segments[si]; + // write values, on exception all is unchanged + this.curSegmentIndex = si; + this.curSegment = seg; + } + this.curPosition = Objects.checkIndex(pos & chunkSizeMask, curSegment.byteSize() + 1); + } catch (IndexOutOfBoundsException e) { + throw handlePositionalIOOBE(e, "seek", pos); + } + } + + @Override + public byte readByte(long pos) throws IOException { + try { + final int si = (int) (pos >> chunkSizePower); + return segments[si].get(LAYOUT_BYTE, pos & chunkSizeMask); + } catch (IndexOutOfBoundsException ioobe) { + throw handlePositionalIOOBE(ioobe, "read", pos); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + // used only by random access methods to handle reads across boundaries + private void setPos(long pos, int si) throws IOException { + try { + final MemorySegment seg = segments[si]; + // write values, on exception above all is unchanged + this.curPosition = pos & chunkSizeMask; + this.curSegmentIndex = si; + this.curSegment = seg; + } catch (IndexOutOfBoundsException ioobe) { + throw handlePositionalIOOBE(ioobe, "read", pos); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + public short readShort(long pos) throws IOException { + final int si = (int) (pos >> chunkSizePower); + try { + return segments[si].get(LAYOUT_LE_SHORT, pos & chunkSizeMask); + } catch ( + @SuppressWarnings("unused") + IndexOutOfBoundsException ioobe) { + // either it's a boundary, or read past EOF, fall back: + setPos(pos, si); + return readShort(); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + public int readInt(long pos) throws IOException { + final int si = (int) (pos >> chunkSizePower); + try { + return segments[si].get(LAYOUT_LE_INT, pos & chunkSizeMask); + } catch ( + @SuppressWarnings("unused") + IndexOutOfBoundsException ioobe) { + // either it's a boundary, or read past EOF, fall back: + setPos(pos, si); + return readInt(); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + public long readLong(long pos) throws IOException { + final int si = (int) (pos >> chunkSizePower); + try { + return segments[si].get(LAYOUT_LE_LONG, pos & chunkSizeMask); + } catch ( + @SuppressWarnings("unused") + IndexOutOfBoundsException ioobe) { + // either it's a boundary, or read past EOF, fall back: + setPos(pos, si); + return readLong(); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + public final long length() { + return length; + } + + @Override + public final MemorySegmentIndexInput clone() { + final MemorySegmentIndexInput clone = buildSlice((String) null, 0L, this.length); + try { + clone.seek(getFilePointer()); + } catch (IOException ioe) { + throw new AssertionError(ioe); + } + + return clone; + } + + /** + * Creates a slice of this index input, with the given description, offset, and length. The slice + * is seeked to the beginning. + */ + @Override + public final MemorySegmentIndexInput slice(String sliceDescription, long offset, long length) { + if (offset < 0 || length < 0 || offset + length > this.length) { + throw new IllegalArgumentException( + "slice() " + + sliceDescription + + " out of bounds: offset=" + + offset + + ",length=" + + length + + ",fileLength=" + + this.length + + ": " + + this); + } + + return buildSlice(sliceDescription, offset, length); + } + + /** Builds the actual sliced IndexInput (may apply extra offset in subclasses). * */ + MemorySegmentIndexInput buildSlice(String sliceDescription, long offset, long length) { + ensureOpen(); + + final long sliceEnd = offset + length; + final int startIndex = (int) (offset >>> chunkSizePower); + final int endIndex = (int) (sliceEnd >>> chunkSizePower); + + // we always allocate one more slice, the last one may be a 0 byte one after truncating with + // asSlice(): + final MemorySegment slices[] = ArrayUtil.copyOfSubArray(segments, startIndex, endIndex + 1); + + // set the last segment's limit for the sliced view. + slices[slices.length - 1] = slices[slices.length - 1].asSlice(0L, sliceEnd & chunkSizeMask); + + offset = offset & chunkSizeMask; + + final String newResourceDescription = getFullSliceDescription(sliceDescription); + if (slices.length == 1) { + return new SingleSegmentImpl( + newResourceDescription, + null, // clones don't have an Arena, as they can't close) + slices[0].asSlice(offset, length), + length, + chunkSizePower); + } else { + return new MultiSegmentImpl( + newResourceDescription, + null, // clones don't have an Arena, as they can't close) + slices, + offset, + length, + chunkSizePower); + } + } + + @Override + public final void close() throws IOException { + if (curSegment == null) { + return; + } + + // make sure all accesses to this IndexInput instance throw NPE: + curSegment = null; + Arrays.fill(segments, null); + + // the master IndexInput has an Arena and is able + // to release all resources (unmap segments) - a + // side effect is that other threads still using clones + // will throw IllegalStateException + if (arena != null) { + arena.close(); + } + } + + /** Optimization of MemorySegmentIndexInput for when there is only one segment. */ + static final class SingleSegmentImpl extends MemorySegmentIndexInput { + + SingleSegmentImpl( + String resourceDescription, + Arena arena, + MemorySegment segment, + long length, + int chunkSizePower) { + super(resourceDescription, arena, new MemorySegment[] {segment}, length, chunkSizePower); + this.curSegmentIndex = 0; + } + + @Override + public void seek(long pos) throws IOException { + ensureOpen(); + try { + curPosition = Objects.checkIndex(pos, length + 1); + } catch (IndexOutOfBoundsException e) { + throw handlePositionalIOOBE(e, "seek", pos); + } + } + + @Override + public long getFilePointer() { + ensureOpen(); + return curPosition; + } + + @Override + public byte readByte(long pos) throws IOException { + try { + return curSegment.get(LAYOUT_BYTE, pos); + } catch (IndexOutOfBoundsException e) { + throw handlePositionalIOOBE(e, "read", pos); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + public short readShort(long pos) throws IOException { + try { + return curSegment.get(LAYOUT_LE_SHORT, pos); + } catch (IndexOutOfBoundsException e) { + throw handlePositionalIOOBE(e, "read", pos); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + public int readInt(long pos) throws IOException { + try { + return curSegment.get(LAYOUT_LE_INT, pos); + } catch (IndexOutOfBoundsException e) { + throw handlePositionalIOOBE(e, "read", pos); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + public long readLong(long pos) throws IOException { + try { + return curSegment.get(LAYOUT_LE_LONG, pos); + } catch (IndexOutOfBoundsException e) { + throw handlePositionalIOOBE(e, "read", pos); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + } + + /** This class adds offset support to MemorySegmentIndexInput, which is needed for slices. */ + static final class MultiSegmentImpl extends MemorySegmentIndexInput { + private final long offset; + + MultiSegmentImpl( + String resourceDescription, + Arena arena, + MemorySegment[] segments, + long offset, + long length, + int chunkSizePower) { + super(resourceDescription, arena, segments, length, chunkSizePower); + this.offset = offset; + try { + seek(0L); + } catch (IOException ioe) { + throw new AssertionError(ioe); + } + assert curSegment != null && curSegmentIndex >= 0; + } + + @Override + RuntimeException handlePositionalIOOBE(RuntimeException unused, String action, long pos) + throws IOException { + return super.handlePositionalIOOBE(unused, action, pos - offset); + } + + @Override + public void seek(long pos) throws IOException { + assert pos >= 0L : "negative position"; + super.seek(pos + offset); + } + + @Override + public long getFilePointer() { + return super.getFilePointer() - offset; + } + + @Override + public byte readByte(long pos) throws IOException { + return super.readByte(pos + offset); + } + + @Override + public short readShort(long pos) throws IOException { + return super.readShort(pos + offset); + } + + @Override + public int readInt(long pos) throws IOException { + return super.readInt(pos + offset); + } + + @Override + public long readLong(long pos) throws IOException { + return super.readLong(pos + offset); + } + + @Override + MemorySegmentIndexInput buildSlice(String sliceDescription, long ofs, long length) { + return super.buildSlice(sliceDescription, this.offset + ofs, length); + } + } +} diff --git a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInputProvider.java b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInputProvider.java new file mode 100644 index 000000000000..e994c2dddfff --- /dev/null +++ b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInputProvider.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.store; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.nio.channels.FileChannel; +import java.nio.channels.FileChannel.MapMode; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.logging.Logger; +import org.apache.lucene.util.Constants; +import org.apache.lucene.util.Unwrappable; + +@SuppressWarnings("preview") +final class MemorySegmentIndexInputProvider implements MMapDirectory.MMapIndexInputProvider { + + public MemorySegmentIndexInputProvider() { + var log = Logger.getLogger(getClass().getName()); + log.info( + "Using MemorySegmentIndexInput with Java 21; to disable start with -D" + + MMapDirectory.ENABLE_MEMORY_SEGMENTS_SYSPROP + + "=false"); + } + + @Override + public IndexInput openInput(Path path, IOContext context, int chunkSizePower, boolean preload) + throws IOException { + final String resourceDescription = "MemorySegmentIndexInput(path=\"" + path.toString() + "\")"; + + // Work around for JDK-8259028: we need to unwrap our test-only file system layers + path = Unwrappable.unwrapAll(path); + + boolean success = false; + final Arena arena = Arena.ofShared(); + try (var fc = FileChannel.open(path, StandardOpenOption.READ)) { + final long fileSize = fc.size(); + final IndexInput in = + MemorySegmentIndexInput.newInstance( + resourceDescription, + arena, + map(arena, resourceDescription, fc, chunkSizePower, preload, fileSize), + fileSize, + chunkSizePower); + success = true; + return in; + } finally { + if (success == false) { + arena.close(); + } + } + } + + @Override + public long getDefaultMaxChunkSize() { + return Constants.JRE_IS_64BIT ? (1L << 34) : (1L << 28); + } + + @Override + public boolean isUnmapSupported() { + return true; + } + + @Override + public String getUnmapNotSupportedReason() { + return null; + } + + private final MemorySegment[] map( + Arena arena, + String resourceDescription, + FileChannel fc, + int chunkSizePower, + boolean preload, + long length) + throws IOException { + if ((length >>> chunkSizePower) >= Integer.MAX_VALUE) + throw new IllegalArgumentException("File too big for chunk size: " + resourceDescription); + + final long chunkSize = 1L << chunkSizePower; + + // we always allocate one more segments, the last one may be a 0 byte one + final int nrSegments = (int) (length >>> chunkSizePower) + 1; + + final MemorySegment[] segments = new MemorySegment[nrSegments]; + + long startOffset = 0L; + for (int segNr = 0; segNr < nrSegments; segNr++) { + final long segSize = + (length > (startOffset + chunkSize)) ? chunkSize : (length - startOffset); + final MemorySegment segment; + try { + segment = fc.map(MapMode.READ_ONLY, startOffset, segSize, arena); + } catch (IOException ioe) { + throw convertMapFailedIOException(ioe, resourceDescription, segSize); + } + if (preload) { + segment.load(); + } + segments[segNr] = segment; + startOffset += segSize; + } + return segments; + } +} diff --git a/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java index dc638f6a529e..5597161a3a98 100644 --- a/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java +++ b/lucene/core/src/test/org/apache/lucene/store/TestMmapDirectory.java @@ -48,9 +48,9 @@ private static boolean isMemorySegmentImpl() { public void testCorrectImplementation() { final int runtimeVersion = Runtime.version().feature(); - if (runtimeVersion == 19 || runtimeVersion == 20) { + if (runtimeVersion >= 19 && runtimeVersion <= 21) { assertTrue( - "on Java 19 and Java 20 we should use MemorySegmentIndexInputProvider to create mmap IndexInputs", + "on Java 19, 20, and 21 we should use MemorySegmentIndexInputProvider to create mmap IndexInputs", isMemorySegmentImpl()); } else { assertSame(MappedByteBufferIndexInputProvider.class, MMapDirectory.PROVIDER.getClass()); diff --git a/lucene/distribution.tests/src/test/org/apache/lucene/distribution/TestModularLayer.java b/lucene/distribution.tests/src/test/org/apache/lucene/distribution/TestModularLayer.java index b3e55f277706..a5b8f6409e93 100644 --- a/lucene/distribution.tests/src/test/org/apache/lucene/distribution/TestModularLayer.java +++ b/lucene/distribution.tests/src/test/org/apache/lucene/distribution/TestModularLayer.java @@ -206,7 +206,7 @@ public void testMultiReleaseJar() { ClassLoader loader = layer.findLoader(coreModuleId); - final Set jarVersions = Set.of(19, 20); + final Set jarVersions = Set.of(19, 20, 21); for (var v : jarVersions) { Assertions.assertThat( loader.getResource(