diff --git a/gradle/generation/extract-jdk-apis.gradle b/gradle/generation/extract-jdk-apis.gradle index 3adde87da838..8a5fbd9e7975 100644 --- a/gradle/generation/extract-jdk-apis.gradle +++ b/gradle/generation/extract-jdk-apis.gradle @@ -20,7 +20,7 @@ def resources = scriptResources(buildscript) configure(project(":lucene:core")) { ext { apijars = layout.projectDirectory.dir("src/generated/jdk") - mrjarJavaVersions = [ 21 ] + mrjarJavaVersions = [ 24 ] } configurations { diff --git a/gradle/generation/extract-jdk-apis/ExtractJdkApis.java b/gradle/generation/extract-jdk-apis/ExtractJdkApis.java index c84c8f16996d..7b4f54e31396 100644 --- a/gradle/generation/extract-jdk-apis/ExtractJdkApis.java +++ b/gradle/generation/extract-jdk-apis/ExtractJdkApis.java @@ -47,14 +47,13 @@ public final class ExtractJdkApis { - private static final FileTime FIXED_FILEDATE = FileTime.from(Instant.parse("2022-01-01T00:00:00Z")); + private static final FileTime FIXED_FILEDATE = FileTime.from(Instant.parse("2025-04-29T00:00:00Z")); - private static final String PATTERN_PANAMA_FOREIGN = "java.base/{java/lang/foreign/*,java/nio/channels/FileChannel}"; private static final String PATTERN_VECTOR_INCUBATOR = "jdk.incubator.vector/jdk/incubator/vector/*"; private static final String PATTERN_VECTOR_VM_INTERNALS = "java.base/jdk/internal/vm/vector/VectorSupport{,$Vector,$VectorMask,$VectorPayload,$VectorShuffle}"; static final Map> CLASSFILE_PATTERNS = Map.of( - 21, List.of(PATTERN_PANAMA_FOREIGN, PATTERN_VECTOR_VM_INTERNALS, PATTERN_VECTOR_INCUBATOR) + 24, List.of(PATTERN_VECTOR_VM_INTERNALS, PATTERN_VECTOR_INCUBATOR) ); public static void main(String... args) throws IOException { diff --git a/gradle/java/core-mrjar.gradle b/gradle/java/core-mrjar.gradle index 89bfc2c6d8cb..8599fc39a106 100644 --- a/gradle/java/core-mrjar.gradle +++ b/gradle/java/core-mrjar.gradle @@ -15,7 +15,7 @@ * limitations under the License. */ -// Produce an MR-JAR with Java 19+ foreign and vector implementations +// Produce an MR-JAR for panama vector implementations configure(project(":lucene:core")) { plugins.withType(JavaPlugin) { @@ -52,7 +52,7 @@ configure(project(":lucene:core")) { mrjarJavaVersions.each { jdkVersion -> // the sourceSet which corresponds to the minimum/base Java version // will copy its output to root of JAR, all other sourceSets will go into MR-JAR folders: - boolean isBaseVersion = (jdkVersion.toString() == rootProject.minJavaVersion.toString()) + boolean isBaseVersion = (jdkVersion.toString() as int <= rootProject.minJavaVersion.toString() as int) into(isBaseVersion ? '' : "META-INF/versions/${jdkVersion}") { from sourceSets["main${jdkVersion}"].output } diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 0a64dce99f43..aa9fab13575f 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -52,6 +52,8 @@ Other * GITHUB#14229: Bump minimum required Java version to 25 +* GITHUB#14564: Remove abstractions from MMapDirectory as we need no MR-JAR anymore. (Uwe Schindler) + ======================= Lucene 10.3.0 ======================= API Changes diff --git a/lucene/core/src/generated/jdk/jdk21.apijar b/lucene/core/src/generated/jdk/jdk21.apijar deleted file mode 100644 index 8120f23b8b46..000000000000 Binary files a/lucene/core/src/generated/jdk/jdk21.apijar and /dev/null differ diff --git a/lucene/core/src/generated/jdk/jdk24.apijar b/lucene/core/src/generated/jdk/jdk24.apijar new file mode 100644 index 000000000000..3fa35f714859 Binary files /dev/null and b/lucene/core/src/generated/jdk/jdk24.apijar differ diff --git a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java index 35b83f77db2b..0fe3810bdc77 100644 --- a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java @@ -19,12 +19,16 @@ import static org.apache.lucene.index.IndexFileNames.CODEC_FILE_PATTERN; import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodType; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; import java.nio.channels.ClosedChannelException; // javadoc @link +import java.nio.channels.FileChannel; +import java.nio.channels.FileChannel.MapMode; import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.Locale; import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Future; import java.util.function.BiFunction; import java.util.function.BiPredicate; @@ -32,6 +36,7 @@ import java.util.logging.Logger; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.util.Constants; +import org.apache.lucene.util.Unwrappable; /** * File-based {@link Directory} implementation that uses mmap for reading, and {@link @@ -56,11 +61,10 @@ * *

This class will use the modern {@link java.lang.foreign.MemorySegment} API available since * Java 21 which allows to safely unmap previously mmapped files after closing the {@link - * IndexInput}s. There is no need to enable the "preview feature" of your Java version; it works out - * of box with some compilation tricks. For more information about the foreign memory API read - * documentation of the {@link java.lang.foreign} package. + * IndexInput}s. For more information about the foreign memory API read documentation of the {@link + * java.lang.foreign} package. * - *

On some platforms like Linux and MacOS X, this class will invoke the syscall {@code madvise()} + *

On some platforms like Linux and macOS, this class will invoke the syscall {@code madvise()} * to advise how OS kernel should handle paging after opening a file. For this to work, Java code * must be able to call native code. If this is not allowed, a warning is logged. To enable native * access for Lucene in a modularized application, pass {@code @@ -130,7 +134,7 @@ public class MMapDirectory extends FSDirectory { return Optional.of(groupKey); }; - private BiFunction> readAdvice = + private BiFunction> readAdviceOverride = (_, _) -> Optional.empty(); private BiPredicate preload = NO_FILES; @@ -142,15 +146,19 @@ public class MMapDirectory extends FSDirectory { *

  • 256 MiBytes for 32 bit JVMs * */ - public static final long DEFAULT_MAX_CHUNK_SIZE; + public static final long DEFAULT_MAX_CHUNK_SIZE = + Constants.JRE_IS_64BIT ? (1L << 34) : (1L << 28); - /** A provider specific context object or null, that will be passed to openInput. */ - final Object attachment = PROVIDER.attachment(); + private static final Optional NATIVE_ACCESS = NativeAccess.getImplementation(); + private static final int SHARED_ARENA_PERMITS = + checkMaxPermits(getSharedArenaMaxPermitsSysprop()); private Function> groupingFunction = GROUP_BY_SEGMENT; final int chunkSizePower; + final ConcurrentHashMap arenas = new ConcurrentHashMap<>(); + /** * Create a new MMapDirectory for the named location. The directory is created at the named * location if it does not yet exist. @@ -195,9 +203,9 @@ public MMapDirectory(Path path, long maxChunkSize) throws IOException { * files cannot be mapped. Using a lower chunk size makes the directory implementation a little * bit slower (as the correct chunk may be resolved on lots of seeks) but the chance is higher * that mmap does not fail. On 64 bit Java platforms, this parameter should always be large (like - * 1 GiBytes, or even larger with recent Java versions), as the address space is big enough. If it - * is larger, fragmentation of address space increases, but number of file handles and mappings is - * lower for huge installations with many open indexes. + * 16 GiBytes), as the address space is big enough. If it is larger, fragmentation of address + * space increases, but number of file handles and mappings is lower for huge installations with + * many open indexes. * *

    Please note: The chunk size is always rounded down to a power of 2. * @@ -242,7 +250,7 @@ public void setPreload(BiPredicate preload) { */ public void setReadAdviceOverride( BiFunction> toReadAdvice) { - this.readAdvice = toReadAdvice; + this.readAdviceOverride = toReadAdvice; } /** @@ -275,82 +283,175 @@ public IndexInput openInput(String name, IOContext context) throws IOException { ensureOpen(); ensureCanRead(name); Path path = directory.resolve(name); - return PROVIDER.openInput( - path, - chunkSizePower, - readAdvice + final String resourceDescription = "MemorySegmentIndexInput(path=\"" + path.toString() + "\")"; + + // Work around for JDK-8259028: we need to unwrap our test-only file system layers + path = Unwrappable.unwrapAll(path); + + boolean success = false; + final boolean confined = context == IOContext.READONCE; + final ReadAdvice readAdvice = + readAdviceOverride .apply(name, context) - .orElseGet(() -> context.readAdvice().orElse(Constants.DEFAULT_READADVICE)), - context == IOContext.READONCE, - preload.test(name, context), - groupingFunction.apply(name), - attachment); + .orElseGet(() -> context.readAdvice().orElse(Constants.DEFAULT_READADVICE)); + final Arena arena = confined ? Arena.ofConfined() : getSharedArena(name, arenas); + try (var fc = FileChannel.open(path, StandardOpenOption.READ)) { + final long fileSize = fc.size(); + final IndexInput in = + MemorySegmentIndexInput.newInstance( + resourceDescription, + arena, + map( + arena, + resourceDescription, + fc, + readAdvice, + chunkSizePower, + preload.test(name, context), + fileSize), + fileSize, + chunkSizePower, + confined); + success = true; + return in; + } finally { + if (success == false) { + arena.close(); + } + } } - // visible for tests: - static final MMapIndexInputProvider PROVIDER; + private final MemorySegment[] map( + Arena arena, + String resourceDescription, + FileChannel fc, + ReadAdvice readAdvice, + int chunkSizePower, + boolean preload, + long length) + throws IOException { + if ((length >>> chunkSizePower) >= Integer.MAX_VALUE) + throw new IllegalArgumentException("File too big for chunk size: " + resourceDescription); + + final long chunkSize = 1L << chunkSizePower; + + // we always allocate one more segments, the last one may be a 0 byte one + final int nrSegments = (int) (length >>> chunkSizePower) + 1; + + final MemorySegment[] segments = new MemorySegment[nrSegments]; + + long startOffset = 0L; + for (int segNr = 0; segNr < nrSegments; segNr++) { + final long segSize = + (length > (startOffset + chunkSize)) ? chunkSize : (length - startOffset); + final MemorySegment segment; + try { + segment = fc.map(MapMode.READ_ONLY, startOffset, segSize, arena); + } catch (IOException ioe) { + throw convertMapFailedIOException(ioe, resourceDescription, segSize); + } + // if preload apply it without madvise. + // skip madvise if the address of our segment is not page-aligned (small segments due to + // internal FileChannel logic) + if (preload) { + segment.load(); + } else if (readAdvice != ReadAdvice.NORMAL + && NATIVE_ACCESS.filter(na -> segment.address() % na.getPageSize() == 0).isPresent()) { + // No need to madvise with ReadAdvice.NORMAL since it is the OS' default read advice. + NATIVE_ACCESS.get().madvise(segment, readAdvice); + } + segments[segNr] = segment; + startOffset += segSize; + } + return segments; + } - interface MMapIndexInputProvider { - IndexInput openInput( - Path path, - int chunkSizePower, - ReadAdvice readAdvice, - boolean readOnce, - boolean preload, - Optional group, - A attachment) - throws IOException; + /** + * Gets an arena for the given filename, potentially aggregating files from the same segment into + * a single ref counted shared arena. A ref counted shared arena, if created will be added to the + * given arenas map. + */ + private Arena getSharedArena( + String name, ConcurrentHashMap arenas) { + final var group = groupingFunction.apply(name); - long getDefaultMaxChunkSize(); + if (group.isEmpty()) { + return Arena.ofShared(); + } - boolean supportsMadvise(); + String key = group.get(); + var refCountedArena = + arenas.computeIfAbsent( + key, s -> new RefCountedSharedArena(s, () -> arenas.remove(s), SHARED_ARENA_PERMITS)); + if (refCountedArena.acquire()) { + return refCountedArena; + } else { + return arenas.compute( + key, + (s, v) -> { + if (v != null && v.acquire()) { + return v; + } else { + v = new RefCountedSharedArena(s, () -> arenas.remove(s), SHARED_ARENA_PERMITS); + v.acquire(); // guaranteed to succeed + return v; + } + }); + } + } - /** An optional attachment of the provider, that will be passed to openInput. */ - default A attachment() { - return null; + private static IOException convertMapFailedIOException( + IOException ioe, String resourceDescription, long bufSize) { + final String originalMessage; + final Throwable originalCause; + if (ioe.getCause() instanceof OutOfMemoryError) { + // nested OOM confuses users, because it's "incorrect", just print a plain message: + originalMessage = "Map failed"; + originalCause = null; + } else { + originalMessage = ioe.getMessage(); + originalCause = ioe.getCause(); + } + final String moreInfo; + if (!Constants.JRE_IS_64BIT) { + moreInfo = + "MMapDirectory should only be used on 64bit platforms, because the address space on 32bit operating systems is too small. "; + } else if (Constants.WINDOWS) { + moreInfo = + "Windows is unfortunately very limited on virtual address space. If your index size is several hundred Gigabytes, consider changing to Linux. "; + } else if (Constants.LINUX) { + moreInfo = + "Please review 'ulimit -v', 'ulimit -m' (both should return 'unlimited'), and 'sysctl vm.max_map_count'. "; + } else { + moreInfo = "Please review 'ulimit -v', 'ulimit -m' (both should return 'unlimited'). "; } + final IOException newIoe = + new IOException( + String.format( + Locale.ENGLISH, + "%s: %s [this may be caused by lack of enough unfragmented virtual address space " + + "or too restrictive virtual memory limits enforced by the operating system, " + + "preventing us to map a chunk of %d bytes. %sMore information: " + + "https://blog.thetaphi.de/2012/07/use-lucenes-mmapdirectory-on-64bit.html]", + originalMessage, + resourceDescription, + bufSize, + moreInfo), + originalCause); + newIoe.setStackTrace(ioe.getStackTrace()); + return newIoe; + } - default IOException convertMapFailedIOException( - IOException ioe, String resourceDescription, long bufSize) { - final String originalMessage; - final Throwable originalCause; - if (ioe.getCause() instanceof OutOfMemoryError) { - // nested OOM confuses users, because it's "incorrect", just print a plain message: - originalMessage = "Map failed"; - originalCause = null; - } else { - originalMessage = ioe.getMessage(); - originalCause = ioe.getCause(); - } - final String moreInfo; - if (!Constants.JRE_IS_64BIT) { - moreInfo = - "MMapDirectory should only be used on 64bit platforms, because the address space on 32bit operating systems is too small. "; - } else if (Constants.WINDOWS) { - moreInfo = - "Windows is unfortunately very limited on virtual address space. If your index size is several hundred Gigabytes, consider changing to Linux. "; - } else if (Constants.LINUX) { - moreInfo = - "Please review 'ulimit -v', 'ulimit -m' (both should return 'unlimited'), and 'sysctl vm.max_map_count'. "; - } else { - moreInfo = "Please review 'ulimit -v', 'ulimit -m' (both should return 'unlimited'). "; - } - final IOException newIoe = - new IOException( - String.format( - Locale.ENGLISH, - "%s: %s [this may be caused by lack of enough unfragmented virtual address space " - + "or too restrictive virtual memory limits enforced by the operating system, " - + "preventing us to map a chunk of %d bytes. %sMore information: " - + "https://blog.thetaphi.de/2012/07/use-lucenes-mmapdirectory-on-64bit.html]", - originalMessage, - resourceDescription, - bufSize, - moreInfo), - originalCause); - newIoe.setStackTrace(ioe.getStackTrace()); - return newIoe; + private static int checkMaxPermits(int maxPermits) { + if (RefCountedSharedArena.validMaxPermits(maxPermits)) { + return maxPermits; } + Logger.getLogger(MMapDirectory.class.getName()) + .warning( + "Invalid value for sysprop " + + MMapDirectory.SHARED_ARENA_MAX_PERMITS_SYSPROP + + ", must be positive and <= 0x07FF. The default value will be used."); + return RefCountedSharedArena.DEFAULT_MAX_PERMITS; } private static int getSharedArenaMaxPermitsSysprop() { @@ -370,41 +471,11 @@ private static int getSharedArenaMaxPermitsSysprop() { return ret; } - private static MMapIndexInputProvider lookupProvider() { - final var maxPermits = getSharedArenaMaxPermitsSysprop(); - final var lookup = MethodHandles.lookup(); - try { - final var cls = lookup.findClass("org.apache.lucene.store.MemorySegmentIndexInputProvider"); - // we use method handles, so we do not need to deal with setAccessible as we have private - // access through the lookup: - final var constr = lookup.findConstructor(cls, MethodType.methodType(void.class, int.class)); - try { - @SuppressWarnings("unchecked") - var res = (MMapIndexInputProvider) constr.invoke(maxPermits); - return res; - } catch (RuntimeException | Error e) { - throw e; - } catch (Throwable th) { - throw new AssertionError(th); - } - } catch (NoSuchMethodException | IllegalAccessException e) { - throw new LinkageError( - "MemorySegmentIndexInputProvider is missing correctly typed constructor", e); - } catch (ClassNotFoundException cnfe) { - throw new LinkageError("MemorySegmentIndexInputProvider is missing in Lucene JAR file", cnfe); - } - } - /** * Returns true, if MMapDirectory uses the platform's {@code madvise()} syscall to advise how OS * kernel should handle paging after opening a file. */ public static boolean supportsMadvise() { - return PROVIDER.supportsMadvise(); - } - - static { - PROVIDER = lookupProvider(); - DEFAULT_MAX_CHUNK_SIZE = PROVIDER.getDefaultMaxChunkSize(); + return NATIVE_ACCESS.isPresent(); } } diff --git a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentAccessInput.java b/lucene/core/src/java/org/apache/lucene/store/MemorySegmentAccessInput.java similarity index 100% rename from lucene/core/src/java21/org/apache/lucene/store/MemorySegmentAccessInput.java rename to lucene/core/src/java/org/apache/lucene/store/MemorySegmentAccessInput.java diff --git a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java b/lucene/core/src/java/org/apache/lucene/store/MemorySegmentIndexInput.java similarity index 97% rename from lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java rename to lucene/core/src/java/org/apache/lucene/store/MemorySegmentIndexInput.java index 5404e1aafc7d..64bed7cc9f1c 100644 --- a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java +++ b/lucene/core/src/java/org/apache/lucene/store/MemorySegmentIndexInput.java @@ -16,6 +16,8 @@ */ package org.apache.lucene.store; +import static java.util.function.Predicate.not; + import java.io.EOFException; import java.io.IOException; import java.lang.foreign.Arena; @@ -37,9 +39,7 @@ *

    For efficiency, this class requires that the segment size are a power-of-two ( * chunkSizePower). */ -@SuppressWarnings("preview") -abstract class MemorySegmentIndexInput extends IndexInput - implements RandomAccessInput, MemorySegmentAccessInput { +abstract class MemorySegmentIndexInput extends IndexInput implements MemorySegmentAccessInput { static final ValueLayout.OfByte LAYOUT_BYTE = ValueLayout.JAVA_BYTE; static final ValueLayout.OfShort LAYOUT_LE_SHORT = ValueLayout.JAVA_SHORT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN); @@ -129,16 +129,9 @@ AlreadyClosedException alreadyClosed(RuntimeException e) { } // in Java 22 or later we can check the isAlive status of all segments // (see https://bugs.openjdk.org/browse/JDK-8310644): - if (Arrays.stream(segments).allMatch(s -> s.scope().isAlive()) == false) { + if (Arrays.stream(segments).anyMatch(not(s -> s.scope().isAlive()))) { return new AlreadyClosedException("Already closed: " + this); } - // fallback for Java 21: ISE can be thrown by MemorySegment and contains "closed" in message: - if (e instanceof IllegalStateException - && e.getMessage() != null - && e.getMessage().contains("closed")) { - // the check is on message only, so preserve original cause for debugging: - return new AlreadyClosedException("Already closed: " + this, e); - } // otherwise rethrow unmodified NPE/ISE (as it possibly a bug with passing a null parameter to // the IndexInput method): throw e; diff --git a/lucene/core/src/java21/org/apache/lucene/store/NativeAccess.java b/lucene/core/src/java/org/apache/lucene/store/NativeAccess.java similarity index 98% rename from lucene/core/src/java21/org/apache/lucene/store/NativeAccess.java rename to lucene/core/src/java/org/apache/lucene/store/NativeAccess.java index affc0e2ac719..838a40f6aa6c 100644 --- a/lucene/core/src/java21/org/apache/lucene/store/NativeAccess.java +++ b/lucene/core/src/java/org/apache/lucene/store/NativeAccess.java @@ -21,7 +21,6 @@ import java.util.Optional; import org.apache.lucene.util.Constants; -@SuppressWarnings("preview") abstract class NativeAccess { /** Invoke the {@code madvise} call for the given {@link MemorySegment}. */ diff --git a/lucene/core/src/java21/org/apache/lucene/store/PosixNativeAccess.java b/lucene/core/src/java/org/apache/lucene/store/PosixNativeAccess.java similarity index 99% rename from lucene/core/src/java21/org/apache/lucene/store/PosixNativeAccess.java rename to lucene/core/src/java/org/apache/lucene/store/PosixNativeAccess.java index 05eb61571188..477383fda32f 100644 --- a/lucene/core/src/java21/org/apache/lucene/store/PosixNativeAccess.java +++ b/lucene/core/src/java/org/apache/lucene/store/PosixNativeAccess.java @@ -27,7 +27,6 @@ import java.util.Optional; import java.util.logging.Logger; -@SuppressWarnings("preview") final class PosixNativeAccess extends NativeAccess { private static final Logger LOG = Logger.getLogger(PosixNativeAccess.class.getName()); diff --git a/lucene/core/src/java21/org/apache/lucene/store/RefCountedSharedArena.java b/lucene/core/src/java/org/apache/lucene/store/RefCountedSharedArena.java similarity index 99% rename from lucene/core/src/java21/org/apache/lucene/store/RefCountedSharedArena.java rename to lucene/core/src/java/org/apache/lucene/store/RefCountedSharedArena.java index e89de280a3cc..2d6dd9b737af 100644 --- a/lucene/core/src/java21/org/apache/lucene/store/RefCountedSharedArena.java +++ b/lucene/core/src/java/org/apache/lucene/store/RefCountedSharedArena.java @@ -38,7 +38,6 @@ * permitted and {@link #acquire()} returns false. This is independent of the actual number of the * ref count. */ -@SuppressWarnings("preview") final class RefCountedSharedArena implements Arena { // default maximum permits diff --git a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInputProvider.java b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInputProvider.java deleted file mode 100644 index f740b1a372b7..000000000000 --- a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInputProvider.java +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.lucene.store; - -import java.io.IOException; -import java.lang.foreign.Arena; -import java.lang.foreign.MemorySegment; -import java.nio.channels.FileChannel; -import java.nio.channels.FileChannel.MapMode; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.logging.Logger; -import org.apache.lucene.util.Constants; -import org.apache.lucene.util.Unwrappable; - -@SuppressWarnings("preview") -final class MemorySegmentIndexInputProvider - implements MMapDirectory.MMapIndexInputProvider< - ConcurrentHashMap> { - - private final Optional nativeAccess; - private final int sharedArenaMaxPermits; - - MemorySegmentIndexInputProvider(int maxPermits) { - this.nativeAccess = NativeAccess.getImplementation(); - this.sharedArenaMaxPermits = checkMaxPermits(maxPermits); - } - - @Override - public IndexInput openInput( - Path path, - int chunkSizePower, - ReadAdvice readAdvice, - boolean confined, - boolean preload, - Optional group, - ConcurrentHashMap arenas) - throws IOException { - final String resourceDescription = "MemorySegmentIndexInput(path=\"" + path.toString() + "\")"; - - // Work around for JDK-8259028: we need to unwrap our test-only file system layers - path = Unwrappable.unwrapAll(path); - - boolean success = false; - final Arena arena = confined ? Arena.ofConfined() : getSharedArena(group, arenas); - try (var fc = FileChannel.open(path, StandardOpenOption.READ)) { - final long fileSize = fc.size(); - final IndexInput in = - MemorySegmentIndexInput.newInstance( - resourceDescription, - arena, - map(arena, resourceDescription, fc, readAdvice, chunkSizePower, preload, fileSize), - fileSize, - chunkSizePower, - confined); - success = true; - return in; - } finally { - if (success == false) { - arena.close(); - } - } - } - - @Override - public long getDefaultMaxChunkSize() { - return Constants.JRE_IS_64BIT ? (1L << 34) : (1L << 28); - } - - @Override - public boolean supportsMadvise() { - return nativeAccess.isPresent(); - } - - private final MemorySegment[] map( - Arena arena, - String resourceDescription, - FileChannel fc, - ReadAdvice readAdvice, - int chunkSizePower, - boolean preload, - long length) - throws IOException { - if ((length >>> chunkSizePower) >= Integer.MAX_VALUE) - throw new IllegalArgumentException("File too big for chunk size: " + resourceDescription); - - final long chunkSize = 1L << chunkSizePower; - - // we always allocate one more segments, the last one may be a 0 byte one - final int nrSegments = (int) (length >>> chunkSizePower) + 1; - - final MemorySegment[] segments = new MemorySegment[nrSegments]; - - long startOffset = 0L; - for (int segNr = 0; segNr < nrSegments; segNr++) { - final long segSize = - (length > (startOffset + chunkSize)) ? chunkSize : (length - startOffset); - final MemorySegment segment; - try { - segment = fc.map(MapMode.READ_ONLY, startOffset, segSize, arena); - } catch (IOException ioe) { - throw convertMapFailedIOException(ioe, resourceDescription, segSize); - } - // if preload apply it without madvise. - // skip madvise if the address of our segment is not page-aligned (small segments due to - // internal FileChannel logic) - if (preload) { - segment.load(); - } else if (readAdvice != ReadAdvice.NORMAL - && nativeAccess.filter(na -> segment.address() % na.getPageSize() == 0).isPresent()) { - // No need to madvise with ReadAdvice.NORMAL since it is the OS' default read advice. - nativeAccess.get().madvise(segment, readAdvice); - } - segments[segNr] = segment; - startOffset += segSize; - } - return segments; - } - - @Override - public ConcurrentHashMap attachment() { - return new ConcurrentHashMap<>(); - } - - private static int checkMaxPermits(int maxPermits) { - if (RefCountedSharedArena.validMaxPermits(maxPermits)) { - return maxPermits; - } - Logger.getLogger(MemorySegmentIndexInputProvider.class.getName()) - .warning( - "Invalid value for sysprop " - + MMapDirectory.SHARED_ARENA_MAX_PERMITS_SYSPROP - + ", must be positive and <= 0x07FF. The default value will be used."); - return RefCountedSharedArena.DEFAULT_MAX_PERMITS; - } - - /** - * Gets an arena for the given group, potentially aggregating files from the same segment into a - * single ref counted shared arena. A ref counted shared arena, if created will be added to the - * given arenas map. - */ - private Arena getSharedArena( - Optional group, ConcurrentHashMap arenas) { - if (group.isEmpty()) { - return Arena.ofShared(); - } - - String key = group.get(); - var refCountedArena = - arenas.computeIfAbsent( - key, s -> new RefCountedSharedArena(s, () -> arenas.remove(s), sharedArenaMaxPermits)); - if (refCountedArena.acquire()) { - return refCountedArena; - } else { - return arenas.compute( - key, - (s, v) -> { - if (v != null && v.acquire()) { - return v; - } else { - v = new RefCountedSharedArena(s, () -> arenas.remove(s), sharedArenaMaxPermits); - v.acquire(); // guaranteed to succeed - return v; - } - }); - } - } -} diff --git a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorer.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorer.java similarity index 100% rename from lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorer.java rename to lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorer.java diff --git a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorerSupplier.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorerSupplier.java similarity index 100% rename from lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorerSupplier.java rename to lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentByteVectorScorerSupplier.java diff --git a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.java similarity index 100% rename from lucene/core/src/java21/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.java rename to lucene/core/src/java24/org/apache/lucene/internal/vectorization/Lucene99MemorySegmentFlatVectorsScorer.java diff --git a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/MemorySegmentPostingDecodingUtil.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/MemorySegmentPostingDecodingUtil.java similarity index 100% rename from lucene/core/src/java21/org/apache/lucene/internal/vectorization/MemorySegmentPostingDecodingUtil.java rename to lucene/core/src/java24/org/apache/lucene/internal/vectorization/MemorySegmentPostingDecodingUtil.java diff --git a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorConstants.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorConstants.java similarity index 100% rename from lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorConstants.java rename to lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorConstants.java diff --git a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java similarity index 100% rename from lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java rename to lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java diff --git a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.java b/lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.java similarity index 100% rename from lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.java rename to lucene/core/src/java24/org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.java diff --git a/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java index 971847a2ec06..a91ae5414ba8 100644 --- a/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java +++ b/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java @@ -25,7 +25,6 @@ import java.util.List; import java.util.Random; import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; @@ -227,10 +226,7 @@ public void testArenas() throws Exception { } } - if (!(dir.attachment instanceof ConcurrentHashMap map)) { - throw new AssertionError("unexpected attachment: " + dir.attachment); - } - assertEquals(0, map.size()); + assertEquals(0, dir.arenas.size()); } } @@ -279,10 +275,7 @@ public void testArenasManySegmentFiles() throws Exception { closeable.close(); } - if (!(dir.attachment instanceof ConcurrentHashMap map)) { - throw new AssertionError("unexpected attachment: " + dir.attachment); - } - assertEquals(0, map.size()); + assertEquals(0, dir.arenas.size()); } } diff --git a/lucene/distribution.tests/src/test/org/apache/lucene/distribution/TestModularLayer.java b/lucene/distribution.tests/src/test/org/apache/lucene/distribution/TestModularLayer.java index 9c412664053c..f19819941db3 100644 --- a/lucene/distribution.tests/src/test/org/apache/lucene/distribution/TestModularLayer.java +++ b/lucene/distribution.tests/src/test/org/apache/lucene/distribution/TestModularLayer.java @@ -207,10 +207,10 @@ public void testMultiReleaseJar() { ClassLoader loader = layer.findLoader(coreModuleId); - final Set mrJarVersions = Set.of(21); - final Integer baseVersion = 21; + final Set mrJarVersions = Set.of(23); + final Integer baseVersion = 23; - // the Java 21 PanamaVectorizationProvider must always be in main section of JAR file: + // the Java 23 PanamaVectorizationProvider must always be in main section of JAR file: final String panamaClassName = "org/apache/lucene/internal/vectorization/PanamaVectorizationProvider.class"; Assertions.assertThat(loader.getResource(panamaClassName)).isNotNull(); @@ -224,11 +224,6 @@ public void testMultiReleaseJar() { loader.getResource("META-INF/versions/" + v + panamaClassName)) .isNotNull(); }); - - // you must be always able to load MemorySegmentIndexInput: - Assertions.assertThat( - loader.loadClass("org.apache.lucene.store.MemorySegmentIndexInput")) - .isNotNull(); }); }