forked from apache/parquet-java
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
PARQUET-4: Use LRU caching for footers in ParquetInputFormat.
Reopening https://github.com/Parquet/parquet-mr/pull/403 against the new Apache repository. Author: Matthieu Martin <[email protected]> Closes #2 from matt-martin/master and squashes the following commits: 99bb5a3 [Matthieu Martin] Minor javadoc and whitespace changes. Also added the FileStatusWrapper class to ParquetInputFormat to make sure that the debugging log statements print out meaningful paths. 250a398 [Matthieu Martin] Be less aggressive about checking whether the underlying file has been appended to/overwritten/deleted in order to minimize the number of namenode interactions. d946445 [Matthieu Martin] Add javadocs to parquet.hadoop.LruCache. Rename cache "entries" as cache "values" to avoid confusion with java.util.Map.Entry (which contains key value pairs whereas our old "entries" really only refer to the values). a363622 [Matthieu Martin] Use LRU caching for footers in ParquetInputFormat. Conflicts: parquet-hadoop/src/test/java/parquet/hadoop/TestInputFormat.java
- Loading branch information
1 parent
7119cb8
commit 1262243
Showing
5 changed files
with
517 additions
and
8 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
181 changes: 181 additions & 0 deletions
181
parquet-hadoop/src/main/java/parquet/hadoop/LruCache.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,181 @@ | ||
| package parquet.hadoop; | ||
|
|
||
| import parquet.Log; | ||
|
|
||
| import java.util.LinkedHashMap; | ||
| import java.util.Map; | ||
|
|
||
| /** | ||
| * A basic implementation of an LRU cache. Besides evicting the least recently | ||
| * used entries (either based on insertion or access order), this class also | ||
| * checks for "stale" entries as entries are inserted or retrieved (note | ||
| * "staleness" is defined by the entries themselves (see | ||
| * {@link parquet.hadoop.LruCache.Value}). | ||
| * | ||
| * @param <K> The key type. Acts as the key in a {@link java.util.LinkedHashMap} | ||
| * @param <V> The value type. Must extend {@link parquet.hadoop.LruCache.Value} | ||
| * so that the "staleness" of the value can be easily determined. | ||
| */ | ||
| final class LruCache<K, V extends LruCache.Value<K, V>> { | ||
| private static final Log LOG = Log.getLog(LruCache.class); | ||
|
|
||
| private static final float DEFAULT_LOAD_FACTOR = 0.75f; | ||
|
|
||
| private final LinkedHashMap<K, V> cacheMap; | ||
|
|
||
| /** | ||
| * Constructs an access-order based LRU cache with {@code maxSize} entries. | ||
| * @param maxSize The maximum number of entries to store in the cache. | ||
| */ | ||
| public LruCache(final int maxSize) { | ||
| this(maxSize, DEFAULT_LOAD_FACTOR, true); | ||
| } | ||
|
|
||
| /** | ||
| * Constructs an LRU cache. | ||
| * | ||
| * @param maxSize The maximum number of entries to store in the cache. | ||
| * @param loadFactor Used to determine the initial capacity. | ||
| * @param accessOrder the ordering mode - {@code true} for access-order, | ||
| * {@code false} for insertion-order | ||
| */ | ||
| public LruCache(final int maxSize, final float loadFactor, final boolean accessOrder) { | ||
| int initialCapacity = Math.round(maxSize / loadFactor); | ||
| cacheMap = | ||
| new LinkedHashMap<K, V>(initialCapacity, loadFactor, accessOrder) { | ||
| @Override | ||
| public boolean removeEldestEntry(final Map.Entry<K, V> eldest) { | ||
| boolean result = size() > maxSize; | ||
| if (result) { | ||
| if (Log.DEBUG) { | ||
| LOG.debug("Removing eldest entry in cache: " | ||
| + eldest.getKey()); | ||
| } | ||
| } | ||
| return result; | ||
| } | ||
| }; | ||
| } | ||
|
|
||
| /** | ||
| * Removes the mapping for the specified key from this cache if present. | ||
| * @param key key whose mapping is to be removed from the cache | ||
| * @return the previous value associated with key, or null if there was no | ||
| * mapping for key. | ||
| */ | ||
| public V remove(final K key) { | ||
| V oldValue = cacheMap.remove(key); | ||
| if (oldValue != null) { | ||
| if (Log.DEBUG) { | ||
| LOG.debug("Removed cache entry for '" + key + "'"); | ||
| } | ||
| } | ||
| return oldValue; | ||
| } | ||
|
|
||
| /** | ||
| * Associates the specified value with the specified key in this cache. The | ||
| * value is only inserted if it is not null and it is considered current. If | ||
| * the cache previously contained a mapping for the key, the old value is | ||
| * replaced only if the new value is "newer" than the old one. | ||
| * @param key key with which the specified value is to be associated | ||
| * @param newValue value to be associated with the specified key | ||
| */ | ||
| public void put(final K key, final V newValue) { | ||
| if (newValue == null || !newValue.isCurrent(key)) { | ||
| if (Log.WARN) { | ||
| LOG.warn("Ignoring new cache entry for '" + key + "' because it is " | ||
| + (newValue == null ? "null" : "not current")); | ||
| } | ||
| return; | ||
| } | ||
|
|
||
| V oldValue = cacheMap.get(key); | ||
| if (oldValue != null && oldValue.isNewerThan(newValue)) { | ||
| if (Log.WARN) { | ||
| LOG.warn("Ignoring new cache entry for '" + key + "' because " | ||
| + "existing cache entry is newer"); | ||
| } | ||
| return; | ||
| } | ||
|
|
||
| // no existing value or new value is newer than old value | ||
| oldValue = cacheMap.put(key, newValue); | ||
| if (Log.DEBUG) { | ||
| if (oldValue == null) { | ||
| LOG.debug("Added new cache entry for '" + key + "'"); | ||
| } else { | ||
| LOG.debug("Overwrote existing cache entry for '" + key + "'"); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Removes all of the mappings from this cache. The cache will be empty | ||
| * after this call returns. | ||
| */ | ||
| public void clear() { | ||
| cacheMap.clear(); | ||
| } | ||
|
|
||
| /** | ||
| * Returns the value to which the specified key is mapped, or null if 1) the | ||
| * value is not current or 2) this cache contains no mapping for the key. | ||
| * @param key the key whose associated value is to be returned | ||
| * @return the value to which the specified key is mapped, or null if 1) the | ||
| * value is not current or 2) this cache contains no mapping for the key | ||
| */ | ||
| public V getCurrentValue(final K key) { | ||
| V value = cacheMap.get(key); | ||
| if (Log.DEBUG) { | ||
| LOG.debug("Value for '" + key + "' " + (value == null ? "not " : "") | ||
| + "in cache"); | ||
| } | ||
| if (value != null && !value.isCurrent(key)) { | ||
| // value is not current; remove it and return null | ||
| remove(key); | ||
| return null; | ||
| } | ||
|
|
||
| return value; | ||
| } | ||
|
|
||
| /** | ||
| * Returns the number of key-value mappings in this cache. | ||
| * @return the number of key-value mappings in this cache. | ||
| */ | ||
| public int size() { | ||
| return cacheMap.size(); | ||
| } | ||
|
|
||
| /** | ||
| * {@link parquet.hadoop.LruCache} expects all values to follow this | ||
| * interface so the cache can determine 1) whether values are current (e.g. | ||
| * the referenced data has not been modified/updated in such a way that the | ||
| * value is no longer useful) and 2) whether a value is strictly "newer" | ||
| * than another value. | ||
| * | ||
| * @param <K> The key type. | ||
| * @param <V> Provides a bound for the {@link #isNewerThan(V)} method | ||
| */ | ||
| interface Value<K, V> { | ||
| /** | ||
| * Is the value still current (e.g. has the referenced data been | ||
| * modified/updated in such a way that the value is no longer useful) | ||
| * @param key the key associated with this value | ||
| * @return {@code true} the value is still current, {@code false} the value | ||
| * is no longer useful | ||
| */ | ||
| boolean isCurrent(K key); | ||
|
|
||
| /** | ||
| * Compares this value with the specified value to check for relative age. | ||
| * @param otherValue the value to be compared. | ||
| * @return {@code true} the value is strictly newer than the other value, | ||
| * {@code false} the value is older or just | ||
| * as new as the other value. | ||
| */ | ||
| boolean isNewerThan(V otherValue); | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.