Skip to content
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion java/lance-jni/src/file_reader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ use lance::io::ObjectStore;
use lance_core::cache::LanceCache;
use lance_core::datatypes::Schema;
use lance_encoding::decoder::{DecoderPlugins, FilterExpression};
use lance_file::v2::reader::{FileReader, FileReaderOptions, ReaderProjection};
use lance_file::reader::{FileReader, FileReaderOptions, ReaderProjection};

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a breaking rust-level change. There are potentially a few (maybe not very many) Rust users who use FileReader directly. I think we can still proceed with the change (we haven't ever claimed the Rust API is stable) but we should maybe mark the PR as breaking.

use lance_io::object_store::{ObjectStoreParams, ObjectStoreRegistry};
use lance_io::{
scheduler::{ScanScheduler, SchedulerConfig},
Expand Down
2 changes: 1 addition & 1 deletion java/lance-jni/src/file_writer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ use jni::{
};
use lance::io::ObjectStore;
use lance_file::{
v2::writer::{FileWriter, FileWriterOptions},
version::LanceFileVersion,
writer::{FileWriter, FileWriterOptions},
};
use lance_io::object_store::{ObjectStoreParams, ObjectStoreRegistry};

Expand Down
2 changes: 1 addition & 1 deletion python/src/dataset.rs
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ use lance_arrow::as_fixed_size_list_array;
use lance_core::Error;
use lance_datafusion::utils::reader_to_stream;
use lance_encoding::decoder::DecoderConfig;
use lance_file::v2::reader::FileReaderOptions;
use lance_file::reader::FileReaderOptions;
use lance_index::scalar::inverted::query::{
BooleanQuery, BoostQuery, FtsQuery, MatchQuery, MultiMatchQuery, Operator, PhraseQuery,
};
Expand Down
17 changes: 6 additions & 11 deletions python/src/file.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,17 +21,12 @@ use futures::stream::StreamExt;
use lance::io::{ObjectStore, RecordBatchStream};
use lance_core::cache::LanceCache;
use lance_encoding::decoder::{DecoderPlugins, FilterExpression};
use lance_file::v2::reader::ReaderProjection;
use lance_file::v2::LanceEncodingsIo;
use lance_file::{
v2::{
reader::{
BufferDescriptor, CachedFileMetadata, FileReader, FileReaderOptions, FileStatistics,
},
writer::{FileWriter, FileWriterOptions},
},
version::LanceFileVersion,
use lance_file::reader::{
BufferDescriptor, CachedFileMetadata, FileReader, FileReaderOptions, FileStatistics,
ReaderProjection,
};
use lance_file::writer::{FileWriter, FileWriterOptions};
use lance_file::{version::LanceFileVersion, LanceEncodingsIo};
use lance_io::object_store::ObjectStoreParams;
use lance_io::{
scheduler::{ScanScheduler, SchedulerConfig},
Expand Down Expand Up @@ -91,7 +86,7 @@ impl LancePageMetadata {
.collect();
Self {
buffers,
encoding: lance_file::v2::reader::describe_encoding(inner),
encoding: lance_file::reader::describe_encoding(inner),
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions python/src/utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ use arrow_schema::DataType;
use lance::datatypes::Schema;
use lance::Result;
use lance_arrow::FixedSizeListArrayExt;
use lance_file::writer::FileWriter;
use lance_file::previous::writer::FileWriter as PreviousFileWriter;
use lance_index::scalar::IndexWriter;
use lance_index::vector::hnsw::{builder::HnswBuildParams, HNSW};
use lance_index::vector::kmeans::{
Expand Down Expand Up @@ -223,7 +223,7 @@ impl Hnsw {
let mut writer = rt()
.block_on(
Some(py),
FileWriter::<ManifestDescribing>::try_new(
PreviousFileWriter::<ManifestDescribing>::try_new(
&object_store,
&path,
Schema::try_from(HNSW::schema().as_ref())
Expand Down
8 changes: 3 additions & 5 deletions rust/lance-file/benches/reader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,10 @@ use futures::{FutureExt, StreamExt};
use lance_datagen::ArrayGeneratorExt;
use lance_encoding::decoder::{DecoderPlugins, FilterExpression};
use lance_file::{
v2::{
reader::{FileReader, FileReaderOptions},
testing::test_cache,
writer::{FileWriter, FileWriterOptions},
},
reader::{FileReader, FileReaderOptions},
testing::test_cache,
version::LanceFileVersion,
writer::{FileWriter, FileWriterOptions},
};
use lance_io::{
object_store::ObjectStore,
Expand Down
2 changes: 0 additions & 2 deletions rust/lance-file/src/format.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,6 @@ pub mod pbfile {
include!(concat!(env!("OUT_DIR"), "/lance.file.v2.rs"));
}

pub mod metadata;

/// These version/magic values are written at the end of Lance files (e.g. versions/1.version)
pub const MAJOR_VERSION: i16 = 0;
pub const MINOR_VERSION: i16 = 2;
Expand Down
File renamed without changes.
7 changes: 5 additions & 2 deletions rust/lance-file/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,14 @@

pub mod datatypes;
pub mod format;
pub mod page_table;
pub(crate) mod io;
pub mod previous;
pub mod reader;
pub mod v2;
pub mod testing;
pub mod writer;

pub use io::LanceEncodingsIo;

use format::MAGIC;
pub use lance_encoding::version;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ use lance_core::datatypes::Schema;
use lance_core::{Error, Result};
use lance_io::traits::ProtoStruct;
use snafu::location;

/// Data File Metadata
#[derive(Debug, Default, DeepSizeOf, PartialEq)]
pub struct Metadata {
Expand Down
4 changes: 4 additions & 0 deletions rust/lance-file/src/previous/format/mod.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The Lance Authors

pub mod metadata;
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The Lance Authors

pub(crate) mod io;
//! Legacy Lance file v1 implementation kept for backwards compatibility.

pub mod format;
pub mod page_table;
pub mod reader;
pub mod testing;
pub mod writer;

pub use io::LanceEncodingsIo;
Loading
Loading