Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion java/lance-jni/src/file_reader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ use lance::io::ObjectStore;
use lance_core::cache::LanceCache;
use lance_core::datatypes::Schema;
use lance_encoding::decoder::{DecoderPlugins, FilterExpression};
use lance_file::v2::reader::{FileReader, FileReaderOptions, ReaderProjection};
use lance_file::reader::{FileReader, FileReaderOptions, ReaderProjection};
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a breaking rust-level change. There are potentially a few (maybe not very many) Rust users who use FileReader directly. I think we can still proceed with the change (we haven't ever claimed the Rust API is stable) but we should maybe mark the PR as breaking.

use lance_io::object_store::{ObjectStoreParams, ObjectStoreRegistry};
use lance_io::{
scheduler::{ScanScheduler, SchedulerConfig},
Expand Down
2 changes: 1 addition & 1 deletion java/lance-jni/src/file_writer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ use jni::{
};
use lance::io::ObjectStore;
use lance_file::{
v2::writer::{FileWriter, FileWriterOptions},
version::LanceFileVersion,
writer::{FileWriter, FileWriterOptions},
};
use lance_io::object_store::{ObjectStoreParams, ObjectStoreRegistry};

Expand Down
2 changes: 1 addition & 1 deletion python/src/dataset.rs
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ use lance_arrow::as_fixed_size_list_array;
use lance_core::Error;
use lance_datafusion::utils::reader_to_stream;
use lance_encoding::decoder::DecoderConfig;
use lance_file::v2::reader::FileReaderOptions;
use lance_file::reader::FileReaderOptions;
use lance_index::scalar::inverted::query::{
BooleanQuery, BoostQuery, FtsQuery, MatchQuery, MultiMatchQuery, Operator, PhraseQuery,
};
Expand Down
17 changes: 6 additions & 11 deletions python/src/file.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,17 +21,12 @@ use futures::stream::StreamExt;
use lance::io::{ObjectStore, RecordBatchStream};
use lance_core::cache::LanceCache;
use lance_encoding::decoder::{DecoderPlugins, FilterExpression};
use lance_file::v2::reader::ReaderProjection;
use lance_file::v2::LanceEncodingsIo;
use lance_file::{
v2::{
reader::{
BufferDescriptor, CachedFileMetadata, FileReader, FileReaderOptions, FileStatistics,
},
writer::{FileWriter, FileWriterOptions},
},
version::LanceFileVersion,
use lance_file::reader::{
BufferDescriptor, CachedFileMetadata, FileReader, FileReaderOptions, FileStatistics,
ReaderProjection,
};
use lance_file::writer::{FileWriter, FileWriterOptions};
use lance_file::{version::LanceFileVersion, LanceEncodingsIo};
use lance_io::object_store::ObjectStoreParams;
use lance_io::{
scheduler::{ScanScheduler, SchedulerConfig},
Expand Down Expand Up @@ -91,7 +86,7 @@ impl LancePageMetadata {
.collect();
Self {
buffers,
encoding: lance_file::v2::reader::describe_encoding(inner),
encoding: lance_file::reader::describe_encoding(inner),
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions python/src/utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ use arrow_schema::DataType;
use lance::datatypes::Schema;
use lance::Result;
use lance_arrow::FixedSizeListArrayExt;
use lance_file::writer::FileWriter;
use lance_file::previous::writer::FileWriter as PreviousFileWriter;
use lance_index::scalar::IndexWriter;
use lance_index::vector::hnsw::{builder::HnswBuildParams, HNSW};
use lance_index::vector::kmeans::{
Expand Down Expand Up @@ -223,7 +223,7 @@ impl Hnsw {
let mut writer = rt()
.block_on(
Some(py),
FileWriter::<ManifestDescribing>::try_new(
PreviousFileWriter::<ManifestDescribing>::try_new(
&object_store,
&path,
Schema::try_from(HNSW::schema().as_ref())
Expand Down
8 changes: 3 additions & 5 deletions rust/lance-file/benches/reader.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,10 @@ use futures::{FutureExt, StreamExt};
use lance_datagen::ArrayGeneratorExt;
use lance_encoding::decoder::{DecoderPlugins, FilterExpression};
use lance_file::{
v2::{
reader::{FileReader, FileReaderOptions},
testing::test_cache,
writer::{FileWriter, FileWriterOptions},
},
reader::{FileReader, FileReaderOptions},
testing::test_cache,
version::LanceFileVersion,
writer::{FileWriter, FileWriterOptions},
};
use lance_io::{
object_store::ObjectStore,
Expand Down
2 changes: 0 additions & 2 deletions rust/lance-file/src/format.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,6 @@ pub mod pbfile {
include!(concat!(env!("OUT_DIR"), "/lance.file.v2.rs"));
}

pub mod metadata;

/// These version/magic values are written at the end of Lance files (e.g. versions/1.version)
pub const MAJOR_VERSION: i16 = 0;
pub const MINOR_VERSION: i16 = 2;
Expand Down
File renamed without changes.
7 changes: 5 additions & 2 deletions rust/lance-file/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,14 @@

pub mod datatypes;
pub mod format;
pub mod page_table;
pub(crate) mod io;
pub mod previous;
pub mod reader;
pub mod v2;
pub mod testing;
pub mod writer;

pub use io::LanceEncodingsIo;

use format::MAGIC;
pub use lance_encoding::version;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ use lance_core::datatypes::Schema;
use lance_core::{Error, Result};
use lance_io::traits::ProtoStruct;
use snafu::location;

/// Data File Metadata
#[derive(Debug, Default, DeepSizeOf, PartialEq)]
pub struct Metadata {
Expand Down
4 changes: 4 additions & 0 deletions rust/lance-file/src/previous/format/mod.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The Lance Authors

pub mod metadata;
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The Lance Authors

pub(crate) mod io;
//! Legacy Lance file v1 implementation kept for backwards compatibility.

pub mod format;
pub mod page_table;
pub mod reader;
pub mod testing;
pub mod writer;

pub use io::LanceEncodingsIo;
Loading
Loading