Skip to content

ARROW-5510: [C++][Python][R][GLib] Implement Feather "V2" using Arrow IPC file format #6694

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wesm wants to merge 15 commits into from
Closed

ARROW-5510: [C++][Python][R][GLib] Implement Feather "V2" using Arrow IPC file format #6694

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
a70b8d5
FeatherV2
wesm Feb 18, 2020
bc0b514
Add lz4 support
wesm Mar 24, 2020
c1a352d
Expose configurable chunksize
wesm Mar 26, 2020
21c16d2
Set default Feather compression to LZ4 unless it is unavailable
wesm Mar 27, 2020
7854e2c
Follow Feather changes in GLib and Ruby bindings
wesm Mar 27, 2020
1a15fd8
Wire up Feather write options in R, add tests all around, and update …
nealrichardson Mar 27, 2020
b8f942d
Have I mentioned lately how I feel about automating lint and style fi…
nealrichardson Mar 27, 2020
7526030
Don't allow compression_level unless using zstd
nealrichardson Mar 27, 2020
6212d88
[GLib][Ruby] Improve writer API
kou Mar 28, 2020
b312d17
Enable zstd
kou Mar 28, 2020
a72a9a5
Use pointer instead of reference for consistency
kou Mar 28, 2020
d41807f
Code review comments, check unsupported V1 options more thoroughly in…
wesm Mar 29, 2020
5ce53cb
Better error message when trying to write unsupported Arrow type with…
wesm Mar 29, 2020
5b16bb3
[R] Accept either RecordBatch or Table in arrow::write_feather. Use L…
wesm Mar 29, 2020
4ebac36
[R] more concise conversion to Table
wesm Mar 29, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/ruby.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ jobs:
ARROW_WITH_LZ4: ON
ARROW_WITH_SNAPPY: ON
ARROW_WITH_ZLIB: ON
ARROW_WITH_ZSTD: ON
XML_CATALOG_FILES: /usr/local/etc/xml/catalog
steps:
- name: Checkout Arrow
Expand Down
49 changes: 25 additions & 24 deletions c_glib/arrow-glib/file-system.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,27 +85,27 @@ garrow_file_info_set_property(GObject *object,
const GValue *value,
GParamSpec *pspec)
{
auto &arrow_file_info = garrow_file_info_get_raw(GARROW_FILE_INFO(object));
auto arrow_file_info = garrow_file_info_get_raw(GARROW_FILE_INFO(object));

switch (prop_id) {
case PROP_FILE_INFO_TYPE:
{
auto arrow_file_type =
static_cast<arrow::fs::FileType>(g_value_get_enum(value));
arrow_file_info.set_type(arrow_file_type);
arrow_file_info->set_type(arrow_file_type);
}
break;
case PROP_FILE_INFO_PATH:
arrow_file_info.set_path(g_value_get_string(value));
arrow_file_info->set_path(g_value_get_string(value));
break;
case PROP_FILE_INFO_SIZE:
arrow_file_info.set_size(g_value_get_int64(value));
arrow_file_info->set_size(g_value_get_int64(value));
break;
case PROP_FILE_INFO_MTIME:
{
const gint64 mtime = g_value_get_int64(value);
const arrow::fs::TimePoint::duration duration(mtime);
arrow_file_info.set_mtime(arrow::fs::TimePoint(duration));
arrow_file_info->set_mtime(arrow::fs::TimePoint(duration));
}
break;
default:
Expand All @@ -120,35 +120,35 @@ garrow_file_info_get_property(GObject *object,
GValue *value,
GParamSpec *pspec)
{
const auto &arrow_file_info =
const auto arrow_file_info =
garrow_file_info_get_raw(GARROW_FILE_INFO(object));

switch (prop_id) {
case PROP_FILE_INFO_TYPE:
{
const auto arrow_file_type = arrow_file_info.type();
const auto arrow_file_type = arrow_file_info->type();
const auto file_type = static_cast<GArrowFileType>(arrow_file_type);
g_value_set_enum(value, file_type);
}
break;
case PROP_FILE_INFO_PATH:
g_value_set_string(value, arrow_file_info.path().c_str());
g_value_set_string(value, arrow_file_info->path().c_str());
break;
case PROP_FILE_INFO_BASE_NAME:
g_value_set_string(value, arrow_file_info.base_name().c_str());
g_value_set_string(value, arrow_file_info->base_name().c_str());
break;
case PROP_FILE_INFO_DIR_NAME:
g_value_set_string(value, arrow_file_info.dir_name().c_str());
g_value_set_string(value, arrow_file_info->dir_name().c_str());
break;
case PROP_FILE_INFO_EXTENSION:
g_value_set_string(value, arrow_file_info.extension().c_str());
g_value_set_string(value, arrow_file_info->extension().c_str());
break;
case PROP_FILE_INFO_SIZE:
g_value_set_int64(value, arrow_file_info.size());
g_value_set_int64(value, arrow_file_info->size());
break;
case PROP_FILE_INFO_MTIME:
{
const auto arrow_mtime = arrow_file_info.mtime();
const auto arrow_mtime = arrow_file_info->mtime();
const auto mtime = arrow_mtime.time_since_epoch().count();
g_value_set_int64(value, mtime);
}
Expand Down Expand Up @@ -317,9 +317,9 @@ gboolean
garrow_file_info_equal(GArrowFileInfo *file_info,
GArrowFileInfo *other_file_info)
{
const auto &arrow_file_info = garrow_file_info_get_raw(file_info);
const auto &arrow_other_file_info = garrow_file_info_get_raw(other_file_info);
return arrow_file_info.Equals(arrow_other_file_info);
const auto arrow_file_info = garrow_file_info_get_raw(file_info);
const auto arrow_other_file_info = garrow_file_info_get_raw(other_file_info);
return arrow_file_info->Equals(*arrow_other_file_info);
}

/**
Expand All @@ -333,8 +333,8 @@ garrow_file_info_equal(GArrowFileInfo *file_info,
gboolean
garrow_file_info_is_file(GArrowFileInfo *file_info)
{
const auto &arrow_file_info = garrow_file_info_get_raw(file_info);
return arrow_file_info.IsFile();
const auto arrow_file_info = garrow_file_info_get_raw(file_info);
return arrow_file_info->IsFile();
}

/**
Expand All @@ -348,8 +348,8 @@ garrow_file_info_is_file(GArrowFileInfo *file_info)
gboolean
garrow_file_info_is_dir(GArrowFileInfo *file_info)
{
const auto &arrow_file_info = garrow_file_info_get_raw(file_info);
return arrow_file_info.IsDirectory();
const auto arrow_file_info = garrow_file_info_get_raw(file_info);
return arrow_file_info->IsDirectory();
}

/**
Expand All @@ -365,8 +365,8 @@ garrow_file_info_is_dir(GArrowFileInfo *file_info)
gchar *
garrow_file_info_to_string(GArrowFileInfo *file_info)
{
const auto &arrow_file_info = garrow_file_info_get_raw(file_info);
auto string = arrow_file_info.ToString();
const auto arrow_file_info = garrow_file_info_get_raw(file_info);
auto string = arrow_file_info->ToString();
return g_strndup(string.data(), string.size());
}

Expand Down Expand Up @@ -1308,10 +1308,11 @@ garrow_file_info_new_raw(const arrow::fs::FileInfo &arrow_file_info)
return file_info;
}

arrow::fs::FileInfo &
arrow::fs::FileInfo *
garrow_file_info_get_raw(GArrowFileInfo *file_info)
{
return GARROW_FILE_INFO_GET_PRIVATE(file_info)->file_info;
auto priv = GARROW_FILE_INFO_GET_PRIVATE(file_info);
return &(priv->file_info);
}

std::shared_ptr<arrow::fs::FileSystem>
Expand Down
2 changes: 1 addition & 1 deletion c_glib/arrow-glib/file-system.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
GArrowFileInfo *
garrow_file_info_new_raw(const arrow::fs::FileInfo &arrow_file_info);

arrow::fs::FileInfo &
arrow::fs::FileInfo *
garrow_file_info_get_raw(GArrowFileInfo *file_info);

std::shared_ptr<arrow::fs::FileSystem>
Expand Down
Loading