Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,9 @@ class Iterator : public DatasetIterator<Dataset> {
filters.push_back(std::move(f));
}

return cbt::Filter::InterleaveFromRange(filters.begin(), filters.end());
return filters.size() > 1 ? cbt::Filter::InterleaveFromRange(
filters.begin(), filters.end())
: filters[0];
}

static std::pair<std::string, std::string> ColumnToFamilyAndQualifier(
Expand Down
37 changes: 22 additions & 15 deletions tensorflow_io/python/ops/bigtable/bigtable_dataset_ops.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ def read_rows(
self,
columns: List[str],
row_set: bigtable_row_set.RowSet,
filter: filters.BigtableFilter = filters.latest(),
filter: filters.BigtableFilter = None,
output_type=tf.string,
):
"""Retrieves values from Google Bigtable sorted by RowKeys.
Expand All @@ -69,23 +69,24 @@ def read_rows(
Returns:
A `tf.data.Dataset` returning the cell contents.
"""

# Python initializes the default arguments once at the start of the
# program. If the fork happens after that (for instance when we run
# tests using xdist) the program deadlocks and hangs. That is why we
# have to make sure, all default arguments are initialized on each
# invocation.
if filter is None:
filter = filters.latest()
return _BigtableDataset(
self._client_resource,
self._table_id,
columns,
row_set,
filter,
output_type,
self._client_resource, self._table_id, columns, row_set, filter, output_type
)

def parallel_read_rows(
self,
columns: List[str],
num_parallel_calls=tf.data.AUTOTUNE,
row_set: bigtable_row_set.RowSet = bigtable_row_set.from_rows_or_ranges(
bigtable_row_range.infinite()
),
filter: filters.BigtableFilter = filters.latest(),
row_set: bigtable_row_set.RowSet = None,
filter: filters.BigtableFilter = None,
output_type=tf.string,
):
"""Retrieves values from Google Bigtable in parallel. The ammount of work
Expand All @@ -101,11 +102,17 @@ def parallel_read_rows(
A `tf.data.Dataset` returning the cell contents.
"""

# We have to make sure that all the default arguments are initialized
# on each invocation. For more info see read_rows method.
if row_set is None:
row_set = bigtable_row_set.from_rows_or_ranges(
bigtable_row_range.infinite()
)
if filter is None:
filter = filters.latest()

samples = core_ops.bigtable_split_row_set_evenly(
self._client_resource,
row_set._impl,
self._table_id,
num_parallel_calls,
self._client_resource, row_set._impl, self._table_id, num_parallel_calls
)

def map_func(idx):
Expand Down
2 changes: 2 additions & 0 deletions tests/test_bigtable/bigtable_emulator.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,14 @@
CBT_EMULATOR_SEARCH_PATHS = [
"/usr/lib/google-cloud-sdk/platform/bigtable-emulator/cbtemulator",
"/usr/local/google-cloud-sdk/platform/bigtable-emulator/cbtemulator",
"/v/google-cloud-sdk/platform/bigtable-emulator/cbtemulator",
"cbtemulator",
]

CBT_CLI_SEARCH_PATHS = [
"/usr/local/google-cloud-sdk/bin/cbt",
"/usr/bin/cbt",
"/v/google-cloud-sdk/bin/cbt",
"cbt",
]

Expand Down
2 changes: 1 addition & 1 deletion tests/test_gcloud/test_pubsub_bigtable.sh
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ fi
curl -sSOL https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-236.0.0-darwin-x86_64.tar.gz
tar -xzf google-cloud-sdk-236.0.0-darwin-x86_64.tar.gz
google-cloud-sdk/install.sh -q
google-cloud-sdk/bin/gcloud -q components install beta
google-cloud-sdk/bin/gcloud -q components install beta bigtable cbt
google-cloud-sdk/bin/gcloud -q components install pubsub-emulator
google-cloud-sdk/bin/gcloud -q components update beta
google-cloud-sdk/bin/gcloud -q beta emulators pubsub start &
Expand Down