From 746d2217f789bc4d6b8345263bcd74397fe8a3c0 Mon Sep 17 00:00:00 2001 From: Brian Dubois Date: Wed, 18 Oct 2023 13:02:56 -0400 Subject: [PATCH] TensorBoard.dev turndown: Remove dataframe api. --- tensorboard/BUILD | 1 - tensorboard/__init__.py | 7 - tensorboard/data/BUILD | 10 - tensorboard/data/__init__.py | 3 - tensorboard/data/experimental/BUILD | 75 +----- tensorboard/data/experimental/__init__.py | 19 -- .../data/experimental/base_experiment.py | 76 ------ .../data/experimental/experiment_from_dev.py | 159 ----------- .../experimental/experiment_from_dev_test.py | 250 ------------------ tensorboard/data/experimental/test_binary.py | 62 ----- tensorboard/data/experimental/utils.py | 64 ----- tensorboard/pip_package/BUILD | 1 - 12 files changed, 2 insertions(+), 725 deletions(-) delete mode 100644 tensorboard/data/experimental/__init__.py delete mode 100644 tensorboard/data/experimental/base_experiment.py delete mode 100644 tensorboard/data/experimental/experiment_from_dev.py delete mode 100644 tensorboard/data/experimental/experiment_from_dev_test.py delete mode 100644 tensorboard/data/experimental/test_binary.py delete mode 100644 tensorboard/data/experimental/utils.py diff --git a/tensorboard/BUILD b/tensorboard/BUILD index d03a9e14529..7c8c1252dfc 100644 --- a/tensorboard/BUILD +++ b/tensorboard/BUILD @@ -89,7 +89,6 @@ py_library( ":lib_init_only", ":notebook", ":program", - "//tensorboard/data:lib_init_only", "//tensorboard/summary", "//tensorboard/summary:summary_v1", "//tensorboard/summary:summary_v2", diff --git a/tensorboard/__init__.py b/tensorboard/__init__.py index 0275d36555b..95bc174e64d 100644 --- a/tensorboard/__init__.py +++ b/tensorboard/__init__.py @@ -71,13 +71,6 @@ # additional discussion. -@_lazy.lazy_load("tensorboard.data") -def data(): - import importlib - - return importlib.import_module("tensorboard.data") - - @_lazy.lazy_load("tensorboard.errors") def errors(): import importlib diff --git a/tensorboard/data/BUILD b/tensorboard/data/BUILD index 22154fd938e..0a3337af1a1 100644 --- a/tensorboard/data/BUILD +++ b/tensorboard/data/BUILD @@ -8,16 +8,6 @@ package(default_visibility = ["//tensorboard:internal"]) licenses(["notice"]) -py_library( - name = "lib_init_only", - srcs = ["__init__.py"], - srcs_version = "PY3", - visibility = ["//tensorboard:internal"], - deps = [ - "//tensorboard/data/experimental:lib_init_only", - ], -) - py_library( name = "provider", srcs = ["provider.py"], diff --git a/tensorboard/data/__init__.py b/tensorboard/data/__init__.py index dfcbc38f92a..931c2ef11db 100644 --- a/tensorboard/data/__init__.py +++ b/tensorboard/data/__init__.py @@ -12,6 +12,3 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== - - -from tensorboard.data import experimental # noqa: F401 diff --git a/tensorboard/data/experimental/BUILD b/tensorboard/data/experimental/BUILD index 8d49db14e56..da1d82e6a29 100644 --- a/tensorboard/data/experimental/BUILD +++ b/tensorboard/data/experimental/BUILD @@ -1,78 +1,7 @@ -# Description: -# Experiment Data Access API. -load("@rules_python//python:py_binary.bzl", "py_binary") -load("@rules_python//python:py_library.bzl", "py_library") -load("@rules_python//python:py_test.bzl", "py_test") - +# This is a stub BUILD file that remains after the deletion of the experimental +# data frame API. We keep it (temporarily) to allow copybara imports to succeed. package(default_visibility = ["//tensorboard:internal"]) licenses(["notice"]) exports_files(["LICENSE"]) - -py_library( - name = "base_experiment", - srcs = ["base_experiment.py"], - srcs_version = "PY3", -) - -py_library( - name = "experiment_from_dev", - srcs = ["experiment_from_dev.py"], - srcs_version = "PY3", - deps = [ - ":base_experiment", - ":utils", - "//tensorboard:expect_grpc_installed", - "//tensorboard:expect_pandas_installed", - "//tensorboard/uploader:auth", - "//tensorboard/uploader:server_info", - "//tensorboard/uploader:util", - "//tensorboard/uploader/proto:protos_all_py_pb2", - "//tensorboard/uploader/proto:protos_all_py_pb2_grpc", - "//tensorboard/util:grpc_util", - ], -) - -py_test( - name = "experiment_from_dev_test", - srcs = ["experiment_from_dev_test.py"], - srcs_version = "PY3", - deps = [ - ":experiment_from_dev", - "//tensorboard:expect_numpy_installed", - "//tensorboard:expect_pandas_installed", - "//tensorboard:test", - "//tensorboard/compat/proto:protos_all_py_pb2", - "//tensorboard/uploader:test_util", - "//tensorboard/uploader/proto:protos_all_py_pb2", - "//tensorboard/util:grpc_util", - ], -) - -py_library( - name = "lib_init_only", - srcs = ["__init__.py"], - srcs_version = "PY3", - visibility = ["//tensorboard:internal"], - deps = [ - ":experiment_from_dev", - ], -) - -py_binary( - name = "test_binary", - srcs = ["test_binary.py"], - srcs_version = "PY3", - deps = ["//tensorboard/data/experimental:experiment_from_dev"], -) - -py_library( - name = "utils", - srcs = ["utils.py"], - srcs_version = "PY3", - visibility = ["//tensorboard:internal"], - deps = [ - "//tensorboard:expect_numpy_installed", - ], -) diff --git a/tensorboard/data/experimental/__init__.py b/tensorboard/data/experimental/__init__.py deleted file mode 100644 index 3257ac3e288..00000000000 --- a/tensorboard/data/experimental/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright 2020 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - - -from tensorboard.data.experimental.experiment_from_dev import ( # noqa: F401 - ExperimentFromDev, -) diff --git a/tensorboard/data/experimental/base_experiment.py b/tensorboard/data/experimental/base_experiment.py deleted file mode 100644 index eb6399673f9..00000000000 --- a/tensorboard/data/experimental/base_experiment.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright 2020 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Base Class of Experiment Data Access API.""" - - -import abc - - -class BaseExperiment(metaclass=abc.ABCMeta): - """Base class for experiment data access.""" - - # TODO(cais): Add list_scalar_runs(). - # TODO(cais): Add list_scalar_tags(). - - @abc.abstractmethod - def get_scalars( - self, - runs_filter=None, - tags_filter=None, - pivot=False, - include_wall_time=False, - ): - """Export scalar data as a pandas.DataFrame. - - Args: - runs_filter: A regex filter for runs (e.g., r'run_[2-4]'). Operates in - logical AND relation with `tags_filter`. - tags_filter: A regex filter for tags (e.g., r'.*loss.*'). Operates in - logical AND related with `runs_filter`. - pivot: Whether to returned DataFrame will be pivoted (via pandas’ - `pivot_data()` method to a “wide” format wherein the tags of a - given run and a given step are all collected in a single row. - Setting `pivot` to `True` stipulates that the sets of step values - are identical among all tags in every run of the experiment (after - any run and tag filtering), so that the pivoting operation will not - introduce missing values in the resultant DataFrame. Failing to meet - this condition will cause `pivot=True` to raise a `ValueError`. - If not provided, defaults to `False`. - include_wall_time: Include wall_time (timestamps in nanoseconds since - the epoch in float64) as a column in the returned DataFrame. - If not provided, defaults to `False`. - - Returns: - If `pivot` (default): - A pivoted DataFrame with the indexing columns of - - run - - step - And value columns that correspond to the tags. - Duplicate entries for each run-step combination will be aggregated - with `numpy.stack`. This format is more friendly to manipulation and - plotting and hence io chosen as the default. When certain rows have - missing values, a warning message will be displayed and advise the - user to use the `pivot=False` if steps have different meanings in - the experiment. - If `not pivot`: - A DataFrame with the following columns. - - run: (non-null object) - - tag: (non-null object) - - steps: (non-null int64) - - wall_time: (non-null object) - - value: (non-null float32) - """ - # TODO(cais): Add description about sorting order. - pass diff --git a/tensorboard/data/experimental/experiment_from_dev.py b/tensorboard/data/experimental/experiment_from_dev.py deleted file mode 100644 index 70bd98efdfa..00000000000 --- a/tensorboard/data/experimental/experiment_from_dev.py +++ /dev/null @@ -1,159 +0,0 @@ -# Copyright 2020 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Experiment Data Access API for tensorboard.dev.""" - - -import sys -import time - -import grpc - -from tensorboard.data.experimental import base_experiment -from tensorboard.data.experimental import utils as experimental_utils -from tensorboard.uploader import auth -from tensorboard.uploader import util -from tensorboard.uploader import server_info as server_info_lib -from tensorboard.uploader.proto import export_service_pb2 -from tensorboard.uploader.proto import export_service_pb2_grpc -from tensorboard.uploader.proto import server_info_pb2 -from tensorboard.util import grpc_util - - -DEFAULT_ORIGIN = "https://tensorboard.dev" - - -def import_pandas(): - """Import pandas, guarded by a user-friendly error message on failure.""" - try: - import pandas - except ImportError: - raise ImportError( - "The get_scalars() feature requires the pandas package, " - "which does not seem to be available in your Python " - "environment. You can install it with command:\n\n" - " pip install pandas\n" - ) - return pandas - - -class ExperimentFromDev(base_experiment.BaseExperiment): - """Implementation of BaseExperiment, specialized for tensorboard.dev.""" - - def __init__(self, experiment_id, api_endpoint=None): - """Constructor of ExperimentFromDev. - - Args: - experiment_id: String ID of the experiment on tensorboard.dev (e.g., - "AdYd1TgeTlaLWXx6I8JUbA"). - api_endpoint: Optional override value for API endpoint. Used for - development only. - """ - super().__init__() - self._experiment_id = experiment_id - self._api_client = get_api_client(api_endpoint=api_endpoint) - - def get_scalars( - self, - runs_filter=None, - tags_filter=None, - pivot=False, - include_wall_time=False, - ): - # NOTE(#3650): Import pandas early in this method, so if the - # Python environment does not have pandas installed, an error can be - # raised early, before any rpc call is made. - pandas = import_pandas() - if runs_filter is not None: - raise NotImplementedError( - "runs_filter support for get_scalars() is not implemented yet." - ) - if tags_filter is not None: - raise NotImplementedError( - "tags_filter support for get_scalars() is not implemented yet." - ) - - request = export_service_pb2.StreamExperimentDataRequest() - request.experiment_id = self._experiment_id - read_time = time.time() - util.set_timestamp(request.read_timestamp, read_time) - # TODO(cais, wchargin): Use another rpc to check for staleness and avoid - # a new StreamExperimentData rpc request if data is not stale. - stream = self._api_client.StreamExperimentData( - request, metadata=grpc_util.version_metadata() - ) - - runs = [] - tags = [] - steps = [] - wall_times = [] - values = [] - for response in stream: - # TODO(cais, wchargin): Display progress bar during data loading. - num_values = len(response.points.values) - runs.extend([response.run_name] * num_values) - tags.extend([response.tag_name] * num_values) - steps.extend(list(response.points.steps)) - wall_times.extend( - [t.ToNanoseconds() / 1e9 for t in response.points.wall_times] - ) - values.extend(list(response.points.values)) - - data = { - "run": runs, - "tag": tags, - "step": steps, - "value": values, - } - if include_wall_time: - data["wall_time"] = wall_times - dataframe = pandas.DataFrame(data) - if pivot: - dataframe = experimental_utils.pivot_dataframe(dataframe) - return dataframe - - -def get_api_client(api_endpoint=None): - server_info = _get_server_info(api_endpoint=api_endpoint) - _handle_server_info(server_info) - channel_creds = grpc.ssl_channel_credentials() - credentials = auth.CredentialsStore().read_credentials() - if credentials: - channel_creds = grpc.composite_channel_credentials( - channel_creds, auth.id_token_call_credentials(credentials) - ) - channel = grpc.secure_channel( - server_info.api_server.endpoint, channel_creds - ) - return export_service_pb2_grpc.TensorBoardExporterServiceStub(channel) - - -def _get_server_info(api_endpoint=None): - # TODO(cais): Add more plugins to the list when more plugin/data types - # are supported - plugins = ["scalars"] - if api_endpoint: - return server_info_lib.create_server_info( - DEFAULT_ORIGIN, api_endpoint, plugins - ) - return server_info_lib.fetch_server_info(DEFAULT_ORIGIN, plugins) - - -def _handle_server_info(info): - compat = info.compatibility - if compat.verdict == server_info_pb2.VERDICT_WARN: - sys.stderr.write("Warning [from server]: %s\n" % compat.details) - sys.stderr.flush() - elif compat.verdict == server_info_pb2.VERDICT_ERROR: - raise ValueError("Error [from server]: %s" % compat.details) diff --git a/tensorboard/data/experimental/experiment_from_dev_test.py b/tensorboard/data/experimental/experiment_from_dev_test.py deleted file mode 100644 index 13f5ce2de95..00000000000 --- a/tensorboard/data/experimental/experiment_from_dev_test.py +++ /dev/null @@ -1,250 +0,0 @@ -# Copyright 2020 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Tests for tensorboard.uploader.exporter.""" - - -from unittest import mock - -import numpy as np -import pandas - -from tensorboard import test as tb_test -from tensorboard.data.experimental import experiment_from_dev -from tensorboard.uploader import test_util -from tensorboard.uploader.proto import export_service_pb2 -from tensorboard.util import grpc_util - - -class ExperimentFromDevTest(tb_test.TestCase): - def test_get_scalars_works(self): - mock_api_client = mock.Mock() - - def stream_experiment_data(request, **kwargs): - self.assertEqual(request.experiment_id, "789") - self.assertEqual(kwargs["metadata"], grpc_util.version_metadata()) - for run in ("train", "test"): - for tag in ("accuracy", "loss"): - response = export_service_pb2.StreamExperimentDataResponse() - response.run_name = run - response.tag_name = tag - display_name = "%s:%s" % (request.experiment_id, tag) - response.tag_metadata.CopyFrom( - test_util.scalar_metadata(display_name) - ) - for step in range(10): - response.points.steps.append(step) - if tag == "loss": - if run == "train": - value = 1.0 / (step + 1) - seconds = step - else: - value = -1.0 / (step + 1) - seconds = 600 + step - else: # "accuracy" - if run == "train": - value = 1.0 / (10 - step) - seconds = step * 2 - else: - value = -1.0 / (10 - step) - seconds = 600 + step * 2 - response.points.values.append(value) - response.points.wall_times.add(seconds=seconds, nanos=0) - yield response - - mock_api_client.StreamExperimentData = mock.Mock( - wraps=stream_experiment_data - ) - - with mock.patch.object( - experiment_from_dev, - "get_api_client", - lambda api_endpoint: mock_api_client, - ): - experiment = experiment_from_dev.ExperimentFromDev("789") - for pivot in (False, True): - for include_wall_time in (False, True): - with self.subTest( - "pivot=%s; include_wall_time=%s" - % (pivot, include_wall_time) - ): - dataframe = experiment.get_scalars( - pivot=pivot, include_wall_time=include_wall_time - ) - - if pivot: - run_key = ( - ("run", "") if include_wall_time else "run" - ) - step_key = ( - ("step", "") if include_wall_time else "step" - ) - accuracy_value_key = ( - ("value", "accuracy") - if include_wall_time - else "accuracy" - ) - loss_value_key = ( - ("value", "loss") - if include_wall_time - else "loss" - ) - data = { - run_key: ["test"] * 10 + ["train"] * 10, - step_key: np.concatenate( - [np.arange(0, 10), np.arange(0, 10)] - ), - accuracy_value_key: np.concatenate( - [ - -1.0 / (10.0 - np.arange(0, 10)), - 1.0 / (10.0 - np.arange(0, 10)), - ], - ), - loss_value_key: np.concatenate( - [ - -1.0 / (1.0 + np.arange(0, 10)), - 1.0 / (1.0 + np.arange(0, 10)), - ], - ), - } - if include_wall_time: - data[ - ("wall_time", "accuracy") - ] = np.concatenate( - [ - 600.0 + 2.0 * np.arange(0, 10), - 2.0 * np.arange(0, 10), - ] - ) - data[("wall_time", "loss")] = np.concatenate( - [ - 600.0 + np.arange(0, 10), - 1.0 * np.arange(0, 10), - ] - ) - expected = pandas.DataFrame(data) - else: # No pivot_table. - data = { - "run": ["train"] * 20 + ["test"] * 20, - "tag": (["accuracy"] * 10 + ["loss"] * 10) * 2, - "step": list(np.arange(0, 10)) * 4, - "value": np.concatenate( - [ - 1.0 / (10.0 - np.arange(0, 10)), - 1.0 / (1.0 + np.arange(0, 10)), - -1.0 / (10.0 - np.arange(0, 10)), - -1.0 / (1.0 + np.arange(0, 10)), - ] - ), - } - if include_wall_time: - data["wall_time"] = np.concatenate( - [ - 2.0 * np.arange(0, 10), - 1.0 * np.arange(0, 10), - 600.0 + 2.0 * np.arange(0, 10), - 600.0 + np.arange(0, 10), - ] - ) - expected = pandas.DataFrame(data) - - pandas.testing.assert_frame_equal( - dataframe, - expected, - check_names=True, - ) - - def test_get_scalars_with_pivot_table_with_missing_value(self): - mock_api_client = mock.Mock() - - def stream_experiment_data(request, **kwargs): - self.assertEqual(request.experiment_id, "789") - self.assertEqual(kwargs["metadata"], grpc_util.version_metadata()) - response = export_service_pb2.StreamExperimentDataResponse() - response.run_name = "train" - response.tag_name = "batch_loss" - response.points.steps.append(0) - response.points.values.append(0.5) - response.points.wall_times.add(seconds=0, nanos=0) - response.points.steps.append(1) - response.points.values.append(0.25) - response.points.wall_times.add(seconds=1, nanos=0) - yield response - response = export_service_pb2.StreamExperimentDataResponse() - response.run_name = "train" - response.tag_name = "epoch_loss" - response.points.steps.append(0) - response.points.values.append(0.375) - response.points.wall_times.add(seconds=2, nanos=0) - yield response - - mock_api_client.StreamExperimentData = mock.Mock( - wraps=stream_experiment_data - ) - - with mock.patch.object( - experiment_from_dev, - "get_api_client", - lambda api_endpoint: mock_api_client, - ): - experiment = experiment_from_dev.ExperimentFromDev("789") - with self.assertRaisesRegex( - ValueError, - r"contains missing value\(s\).*different sets of " - r"steps.*pivot=False", - ): - experiment.get_scalars(pivot=True) - - def test_get_scalars_with_actual_inf_and_nan(self): - """Test for get_scalars() call that involve inf and nan in user data.""" - mock_api_client = mock.Mock() - - def stream_experiment_data(request, **kwargs): - self.assertEqual(request.experiment_id, "789") - self.assertEqual(kwargs["metadata"], grpc_util.version_metadata()) - response = export_service_pb2.StreamExperimentDataResponse() - response.run_name = "train" - response.tag_name = "batch_loss" - response.points.steps.append(0) - response.points.values.append(np.nan) - response.points.wall_times.add(seconds=0, nanos=0) - response.points.steps.append(1) - response.points.values.append(np.inf) - response.points.wall_times.add(seconds=10, nanos=0) - yield response - - mock_api_client.StreamExperimentData = mock.Mock( - wraps=stream_experiment_data - ) - - with mock.patch.object( - experiment_from_dev, - "get_api_client", - lambda api_endpoint: mock_api_client, - ): - experiment = experiment_from_dev.ExperimentFromDev("789") - dataframe = experiment.get_scalars(pivot=True) - - expected = pandas.DataFrame( - { - "run": ["train"] * 2, - "step": [0, 1], - "batch_loss": [np.nan, np.inf], - } - ) - pandas.testing.assert_frame_equal(dataframe, expected, check_names=True) - - -if __name__ == "__main__": - tb_test.main() diff --git a/tensorboard/data/experimental/test_binary.py b/tensorboard/data/experimental/test_binary.py deleted file mode 100644 index 2ec9e57cc2d..00000000000 --- a/tensorboard/data/experimental/test_binary.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright 2020 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""A test binary that can be used to test ExperimentFromDev features.""" - - -import argparse - -from tensorboard.data.experimental import experiment_from_dev - - -def parse_args(): - parser = argparse.ArgumentParser("Test run of ExperimentFromDev") - parser.add_argument( - "--experiment_id", - type=str, - default="AdYd1TgeTlaLWXx6I8JUbA", - help="Experiment ID", - ) - parser.add_argument( - "--api_endpoint", - type=str, - default=None, - help="Optional API endpoint used to override the default", - ) - parser.add_argument( - "--pivot", - action="store_true", - help="Pivot the DataFrame, so that the tags become columns " - "of the DataFrame.", - ) - parser.add_argument( - "--include_wall_time", - action="store_true", - help="Include wall_time column(s) in the DataFrame", - ) - return parser.parse_args() - - -def main(args): - experiment = experiment_from_dev.ExperimentFromDev( - args.experiment_id, api_endpoint=args.api_endpoint - ) - dataframe = experiment.get_scalars( - pivot=args.pivot, include_wall_time=args.include_wall_time - ) - print(dataframe) - - -if __name__ == "__main__": - main(parse_args()) diff --git a/tensorboard/data/experimental/utils.py b/tensorboard/data/experimental/utils.py deleted file mode 100644 index a093e17ad60..00000000000 --- a/tensorboard/data/experimental/utils.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2020 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""Utility methods for working with the Experiment Data Access API.""" - -import numpy as np - - -def pivot_dataframe(dataframe): - """Gets a pivoted wide-form pandas dataframe. - - The wide-form DataFrame has all its tags included as columns of the - DataFrame, which is more convenient to work. If the condition of having - uniform sets of step values across all tags in all runs is not met, - this will error. - - Args: - dataframe: pandas dataframe to pivot. - - Returns: - Pivoted wide-form pandas dataframe. - Raises: - ValueError if step values across all tags are not uniform. - """ - num_missing_0 = np.count_nonzero(dataframe.isnull().values) - dataframe = dataframe.pivot_table( - values=( - ["value", "wall_time"] - if "wall_time" in dataframe.columns - else "value" - ), - index=["run", "step"], - columns="tag", - dropna=False, - ) - num_missing_1 = np.count_nonzero(dataframe.isnull().values) - if num_missing_1 > num_missing_0: - raise ValueError( - "pivoted DataFrame contains missing value(s). " - "This is likely due to two timeseries having different " - "sets of steps in your experiment. " - "You can avoid this error by calling `get_scalars()` with " - "`pivot=False` to disable the DataFrame pivoting." - ) - # `reset_index()` removes the MultiIndex structure of the pivoted - # DataFrame. Before the call, the DataFrame consits of two levels - # of index: "run" and "step". After the call, the index become a - # single range index (e.g,. `dataframe[:2]` works). - dataframe = dataframe.reset_index() - # Remove the columns name "tag". - dataframe.columns.name = None - dataframe.columns.names = [None for name in dataframe.columns.names] - return dataframe diff --git a/tensorboard/pip_package/BUILD b/tensorboard/pip_package/BUILD index 8baa65b0fed..9e71cad7eff 100644 --- a/tensorboard/pip_package/BUILD +++ b/tensorboard/pip_package/BUILD @@ -42,7 +42,6 @@ sh_binary( "//tensorboard", # Main tensorboard binary and everything it uses "//tensorboard:lib", # User-facing overall TensorBoard API "//tensorboard:version", # Version module (read by setup.py) - "//tensorboard/data/experimental:experiment_from_dev", "//tensorboard/plugins/hparams", # User-facing hparams API "//tensorboard/plugins/mesh", # User-facing mesh API "//tensorboard/plugins/projector", # User-facing projector API