From 2e98c026c5bb650a805aa86b395f674cfec8d3d7 Mon Sep 17 00:00:00 2001 From: Riley Bauer Date: Tue, 14 Aug 2018 15:59:30 -0700 Subject: [PATCH 1/4] Importing google.datalab.kernel is working --- google/datalab/stackdriver/monitoring/__init__.py | 6 +++++- google/datalab/stackdriver/monitoring/_query.py | 12 ++++++------ setup.py | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/google/datalab/stackdriver/monitoring/__init__.py b/google/datalab/stackdriver/monitoring/__init__.py index 824073932..81139bbe1 100644 --- a/google/datalab/stackdriver/monitoring/__init__.py +++ b/google/datalab/stackdriver/monitoring/__init__.py @@ -14,7 +14,11 @@ from __future__ import absolute_import -from google.cloud.monitoring import Aligner, Reducer +from google.cloud.monitoring import enums + +Aligner = enums.Aggregation.Aligner +Reducer = enums.Aggregation.Reducer + from ._group import Groups from ._metric import MetricDescriptors from ._query import Query diff --git a/google/datalab/stackdriver/monitoring/_query.py b/google/datalab/stackdriver/monitoring/_query.py index 8faa4b539..6294e6247 100644 --- a/google/datalab/stackdriver/monitoring/_query.py +++ b/google/datalab/stackdriver/monitoring/_query.py @@ -14,17 +14,17 @@ from __future__ import absolute_import -import google.cloud.monitoring +import google.cloud.monitoring_v3.query from . import _query_metadata from . import _utils -class Query(google.cloud.monitoring.Query): +class Query(google.cloud.monitoring_v3.query.Query): """Query object for retrieving metric data.""" def __init__(self, - metric_type=google.cloud.monitoring.Query.DEFAULT_METRIC_TYPE, + metric_type=google.cloud.monitoring_v3.query.Query.DEFAULT_METRIC_TYPE, end_time=None, days=0, hours=0, minutes=0, context=None): """Initializes the core query parameters. @@ -33,13 +33,13 @@ def __init__(self, the resulting duration from the end time. It is also allowed to omit the end time and duration here, - in which case :meth:`~google.cloud.monitoring.query.Query.select_interval` + in which case :meth:`~google.cloud.monitoring_v3.query.Query.select_interval` must be called before the query is executed. Args: metric_type: The metric type name. The default value is :data:`Query.DEFAULT_METRIC_TYPE - `, but + `, but please note that this default value is provided only for demonstration purposes and is subject to change. end_time: The end time (inclusive) of the time interval for which @@ -54,7 +54,7 @@ def __init__(self, ValueError: ``end_time`` was specified but ``days``, ``hours``, and ``minutes`` are all zero. If you really want to specify a point in time, use - :meth:`~google.cloud.monitoring.query.Query.select_interval`. + :meth:`~google.cloud.monitoring_v3.query.Query.select_interval`. """ client = _utils.make_client(context) super(Query, self).__init__(client, metric_type, diff --git a/setup.py b/setup.py index 9112f2780..9e4fd2baa 100644 --- a/setup.py +++ b/setup.py @@ -93,7 +93,7 @@ 'configparser>=3.5.0', 'mock>=2.0.0', 'future>=0.16.0', - 'google-cloud>=0.30.0', + 'google-cloud-monitoring>=0.30.1', 'google-api-python-client>=1.6.2', 'seaborn>=0.7.0', 'plotly>=1.12.5', From 1fca88250c33dfd0844476cc1b8faca9194eedba Mon Sep 17 00:00:00 2001 From: Riley Bauer Date: Tue, 14 Aug 2018 16:42:56 -0700 Subject: [PATCH 2/4] Fix additional imports, add google-cloud-logging --- setup.py | 1 + tests/stackdriver/monitoring/query_metadata_tests.py | 6 +++--- tests/stackdriver/monitoring/query_tests.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 9e4fd2baa..50264feb4 100644 --- a/setup.py +++ b/setup.py @@ -93,6 +93,7 @@ 'configparser>=3.5.0', 'mock>=2.0.0', 'future>=0.16.0', + 'google-cloud-logging>=1.6.0', 'google-cloud-monitoring>=0.30.1', 'google-api-python-client>=1.6.2', 'seaborn>=0.7.0', diff --git a/tests/stackdriver/monitoring/query_metadata_tests.py b/tests/stackdriver/monitoring/query_metadata_tests.py index d5cce0660..0d0388461 100644 --- a/tests/stackdriver/monitoring/query_metadata_tests.py +++ b/tests/stackdriver/monitoring/query_metadata_tests.py @@ -14,9 +14,9 @@ import mock import unittest -from google.cloud.monitoring import Resource -from google.cloud.monitoring import Metric -from google.cloud.monitoring import TimeSeries +from google.cloud.logging.resource import Resource +from google.cloud.monitoring_v3.types import Metric +from google.cloud.monitoring_v3.types import TimeSeries import google.auth import google.datalab diff --git a/tests/stackdriver/monitoring/query_tests.py b/tests/stackdriver/monitoring/query_tests.py index 7623c40ef..73c288573 100644 --- a/tests/stackdriver/monitoring/query_tests.py +++ b/tests/stackdriver/monitoring/query_tests.py @@ -15,7 +15,7 @@ import mock import unittest -from google.cloud.monitoring import Query as BaseQuery +from google.cloud.monitoring_v3.query import Query as BaseQuery import google.auth import google.datalab From 4c9a0482cc107b09468627653ce008b40c91fbc2 Mon Sep 17 00:00:00 2001 From: Riley Bauer Date: Tue, 14 Aug 2018 17:10:49 -0700 Subject: [PATCH 3/4] More broken imports --- .../datalab/stackdriver/monitoring/_query_metadata.py | 4 ++-- google/datalab/stackdriver/monitoring/_utils.py | 4 ++-- tests/stackdriver/monitoring/metric_tests.py | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/google/datalab/stackdriver/monitoring/_query_metadata.py b/google/datalab/stackdriver/monitoring/_query_metadata.py index 2cfe99fbe..4a9ca3a6c 100644 --- a/google/datalab/stackdriver/monitoring/_query_metadata.py +++ b/google/datalab/stackdriver/monitoring/_query_metadata.py @@ -17,7 +17,7 @@ from __future__ import unicode_literals from builtins import object -import google.cloud.monitoring +from google.cloud.monitoring_v3 import _dataframe import pandas @@ -78,7 +78,7 @@ def as_dataframe(self, max_rows=None): for col in dataframe.columns]) # Re-order the columns. - resource_keys = google.cloud.monitoring._dataframe._sorted_resource_labels( + resource_keys = _dataframe._sorted_resource_labels( dataframe['resource.labels'].columns) sorted_columns = [('resource.type', '')] sorted_columns += [('resource.labels', key) for key in resource_keys] diff --git a/google/datalab/stackdriver/monitoring/_utils.py b/google/datalab/stackdriver/monitoring/_utils.py index a416396a5..d38e5cb9c 100644 --- a/google/datalab/stackdriver/monitoring/_utils.py +++ b/google/datalab/stackdriver/monitoring/_utils.py @@ -14,14 +14,14 @@ from __future__ import absolute_import -import google.cloud.monitoring +from google.cloud.logging.client import Client import google.datalab def make_client(context=None): context = context or google.datalab.Context.default() - client = google.cloud.monitoring.Client( + client = Client( project=context.project_id, credentials=context.credentials, ) diff --git a/tests/stackdriver/monitoring/metric_tests.py b/tests/stackdriver/monitoring/metric_tests.py index a2362772e..4f990902a 100644 --- a/tests/stackdriver/monitoring/metric_tests.py +++ b/tests/stackdriver/monitoring/metric_tests.py @@ -14,7 +14,7 @@ import mock import unittest -import google.cloud.monitoring +import google.cloud.logging.client import google.auth import google.datalab @@ -70,7 +70,7 @@ def test_constructor_maximal(self): self.assertEqual(descriptors._type_prefix, TYPE_PREFIX) self.assertIsNone(descriptors._descriptors) - @mock.patch('google.cloud.monitoring.Client.list_metric_descriptors') + @mock.patch('google.cloud.logging.client.Client.list_metric_descriptors') def test_list(self, mock_gcloud_list_descriptors): mock_gcloud_list_descriptors.return_value = self._list_metrics_get_result( context=self.context) @@ -83,7 +83,7 @@ def test_list(self, mock_gcloud_list_descriptors): self.assertEqual(metric_descriptor_list[0].type, METRIC_TYPES[0]) self.assertEqual(metric_descriptor_list[1].type, METRIC_TYPES[1]) - @mock.patch('google.cloud.monitoring.Client.list_metric_descriptors') + @mock.patch('google.cloud.logging.client.Client.list_metric_descriptors') def test_list_w_api_filter(self, mock_gcloud_list_descriptors): mock_gcloud_list_descriptors.return_value = self._list_metrics_get_result( context=self.context) @@ -99,7 +99,7 @@ def test_list_w_api_filter(self, mock_gcloud_list_descriptors): self.assertEqual(metric_descriptor_list[0].type, METRIC_TYPES[0]) self.assertEqual(metric_descriptor_list[1].type, METRIC_TYPES[1]) - @mock.patch('google.cloud.monitoring.Client.list_metric_descriptors') + @mock.patch('google.cloud.logging.client.Client.list_metric_descriptors') def test_list_w_pattern_match(self, mock_gcloud_list_descriptors): mock_gcloud_list_descriptors.return_value = self._list_metrics_get_result( context=self.context) @@ -111,7 +111,7 @@ def test_list_w_pattern_match(self, mock_gcloud_list_descriptors): self.assertEqual(len(metric_descriptor_list), 1) self.assertEqual(metric_descriptor_list[0].type, METRIC_TYPES[1]) - @mock.patch('google.cloud.monitoring.Client.list_metric_descriptors') + @mock.patch('google.cloud.logging.client.Client.list_metric_descriptors') def test_list_caching(self, mock_gcloud_list_descriptors): mock_gcloud_list_descriptors.return_value = self._list_metrics_get_result( context=self.context) From 22b4efd9838571a7f641fcc2de283f2dd8e75af2 Mon Sep 17 00:00:00 2001 From: Riley Bauer Date: Wed, 15 Aug 2018 10:38:36 -0700 Subject: [PATCH 4/4] Further fixes. Going to put this branch on the backburner for a bit because it seems the google.cloud.monitoring library made a significant change from a single Client class to a series of smaller ServiceClients, and our code seems to assume we have a single object with the functionality of multiple of these ServiceClients in various places. --- google/datalab/stackdriver/monitoring/_query.py | 4 +++- google/datalab/stackdriver/monitoring/_utils.py | 11 +++++++---- tests/stackdriver/monitoring/metric_tests.py | 14 +++++++------- tests/stackdriver/monitoring/resource_tests.py | 14 +++++++------- 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/google/datalab/stackdriver/monitoring/_query.py b/google/datalab/stackdriver/monitoring/_query.py index 6294e6247..ff79fa74e 100644 --- a/google/datalab/stackdriver/monitoring/_query.py +++ b/google/datalab/stackdriver/monitoring/_query.py @@ -57,7 +57,9 @@ def __init__(self, :meth:`~google.cloud.monitoring_v3.query.Query.select_interval`. """ client = _utils.make_client(context) - super(Query, self).__init__(client, metric_type, + super(Query, self).__init__(client, + project=context.project_id, + metric_type=metric_type, end_time=end_time, days=days, hours=hours, minutes=minutes) diff --git a/google/datalab/stackdriver/monitoring/_utils.py b/google/datalab/stackdriver/monitoring/_utils.py index d38e5cb9c..60a3dea9a 100644 --- a/google/datalab/stackdriver/monitoring/_utils.py +++ b/google/datalab/stackdriver/monitoring/_utils.py @@ -14,15 +14,18 @@ from __future__ import absolute_import -from google.cloud.logging.client import Client +# google.cloud.monitoring used to have a single Client, but +# it has now been broken up into multiple subclients. Most +# of the usages I found aligned with the MetricServiceClient, +# but other uses, such as those in group_tests.py, require a +# GroupServiceClient, and there are potentially others. +from google.cloud.monitoring_v3 import MetricServiceClient import google.datalab - def make_client(context=None): context = context or google.datalab.Context.default() - client = Client( - project=context.project_id, + client = MetricServiceClient( credentials=context.credentials, ) client._connection.USER_AGENT = 'pydatalab/v0' diff --git a/tests/stackdriver/monitoring/metric_tests.py b/tests/stackdriver/monitoring/metric_tests.py index 4f990902a..ecefe0fb5 100644 --- a/tests/stackdriver/monitoring/metric_tests.py +++ b/tests/stackdriver/monitoring/metric_tests.py @@ -14,7 +14,7 @@ import mock import unittest -import google.cloud.logging.client +import google.cloud.monitoring_v3 import google.auth import google.datalab @@ -70,7 +70,7 @@ def test_constructor_maximal(self): self.assertEqual(descriptors._type_prefix, TYPE_PREFIX) self.assertIsNone(descriptors._descriptors) - @mock.patch('google.cloud.logging.client.Client.list_metric_descriptors') + @mock.patch('google.cloud.monitoring_v3.MetricServiceClient.list_metric_descriptors') def test_list(self, mock_gcloud_list_descriptors): mock_gcloud_list_descriptors.return_value = self._list_metrics_get_result( context=self.context) @@ -83,7 +83,7 @@ def test_list(self, mock_gcloud_list_descriptors): self.assertEqual(metric_descriptor_list[0].type, METRIC_TYPES[0]) self.assertEqual(metric_descriptor_list[1].type, METRIC_TYPES[1]) - @mock.patch('google.cloud.logging.client.Client.list_metric_descriptors') + @mock.patch('google.cloud.monitoring_v3.MetricServiceClient.list_metric_descriptors') def test_list_w_api_filter(self, mock_gcloud_list_descriptors): mock_gcloud_list_descriptors.return_value = self._list_metrics_get_result( context=self.context) @@ -99,7 +99,7 @@ def test_list_w_api_filter(self, mock_gcloud_list_descriptors): self.assertEqual(metric_descriptor_list[0].type, METRIC_TYPES[0]) self.assertEqual(metric_descriptor_list[1].type, METRIC_TYPES[1]) - @mock.patch('google.cloud.logging.client.Client.list_metric_descriptors') + @mock.patch('google.cloud.monitoring_v3.MetricServiceClient.list_metric_descriptors') def test_list_w_pattern_match(self, mock_gcloud_list_descriptors): mock_gcloud_list_descriptors.return_value = self._list_metrics_get_result( context=self.context) @@ -111,7 +111,7 @@ def test_list_w_pattern_match(self, mock_gcloud_list_descriptors): self.assertEqual(len(metric_descriptor_list), 1) self.assertEqual(metric_descriptor_list[0].type, METRIC_TYPES[1]) - @mock.patch('google.cloud.logging.client.Client.list_metric_descriptors') + @mock.patch('google.cloud.monitoring_v3.MetricServiceClient.list_metric_descriptors') def test_list_caching(self, mock_gcloud_list_descriptors): mock_gcloud_list_descriptors.return_value = self._list_metrics_get_result( context=self.context) @@ -164,10 +164,10 @@ def _create_context(project_id): @staticmethod def _list_metrics_get_result(context): client = gcm._utils.make_client(context=context) - all_labels = [google.cloud.monitoring.LabelDescriptor(**labels) + all_labels = [google.cloud.monitoring_v3.enums.LabelDescriptor(**labels) for labels in LABELS] descriptors = [ - client.metric_descriptor( + client.create_metric_descriptor( metric_type, metric_kind=METRIC_KIND, value_type=VALUE_TYPE, unit=UNIT, display_name=display_name, labels=all_labels, ) diff --git a/tests/stackdriver/monitoring/resource_tests.py b/tests/stackdriver/monitoring/resource_tests.py index b394d7abe..4be746fae 100644 --- a/tests/stackdriver/monitoring/resource_tests.py +++ b/tests/stackdriver/monitoring/resource_tests.py @@ -14,7 +14,7 @@ import mock import unittest -import google.cloud.monitoring +import google.cloud.monitoring_v3 import google.auth import google.datalab @@ -59,7 +59,7 @@ def test_constructor_maximal(self): self.assertEqual(descriptors._filter_string, FILTER_STRING) self.assertIsNone(descriptors._descriptors) - @mock.patch('google.cloud.monitoring.Client.list_resource_descriptors') + @mock.patch('google.cloud.monitoring_v3.MetricServiceClient.list_monitored_resource_descriptors') def test_list(self, mock_api_list_descriptors): mock_api_list_descriptors.return_value = self._list_resources_get_result() @@ -70,7 +70,7 @@ def test_list(self, mock_api_list_descriptors): self.assertEqual(resource_descriptor_list[0].type, RESOURCE_TYPES[0]) self.assertEqual(resource_descriptor_list[1].type, RESOURCE_TYPES[1]) - @mock.patch('google.cloud.monitoring.Client.list_resource_descriptors') + @mock.patch('google.cloud.monitoring_v3.MetricServiceClient.list_monitored_resource_descriptors') def test_list_w_api_filter(self, mock_api_list_descriptors): mock_api_list_descriptors.return_value = self._list_resources_get_result() @@ -84,7 +84,7 @@ def test_list_w_api_filter(self, mock_api_list_descriptors): self.assertEqual(resource_descriptor_list[0].type, RESOURCE_TYPES[0]) self.assertEqual(resource_descriptor_list[1].type, RESOURCE_TYPES[1]) - @mock.patch('google.cloud.monitoring.Client.list_resource_descriptors') + @mock.patch('google.cloud.monitoring_v3.MetricServiceClient.list_monitored_resource_descriptors') def test_list_w_pattern_match(self, mock_api_list_descriptors): mock_api_list_descriptors.return_value = self._list_resources_get_result() @@ -94,7 +94,7 @@ def test_list_w_pattern_match(self, mock_api_list_descriptors): self.assertEqual(len(resource_descriptor_list), 1) self.assertEqual(resource_descriptor_list[0].type, RESOURCE_TYPES[1]) - @mock.patch('google.cloud.monitoring.Client.list_resource_descriptors') + @mock.patch('google.cloud.monitoring_v3.MetricServiceClient.list_monitored_resource_descriptors') def test_list_caching(self, mock_gcloud_list_descriptors): mock_gcloud_list_descriptors.return_value = ( self._list_resources_get_result()) @@ -144,10 +144,10 @@ def _create_context(project_id): @staticmethod def _list_resources_get_result(): - all_labels = [google.cloud.monitoring.LabelDescriptor(**labels) + all_labels = [google.cloud.monitoring_v3.enums.LabelDescriptor(**labels) for labels in LABELS] descriptors = [ - google.cloud.monitoring.ResourceDescriptor( + google.cloud.monitoring_v3.types.MonitoredResourceDescriptor( name=None, type_=resource_type, display_name=display_name, description=None, labels=all_labels, )