Skip to content

Druid v0.6 compatibility #2331

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
dimaslv wants to merge 5 commits into from
Closed

Druid v0.6 compatibility #2331

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a9e832e
Do not fail on version check on old Druid
Mar 2, 2017
246c62c
Fix metrics for Druid v0.6
Mar 3, 2017
9e5678d
Compare Druid version with something more reasonable
Mar 9, 2017
18ba45b
Style fixes
Mar 9, 2017
cfa0f58
Merge branch 'master' into master
dimaslv Mar 9, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 51 additions & 15 deletions superset/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1650,7 +1650,11 @@ def get_druid_version(self):
endpoint = (
"http://{obj.coordinator_host}:{obj.coordinator_port}/status"
).format(obj=self)
return json.loads(requests.get(endpoint).text)['version']
ver = json.loads(requests.get(endpoint).text)['version']
Copy link
Member

@mistercrunch mistercrunch Mar 3, 2017 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd prefer an approach where we can specify the druid_version as a configuration element. I would have to be associated with the DruidCluster model. Here we'd use the information defined manually and fallback on this current approach when not specified.

Copy link
Author

@dimaslv dimaslv Mar 9, 2017 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My programmer skills are quite weak and my knowledge about superset architecture is even more weak. Can you point me on how do I do that? Or maybe you can help me by coding this?

if ver is None:
return "0"
else:
return ver

def refresh_datasources(self, datasource_name=None, merge_flag=False):
"""Refresh metadata of all datasources in the cluster
Expand Down Expand Up @@ -1715,7 +1719,7 @@ def dimension_spec(self):
if self.dimension_spec_json:
return json.loads(self.dimension_spec_json)

def generate_metrics(self):
def generate_metrics(self, ver):
"""Generate metrics based on the column metadata"""
M = DruidMetric # noqa
metrics = []
Expand Down Expand Up @@ -1743,18 +1747,42 @@ def generate_metrics(self):
))

if self.avg and self.is_num:
mt = corrected_type.lower() + 'Avg'
name = 'avg__' + self.column_name
metrics.append(DruidMetric(
metric_name=name,
metric_type='avg',
verbose_name='AVG({})'.format(self.column_name),
json=json.dumps({
'type': mt, 'name': name, 'fieldName': self.column_name})
))
if ver >= '0.7.':
mt = corrected_type.lower() + 'Avg'
name = 'avg__' + self.column_name
metrics.append(DruidMetric(
metric_name=name,
metric_type='avg',
verbose_name='AVG({})'.format(self.column_name),
json=json.dumps({
'type': mt,
'name': name,
'fieldName': self.column_name})
))
else:
name = 'avg__' + self.column_name
metrics.append(DruidMetric(
metric_name=name,
metric_type='postagg',
verbose_name='AVG({})'.format(self.column_name),
json=json.dumps({
'type': 'arithmetic',
'name': name,
'fn': '/',
'fields': [
{'type': 'fieldAccess',
'fieldName': 'sum__' + self.column_name},
{'type': 'fieldAccess',
'fieldName': 'count'}
]
})
))

if self.min and self.is_num:
mt = corrected_type.lower() + 'Min'
if ver >= '0.7.':
mt = corrected_type.lower() + 'Min'
else:
mt = 'min'
name = 'min__' + self.column_name
metrics.append(DruidMetric(
metric_name=name,
Expand All @@ -1764,7 +1792,10 @@ def generate_metrics(self):
'type': mt, 'name': name, 'fieldName': self.column_name})
))
if self.max and self.is_num:
mt = corrected_type.lower() + 'Max'
if ver >= '0.7.':
mt = corrected_type.lower() + 'Max'
else:
mt = 'max'
name = 'max__' + self.column_name
metrics.append(DruidMetric(
metric_name=name,
Expand Down Expand Up @@ -2093,7 +2124,7 @@ def latest_metadata(self):

def generate_metrics(self):
for col in self.columns:
col.generate_metrics()
col.generate_metrics(self.cluster.druid_version)

@classmethod
def sync_to_db_from_config(cls, druid_config, user, cluster):
Expand Down Expand Up @@ -2207,9 +2238,14 @@ def sync_to_db(cls, name, cluster, merge):
col_obj.count_distinct = True
if col_obj:
col_obj.type = cols[col]['type']
if datatype != "STRING":
col_obj.avg = "1"
col_obj.min = "1"
col_obj.max = "1"
col_obj.sum = "1"
session.flush()
col_obj.datasource = datasource
col_obj.generate_metrics()
col_obj.generate_metrics(cluster.druid_version)
session.flush()

@staticmethod
Expand Down