diff --git a/docs/_includes/themes/zeppelin/_navigation.html b/docs/_includes/themes/zeppelin/_navigation.html index 4e49a1acb3a..f7091c60d77 100644 --- a/docs/_includes/themes/zeppelin/_navigation.html +++ b/docs/_includes/themes/zeppelin/_navigation.html @@ -88,6 +88,7 @@
  • Text
  • Html
  • Table
    • +
  • Network
  • Angular API
  • Angular (backend API)
    • diff --git a/docs/assets/themes/zeppelin/img/screenshots/display_complex_network.png b/docs/assets/themes/zeppelin/img/screenshots/display_complex_network.png new file mode 100644 index 00000000000..c788c26004d Binary files /dev/null and b/docs/assets/themes/zeppelin/img/screenshots/display_complex_network.png differ diff --git a/docs/assets/themes/zeppelin/img/screenshots/display_network.png b/docs/assets/themes/zeppelin/img/screenshots/display_network.png new file mode 100644 index 00000000000..516c85332f5 Binary files /dev/null and b/docs/assets/themes/zeppelin/img/screenshots/display_network.png differ diff --git a/docs/assets/themes/zeppelin/img/screenshots/display_network_flatten.png b/docs/assets/themes/zeppelin/img/screenshots/display_network_flatten.png new file mode 100644 index 00000000000..743e6666c6d Binary files /dev/null and b/docs/assets/themes/zeppelin/img/screenshots/display_network_flatten.png differ diff --git a/docs/assets/themes/zeppelin/img/screenshots/display_simple_network.png b/docs/assets/themes/zeppelin/img/screenshots/display_simple_network.png new file mode 100644 index 00000000000..86a08fe02a0 Binary files /dev/null and b/docs/assets/themes/zeppelin/img/screenshots/display_simple_network.png differ diff --git a/docs/displaysystem/basicdisplaysystem.md b/docs/displaysystem/basicdisplaysystem.md index 7c4243208e4..15cefefbb51 100644 --- a/docs/displaysystem/basicdisplaysystem.md +++ b/docs/displaysystem/basicdisplaysystem.md @@ -61,3 +61,104 @@ If table contents start with `%html`, it is interpreted as an HTML. > **Note :** Display system is backend independent. + +## Network + +With the `%network` directive, Zeppelin treats your output as a graph. Zeppelin can leverage the Property Graph Model. + +### What is the Labelled Property Graph Model? + +A [Property Graph](https://github.com/tinkerpop/gremlin/wiki/Defining-a-Property-Graph) is a graph that has these elements: + +* a set of vertices + * each vertex has a unique identifier. + * each vertex has a set of outgoing edges. + * each vertex has a set of incoming edges. + * each vertex has a collection of properties defined by a map from key to value +* a set of edges + * each edge has a unique identifier. + * each edge has an outgoing tail vertex. + * each edge has an incoming head vertex. + * each edge has a label that denotes the type of relationship between its two vertices. + * each edge has a collection of properties defined by a map from key to value. + + + +A [Labelled Property Graph](https://neo4j.com/developer/graph-database/#property-graph) is a Property Graph where the nodes can be tagged with **labels** representing their different roles in the graph model + + + +### What are the APIs? + +The new NETWORK visualization is based on json with the following params: + +* "nodes" (mandatory): list of nodes of the graph every node can have the following params: + * "id" (mandatory): the id of the node (must be unique); + * "label": the main Label of the node; + * "labels": the list of the labels of the node; + * "data": the data attached to the node; +* "edges": list of the edges of the graph; + * "id" (mandatory): the id of the edge (must be unique); + * "source" (mandatory): the id of source node of the edge; + * "target" (mandatory): the id of target node of the edge; + * "label": the main type of the edge; + * "data": the data attached to the edge; +* "labels": a map (K, V) where K is the node label and V is the color of the node; +* "directed": (true/false, default false) wich tells if is directed graph or not; +* "types": a *distinct* list of the edge types of the graph + +If you click on a node or edge on the bottom of the paragraph you find a list of entity properties + + + +This kind of graph can be easily *flatten* in order to support other visualization formats provided by Zeppelin. + + + +### How to use it? + +An example of a simple graph + +``` +%spark +print(s""" +%network { + "nodes": [ + {"id": 1}, + {"id": 2}, + {"id": 3} + ], + "edges": [ + {"source": 1, "target": 2, "id" : 1}, + {"source": 2, "target": 3, "id" : 2}, + {"source": 1, "target": 2, "id" : 3}, + {"source": 1, "target": 2, "id" : 4}, + {"source": 2, "target": 1, "id" : 5}, + {"source": 2, "target": 1, "id" : 6} + ] +} +""") +``` + +that will look like: + + + +A little more complex graph: + +``` +%spark +print(s""" +%network { + "nodes": [{"id": 1, "label": "User", "data": {"fullName":"Andrea Santurbano"}},{"id": 2, "label": "User", "data": {"fullName":"Lee Moon Soo"}},{"id": 3, "label": "Project", "data": {"name":"Zeppelin"}}], + "edges": [{"source": 2, "target": 1, "id" : 1, "label": "HELPS"},{"source": 2, "target": 3, "id" : 2, "label": "CREATE"},{"source": 1, "target": 3, "id" : 3, "label": "CONTRIBUTE_TO", "data": {"oldPR": "https://github.com/apache/zeppelin/pull/1582"}}], + "labels": {"User": "#8BC34A", "Project": "#3071A9"}, + "directed": true, + "types": ["HELPS", "CREATE", "CONTRIBUTE_TO"] +} +""") +``` + +that will look like: + + \ No newline at end of file diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResult.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResult.java index 23164906626..7aab8a3e1e8 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResult.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterResult.java @@ -50,7 +50,8 @@ public static enum Type { TABLE, IMG, SVG, - NULL + NULL, + NETWORK } Code code; diff --git a/zeppelin-web/.eslintrc b/zeppelin-web/.eslintrc index 1fe3fa5e7de..6dca5c8982b 100644 --- a/zeppelin-web/.eslintrc +++ b/zeppelin-web/.eslintrc @@ -58,6 +58,7 @@ "no-undef": 2, "no-unused-vars": [2, { "vars": "local", "args": "none" }], "strict": [2, "global"], - "max-len": [2, {"code": 120, "ignoreComments": true, "ignoreRegExpLiterals": true}] + "max-len": [2, {"code": 120, "ignoreComments": true, "ignoreRegExpLiterals": true}], + "linebreak-style": 0 } } diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.css b/zeppelin-web/src/app/notebook/paragraph/paragraph.css index b17acf74fb3..b63794d2bc2 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.css +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.css @@ -557,3 +557,7 @@ table.table-striped { .markdown-body h4 { font-size: 16px; } + +.network-labels { + margin: 0.2em; +} diff --git a/zeppelin-web/src/app/notebook/paragraph/result/result-chart-selector.html b/zeppelin-web/src/app/notebook/paragraph/result/result-chart-selector.html index 4becdc6f22a..9592d809d34 100644 --- a/zeppelin-web/src/app/notebook/paragraph/result/result-chart-selector.html +++ b/zeppelin-web/src/app/notebook/paragraph/result/result-chart-selector.html @@ -13,12 +13,13 @@ -->
    -
    +
    + +
    + +
    + +
    + +
    \ No newline at end of file diff --git a/zeppelin-web/src/app/tabledata/networkdata.js b/zeppelin-web/src/app/tabledata/networkdata.js new file mode 100644 index 00000000000..7983d827265 --- /dev/null +++ b/zeppelin-web/src/app/tabledata/networkdata.js @@ -0,0 +1,145 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import TableData from './tabledata' +import {DatasetType} from './dataset' + +/** + * Create network data object from paragraph graph type result + */ +export default class NetworkData extends TableData { + constructor(graph) { + super() + this.graph = graph || {} + if (this.graph.nodes) { + this.loadParagraphResult({msg: JSON.stringify(graph), type: DatasetType.NETWORK}) + } + } + + loadParagraphResult(paragraphResult) { + if (!paragraphResult || paragraphResult.type !== DatasetType.NETWORK) { + console.log('Can not load paragraph result') + return + } + + this.graph = JSON.parse(paragraphResult.msg.trim() || '{}') + + if (!this.graph.nodes) { + console.log('Graph result is empty') + return + } + + this.setNodesDefaults() + this.setEdgesDefaults() + + this.networkNodes = angular.equals({}, this.graph.labels || {}) + ? null : {count: this.graph.nodes.length, labels: this.graph.labels} + this.networkRelationships = angular.equals([], this.graph.types || []) + ? null : {count: this.graph.edges.length, types: this.graph.types} + + let rows = [] + let comment = '' + let entities = this.graph.nodes.concat(this.graph.edges) + let baseColumnNames = [{name: 'id', index: 0, aggr: 'sum'}, + {name: 'label', index: 1, aggr: 'sum'}] + let internalFieldsToJump = ['count', 'size', 'totalCount', + 'data', 'x', 'y', 'labels'] + let baseCols = _.map(baseColumnNames, function(col) { return col.name }) + let keys = _.map(entities, function(elem) { return Object.keys(elem.data || {}) }) + keys = _.flatten(keys) + keys = _.uniq(keys).filter(function(key) { + return baseCols.indexOf(key) === -1 + }) + let columnNames = baseColumnNames.concat(_.map(keys, function(elem, i) { + return {name: elem, index: i + baseColumnNames.length, aggr: 'sum'} + })) + for (let i = 0; i < entities.length; i++) { + let entity = entities[i] + let col = [] + let col2 = [] + entity.data = entity.data || {} + for (let j = 0; j < columnNames.length; j++) { + let name = columnNames[j].name + let value = name in entity && internalFieldsToJump.indexOf(name) === -1 + ? entity[name] : entity.data[name] + let parsedValue = value === null || value === undefined ? '' : value + col.push(parsedValue) + col2.push({key: name, value: parsedValue}) + } + rows.push(col) + } + + this.comment = comment + this.columns = columnNames + this.rows = rows + } + + setNodesDefaults() { + } + + setEdgesDefaults() { + this.graph.edges + .sort((a, b) => { + if (a.source > b.source) { + return 1 + } else if (a.source < b.source) { + return -1 + } else if (a.target > b.target) { + return 1 + } else if (a.target < b.target) { + return -1 + } else { + return 0 + } + }) + this.graph.edges + .forEach((edge, index) => { + let prevEdge = this.graph.edges[index - 1] + edge.count = (index > 0 && +edge.source === +prevEdge.source && +edge.target === +prevEdge.target + ? prevEdge.count : 0) + 1 + edge.totalCount = this.graph.edges + .filter((innerEdge) => +edge.source === +innerEdge.source && +edge.target === +innerEdge.target) + .length + }) + this.graph.edges + .forEach((edge) => { + if (typeof +edge.source === 'number') { + edge.source = this.graph.nodes.filter((node) => +edge.source === +node.id)[0] || null + } + if (typeof +edge.target === 'number') { + edge.target = this.graph.nodes.filter((node) => +edge.target === +node.id)[0] || null + } + }) + } + + getNetworkProperties() { + let baseCols = ['id', 'label'] + let properties = {} + this.graph.nodes.forEach(function(node) { + let hasLabel = 'label' in node && node.label !== '' + if (!hasLabel) { + return + } + let label = node.label + let hasKey = hasLabel && label in properties + let keys = _.uniq(Object.keys(node.data || {}) + .concat(hasKey ? properties[label].keys : baseCols)) + if (!hasKey) { + properties[label] = {selected: 'label'} + } + properties[label].keys = keys + }) + return properties + } +} diff --git a/zeppelin-web/src/app/tabledata/networkdata.test.js b/zeppelin-web/src/app/tabledata/networkdata.test.js new file mode 100644 index 00000000000..f8d98a89a3f --- /dev/null +++ b/zeppelin-web/src/app/tabledata/networkdata.test.js @@ -0,0 +1,46 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import NetworkData from './networkdata.js' +import {DatasetType} from './dataset.js' + +describe('NetworkData build', function() { + let nd + + beforeEach(function() { + nd = new NetworkData() + }) + + it('should initialize the default value', function() { + expect(nd.columns.length).toBe(0) + expect(nd.rows.length).toBe(0) + expect(nd.graph).toEqual({}) + }) + + it('should able to create NetowkData from paragraph result', function() { + let jsonExpected = {nodes: [{id: 1}, {id: 2}], edges: [{source: 2, target: 1, id: 1}]} + nd.loadParagraphResult({ + type: DatasetType.NETWORK, + msg: JSON.stringify(jsonExpected) + }) + + expect(nd.columns.length).toBe(2) + expect(nd.rows.length).toBe(3) + expect(nd.graph.nodes[0].id).toBe(jsonExpected.nodes[0].id) + expect(nd.graph.nodes[1].id).toBe(jsonExpected.nodes[1].id) + expect(nd.graph.edges[0].id).toBe(jsonExpected.edges[0].id) + expect(nd.graph.edges[0].source.id).toBe(jsonExpected.nodes[1].id) + expect(nd.graph.edges[0].target.id).toBe(jsonExpected.nodes[0].id) + }) +}) diff --git a/zeppelin-web/src/app/tabledata/tabledata.js b/zeppelin-web/src/app/tabledata/tabledata.js index 8e4e6b6625e..3fe01b7791c 100644 --- a/zeppelin-web/src/app/tabledata/tabledata.js +++ b/zeppelin-web/src/app/tabledata/tabledata.js @@ -11,19 +11,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import {Dataset, DatasetType} from './dataset' /** * Create table data object from paragraph table type result */ -export default class TableData { +export default class TableData extends Dataset { constructor (columns, rows, comment) { + super() this.columns = columns || [] this.rows = rows || [] this.comment = comment || '' } loadParagraphResult (paragraphResult) { - if (!paragraphResult || paragraphResult.type !== 'TABLE') { + if (!paragraphResult || paragraphResult.type !== DatasetType.TABLE) { console.log('Can not load paragraph result') return } diff --git a/zeppelin-web/src/app/visualization/builtins/visualization-d3network.js b/zeppelin-web/src/app/visualization/builtins/visualization-d3network.js new file mode 100644 index 00000000000..506b1c5f186 --- /dev/null +++ b/zeppelin-web/src/app/visualization/builtins/visualization-d3network.js @@ -0,0 +1,263 @@ +/* + * Licensed under the Apache License, Version 2.0 (the 'License'); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an 'AS IS' BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Visualization from '../visualization' +import NetworkTransformation from '../../tabledata/network' + +/** + * Visualize data in network format + */ +export default class NetworkVisualization extends Visualization { + constructor(targetEl, config) { + super(targetEl, config) + console.log('Init network viz') + if (!config.properties) { + config.properties = {} + } + if (!config.d3Graph) { + config.d3Graph = { + forceLayout: { + timeout: 10000, + charge: -120, + linkDistance: 80, + }, + zoom: { + minScale: 1.3 + } + } + } + this.targetEl.addClass('network') + this.containerId = this.targetEl.prop('id') + this.force = null + this.svg = null + this.$timeout = angular.injector(['ng']).get('$timeout') + this.$interpolate = angular.injector(['ng']).get('$interpolate') + this.transformation = new NetworkTransformation(config) + } + + refresh() { + console.log('refresh') + } + + render(networkData) { + if (!('graph' in networkData)) { + console.log('graph not found') + return + } + console.log('Render Graph Visualization') + + let transformationConfig = this.transformation.getSetting().scope.config + console.log('cfg', transformationConfig) + if (transformationConfig && angular.equals({}, transformationConfig.properties)) { + transformationConfig.properties = networkData.getNetworkProperties() + } + + this.targetEl.empty().append('') + + let width = this.targetEl.width() + let height = this.targetEl.height() + let self = this + let defaultOpacity = 0 + let nodeSize = 10 + let textOffset = 3 + let linkSize = 10 + + let arcPath = (leftHand, d) => { + let start = leftHand ? d.source : d.target + let end = leftHand ? d.target : d.source + let dx = end.x - start.x + let dy = end.y - start.y + let dr = d.totalCount === 1 + ? 0 : Math.sqrt(Math.pow(dx, 2) + Math.pow(dy, 2)) / (1 + (1 / d.totalCount) * (d.count - 1)) + let sweep = leftHand ? 0 : 1 + return `M${start.x},${start.y}A${dr},${dr} 0 0,${sweep} ${end.x},${end.y}` + } + // Use elliptical arc path segments to doubly-encode directionality. + let tick = () => { + // Links + linkPath.attr('d', function(d) { + return arcPath(true, d) + }) + textPath.attr('d', function(d) { + return arcPath(d.source.x < d.target.x, d) + }) + // Nodes + circle.attr('transform', (d) => `translate(${d.x},${d.y})`) + text.attr('transform', (d) => `translate(${d.x},${d.y})`) + } + + let setOpacity = (scale) => { + let opacity = scale >= +transformationConfig.d3Graph.zoom.minScale ? 1 : 0 + this.svg.selectAll('.nodeLabel') + .style('opacity', opacity) + this.svg.selectAll('textPath') + .style('opacity', opacity) + } + + let zoom = d3.behavior.zoom() + .scaleExtent([1, 10]) + .on('zoom', () => { + console.log('zoom') + setOpacity(d3.event.scale) + container.attr('transform', `translate(${d3.event.translate})scale(${d3.event.scale})`) + }) + + this.svg = d3.select(`#${this.containerId} svg`) + .attr('width', width) + .attr('height', height) + .call(zoom) + + this.force = d3.layout.force() + .charge(transformationConfig.d3Graph.forceLayout.charge) + .linkDistance(transformationConfig.d3Graph.forceLayout.linkDistance) + .on('tick', tick) + .nodes(networkData.graph.nodes) + .links(networkData.graph.edges) + .size([width, height]) + .on('start', () => { + console.log('force layout start') + this.$timeout(() => { this.force.stop() }, transformationConfig.d3Graph.forceLayout.timeout) + }) + .on('end', () => { + console.log('force layout stop') + setOpacity(zoom.scale()) + }) + .start() + + let renderFooterOnClick = (entity, type) => { + let footerId = this.containerId + '_footer' + let obj = {id: entity.id, label: entity.defaultLabel || entity.label, type: type} + let html = [this.$interpolate(['
  • {{type}}_id: {{id}}
    • ', + '
  • {{type}}_type: {{label}}
    • '].join(''))(obj)] + html = html.concat(_.map(entity.data, (v, k) => { + return this.$interpolate('
  • {{field}}: {{value}}
    • ')({field: k, value: v}) + })) + angular.element('#' + footerId) + .find('.list-inline') + .empty() + .append(html.join('')) + } + + let drag = d3.behavior.drag() + .origin((d) => d) + .on('dragstart', function(d) { + console.log('dragstart') + d3.event.sourceEvent.stopPropagation() + d3.select(this).classed('dragging', true) + self.force.stop() + }) + .on('drag', function(d) { + console.log('drag') + d.px += d3.event.dx + d.py += d3.event.dy + d.x += d3.event.dx + d.y += d3.event.dy + }) + .on('dragend', function(d) { + console.log('dragend') + d.fixed = true + d3.select(this).classed('dragging', false) + self.force.resume() + }) + + let container = this.svg.append('g') + if (networkData.graph.directed) { + container.append('svg:defs').selectAll('marker') + .data(['arrowMarker-' + this.containerId]) + .enter() + .append('svg:marker') + .attr('id', String) + .attr('viewBox', '0 -5 10 10') + .attr('refX', 16) + .attr('refY', 0) + .attr('markerWidth', 4) + .attr('markerHeight', 4) + .attr('orient', 'auto') + .append('svg:path') + .attr('d', 'M0,-5L10,0L0,5') + } + // Links + let link = container.append('svg:g') + .on('click', () => { + renderFooterOnClick(d3.select(d3.event.target).datum(), 'edge') + }) + .selectAll('g.link') + .data(self.force.links()) + .enter() + .append('g') + let getPathId = (d) => this.containerId + '_' + d.source.index + '_' + d.target.index + '_' + d.count + let showLabel = (d) => this._showNodeLabel(d) + let linkPath = link.append('svg:path') + .attr('class', 'link') + .attr('size', linkSize) + .attr('marker-end', `url(#arrowMarker-${this.containerId})`) + let textPath = link.append('svg:path') + .attr('id', getPathId) + .attr('class', 'textpath') + container.append('svg:g') + .selectAll('.pathLabel') + .data(self.force.links()) + .enter() + .append('svg:text') + .attr('class', 'pathLabel') + .append('svg:textPath') + .attr('startOffset', '50%') + .attr('text-anchor', 'middle') + .attr('xlink:href', (d) => '#' + getPathId(d)) + .text((d) => d.label) + .style('opacity', defaultOpacity) + // Nodes + let circle = container.append('svg:g') + .on('click', () => { + renderFooterOnClick(d3.select(d3.event.target).datum(), 'node') + }) + .selectAll('circle') + .data(self.force.nodes()) + .enter().append('svg:circle') + .attr('r', (d) => nodeSize) + .attr('fill', (d) => networkData.graph.labels && d.label in networkData.graph.labels + ? networkData.graph.labels[d.label] : '#000000') + .call(drag) + let text = container.append('svg:g').selectAll('g') + .data(self.force.nodes()) + .enter().append('svg:g') + text.append('svg:text') + .attr('x', (d) => nodeSize + textOffset) + .attr('size', nodeSize) + .attr('y', '.31em') + .attr('class', (d) => 'nodeLabel shadow label-' + d.label) + .text(showLabel) + .style('opacity', defaultOpacity) + text.append('svg:text') + .attr('x', (d) => nodeSize + textOffset) + .attr('size', nodeSize) + .attr('y', '.31em') + .attr('class', (d) => 'nodeLabel label-' + d.label) + .text(showLabel) + .style('opacity', defaultOpacity) + } + + destroy() { + } + + _showNodeLabel(d) { + let transformationConfig = this.transformation.getSetting().scope.config + let selectedLabel = (transformationConfig.properties[d.label] || {selected: 'label'}).selected + return d.data[selectedLabel] || d[selectedLabel] + } + + getTransformation() { + return this.transformation + } +} diff --git a/zeppelin-web/src/components/resizable/resizable.directive.js b/zeppelin-web/src/components/resizable/resizable.directive.js index f0eed7604bf..7bf84772945 100644 --- a/zeppelin-web/src/components/resizable/resizable.directive.js +++ b/zeppelin-web/src/components/resizable/resizable.directive.js @@ -35,7 +35,7 @@ function resizable () { let colStep = window.innerWidth / 12 elem.off('resizestop') let conf = angular.copy(resizableConfig) - if (resize.graphType === 'TABLE' || resize.graphType === 'TEXT') { + if (resize.graphType === 'TABLE' || resize.graphType === 'NETWORK' || resize.graphType === 'TEXT') { conf.grid = [colStep, 10] conf.minHeight = 100 } else {