From 7424af247b08afabbd9d080a288a88a7de4b7a1c Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Fri, 4 Mar 2016 22:08:19 -0800 Subject: [PATCH 01/69] Remove resource on note / paragraph removal --- .../thrift/RemoteInterpreterContext.java | 17 + .../thrift/RemoteInterpreterEvent.java | 17 + .../thrift/RemoteInterpreterEventType.java | 17 + .../thrift/RemoteInterpreterResult.java | 17 + .../thrift/RemoteInterpreterService.java | 997 ++++++++++++++++++ .../zeppelin/resource/ResourcePoolUtils.java | 1 + .../interpreter/mock/MockInterpreter1.java | 70 +- 7 files changed, 1101 insertions(+), 35 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterContext.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterContext.java index 889e45d7d99..097f762ce7d 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterContext.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterContext.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ /** * Autogenerated by Thrift Compiler (0.9.2) * diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java index c89a287abbe..e42214f7026 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ /** * Autogenerated by Thrift Compiler (0.9.2) * diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java index 8db330a465c..66631d2f2a8 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ /** * Autogenerated by Thrift Compiler (0.9.2) * diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterResult.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterResult.java index 7ed20f6b697..082df0525b4 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterResult.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterResult.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ /** * Autogenerated by Thrift Compiler (0.9.2) * diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterService.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterService.java index 3f26b794e36..84ede734ea1 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterService.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterService.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ /** * Autogenerated by Thrift Compiler (0.9.2) * @@ -14112,6 +14129,54 @@ public String getResourceName() { return this.resourceName; } + public resourceGet_args setNoteId(String noteId) { + this.noteId = noteId; + return this; + } + + public void unsetNoteId() { + this.noteId = null; + } + + /** Returns true if field noteId is set (has been assigned a value) and false otherwise */ + public boolean isSetNoteId() { + return this.noteId != null; + } + + public void setNoteIdIsSet(boolean value) { + if (!value) { + this.noteId = null; + } + } + + public String getParagraphId() { + return this.paragraphId; + } + + public resourceGet_args setParagraphId(String paragraphId) { + this.paragraphId = paragraphId; + return this; + } + + public void unsetParagraphId() { + this.paragraphId = null; + } + + /** Returns true if field paragraphId is set (has been assigned a value) and false otherwise */ + public boolean isSetParagraphId() { + return this.paragraphId != null; + } + + public void setParagraphIdIsSet(boolean value) { + if (!value) { + this.paragraphId = null; + } + } + + public String getResourceName() { + return this.resourceName; + } + public resourceGet_args setResourceName(String resourceName) { this.resourceName = resourceName; return this; @@ -15799,6 +15864,938 @@ public void read(org.apache.thrift.protocol.TProtocol prot, resourceRemove_resul } + public static class angularObjectUpdate_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("angularObjectUpdate_args"); + + private static final org.apache.thrift.protocol.TField NOTE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("noteId", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField PARAGRAPH_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("paragraphId", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField RESOURCE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("resourceName", org.apache.thrift.protocol.TType.STRING, (short)3); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new resourceRemove_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new resourceRemove_argsTupleSchemeFactory()); + } + + public String noteId; // required + public String paragraphId; // required + public String resourceName; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + NOTE_ID((short)1, "noteId"), + PARAGRAPH_ID((short)2, "paragraphId"), + RESOURCE_NAME((short)3, "resourceName"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // NOTE_ID + return NOTE_ID; + case 2: // PARAGRAPH_ID + return PARAGRAPH_ID; + case 3: // RESOURCE_NAME + return RESOURCE_NAME; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.NOTE_ID, new org.apache.thrift.meta_data.FieldMetaData("noteId", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.PARAGRAPH_ID, new org.apache.thrift.meta_data.FieldMetaData("paragraphId", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.RESOURCE_NAME, new org.apache.thrift.meta_data.FieldMetaData("resourceName", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(resourceRemove_args.class, metaDataMap); + } + + public resourceRemove_args() { + } + + public resourceRemove_args( + String noteId, + String paragraphId, + String resourceName) + { + this(); + this.noteId = noteId; + this.paragraphId = paragraphId; + this.resourceName = resourceName; + } + + /** + * Performs a deep copy on other. + */ + public resourceRemove_args(resourceRemove_args other) { + if (other.isSetNoteId()) { + this.noteId = other.noteId; + } + if (other.isSetParagraphId()) { + this.paragraphId = other.paragraphId; + } + if (other.isSetResourceName()) { + this.resourceName = other.resourceName; + } + } + + public resourceRemove_args deepCopy() { + return new resourceRemove_args(this); + } + + @Override + public void clear() { + this.noteId = null; + this.paragraphId = null; + this.resourceName = null; + } + + public String getNoteId() { + return this.noteId; + } + + public resourceRemove_args setNoteId(String noteId) { + this.noteId = noteId; + return this; + } + + public void unsetNoteId() { + this.noteId = null; + } + + /** Returns true if field noteId is set (has been assigned a value) and false otherwise */ + public boolean isSetNoteId() { + return this.noteId != null; + } + + public void setNoteIdIsSet(boolean value) { + if (!value) { + this.noteId = null; + } + } + + public String getParagraphId() { + return this.paragraphId; + } + + public resourceRemove_args setParagraphId(String paragraphId) { + this.paragraphId = paragraphId; + return this; + } + + public void unsetParagraphId() { + this.paragraphId = null; + } + + /** Returns true if field paragraphId is set (has been assigned a value) and false otherwise */ + public boolean isSetParagraphId() { + return this.paragraphId != null; + } + + public void setParagraphIdIsSet(boolean value) { + if (!value) { + this.paragraphId = null; + } + } + + public String getResourceName() { + return this.resourceName; + } + + public resourceRemove_args setResourceName(String resourceName) { + this.resourceName = resourceName; + return this; + } + + public void unsetResourceName() { + this.resourceName = null; + } + + /** Returns true if field resourceName is set (has been assigned a value) and false otherwise */ + public boolean isSetResourceName() { + return this.resourceName != null; + } + + public void setResourceNameIsSet(boolean value) { + if (!value) { + this.resourceName = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case NOTE_ID: + if (value == null) { + unsetNoteId(); + } else { + setNoteId((String)value); + } + break; + + case PARAGRAPH_ID: + if (value == null) { + unsetParagraphId(); + } else { + setParagraphId((String)value); + } + break; + + case RESOURCE_NAME: + if (value == null) { + unsetResourceName(); + } else { + setResourceName((String)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case NOTE_ID: + return getNoteId(); + + case PARAGRAPH_ID: + return getParagraphId(); + + case RESOURCE_NAME: + return getResourceName(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case NOTE_ID: + return isSetNoteId(); + case PARAGRAPH_ID: + return isSetParagraphId(); + case RESOURCE_NAME: + return isSetResourceName(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof resourceRemove_args) + return this.equals((resourceRemove_args)that); + return false; + } + + public boolean equals(resourceRemove_args that) { + if (that == null) + return false; + + boolean this_present_noteId = true && this.isSetNoteId(); + boolean that_present_noteId = true && that.isSetNoteId(); + if (this_present_noteId || that_present_noteId) { + if (!(this_present_noteId && that_present_noteId)) + return false; + if (!this.noteId.equals(that.noteId)) + return false; + } + + boolean this_present_paragraphId = true && this.isSetParagraphId(); + boolean that_present_paragraphId = true && that.isSetParagraphId(); + if (this_present_paragraphId || that_present_paragraphId) { + if (!(this_present_paragraphId && that_present_paragraphId)) + return false; + if (!this.paragraphId.equals(that.paragraphId)) + return false; + } + + boolean this_present_resourceName = true && this.isSetResourceName(); + boolean that_present_resourceName = true && that.isSetResourceName(); + if (this_present_resourceName || that_present_resourceName) { + if (!(this_present_resourceName && that_present_resourceName)) + return false; + if (!this.resourceName.equals(that.resourceName)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + List list = new ArrayList(); + + boolean present_noteId = true && (isSetNoteId()); + list.add(present_noteId); + if (present_noteId) + list.add(noteId); + + boolean present_paragraphId = true && (isSetParagraphId()); + list.add(present_paragraphId); + if (present_paragraphId) + list.add(paragraphId); + + boolean present_resourceName = true && (isSetResourceName()); + list.add(present_resourceName); + if (present_resourceName) + list.add(resourceName); + + return list.hashCode(); + } + + @Override + public int compareTo(resourceRemove_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = Boolean.valueOf(isSetNoteId()).compareTo(other.isSetNoteId()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetNoteId()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.noteId, other.noteId); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetParagraphId()).compareTo(other.isSetParagraphId()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetParagraphId()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.paragraphId, other.paragraphId); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetResourceName()).compareTo(other.isSetResourceName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetResourceName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.resourceName, other.resourceName); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("resourceRemove_args("); + boolean first = true; + + sb.append("noteId:"); + if (this.noteId == null) { + sb.append("null"); + } else { + sb.append(this.noteId); + } + first = false; + if (!first) sb.append(", "); + sb.append("paragraphId:"); + if (this.paragraphId == null) { + sb.append("null"); + } else { + sb.append(this.paragraphId); + } + first = false; + if (!first) sb.append(", "); + sb.append("resourceName:"); + if (this.resourceName == null) { + sb.append("null"); + } else { + sb.append(this.resourceName); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class resourceRemove_argsStandardSchemeFactory implements SchemeFactory { + public resourceRemove_argsStandardScheme getScheme() { + return new resourceRemove_argsStandardScheme(); + } + } + + private static class resourceRemove_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, resourceRemove_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // NOTE_ID + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.noteId = iprot.readString(); + struct.setNoteIdIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // PARAGRAPH_ID + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.paragraphId = iprot.readString(); + struct.setParagraphIdIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // RESOURCE_NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.resourceName = iprot.readString(); + struct.setResourceNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, resourceRemove_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.noteId != null) { + oprot.writeFieldBegin(NOTE_ID_FIELD_DESC); + oprot.writeString(struct.noteId); + oprot.writeFieldEnd(); + } + if (struct.paragraphId != null) { + oprot.writeFieldBegin(PARAGRAPH_ID_FIELD_DESC); + oprot.writeString(struct.paragraphId); + oprot.writeFieldEnd(); + } + if (struct.resourceName != null) { + oprot.writeFieldBegin(RESOURCE_NAME_FIELD_DESC); + oprot.writeString(struct.resourceName); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class resourceRemove_argsTupleSchemeFactory implements SchemeFactory { + public resourceRemove_argsTupleScheme getScheme() { + return new resourceRemove_argsTupleScheme(); + } + } + + private static class resourceRemove_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, resourceRemove_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetNoteId()) { + optionals.set(0); + } + if (struct.isSetParagraphId()) { + optionals.set(1); + } + if (struct.isSetResourceName()) { + optionals.set(2); + } + oprot.writeBitSet(optionals, 3); + if (struct.isSetNoteId()) { + oprot.writeString(struct.noteId); + } + if (struct.isSetParagraphId()) { + oprot.writeString(struct.paragraphId); + } + if (struct.isSetResourceName()) { + oprot.writeString(struct.resourceName); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, resourceRemove_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(3); + if (incoming.get(0)) { + struct.noteId = iprot.readString(); + struct.setNoteIdIsSet(true); + } + if (incoming.get(1)) { + struct.paragraphId = iprot.readString(); + struct.setParagraphIdIsSet(true); + } + if (incoming.get(2)) { + struct.resourceName = iprot.readString(); + struct.setResourceNameIsSet(true); + } + } + } + + } + + public static class resourceRemove_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("resourceRemove_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.BOOL, (short)0); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new resourceRemove_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new resourceRemove_resultTupleSchemeFactory()); + } + + public boolean success; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __SUCCESS_ISSET_ID = 0; + private byte __isset_bitfield = 0; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(resourceRemove_result.class, metaDataMap); + } + + public resourceRemove_result() { + } + + public resourceRemove_result( + boolean success) + { + this(); + this.success = success; + setSuccessIsSet(true); + } + + /** + * Performs a deep copy on other. + */ + public resourceRemove_result(resourceRemove_result other) { + __isset_bitfield = other.__isset_bitfield; + this.success = other.success; + } + + public resourceRemove_result deepCopy() { + return new resourceRemove_result(this); + } + + @Override + public void clear() { + setSuccessIsSet(false); + this.success = false; + } + + public boolean isSuccess() { + return this.success; + } + + public resourceRemove_result setSuccess(boolean success) { + this.success = success; + setSuccessIsSet(true); + return this; + } + + public void unsetSuccess() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __SUCCESS_ISSET_ID); + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return EncodingUtils.testBit(__isset_bitfield, __SUCCESS_ISSET_ID); + } + + public void setSuccessIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __SUCCESS_ISSET_ID, value); + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((Boolean)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return Boolean.valueOf(isSuccess()); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof resourceRemove_result) + return this.equals((resourceRemove_result)that); + return false; + } + + public boolean equals(resourceRemove_result that) { + if (that == null) + return false; + + boolean this_present_success = true; + boolean that_present_success = true; + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (this.success != that.success) + return false; + } + + return true; + } + + @Override + public int hashCode() { + List list = new ArrayList(); + + boolean present_success = true; + list.add(present_success); + if (present_success) + list.add(success); + + return list.hashCode(); + } + + @Override + public int compareTo(resourceRemove_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("resourceRemove_result("); + boolean first = true; + + sb.append("success:"); + sb.append(this.success); + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bitfield = 0; + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class resourceRemove_resultStandardSchemeFactory implements SchemeFactory { + public resourceRemove_resultStandardScheme getScheme() { + return new resourceRemove_resultStandardScheme(); + } + } + + private static class resourceRemove_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, resourceRemove_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) { + struct.success = iprot.readBool(); + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, resourceRemove_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.isSetSuccess()) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + oprot.writeBool(struct.success); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class resourceRemove_resultTupleSchemeFactory implements SchemeFactory { + public resourceRemove_resultTupleScheme getScheme() { + return new resourceRemove_resultTupleScheme(); + } + } + + private static class resourceRemove_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, resourceRemove_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetSuccess()) { + oprot.writeBool(struct.success); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, resourceRemove_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.success = iprot.readBool(); + struct.setSuccessIsSet(true); + } + } + } + + } + public static class angularObjectUpdate_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("angularObjectUpdate_args"); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java index 1825bfed217..3ad8ff95624 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java @@ -134,3 +134,4 @@ public static void removeResourcesBelongsToParagraph(String noteId, String parag } } } + diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java index cf0a61383cc..0333f7b07bc 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/mock/MockInterpreter1.java @@ -1,19 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ package org.apache.zeppelin.interpreter.mock; @@ -29,30 +29,30 @@ import org.apache.zeppelin.scheduler.SchedulerFactory; public class MockInterpreter1 extends Interpreter{ - Map vars = new HashMap(); +Map vars = new HashMap(); - public MockInterpreter1(Properties property) { - super(property); - } +public MockInterpreter1(Properties property) { + super(property); +} - @Override - public void open() { - } +@Override +public void open() { +} - @Override - public void close() { - } +@Override +public void close() { +} - @Override - public InterpreterResult interpret(String st, InterpreterContext context) { - InterpreterResult result; - - if ("getId".equals(st)) { - // get unique id of this interpreter instance - result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "" + this.hashCode()); - } else { - result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl1: " + st); - } +@Override +public InterpreterResult interpret(String st, InterpreterContext context) { + InterpreterResult result; + + if ("getId".equals(st)) { + // get unique id of this interpreter instance + result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "" + this.hashCode()); + } else { + result = new InterpreterResult(InterpreterResult.Code.SUCCESS, "repl1: " + st); + } if (context.getResourcePool() != null) { context.getResourcePool().put(context.getNoteId(), context.getParagraphId(), "result", result); From 568ee541db31133ccd8f2d99d16ede60cb9433c1 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sun, 6 Mar 2016 18:55:17 -0800 Subject: [PATCH 02/69] ApplicationLoader --- .../apache/zeppelin/helium/Application.java | 53 ++++ .../zeppelin/helium/ApplicationContext.java | 38 +++ .../zeppelin/helium/ApplicationException.java | 34 +++ .../zeppelin/helium/ApplicationLoader.java | 266 ++++++++++++++++++ .../helium/ClassLoaderApplication.java | 56 ++++ .../zeppelin/helium/HeliumPackageInfo.java | 89 ++++++ .../zeppelin/helium/HeliumRegistry.java | 35 +++ .../helium/ApplicationLoaderTest.java | 93 ++++++ .../zeppelin/helium/MockApplication1.java | 52 ++++ 9 files changed, 716 insertions(+) create mode 100644 zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/Application.java create mode 100644 zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationContext.java create mode 100644 zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationException.java create mode 100644 zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java create mode 100644 zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ClassLoaderApplication.java create mode 100644 zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackageInfo.java create mode 100644 zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumRegistry.java create mode 100644 zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/ApplicationLoaderTest.java create mode 100644 zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/MockApplication1.java diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/Application.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/Application.java new file mode 100644 index 00000000000..bee1397bab8 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/Application.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.resource.ResourceSet; + +/** + * Zeppelin Application base + */ +public abstract class Application { + private final ResourceSet args; + private final ApplicationContext context; + + public Application(ResourceSet args, ApplicationContext context) throws ApplicationException { + this.args = args; + this.context = context; + } + + public ResourceSet args() { + return args; + } + + public ApplicationContext context() { + return context; + } + + /** + * This method can be invoked multiple times before unload(), + * Either just after application selected or when paragraph re-run after application load + */ + public abstract void run() throws ApplicationException; + + + /** + * this method is invoked just before application is removed + */ + public abstract void unload() throws ApplicationException; +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationContext.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationContext.java new file mode 100644 index 00000000000..58a2d49fe2e --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationContext.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +/** + * ApplicationContext + */ +public class ApplicationContext { + private final String noteId; + private final String paragraphId; + + public ApplicationContext(String noteId, String paragraphId) { + this.noteId = noteId; + this.paragraphId = paragraphId; + } + + public String getNoteId() { + return noteId; + } + + public String getParagraphId() { + return paragraphId; + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationException.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationException.java new file mode 100644 index 00000000000..4bf0ac28b94 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationException.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +/** + * Application exception + */ +public class ApplicationException extends Exception { + public ApplicationException(String s) { + super(s); + } + + public ApplicationException(Exception e) { + super(e); + } + + public ApplicationException() { + + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java new file mode 100644 index 00000000000..0e3daf6ed65 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java @@ -0,0 +1,266 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import org.apache.zeppelin.dep.DependencyResolver; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.resource.DistributedResourcePool; +import org.apache.zeppelin.resource.Resource; +import org.apache.zeppelin.resource.ResourcePool; +import org.apache.zeppelin.resource.ResourceSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.lang.reflect.Constructor; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.*; + +/** + * Load application + */ +public class ApplicationLoader { + Logger logger = LoggerFactory.getLogger(ApplicationLoader.class); + + private final DependencyResolver depResolver; + private final ResourcePool resourcePool; + private final Map> cached; + private final Map runningApplications; + + public ApplicationLoader(ResourcePool resourcePool, DependencyResolver depResolver) { + this.depResolver = depResolver; + this.resourcePool = resourcePool; + cached = Collections.synchronizedMap( + new HashMap>()); + runningApplications = new HashMap(); + } + + /** + * Information of loaded application + */ + private static class RunningApplication { + HeliumPackageInfo packageInfo; + String noteId; + String paragraphId; + + public RunningApplication(HeliumPackageInfo packageInfo, String noteId, String paragraphId) { + this.packageInfo = packageInfo; + this.noteId = noteId; + this.paragraphId = paragraphId; + } + + public HeliumPackageInfo getPackageInfo() { + return packageInfo; + } + + public String getNoteId() { + return noteId; + } + + public String getParagraphId() { + return paragraphId; + } + + @Override + public int hashCode() { + return (paragraphId + noteId + packageInfo.getArtifact() + packageInfo.getClassName()) + .hashCode(); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof RunningApplication)) { + return false; + } + + RunningApplication r = (RunningApplication) o; + return packageInfo.equals(r.getPackageInfo()) && paragraphId.equals(r.getParagraphId()) && + noteId.equals(r.getNoteId()); + } + } + + /** + * + * Instantiate application + * + * @param packageInfo + * @param context + * @return + * @throws Exception + */ + public Application load(HeliumPackageInfo packageInfo, ApplicationContext context) + throws Exception { + if (packageInfo.getType() != HeliumPackageInfo.Type.APPLICATION) { + throw new ApplicationException( + "Can't instantiate " + packageInfo.getType() + " package using ApplicationLoader"); + } + + // check if already loaded + RunningApplication key = + new RunningApplication(packageInfo, context.getNoteId(), context.getParagraphId()); + synchronized (runningApplications) { + if (runningApplications.containsKey(key)) { + return runningApplications.get(key); + } + } + + // get resource required by this package + ResourceSet resources = findRequiredResourceSet(packageInfo.getResources(), + context.getNoteId(), context.getParagraphId()); + + // load class + Class appClass = loadClass(packageInfo); + + // instantiate + ClassLoader oldcl = Thread.currentThread().getContextClassLoader(); + ClassLoader cl = appClass.getClassLoader(); + Thread.currentThread().setContextClassLoader(cl); + try { + Constructor constructor = + appClass.getConstructor(ResourceSet.class, ApplicationContext.class); + + synchronized (runningApplications) { + if (!runningApplications.containsKey(key)) { + logger.info("Load {} {} from note {} paragraph {}", + packageInfo.getArtifact(), + packageInfo.getClassName(), + context.getNoteId(), + context.getParagraphId()); + + Application app = new ClassLoaderApplication( + constructor.newInstance(resources, context), + cl); + runningApplications.put(key, app); + return app; + } else { + return runningApplications.get(key); + } + } + } catch (Exception e) { + throw new ApplicationException(e); + } finally { + Thread.currentThread().setContextClassLoader(oldcl); + } + } + + public void unload(HeliumPackageInfo packageInfo, ApplicationContext context) + throws ApplicationException { + Application appToUnload = null; + synchronized (runningApplications) { + RunningApplication key + = new RunningApplication(packageInfo, context.getNoteId(), context.getParagraphId()); + + if (runningApplications.containsKey(key)) { + appToUnload = runningApplications.remove(key); + } + } + + if (appToUnload != null) { + logger.info("Unload {} {} from note {} paragraph {}", + packageInfo.getArtifact(), + packageInfo.getClassName(), + context.getNoteId(), + context.getParagraphId()); + appToUnload.unload(); + } + } + + public Application get(HeliumPackageInfo packageInfo, ApplicationContext context) { + synchronized (runningApplications) { + RunningApplication key + = new RunningApplication(packageInfo, context.getNoteId(), context.getParagraphId()); + return runningApplications.get(key); + } + } + + private ResourceSet findRequiredResourceSet( + String [][] requiredResources, String noteId, String paragraphId) + throws ApplicationException { + if (requiredResources == null || requiredResources.length == 0) { + return new ResourceSet(); + } + + String localResourcePoolId = resourcePool.id(); + ResourceSet args = new ResourceSet(); + ResourceSet allResources; + if (resourcePool instanceof DistributedResourcePool) { + allResources = ((DistributedResourcePool) resourcePool).getAll(false); + } else { + allResources = resourcePool.getAll(); + } + + allResources = allResources.filterByNoteId(noteId).filterByParagraphId(paragraphId); + + for (String [] requires : requiredResources) { + args.clear(); + + for (String require : requires) { + boolean found = false; + + for (Resource r : allResources) { + if (r.getClassName().equals(require)) { + args.add(r); + found = true; + break; + } + } + + if (found == false) { + break; + } + } + + if (args.size() == requires.length) { + return args; + } + } + + throw new ApplicationException("Can not find available resources"); + } + + + private Class loadClass(HeliumPackageInfo packageInfo) throws Exception { + if (cached.containsKey(packageInfo)) { + return cached.get(packageInfo); + } + + // Create Application classloader + List urlList = new LinkedList(); + + // load artifact + if (packageInfo.getArtifact() != null) { + List paths = depResolver.load(packageInfo.getArtifact()); + + if (paths != null) { + + for (File path : paths) { + urlList.add(path.toURI().toURL()); + } + } + } + URLClassLoader applicationClassLoader = + new URLClassLoader( + urlList.toArray(new URL[]{}), + Thread.currentThread().getContextClassLoader()); + + Class cls = + (Class) applicationClassLoader.loadClass(packageInfo.getClassName()); + cached.put(packageInfo, cls); + return cls; + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ClassLoaderApplication.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ClassLoaderApplication.java new file mode 100644 index 00000000000..7e3f61310d7 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ClassLoaderApplication.java @@ -0,0 +1,56 @@ +package org.apache.zeppelin.helium; + +import org.apache.zeppelin.resource.ResourceSet; + +/** + * Application wrapper + */ +public class ClassLoaderApplication extends Application { + Application app; + ClassLoader cl; + public ClassLoaderApplication(Application app, ClassLoader cl) throws ApplicationException { + super(null, null); + this.app = app; + this.cl = cl; + } + + @Override + public void run() throws ApplicationException { + // instantiate + ClassLoader oldcl = Thread.currentThread().getContextClassLoader(); + Thread.currentThread().setContextClassLoader(cl); + try { + app.run(); + } catch (ApplicationException e) { + throw e; + } catch (Exception e) { + throw new ApplicationException(e); + } finally { + Thread.currentThread().setContextClassLoader(oldcl); + } + } + + @Override + public void unload() throws ApplicationException { + // instantiate + ClassLoader oldcl = Thread.currentThread().getContextClassLoader(); + Thread.currentThread().setContextClassLoader(cl); + try { + app.unload(); + } catch (ApplicationException e) { + throw e; + } catch (Exception e) { + throw new ApplicationException(e); + } finally { + Thread.currentThread().setContextClassLoader(oldcl); + } + } + + public ClassLoader getClassLoader() { + return cl; + } + + public Application getInnerApplication() { + return app; + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackageInfo.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackageInfo.java new file mode 100644 index 00000000000..70d4fbaa0a6 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackageInfo.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +/** + * Helium package information + */ +public class HeliumPackageInfo { + private Type type; + private String name; // user friendly name of this application + private String description; // description + private String artifact; // artifact name e.g) groupId:artifactId:versionId + private String className; // entry point + private String [][] resources; // resource classnames that requires + // [[ .. and .. and .. ] or [ .. and .. and ..] ..] + + public static enum Type { + INTERPRETER, + NOTEBOOK_REPO, + APPLICATION + } + + public HeliumPackageInfo(Type type, + String name, + String description, + String artifact, + String className, + String[][] resources) { + this.type = type; + this.name = name; + this.description = description; + this.artifact = artifact; + this.className = className; + this.resources = resources; + } + + @Override + public int hashCode() { + return (type.toString() + artifact + className).hashCode(); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof HeliumPackageInfo)) { + return false; + } + + HeliumPackageInfo info = (HeliumPackageInfo) o; + return type == info.type && artifact.equals(info.artifact) && className.equals(info.className); + } + + public Type getType() { + return type; + } + + public String getName() { + return name; + } + + public String getDescription() { + return description; + } + + public String getArtifact() { + return artifact; + } + + public String getClassName() { + return className; + } + + public String[][] getResources() { + return resources; + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumRegistry.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumRegistry.java new file mode 100644 index 00000000000..a893df6be95 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumRegistry.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import java.net.URI; + +/** + * Helium registry + */ +public abstract class HeliumRegistry { + private final URI uri; + + public HeliumRegistry(URI uri) { + this.uri = uri; + } + + public URI uri() { + return uri; + } + +} diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/ApplicationLoaderTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/ApplicationLoaderTest.java new file mode 100644 index 00000000000..eac4f23f187 --- /dev/null +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/ApplicationLoaderTest.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.helium; + +import org.apache.commons.io.FileUtils; +import org.apache.zeppelin.dep.DependencyResolver; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.resource.LocalResourcePool; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; + +import static org.junit.Assert.*; + +public class ApplicationLoaderTest { + private File tmpDir; + + @Before + public void setUp() { + tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis()); + tmpDir.mkdirs(); + } + + @After + public void tearDown() throws IOException { + FileUtils.deleteDirectory(tmpDir); + } + + @Test + public void loadUnloadApplication() throws Exception { + LocalResourcePool resourcePool = new LocalResourcePool("pool1"); + DependencyResolver dep = new DependencyResolver(tmpDir.getAbsolutePath()); + ApplicationLoader appLoader = new ApplicationLoader(resourcePool, dep); + + HeliumPackageInfo pkg1 = createPackageInfo(MockApplication1.class.getName(), "artifact1"); + ApplicationContext context1 = createContext("note1", "paragraph1"); + + // app not loaded yet + assertEquals(null, appLoader.get(pkg1, context1)); + + // load application + MockApplication1 app = (MockApplication1) ((ClassLoaderApplication) + appLoader.load(pkg1, context1)).getInnerApplication(); + + // then + assertFalse(app.isUnloaded()); + assertEquals(0, app.getNumRun()); + assertNotNull(appLoader.get(pkg1, context1)); + + // unload application + appLoader.unload(pkg1, context1); + + // then + assertTrue(app.isUnloaded()); + assertEquals(0, app.getNumRun()); + } + + public HeliumPackageInfo createPackageInfo(String className, String artifact) { + HeliumPackageInfo app1 = new HeliumPackageInfo( + HeliumPackageInfo.Type.APPLICATION, + "name1", + "desc1", + artifact, + className, + new String[][]{{}}); + return app1; + } + + public ApplicationContext createContext(String noteId, String paragraphId) { + ApplicationContext context1 = new ApplicationContext( + noteId, + paragraphId); + return context1; + } +} diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/MockApplication1.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/MockApplication1.java new file mode 100644 index 00000000000..90cf63a7c68 --- /dev/null +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/MockApplication1.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.resource.ResourceSet; + +/** + * Mock application + */ +public class MockApplication1 extends Application { + boolean unloaded; + int run; + + public MockApplication1(ResourceSet args, ApplicationContext context) throws ApplicationException { + super(args, context); + unloaded = false; + run = 0; + } + + @Override + public void run() { + run++; + } + + @Override + public void unload() { + unloaded = true; + } + + public boolean isUnloaded() { + return unloaded; + } + + public int getNumRun() { + return run; + } +} From b891b98ee5a3bd9dde8c044b32c0dc09074cdcb0 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sun, 13 Mar 2016 16:13:49 -0700 Subject: [PATCH 03/69] HeliumRegistry --- .../zeppelin/helium/ApplicationLoader.java | 73 +- ...iumPackageInfo.java => HeliumPackage.java} | 20 +- .../remote/RemoteInterpreterServer.java | 90 +- .../thrift/RemoteApplicationResult.java | 518 +++ .../thrift/RemoteInterpreterContext.java | 2 +- .../thrift/RemoteInterpreterEvent.java | 2 +- .../thrift/RemoteInterpreterResult.java | 2 +- .../thrift/RemoteInterpreterService.java | 3932 ++++++++++++----- .../thrift/RemoteInterpreterService.thrift | 9 + .../helium/ApplicationLoaderTest.java | 20 +- .../org/apache/zeppelin/helium/Helium.java | 119 + .../apache/zeppelin/helium/HeliumConf.java | 35 + .../zeppelin/helium/HeliumLocalRegistry.java | 76 + .../helium/HeliumPackageSearchResult.java | 43 + .../zeppelin/helium/HeliumRegistry.java | 14 +- .../helium/HeliumRegistrySerializer.java | 75 + .../helium/HeliumLocalRegistryTest.java | 64 + .../apache/zeppelin/helium/HeliumTest.java | 98 + .../zeppelin/helium/HeliumTestRegistry.java | 39 + 19 files changed, 4129 insertions(+), 1102 deletions(-) rename zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/{HeliumPackageInfo.java => HeliumPackage.java} (83%) create mode 100644 zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteApplicationResult.java create mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java create mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumConf.java create mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumLocalRegistry.java create mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumPackageSearchResult.java rename {zeppelin-interpreter => zeppelin-zengine}/src/main/java/org/apache/zeppelin/helium/HeliumRegistry.java (77%) create mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumRegistrySerializer.java create mode 100644 zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumLocalRegistryTest.java create mode 100644 zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java create mode 100644 zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTestRegistry.java diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java index 0e3daf6ed65..ae8889c6c9a 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java @@ -17,7 +17,6 @@ package org.apache.zeppelin.helium; import org.apache.zeppelin.dep.DependencyResolver; -import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.resource.DistributedResourcePool; import org.apache.zeppelin.resource.Resource; import org.apache.zeppelin.resource.ResourcePool; @@ -39,32 +38,30 @@ public class ApplicationLoader { private final DependencyResolver depResolver; private final ResourcePool resourcePool; - private final Map> cached; - private final Map runningApplications; + private final Map> cached; public ApplicationLoader(ResourcePool resourcePool, DependencyResolver depResolver) { this.depResolver = depResolver; this.resourcePool = resourcePool; cached = Collections.synchronizedMap( - new HashMap>()); - runningApplications = new HashMap(); + new HashMap>()); } /** * Information of loaded application */ private static class RunningApplication { - HeliumPackageInfo packageInfo; + HeliumPackage packageInfo; String noteId; String paragraphId; - public RunningApplication(HeliumPackageInfo packageInfo, String noteId, String paragraphId) { + public RunningApplication(HeliumPackage packageInfo, String noteId, String paragraphId) { this.packageInfo = packageInfo; this.noteId = noteId; this.paragraphId = paragraphId; } - public HeliumPackageInfo getPackageInfo() { + public HeliumPackage getPackageInfo() { return packageInfo; } @@ -103,9 +100,9 @@ public boolean equals(Object o) { * @return * @throws Exception */ - public Application load(HeliumPackageInfo packageInfo, ApplicationContext context) + public Application load(HeliumPackage packageInfo, ApplicationContext context) throws Exception { - if (packageInfo.getType() != HeliumPackageInfo.Type.APPLICATION) { + if (packageInfo.getType() != HeliumPackage.Type.APPLICATION) { throw new ApplicationException( "Can't instantiate " + packageInfo.getType() + " package using ApplicationLoader"); } @@ -113,11 +110,6 @@ public Application load(HeliumPackageInfo packageInfo, ApplicationContext contex // check if already loaded RunningApplication key = new RunningApplication(packageInfo, context.getNoteId(), context.getParagraphId()); - synchronized (runningApplications) { - if (runningApplications.containsKey(key)) { - return runningApplications.get(key); - } - } // get resource required by this package ResourceSet resources = findRequiredResourceSet(packageInfo.getResources(), @@ -134,23 +126,8 @@ public Application load(HeliumPackageInfo packageInfo, ApplicationContext contex Constructor constructor = appClass.getConstructor(ResourceSet.class, ApplicationContext.class); - synchronized (runningApplications) { - if (!runningApplications.containsKey(key)) { - logger.info("Load {} {} from note {} paragraph {}", - packageInfo.getArtifact(), - packageInfo.getClassName(), - context.getNoteId(), - context.getParagraphId()); - - Application app = new ClassLoaderApplication( - constructor.newInstance(resources, context), - cl); - runningApplications.put(key, app); - return app; - } else { - return runningApplications.get(key); - } - } + Application app = new ClassLoaderApplication(constructor.newInstance(resources, context), cl); + return app; } catch (Exception e) { throw new ApplicationException(e); } finally { @@ -158,36 +135,6 @@ public Application load(HeliumPackageInfo packageInfo, ApplicationContext contex } } - public void unload(HeliumPackageInfo packageInfo, ApplicationContext context) - throws ApplicationException { - Application appToUnload = null; - synchronized (runningApplications) { - RunningApplication key - = new RunningApplication(packageInfo, context.getNoteId(), context.getParagraphId()); - - if (runningApplications.containsKey(key)) { - appToUnload = runningApplications.remove(key); - } - } - - if (appToUnload != null) { - logger.info("Unload {} {} from note {} paragraph {}", - packageInfo.getArtifact(), - packageInfo.getClassName(), - context.getNoteId(), - context.getParagraphId()); - appToUnload.unload(); - } - } - - public Application get(HeliumPackageInfo packageInfo, ApplicationContext context) { - synchronized (runningApplications) { - RunningApplication key - = new RunningApplication(packageInfo, context.getNoteId(), context.getParagraphId()); - return runningApplications.get(key); - } - } - private ResourceSet findRequiredResourceSet( String [][] requiredResources, String noteId, String paragraphId) throws ApplicationException { @@ -234,7 +181,7 @@ private ResourceSet findRequiredResourceSet( } - private Class loadClass(HeliumPackageInfo packageInfo) throws Exception { + private Class loadClass(HeliumPackage packageInfo) throws Exception { if (cached.containsKey(packageInfo)) { return cached.get(packageInfo); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackageInfo.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackage.java similarity index 83% rename from zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackageInfo.java rename to zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackage.java index 70d4fbaa0a6..abd63f4c221 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackageInfo.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackage.java @@ -17,9 +17,9 @@ package org.apache.zeppelin.helium; /** - * Helium package information + * Helium package definition */ -public class HeliumPackageInfo { +public class HeliumPackage { private Type type; private String name; // user friendly name of this application private String description; // description @@ -34,12 +34,12 @@ public static enum Type { APPLICATION } - public HeliumPackageInfo(Type type, - String name, - String description, - String artifact, - String className, - String[][] resources) { + public HeliumPackage(Type type, + String name, + String description, + String artifact, + String className, + String[][] resources) { this.type = type; this.name = name; this.description = description; @@ -55,11 +55,11 @@ public int hashCode() { @Override public boolean equals(Object o) { - if (!(o instanceof HeliumPackageInfo)) { + if (!(o instanceof HeliumPackage)) { return false; } - HeliumPackageInfo info = (HeliumPackageInfo) o; + HeliumPackage info = (HeliumPackage) o; return type == info.type && artifact.equals(info.artifact) && className.equals(info.className); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java index 6e369c0694a..7cd378b76b4 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java @@ -29,13 +29,12 @@ import org.apache.thrift.server.TThreadPoolServer; import org.apache.thrift.transport.TServerSocket; import org.apache.thrift.transport.TTransportException; +import org.apache.zeppelin.dep.DependencyResolver; import org.apache.zeppelin.display.*; +import org.apache.zeppelin.helium.*; import org.apache.zeppelin.interpreter.*; import org.apache.zeppelin.interpreter.InterpreterResult.Code; -import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterContext; -import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEvent; -import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResult; -import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService; +import org.apache.zeppelin.interpreter.thrift.*; import org.apache.zeppelin.resource.*; import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.Job.Status; @@ -61,6 +60,8 @@ public class RemoteInterpreterServer InterpreterGroup interpreterGroup; AngularObjectRegistry angularObjectRegistry; DistributedResourcePool resourcePool; + private ApplicationLoader appLoader; + Gson gson = new Gson(); RemoteInterpreterService.Processor processor; @@ -69,6 +70,11 @@ public class RemoteInterpreterServer private TThreadPoolServer server; RemoteInterpreterEventClient eventClient = new RemoteInterpreterEventClient(); + private DependencyResolver depLoader; + + private final Map runningApplications = + Collections.synchronizedMap(new HashMap()); + public RemoteInterpreterServer(int port) throws TTransportException { this.port = port; @@ -137,14 +143,17 @@ public static void main(String[] args) @Override public void createInterpreter(String interpreterGroupId, String noteId, String - className, - Map properties) throws TException { + className, Map properties) throws TException { if (interpreterGroup == null) { interpreterGroup = new InterpreterGroup(interpreterGroupId); angularObjectRegistry = new AngularObjectRegistry(interpreterGroup.getId(), this); resourcePool = new DistributedResourcePool(interpreterGroup.getId(), eventClient); interpreterGroup.setAngularObjectRegistry(angularObjectRegistry); interpreterGroup.setResourcePool(resourcePool); + + String localRepoPath = properties.get("zeppelin.interpreter.localRepo"); + depLoader = new DependencyResolver(localRepoPath); + appLoader = new ApplicationLoader(resourcePool, depLoader); } try { @@ -696,4 +705,73 @@ public void angularRegistryPush(String registryAsString) throws TException { logger.info("Exception in RemoteInterpreterServer while angularRegistryPush, nolock", e); } } + + private ApplicationContext getApplicationContext( + HeliumPackage packageInfo, String noteId, String paragraphId) { + return new ApplicationContext(noteId, paragraphId); + } + + @Override + public RemoteApplicationResult loadApplication( + String applicationInstanceId, String packageInfo, String noteId, String paragraphId) + throws TException { + if (runningApplications.containsKey(applicationInstanceId)) { + logger.warn("Application instance {} is already running"); + return new RemoteApplicationResult(true, ""); + } + HeliumPackage pkgInfo = gson.fromJson(packageInfo, HeliumPackage.class); + ApplicationContext context = getApplicationContext(pkgInfo, noteId, paragraphId); + try { + Application app = null; + logger.info( + "Loading application {}({}). artifact={}, className={} into note={}, paragraph={}", + pkgInfo.getName(), + applicationInstanceId, + pkgInfo.getArtifact(), + pkgInfo.getClassName(), + noteId, + paragraphId); + app = appLoader.load(pkgInfo, context); + runningApplications.put(applicationInstanceId, app); + return new RemoteApplicationResult(true, ""); + } catch (Exception e) { + logger.error(e.getMessage(), e); + return new RemoteApplicationResult(false, e.getMessage()); + } + } + + @Override + public RemoteApplicationResult unloadApplication(String applicationInstanceId) + throws TException { + Application app = runningApplications.remove(applicationInstanceId); + if (app != null) { + try { + logger.info("Unloading application {}", applicationInstanceId); + app.unload(); + } catch (ApplicationException e) { + logger.error(e.getMessage(), e); + return new RemoteApplicationResult(false, e.getMessage()); + } + } + return new RemoteApplicationResult(true, ""); + } + + @Override + public RemoteApplicationResult runApplication(String applicationInstanceId) + throws TException { + logger.info("run application {}", applicationInstanceId); + + Application app = runningApplications.get(applicationInstanceId); + if (app == null) { + logger.error("Application instance {} not exists", applicationInstanceId); + return new RemoteApplicationResult(false, "Application instance does not exists"); + } else { + try { + app.run(); + return new RemoteApplicationResult(true, ""); + } catch (ApplicationException e) { + return new RemoteApplicationResult(false, e.getMessage()); + } + } + } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteApplicationResult.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteApplicationResult.java new file mode 100644 index 00000000000..2a3945fa406 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteApplicationResult.java @@ -0,0 +1,518 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Autogenerated by Thrift Compiler (0.9.2) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.zeppelin.interpreter.thrift; + +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; + +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.protocol.TTupleProtocol; +import org.apache.thrift.protocol.TProtocolException; +import org.apache.thrift.EncodingUtils; +import org.apache.thrift.TException; +import org.apache.thrift.async.AsyncMethodCallback; +import org.apache.thrift.server.AbstractNonblockingServer.*; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.EnumMap; +import java.util.Set; +import java.util.HashSet; +import java.util.EnumSet; +import java.util.Collections; +import java.util.BitSet; +import java.nio.ByteBuffer; +import java.util.Arrays; +import javax.annotation.Generated; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) +@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2016-3-18") +public class RemoteApplicationResult implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteApplicationResult"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.BOOL, (short)1); + private static final org.apache.thrift.protocol.TField MSG_FIELD_DESC = new org.apache.thrift.protocol.TField("msg", org.apache.thrift.protocol.TType.STRING, (short)2); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new RemoteApplicationResultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new RemoteApplicationResultTupleSchemeFactory()); + } + + public boolean success; // required + public String msg; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)1, "success"), + MSG((short)2, "msg"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // SUCCESS + return SUCCESS; + case 2: // MSG + return MSG; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + private static final int __SUCCESS_ISSET_ID = 0; + private byte __isset_bitfield = 0; + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); + tmpMap.put(_Fields.MSG, new org.apache.thrift.meta_data.FieldMetaData("msg", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(RemoteApplicationResult.class, metaDataMap); + } + + public RemoteApplicationResult() { + } + + public RemoteApplicationResult( + boolean success, + String msg) + { + this(); + this.success = success; + setSuccessIsSet(true); + this.msg = msg; + } + + /** + * Performs a deep copy on other. + */ + public RemoteApplicationResult(RemoteApplicationResult other) { + __isset_bitfield = other.__isset_bitfield; + this.success = other.success; + if (other.isSetMsg()) { + this.msg = other.msg; + } + } + + public RemoteApplicationResult deepCopy() { + return new RemoteApplicationResult(this); + } + + @Override + public void clear() { + setSuccessIsSet(false); + this.success = false; + this.msg = null; + } + + public boolean isSuccess() { + return this.success; + } + + public RemoteApplicationResult setSuccess(boolean success) { + this.success = success; + setSuccessIsSet(true); + return this; + } + + public void unsetSuccess() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __SUCCESS_ISSET_ID); + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return EncodingUtils.testBit(__isset_bitfield, __SUCCESS_ISSET_ID); + } + + public void setSuccessIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __SUCCESS_ISSET_ID, value); + } + + public String getMsg() { + return this.msg; + } + + public RemoteApplicationResult setMsg(String msg) { + this.msg = msg; + return this; + } + + public void unsetMsg() { + this.msg = null; + } + + /** Returns true if field msg is set (has been assigned a value) and false otherwise */ + public boolean isSetMsg() { + return this.msg != null; + } + + public void setMsgIsSet(boolean value) { + if (!value) { + this.msg = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((Boolean)value); + } + break; + + case MSG: + if (value == null) { + unsetMsg(); + } else { + setMsg((String)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return Boolean.valueOf(isSuccess()); + + case MSG: + return getMsg(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + case MSG: + return isSetMsg(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof RemoteApplicationResult) + return this.equals((RemoteApplicationResult)that); + return false; + } + + public boolean equals(RemoteApplicationResult that) { + if (that == null) + return false; + + boolean this_present_success = true; + boolean that_present_success = true; + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (this.success != that.success) + return false; + } + + boolean this_present_msg = true && this.isSetMsg(); + boolean that_present_msg = true && that.isSetMsg(); + if (this_present_msg || that_present_msg) { + if (!(this_present_msg && that_present_msg)) + return false; + if (!this.msg.equals(that.msg)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + List list = new ArrayList(); + + boolean present_success = true; + list.add(present_success); + if (present_success) + list.add(success); + + boolean present_msg = true && (isSetMsg()); + list.add(present_msg); + if (present_msg) + list.add(msg); + + return list.hashCode(); + } + + @Override + public int compareTo(RemoteApplicationResult other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetMsg()).compareTo(other.isSetMsg()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetMsg()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.msg, other.msg); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("RemoteApplicationResult("); + boolean first = true; + + sb.append("success:"); + sb.append(this.success); + first = false; + if (!first) sb.append(", "); + sb.append("msg:"); + if (this.msg == null) { + sb.append("null"); + } else { + sb.append(this.msg); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. + __isset_bitfield = 0; + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class RemoteApplicationResultStandardSchemeFactory implements SchemeFactory { + public RemoteApplicationResultStandardScheme getScheme() { + return new RemoteApplicationResultStandardScheme(); + } + } + + private static class RemoteApplicationResultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, RemoteApplicationResult struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) { + struct.success = iprot.readBool(); + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // MSG + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.msg = iprot.readString(); + struct.setMsgIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, RemoteApplicationResult struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + oprot.writeBool(struct.success); + oprot.writeFieldEnd(); + if (struct.msg != null) { + oprot.writeFieldBegin(MSG_FIELD_DESC); + oprot.writeString(struct.msg); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class RemoteApplicationResultTupleSchemeFactory implements SchemeFactory { + public RemoteApplicationResultTupleScheme getScheme() { + return new RemoteApplicationResultTupleScheme(); + } + } + + private static class RemoteApplicationResultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, RemoteApplicationResult struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + if (struct.isSetMsg()) { + optionals.set(1); + } + oprot.writeBitSet(optionals, 2); + if (struct.isSetSuccess()) { + oprot.writeBool(struct.success); + } + if (struct.isSetMsg()) { + oprot.writeString(struct.msg); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, RemoteApplicationResult struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(2); + if (incoming.get(0)) { + struct.success = iprot.readBool(); + struct.setSuccessIsSet(true); + } + if (incoming.get(1)) { + struct.msg = iprot.readString(); + struct.setMsgIsSet(true); + } + } + } + +} + diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterContext.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterContext.java index 097f762ce7d..5fadb74b49d 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterContext.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterContext.java @@ -51,7 +51,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2016-3-17") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2016-3-18") public class RemoteInterpreterContext implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteInterpreterContext"); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java index e42214f7026..6198ff06cd2 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java @@ -51,7 +51,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2016-3-17") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2016-3-18") public class RemoteInterpreterEvent implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteInterpreterEvent"); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterResult.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterResult.java index 082df0525b4..3078147ea9f 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterResult.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterResult.java @@ -51,7 +51,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2016-3-17") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2016-3-18") public class RemoteInterpreterResult implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteInterpreterResult"); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterService.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterService.java index 84ede734ea1..3b5a9c1384b 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterService.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterService.java @@ -51,7 +51,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2016-3-17") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2016-3-18") public class RemoteInterpreterService { public interface Iface { @@ -96,6 +96,12 @@ public interface Iface { public void angularRegistryPush(String registry) throws org.apache.thrift.TException; + public RemoteApplicationResult loadApplication(String applicationInstanceId, String packageInfo, String noteId, String paragraphId) throws org.apache.thrift.TException; + + public RemoteApplicationResult unloadApplication(String applicationInstanceId) throws org.apache.thrift.TException; + + public RemoteApplicationResult runApplication(String applicationInstanceId) throws org.apache.thrift.TException; + } public interface AsyncIface { @@ -140,6 +146,12 @@ public interface AsyncIface { public void angularRegistryPush(String registry, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + public void loadApplication(String applicationInstanceId, String packageInfo, String noteId, String paragraphId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void unloadApplication(String applicationInstanceId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + + public void runApplication(String applicationInstanceId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException; + } public static class Client extends org.apache.thrift.TServiceClient implements Iface { @@ -616,6 +628,78 @@ public void recv_angularRegistryPush() throws org.apache.thrift.TException return; } + public RemoteApplicationResult loadApplication(String applicationInstanceId, String packageInfo, String noteId, String paragraphId) throws org.apache.thrift.TException + { + send_loadApplication(applicationInstanceId, packageInfo, noteId, paragraphId); + return recv_loadApplication(); + } + + public void send_loadApplication(String applicationInstanceId, String packageInfo, String noteId, String paragraphId) throws org.apache.thrift.TException + { + loadApplication_args args = new loadApplication_args(); + args.setApplicationInstanceId(applicationInstanceId); + args.setPackageInfo(packageInfo); + args.setNoteId(noteId); + args.setParagraphId(paragraphId); + sendBase("loadApplication", args); + } + + public RemoteApplicationResult recv_loadApplication() throws org.apache.thrift.TException + { + loadApplication_result result = new loadApplication_result(); + receiveBase(result, "loadApplication"); + if (result.isSetSuccess()) { + return result.success; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "loadApplication failed: unknown result"); + } + + public RemoteApplicationResult unloadApplication(String applicationInstanceId) throws org.apache.thrift.TException + { + send_unloadApplication(applicationInstanceId); + return recv_unloadApplication(); + } + + public void send_unloadApplication(String applicationInstanceId) throws org.apache.thrift.TException + { + unloadApplication_args args = new unloadApplication_args(); + args.setApplicationInstanceId(applicationInstanceId); + sendBase("unloadApplication", args); + } + + public RemoteApplicationResult recv_unloadApplication() throws org.apache.thrift.TException + { + unloadApplication_result result = new unloadApplication_result(); + receiveBase(result, "unloadApplication"); + if (result.isSetSuccess()) { + return result.success; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "unloadApplication failed: unknown result"); + } + + public RemoteApplicationResult runApplication(String applicationInstanceId) throws org.apache.thrift.TException + { + send_runApplication(applicationInstanceId); + return recv_runApplication(); + } + + public void send_runApplication(String applicationInstanceId) throws org.apache.thrift.TException + { + runApplication_args args = new runApplication_args(); + args.setApplicationInstanceId(applicationInstanceId); + sendBase("runApplication", args); + } + + public RemoteApplicationResult recv_runApplication() throws org.apache.thrift.TException + { + runApplication_result result = new runApplication_result(); + receiveBase(result, "runApplication"); + if (result.isSetSuccess()) { + return result.success; + } + throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "runApplication failed: unknown result"); + } + } public static class AsyncClient extends org.apache.thrift.async.TAsyncClient implements AsyncIface { public static class Factory implements org.apache.thrift.async.TAsyncClientFactory { @@ -1355,6 +1439,111 @@ public void getResult() throws org.apache.thrift.TException { } } + public void loadApplication(String applicationInstanceId, String packageInfo, String noteId, String paragraphId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + loadApplication_call method_call = new loadApplication_call(applicationInstanceId, packageInfo, noteId, paragraphId, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class loadApplication_call extends org.apache.thrift.async.TAsyncMethodCall { + private String applicationInstanceId; + private String packageInfo; + private String noteId; + private String paragraphId; + public loadApplication_call(String applicationInstanceId, String packageInfo, String noteId, String paragraphId, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.applicationInstanceId = applicationInstanceId; + this.packageInfo = packageInfo; + this.noteId = noteId; + this.paragraphId = paragraphId; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("loadApplication", org.apache.thrift.protocol.TMessageType.CALL, 0)); + loadApplication_args args = new loadApplication_args(); + args.setApplicationInstanceId(applicationInstanceId); + args.setPackageInfo(packageInfo); + args.setNoteId(noteId); + args.setParagraphId(paragraphId); + args.write(prot); + prot.writeMessageEnd(); + } + + public RemoteApplicationResult getResult() throws org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_loadApplication(); + } + } + + public void unloadApplication(String applicationInstanceId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + unloadApplication_call method_call = new unloadApplication_call(applicationInstanceId, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class unloadApplication_call extends org.apache.thrift.async.TAsyncMethodCall { + private String applicationInstanceId; + public unloadApplication_call(String applicationInstanceId, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.applicationInstanceId = applicationInstanceId; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("unloadApplication", org.apache.thrift.protocol.TMessageType.CALL, 0)); + unloadApplication_args args = new unloadApplication_args(); + args.setApplicationInstanceId(applicationInstanceId); + args.write(prot); + prot.writeMessageEnd(); + } + + public RemoteApplicationResult getResult() throws org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_unloadApplication(); + } + } + + public void runApplication(String applicationInstanceId, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws org.apache.thrift.TException { + checkReady(); + runApplication_call method_call = new runApplication_call(applicationInstanceId, resultHandler, this, ___protocolFactory, ___transport); + this.___currentMethod = method_call; + ___manager.call(method_call); + } + + public static class runApplication_call extends org.apache.thrift.async.TAsyncMethodCall { + private String applicationInstanceId; + public runApplication_call(String applicationInstanceId, org.apache.thrift.async.AsyncMethodCallback resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException { + super(client, protocolFactory, transport, resultHandler, false); + this.applicationInstanceId = applicationInstanceId; + } + + public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException { + prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("runApplication", org.apache.thrift.protocol.TMessageType.CALL, 0)); + runApplication_args args = new runApplication_args(); + args.setApplicationInstanceId(applicationInstanceId); + args.write(prot); + prot.writeMessageEnd(); + } + + public RemoteApplicationResult getResult() throws org.apache.thrift.TException { + if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) { + throw new IllegalStateException("Method call not finished!"); + } + org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array()); + org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport); + return (new Client(prot)).recv_runApplication(); + } + } + } public static class Processor extends org.apache.thrift.TBaseProcessor implements org.apache.thrift.TProcessor { @@ -1388,6 +1577,9 @@ protected Processor(I iface, Map extends org.apache.thrift.ProcessFunction { + public loadApplication() { + super("loadApplication"); + } + + public loadApplication_args getEmptyArgsInstance() { + return new loadApplication_args(); + } + + protected boolean isOneway() { + return false; + } + + public loadApplication_result getResult(I iface, loadApplication_args args) throws org.apache.thrift.TException { + loadApplication_result result = new loadApplication_result(); + result.success = iface.loadApplication(args.applicationInstanceId, args.packageInfo, args.noteId, args.paragraphId); + return result; + } + } + + public static class unloadApplication extends org.apache.thrift.ProcessFunction { + public unloadApplication() { + super("unloadApplication"); + } + + public unloadApplication_args getEmptyArgsInstance() { + return new unloadApplication_args(); + } + + protected boolean isOneway() { + return false; + } + + public unloadApplication_result getResult(I iface, unloadApplication_args args) throws org.apache.thrift.TException { + unloadApplication_result result = new unloadApplication_result(); + result.success = iface.unloadApplication(args.applicationInstanceId); + return result; + } + } + + public static class runApplication extends org.apache.thrift.ProcessFunction { + public runApplication() { + super("runApplication"); + } + + public runApplication_args getEmptyArgsInstance() { + return new runApplication_args(); + } + + protected boolean isOneway() { + return false; + } + + public runApplication_result getResult(I iface, runApplication_args args) throws org.apache.thrift.TException { + runApplication_result result = new runApplication_result(); + result.success = iface.runApplication(args.applicationInstanceId); + return result; + } + } + } public static class AsyncProcessor extends org.apache.thrift.TBaseAsyncProcessor { @@ -1826,6 +2078,9 @@ protected AsyncProcessor(I iface, Map, java.io.Serializable, Cloneable, Comparable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("createInterpreter_args"); - - private static final org.apache.thrift.protocol.TField INTP_GROUP_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("intpGroupId", org.apache.thrift.protocol.TType.STRING, (short)1); - private static final org.apache.thrift.protocol.TField NOTE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("noteId", org.apache.thrift.protocol.TType.STRING, (short)2); - private static final org.apache.thrift.protocol.TField CLASS_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("className", org.apache.thrift.protocol.TType.STRING, (short)3); - private static final org.apache.thrift.protocol.TField PROPERTIES_FIELD_DESC = new org.apache.thrift.protocol.TField("properties", org.apache.thrift.protocol.TType.MAP, (short)4); - - private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); - static { - schemes.put(StandardScheme.class, new createInterpreter_argsStandardSchemeFactory()); - schemes.put(TupleScheme.class, new createInterpreter_argsTupleSchemeFactory()); - } - - public String intpGroupId; // required - public String noteId; // required - public String className; // required - public Map properties; // required - - /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ - public enum _Fields implements org.apache.thrift.TFieldIdEnum { - INTP_GROUP_ID((short)1, "intpGroupId"), - NOTE_ID((short)2, "noteId"), - CLASS_NAME((short)3, "className"), - PROPERTIES((short)4, "properties"); - - private static final Map byName = new HashMap(); - - static { - for (_Fields field : EnumSet.allOf(_Fields.class)) { - byName.put(field.getFieldName(), field); - } + public static class loadApplication extends org.apache.thrift.AsyncProcessFunction { + public loadApplication() { + super("loadApplication"); } - /** - * Find the _Fields constant that matches fieldId, or null if its not found. - */ - public static _Fields findByThriftId(int fieldId) { - switch(fieldId) { - case 1: // INTP_GROUP_ID - return INTP_GROUP_ID; - case 2: // NOTE_ID - return NOTE_ID; - case 3: // CLASS_NAME - return CLASS_NAME; - case 4: // PROPERTIES - return PROPERTIES; - default: - return null; - } + public loadApplication_args getEmptyArgsInstance() { + return new loadApplication_args(); } - /** - * Find the _Fields constant that matches fieldId, throwing an exception - * if it is not found. - */ - public static _Fields findByThriftIdOrThrow(int fieldId) { - _Fields fields = findByThriftId(fieldId); - if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); - return fields; + public AsyncMethodCallback getResultHandler(final AsyncFrameBuffer fb, final int seqid) { + final org.apache.thrift.AsyncProcessFunction fcall = this; + return new AsyncMethodCallback() { + public void onComplete(RemoteApplicationResult o) { + loadApplication_result result = new loadApplication_result(); + result.success = o; + try { + fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid); + return; + } catch (Exception e) { + LOGGER.error("Exception writing to internal frame buffer", e); + } + fb.close(); + } + public void onError(Exception e) { + byte msgType = org.apache.thrift.protocol.TMessageType.REPLY; + org.apache.thrift.TBase msg; + loadApplication_result result = new loadApplication_result(); + { + msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION; + msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage()); + } + try { + fcall.sendResponse(fb,msg,msgType,seqid); + return; + } catch (Exception ex) { + LOGGER.error("Exception writing to internal frame buffer", ex); + } + fb.close(); + } + }; } - /** - * Find the _Fields constant that matches name, or null if its not found. - */ - public static _Fields findByName(String name) { - return byName.get(name); + protected boolean isOneway() { + return false; } - private final short _thriftId; - private final String _fieldName; - - _Fields(short thriftId, String fieldName) { - _thriftId = thriftId; - _fieldName = fieldName; + public void start(I iface, loadApplication_args args, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws TException { + iface.loadApplication(args.applicationInstanceId, args.packageInfo, args.noteId, args.paragraphId,resultHandler); } + } - public short getThriftFieldId() { - return _thriftId; + public static class unloadApplication extends org.apache.thrift.AsyncProcessFunction { + public unloadApplication() { + super("unloadApplication"); } - public String getFieldName() { - return _fieldName; + public unloadApplication_args getEmptyArgsInstance() { + return new unloadApplication_args(); } - } - // isset id assignments - public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; - static { - Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + public AsyncMethodCallback getResultHandler(final AsyncFrameBuffer fb, final int seqid) { + final org.apache.thrift.AsyncProcessFunction fcall = this; + return new AsyncMethodCallback() { + public void onComplete(RemoteApplicationResult o) { + unloadApplication_result result = new unloadApplication_result(); + result.success = o; + try { + fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid); + return; + } catch (Exception e) { + LOGGER.error("Exception writing to internal frame buffer", e); + } + fb.close(); + } + public void onError(Exception e) { + byte msgType = org.apache.thrift.protocol.TMessageType.REPLY; + org.apache.thrift.TBase msg; + unloadApplication_result result = new unloadApplication_result(); + { + msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION; + msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage()); + } + try { + fcall.sendResponse(fb,msg,msgType,seqid); + return; + } catch (Exception ex) { + LOGGER.error("Exception writing to internal frame buffer", ex); + } + fb.close(); + } + }; + } + + protected boolean isOneway() { + return false; + } + + public void start(I iface, unloadApplication_args args, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws TException { + iface.unloadApplication(args.applicationInstanceId,resultHandler); + } + } + + public static class runApplication extends org.apache.thrift.AsyncProcessFunction { + public runApplication() { + super("runApplication"); + } + + public runApplication_args getEmptyArgsInstance() { + return new runApplication_args(); + } + + public AsyncMethodCallback getResultHandler(final AsyncFrameBuffer fb, final int seqid) { + final org.apache.thrift.AsyncProcessFunction fcall = this; + return new AsyncMethodCallback() { + public void onComplete(RemoteApplicationResult o) { + runApplication_result result = new runApplication_result(); + result.success = o; + try { + fcall.sendResponse(fb,result, org.apache.thrift.protocol.TMessageType.REPLY,seqid); + return; + } catch (Exception e) { + LOGGER.error("Exception writing to internal frame buffer", e); + } + fb.close(); + } + public void onError(Exception e) { + byte msgType = org.apache.thrift.protocol.TMessageType.REPLY; + org.apache.thrift.TBase msg; + runApplication_result result = new runApplication_result(); + { + msgType = org.apache.thrift.protocol.TMessageType.EXCEPTION; + msg = (org.apache.thrift.TBase)new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.INTERNAL_ERROR, e.getMessage()); + } + try { + fcall.sendResponse(fb,msg,msgType,seqid); + return; + } catch (Exception ex) { + LOGGER.error("Exception writing to internal frame buffer", ex); + } + fb.close(); + } + }; + } + + protected boolean isOneway() { + return false; + } + + public void start(I iface, runApplication_args args, org.apache.thrift.async.AsyncMethodCallback resultHandler) throws TException { + iface.runApplication(args.applicationInstanceId,resultHandler); + } + } + + } + + public static class createInterpreter_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("createInterpreter_args"); + + private static final org.apache.thrift.protocol.TField INTP_GROUP_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("intpGroupId", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField NOTE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("noteId", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField CLASS_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("className", org.apache.thrift.protocol.TType.STRING, (short)3); + private static final org.apache.thrift.protocol.TField PROPERTIES_FIELD_DESC = new org.apache.thrift.protocol.TField("properties", org.apache.thrift.protocol.TType.MAP, (short)4); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new createInterpreter_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new createInterpreter_argsTupleSchemeFactory()); + } + + public String intpGroupId; // required + public String noteId; // required + public String className; // required + public Map properties; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + INTP_GROUP_ID((short)1, "intpGroupId"), + NOTE_ID((short)2, "noteId"), + CLASS_NAME((short)3, "className"), + PROPERTIES((short)4, "properties"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // INTP_GROUP_ID + return INTP_GROUP_ID; + case 2: // NOTE_ID + return NOTE_ID; + case 3: // CLASS_NAME + return CLASS_NAME; + case 4: // PROPERTIES + return PROPERTIES; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); tmpMap.put(_Fields.INTP_GROUP_ID, new org.apache.thrift.meta_data.FieldMetaData("intpGroupId", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); tmpMap.put(_Fields.NOTE_ID, new org.apache.thrift.meta_data.FieldMetaData("noteId", org.apache.thrift.TFieldRequirementType.DEFAULT, @@ -14129,54 +14537,6 @@ public String getResourceName() { return this.resourceName; } - public resourceGet_args setNoteId(String noteId) { - this.noteId = noteId; - return this; - } - - public void unsetNoteId() { - this.noteId = null; - } - - /** Returns true if field noteId is set (has been assigned a value) and false otherwise */ - public boolean isSetNoteId() { - return this.noteId != null; - } - - public void setNoteIdIsSet(boolean value) { - if (!value) { - this.noteId = null; - } - } - - public String getParagraphId() { - return this.paragraphId; - } - - public resourceGet_args setParagraphId(String paragraphId) { - this.paragraphId = paragraphId; - return this; - } - - public void unsetParagraphId() { - this.paragraphId = null; - } - - /** Returns true if field paragraphId is set (has been assigned a value) and false otherwise */ - public boolean isSetParagraphId() { - return this.paragraphId != null; - } - - public void setParagraphIdIsSet(boolean value) { - if (!value) { - this.paragraphId = null; - } - } - - public String getResourceName() { - return this.resourceName; - } - public resourceGet_args setResourceName(String resourceName) { this.resourceName = resourceName; return this; @@ -15867,25 +16227,28 @@ public void read(org.apache.thrift.protocol.TProtocol prot, resourceRemove_resul public static class angularObjectUpdate_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("angularObjectUpdate_args"); - private static final org.apache.thrift.protocol.TField NOTE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("noteId", org.apache.thrift.protocol.TType.STRING, (short)1); - private static final org.apache.thrift.protocol.TField PARAGRAPH_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("paragraphId", org.apache.thrift.protocol.TType.STRING, (short)2); - private static final org.apache.thrift.protocol.TField RESOURCE_NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("resourceName", org.apache.thrift.protocol.TType.STRING, (short)3); + private static final org.apache.thrift.protocol.TField NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("name", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField NOTE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("noteId", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField PARAGRAPH_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("paragraphId", org.apache.thrift.protocol.TType.STRING, (short)3); + private static final org.apache.thrift.protocol.TField OBJECT_FIELD_DESC = new org.apache.thrift.protocol.TField("object", org.apache.thrift.protocol.TType.STRING, (short)4); private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); static { - schemes.put(StandardScheme.class, new resourceRemove_argsStandardSchemeFactory()); - schemes.put(TupleScheme.class, new resourceRemove_argsTupleSchemeFactory()); + schemes.put(StandardScheme.class, new angularObjectUpdate_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new angularObjectUpdate_argsTupleSchemeFactory()); } + public String name; // required public String noteId; // required public String paragraphId; // required - public String resourceName; // required + public String object; // required /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { - NOTE_ID((short)1, "noteId"), - PARAGRAPH_ID((short)2, "paragraphId"), - RESOURCE_NAME((short)3, "resourceName"); + NAME((short)1, "name"), + NOTE_ID((short)2, "noteId"), + PARAGRAPH_ID((short)3, "paragraphId"), + OBJECT((short)4, "object"); private static final Map byName = new HashMap(); @@ -15900,12 +16263,14 @@ public enum _Fields implements org.apache.thrift.TFieldIdEnum { */ public static _Fields findByThriftId(int fieldId) { switch(fieldId) { - case 1: // NOTE_ID + case 1: // NAME + return NAME; + case 2: // NOTE_ID return NOTE_ID; - case 2: // PARAGRAPH_ID + case 3: // PARAGRAPH_ID return PARAGRAPH_ID; - case 3: // RESOURCE_NAME - return RESOURCE_NAME; + case 4: // OBJECT + return OBJECT; default: return null; } @@ -15949,61 +16314,93 @@ public String getFieldName() { public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.NAME, new org.apache.thrift.meta_data.FieldMetaData("name", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); tmpMap.put(_Fields.NOTE_ID, new org.apache.thrift.meta_data.FieldMetaData("noteId", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); tmpMap.put(_Fields.PARAGRAPH_ID, new org.apache.thrift.meta_data.FieldMetaData("paragraphId", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); - tmpMap.put(_Fields.RESOURCE_NAME, new org.apache.thrift.meta_data.FieldMetaData("resourceName", org.apache.thrift.TFieldRequirementType.DEFAULT, + tmpMap.put(_Fields.OBJECT, new org.apache.thrift.meta_data.FieldMetaData("object", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); metaDataMap = Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(resourceRemove_args.class, metaDataMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(angularObjectUpdate_args.class, metaDataMap); } - public resourceRemove_args() { + public angularObjectUpdate_args() { } - public resourceRemove_args( + public angularObjectUpdate_args( + String name, String noteId, String paragraphId, - String resourceName) + String object) { this(); + this.name = name; this.noteId = noteId; this.paragraphId = paragraphId; - this.resourceName = resourceName; + this.object = object; } /** * Performs a deep copy on other. */ - public resourceRemove_args(resourceRemove_args other) { + public angularObjectUpdate_args(angularObjectUpdate_args other) { + if (other.isSetName()) { + this.name = other.name; + } if (other.isSetNoteId()) { this.noteId = other.noteId; } if (other.isSetParagraphId()) { this.paragraphId = other.paragraphId; } - if (other.isSetResourceName()) { - this.resourceName = other.resourceName; + if (other.isSetObject()) { + this.object = other.object; } } - public resourceRemove_args deepCopy() { - return new resourceRemove_args(this); + public angularObjectUpdate_args deepCopy() { + return new angularObjectUpdate_args(this); } @Override public void clear() { + this.name = null; this.noteId = null; this.paragraphId = null; - this.resourceName = null; + this.object = null; + } + + public String getName() { + return this.name; + } + + public angularObjectUpdate_args setName(String name) { + this.name = name; + return this; + } + + public void unsetName() { + this.name = null; + } + + /** Returns true if field name is set (has been assigned a value) and false otherwise */ + public boolean isSetName() { + return this.name != null; + } + + public void setNameIsSet(boolean value) { + if (!value) { + this.name = null; + } } public String getNoteId() { return this.noteId; } - public resourceRemove_args setNoteId(String noteId) { + public angularObjectUpdate_args setNoteId(String noteId) { this.noteId = noteId; return this; } @@ -16027,7 +16424,7 @@ public String getParagraphId() { return this.paragraphId; } - public resourceRemove_args setParagraphId(String paragraphId) { + public angularObjectUpdate_args setParagraphId(String paragraphId) { this.paragraphId = paragraphId; return this; } @@ -16047,32 +16444,40 @@ public void setParagraphIdIsSet(boolean value) { } } - public String getResourceName() { - return this.resourceName; + public String getObject() { + return this.object; } - public resourceRemove_args setResourceName(String resourceName) { - this.resourceName = resourceName; + public angularObjectUpdate_args setObject(String object) { + this.object = object; return this; } - public void unsetResourceName() { - this.resourceName = null; + public void unsetObject() { + this.object = null; } - /** Returns true if field resourceName is set (has been assigned a value) and false otherwise */ - public boolean isSetResourceName() { - return this.resourceName != null; + /** Returns true if field object is set (has been assigned a value) and false otherwise */ + public boolean isSetObject() { + return this.object != null; } - public void setResourceNameIsSet(boolean value) { + public void setObjectIsSet(boolean value) { if (!value) { - this.resourceName = null; + this.object = null; } } public void setFieldValue(_Fields field, Object value) { switch (field) { + case NAME: + if (value == null) { + unsetName(); + } else { + setName((String)value); + } + break; + case NOTE_ID: if (value == null) { unsetNoteId(); @@ -16089,11 +16494,11 @@ public void setFieldValue(_Fields field, Object value) { } break; - case RESOURCE_NAME: + case OBJECT: if (value == null) { - unsetResourceName(); + unsetObject(); } else { - setResourceName((String)value); + setObject((String)value); } break; @@ -16102,14 +16507,17 @@ public void setFieldValue(_Fields field, Object value) { public Object getFieldValue(_Fields field) { switch (field) { + case NAME: + return getName(); + case NOTE_ID: return getNoteId(); case PARAGRAPH_ID: return getParagraphId(); - case RESOURCE_NAME: - return getResourceName(); + case OBJECT: + return getObject(); } throw new IllegalStateException(); @@ -16122,12 +16530,14 @@ public boolean isSet(_Fields field) { } switch (field) { + case NAME: + return isSetName(); case NOTE_ID: return isSetNoteId(); case PARAGRAPH_ID: return isSetParagraphId(); - case RESOURCE_NAME: - return isSetResourceName(); + case OBJECT: + return isSetObject(); } throw new IllegalStateException(); } @@ -16136,15 +16546,24 @@ public boolean isSet(_Fields field) { public boolean equals(Object that) { if (that == null) return false; - if (that instanceof resourceRemove_args) - return this.equals((resourceRemove_args)that); + if (that instanceof angularObjectUpdate_args) + return this.equals((angularObjectUpdate_args)that); return false; } - public boolean equals(resourceRemove_args that) { + public boolean equals(angularObjectUpdate_args that) { if (that == null) return false; + boolean this_present_name = true && this.isSetName(); + boolean that_present_name = true && that.isSetName(); + if (this_present_name || that_present_name) { + if (!(this_present_name && that_present_name)) + return false; + if (!this.name.equals(that.name)) + return false; + } + boolean this_present_noteId = true && this.isSetNoteId(); boolean that_present_noteId = true && that.isSetNoteId(); if (this_present_noteId || that_present_noteId) { @@ -16163,12 +16582,12 @@ public boolean equals(resourceRemove_args that) { return false; } - boolean this_present_resourceName = true && this.isSetResourceName(); - boolean that_present_resourceName = true && that.isSetResourceName(); - if (this_present_resourceName || that_present_resourceName) { - if (!(this_present_resourceName && that_present_resourceName)) + boolean this_present_object = true && this.isSetObject(); + boolean that_present_object = true && that.isSetObject(); + if (this_present_object || that_present_object) { + if (!(this_present_object && that_present_object)) return false; - if (!this.resourceName.equals(that.resourceName)) + if (!this.object.equals(that.object)) return false; } @@ -16179,6 +16598,11 @@ public boolean equals(resourceRemove_args that) { public int hashCode() { List list = new ArrayList(); + boolean present_name = true && (isSetName()); + list.add(present_name); + if (present_name) + list.add(name); + boolean present_noteId = true && (isSetNoteId()); list.add(present_noteId); if (present_noteId) @@ -16189,22 +16613,32 @@ public int hashCode() { if (present_paragraphId) list.add(paragraphId); - boolean present_resourceName = true && (isSetResourceName()); - list.add(present_resourceName); - if (present_resourceName) - list.add(resourceName); + boolean present_object = true && (isSetObject()); + list.add(present_object); + if (present_object) + list.add(object); return list.hashCode(); } @Override - public int compareTo(resourceRemove_args other) { + public int compareTo(angularObjectUpdate_args other) { if (!getClass().equals(other.getClass())) { return getClass().getName().compareTo(other.getClass().getName()); } int lastComparison = 0; + lastComparison = Boolean.valueOf(isSetName()).compareTo(other.isSetName()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetName()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.name, other.name); + if (lastComparison != 0) { + return lastComparison; + } + } lastComparison = Boolean.valueOf(isSetNoteId()).compareTo(other.isSetNoteId()); if (lastComparison != 0) { return lastComparison; @@ -16225,12 +16659,12 @@ public int compareTo(resourceRemove_args other) { return lastComparison; } } - lastComparison = Boolean.valueOf(isSetResourceName()).compareTo(other.isSetResourceName()); + lastComparison = Boolean.valueOf(isSetObject()).compareTo(other.isSetObject()); if (lastComparison != 0) { return lastComparison; } - if (isSetResourceName()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.resourceName, other.resourceName); + if (isSetObject()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.object, other.object); if (lastComparison != 0) { return lastComparison; } @@ -16252,9 +16686,17 @@ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache. @Override public String toString() { - StringBuilder sb = new StringBuilder("resourceRemove_args("); + StringBuilder sb = new StringBuilder("angularObjectUpdate_args("); boolean first = true; + sb.append("name:"); + if (this.name == null) { + sb.append("null"); + } else { + sb.append(this.name); + } + first = false; + if (!first) sb.append(", "); sb.append("noteId:"); if (this.noteId == null) { sb.append("null"); @@ -16271,11 +16713,11 @@ public String toString() { } first = false; if (!first) sb.append(", "); - sb.append("resourceName:"); - if (this.resourceName == null) { + sb.append("object:"); + if (this.object == null) { sb.append("null"); } else { - sb.append(this.resourceName); + sb.append(this.object); } first = false; sb.append(")"); @@ -16303,15 +16745,15 @@ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException } } - private static class resourceRemove_argsStandardSchemeFactory implements SchemeFactory { - public resourceRemove_argsStandardScheme getScheme() { - return new resourceRemove_argsStandardScheme(); + private static class angularObjectUpdate_argsStandardSchemeFactory implements SchemeFactory { + public angularObjectUpdate_argsStandardScheme getScheme() { + return new angularObjectUpdate_argsStandardScheme(); } } - private static class resourceRemove_argsStandardScheme extends StandardScheme { + private static class angularObjectUpdate_argsStandardScheme extends StandardScheme { - public void read(org.apache.thrift.protocol.TProtocol iprot, resourceRemove_args struct) throws org.apache.thrift.TException { + public void read(org.apache.thrift.protocol.TProtocol iprot, angularObjectUpdate_args struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TField schemeField; iprot.readStructBegin(); while (true) @@ -16321,7 +16763,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, resourceRemove_args break; } switch (schemeField.id) { - case 1: // NOTE_ID + case 1: // NAME + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.name = iprot.readString(); + struct.setNameIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // NOTE_ID if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { struct.noteId = iprot.readString(); struct.setNoteIdIsSet(true); @@ -16329,7 +16779,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, resourceRemove_args org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; - case 2: // PARAGRAPH_ID + case 3: // PARAGRAPH_ID if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { struct.paragraphId = iprot.readString(); struct.setParagraphIdIsSet(true); @@ -16337,10 +16787,10 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, resourceRemove_args org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; - case 3: // RESOURCE_NAME + case 4: // OBJECT if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.resourceName = iprot.readString(); - struct.setResourceNameIsSet(true); + struct.object = iprot.readString(); + struct.setObjectIsSet(true); } else { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -16356,10 +16806,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, resourceRemove_args struct.validate(); } - public void write(org.apache.thrift.protocol.TProtocol oprot, resourceRemove_args struct) throws org.apache.thrift.TException { + public void write(org.apache.thrift.protocol.TProtocol oprot, angularObjectUpdate_args struct) throws org.apache.thrift.TException { struct.validate(); oprot.writeStructBegin(STRUCT_DESC); + if (struct.name != null) { + oprot.writeFieldBegin(NAME_FIELD_DESC); + oprot.writeString(struct.name); + oprot.writeFieldEnd(); + } if (struct.noteId != null) { oprot.writeFieldBegin(NOTE_ID_FIELD_DESC); oprot.writeString(struct.noteId); @@ -16370,9 +16825,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, resourceRemove_arg oprot.writeString(struct.paragraphId); oprot.writeFieldEnd(); } - if (struct.resourceName != null) { - oprot.writeFieldBegin(RESOURCE_NAME_FIELD_DESC); - oprot.writeString(struct.resourceName); + if (struct.object != null) { + oprot.writeFieldBegin(OBJECT_FIELD_DESC); + oprot.writeString(struct.object); oprot.writeFieldEnd(); } oprot.writeFieldStop(); @@ -16381,76 +16836,84 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, resourceRemove_arg } - private static class resourceRemove_argsTupleSchemeFactory implements SchemeFactory { - public resourceRemove_argsTupleScheme getScheme() { - return new resourceRemove_argsTupleScheme(); + private static class angularObjectUpdate_argsTupleSchemeFactory implements SchemeFactory { + public angularObjectUpdate_argsTupleScheme getScheme() { + return new angularObjectUpdate_argsTupleScheme(); } } - private static class resourceRemove_argsTupleScheme extends TupleScheme { + private static class angularObjectUpdate_argsTupleScheme extends TupleScheme { @Override - public void write(org.apache.thrift.protocol.TProtocol prot, resourceRemove_args struct) throws org.apache.thrift.TException { + public void write(org.apache.thrift.protocol.TProtocol prot, angularObjectUpdate_args struct) throws org.apache.thrift.TException { TTupleProtocol oprot = (TTupleProtocol) prot; BitSet optionals = new BitSet(); - if (struct.isSetNoteId()) { + if (struct.isSetName()) { optionals.set(0); } - if (struct.isSetParagraphId()) { + if (struct.isSetNoteId()) { optionals.set(1); } - if (struct.isSetResourceName()) { + if (struct.isSetParagraphId()) { optionals.set(2); } - oprot.writeBitSet(optionals, 3); + if (struct.isSetObject()) { + optionals.set(3); + } + oprot.writeBitSet(optionals, 4); + if (struct.isSetName()) { + oprot.writeString(struct.name); + } if (struct.isSetNoteId()) { oprot.writeString(struct.noteId); } if (struct.isSetParagraphId()) { oprot.writeString(struct.paragraphId); } - if (struct.isSetResourceName()) { - oprot.writeString(struct.resourceName); + if (struct.isSetObject()) { + oprot.writeString(struct.object); } } @Override - public void read(org.apache.thrift.protocol.TProtocol prot, resourceRemove_args struct) throws org.apache.thrift.TException { + public void read(org.apache.thrift.protocol.TProtocol prot, angularObjectUpdate_args struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; - BitSet incoming = iprot.readBitSet(3); + BitSet incoming = iprot.readBitSet(4); if (incoming.get(0)) { + struct.name = iprot.readString(); + struct.setNameIsSet(true); + } + if (incoming.get(1)) { struct.noteId = iprot.readString(); struct.setNoteIdIsSet(true); } - if (incoming.get(1)) { + if (incoming.get(2)) { struct.paragraphId = iprot.readString(); struct.setParagraphIdIsSet(true); } - if (incoming.get(2)) { - struct.resourceName = iprot.readString(); - struct.setResourceNameIsSet(true); + if (incoming.get(3)) { + struct.object = iprot.readString(); + struct.setObjectIsSet(true); } } } } - public static class resourceRemove_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("resourceRemove_result"); + public static class angularObjectUpdate_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("angularObjectUpdate_result"); - private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.BOOL, (short)0); private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); static { - schemes.put(StandardScheme.class, new resourceRemove_resultStandardSchemeFactory()); - schemes.put(TupleScheme.class, new resourceRemove_resultTupleSchemeFactory()); + schemes.put(StandardScheme.class, new angularObjectUpdate_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new angularObjectUpdate_resultTupleSchemeFactory()); } - public boolean success; // required /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { - SUCCESS((short)0, "success"); +; private static final Map byName = new HashMap(); @@ -16465,8 +16928,6 @@ public enum _Fields implements org.apache.thrift.TFieldIdEnum { */ public static _Fields findByThriftId(int fieldId) { switch(fieldId) { - case 0: // SUCCESS - return SUCCESS; default: return null; } @@ -16505,89 +16966,37 @@ public String getFieldName() { return _fieldName; } } - - // isset id assignments - private static final int __SUCCESS_ISSET_ID = 0; - private byte __isset_bitfield = 0; public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); - tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); metaDataMap = Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(resourceRemove_result.class, metaDataMap); - } - - public resourceRemove_result() { + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(angularObjectUpdate_result.class, metaDataMap); } - public resourceRemove_result( - boolean success) - { - this(); - this.success = success; - setSuccessIsSet(true); + public angularObjectUpdate_result() { } /** * Performs a deep copy on other. */ - public resourceRemove_result(resourceRemove_result other) { - __isset_bitfield = other.__isset_bitfield; - this.success = other.success; + public angularObjectUpdate_result(angularObjectUpdate_result other) { } - public resourceRemove_result deepCopy() { - return new resourceRemove_result(this); + public angularObjectUpdate_result deepCopy() { + return new angularObjectUpdate_result(this); } @Override public void clear() { - setSuccessIsSet(false); - this.success = false; - } - - public boolean isSuccess() { - return this.success; - } - - public resourceRemove_result setSuccess(boolean success) { - this.success = success; - setSuccessIsSet(true); - return this; - } - - public void unsetSuccess() { - __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __SUCCESS_ISSET_ID); - } - - /** Returns true if field success is set (has been assigned a value) and false otherwise */ - public boolean isSetSuccess() { - return EncodingUtils.testBit(__isset_bitfield, __SUCCESS_ISSET_ID); - } - - public void setSuccessIsSet(boolean value) { - __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __SUCCESS_ISSET_ID, value); } public void setFieldValue(_Fields field, Object value) { switch (field) { - case SUCCESS: - if (value == null) { - unsetSuccess(); - } else { - setSuccess((Boolean)value); - } - break; - } } public Object getFieldValue(_Fields field) { switch (field) { - case SUCCESS: - return Boolean.valueOf(isSuccess()); - } throw new IllegalStateException(); } @@ -16599,8 +17008,6 @@ public boolean isSet(_Fields field) { } switch (field) { - case SUCCESS: - return isSetSuccess(); } throw new IllegalStateException(); } @@ -16609,24 +17016,15 @@ public boolean isSet(_Fields field) { public boolean equals(Object that) { if (that == null) return false; - if (that instanceof resourceRemove_result) - return this.equals((resourceRemove_result)that); + if (that instanceof angularObjectUpdate_result) + return this.equals((angularObjectUpdate_result)that); return false; } - public boolean equals(resourceRemove_result that) { + public boolean equals(angularObjectUpdate_result that) { if (that == null) return false; - boolean this_present_success = true; - boolean that_present_success = true; - if (this_present_success || that_present_success) { - if (!(this_present_success && that_present_success)) - return false; - if (this.success != that.success) - return false; - } - return true; } @@ -16634,32 +17032,17 @@ public boolean equals(resourceRemove_result that) { public int hashCode() { List list = new ArrayList(); - boolean present_success = true; - list.add(present_success); - if (present_success) - list.add(success); - return list.hashCode(); } @Override - public int compareTo(resourceRemove_result other) { + public int compareTo(angularObjectUpdate_result other) { if (!getClass().equals(other.getClass())) { return getClass().getName().compareTo(other.getClass().getName()); } int lastComparison = 0; - lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetSuccess()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success); - if (lastComparison != 0) { - return lastComparison; - } - } return 0; } @@ -16677,12 +17060,9 @@ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache. @Override public String toString() { - StringBuilder sb = new StringBuilder("resourceRemove_result("); + StringBuilder sb = new StringBuilder("angularObjectUpdate_result("); boolean first = true; - sb.append("success:"); - sb.append(this.success); - first = false; sb.append(")"); return sb.toString(); } @@ -16702,23 +17082,21 @@ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOExcept private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { try { - // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor. - __isset_bitfield = 0; read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); } catch (org.apache.thrift.TException te) { throw new java.io.IOException(te); } } - private static class resourceRemove_resultStandardSchemeFactory implements SchemeFactory { - public resourceRemove_resultStandardScheme getScheme() { - return new resourceRemove_resultStandardScheme(); + private static class angularObjectUpdate_resultStandardSchemeFactory implements SchemeFactory { + public angularObjectUpdate_resultStandardScheme getScheme() { + return new angularObjectUpdate_resultStandardScheme(); } } - private static class resourceRemove_resultStandardScheme extends StandardScheme { + private static class angularObjectUpdate_resultStandardScheme extends StandardScheme { - public void read(org.apache.thrift.protocol.TProtocol iprot, resourceRemove_result struct) throws org.apache.thrift.TException { + public void read(org.apache.thrift.protocol.TProtocol iprot, angularObjectUpdate_result struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TField schemeField; iprot.readStructBegin(); while (true) @@ -16728,14 +17106,6 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, resourceRemove_resu break; } switch (schemeField.id) { - case 0: // SUCCESS - if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) { - struct.success = iprot.readBool(); - struct.setSuccessIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -16747,57 +17117,39 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, resourceRemove_resu struct.validate(); } - public void write(org.apache.thrift.protocol.TProtocol oprot, resourceRemove_result struct) throws org.apache.thrift.TException { + public void write(org.apache.thrift.protocol.TProtocol oprot, angularObjectUpdate_result struct) throws org.apache.thrift.TException { struct.validate(); oprot.writeStructBegin(STRUCT_DESC); - if (struct.isSetSuccess()) { - oprot.writeFieldBegin(SUCCESS_FIELD_DESC); - oprot.writeBool(struct.success); - oprot.writeFieldEnd(); - } oprot.writeFieldStop(); oprot.writeStructEnd(); } } - private static class resourceRemove_resultTupleSchemeFactory implements SchemeFactory { - public resourceRemove_resultTupleScheme getScheme() { - return new resourceRemove_resultTupleScheme(); + private static class angularObjectUpdate_resultTupleSchemeFactory implements SchemeFactory { + public angularObjectUpdate_resultTupleScheme getScheme() { + return new angularObjectUpdate_resultTupleScheme(); } } - private static class resourceRemove_resultTupleScheme extends TupleScheme { + private static class angularObjectUpdate_resultTupleScheme extends TupleScheme { @Override - public void write(org.apache.thrift.protocol.TProtocol prot, resourceRemove_result struct) throws org.apache.thrift.TException { + public void write(org.apache.thrift.protocol.TProtocol prot, angularObjectUpdate_result struct) throws org.apache.thrift.TException { TTupleProtocol oprot = (TTupleProtocol) prot; - BitSet optionals = new BitSet(); - if (struct.isSetSuccess()) { - optionals.set(0); - } - oprot.writeBitSet(optionals, 1); - if (struct.isSetSuccess()) { - oprot.writeBool(struct.success); - } } @Override - public void read(org.apache.thrift.protocol.TProtocol prot, resourceRemove_result struct) throws org.apache.thrift.TException { + public void read(org.apache.thrift.protocol.TProtocol prot, angularObjectUpdate_result struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; - BitSet incoming = iprot.readBitSet(1); - if (incoming.get(0)) { - struct.success = iprot.readBool(); - struct.setSuccessIsSet(true); - } } } } - public static class angularObjectUpdate_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("angularObjectUpdate_args"); + public static class angularObjectAdd_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("angularObjectAdd_args"); private static final org.apache.thrift.protocol.TField NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("name", org.apache.thrift.protocol.TType.STRING, (short)1); private static final org.apache.thrift.protocol.TField NOTE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("noteId", org.apache.thrift.protocol.TType.STRING, (short)2); @@ -16806,8 +17158,8 @@ public static class angularObjectUpdate_args implements org.apache.thrift.TBase< private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); static { - schemes.put(StandardScheme.class, new angularObjectUpdate_argsStandardSchemeFactory()); - schemes.put(TupleScheme.class, new angularObjectUpdate_argsTupleSchemeFactory()); + schemes.put(StandardScheme.class, new angularObjectAdd_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new angularObjectAdd_argsTupleSchemeFactory()); } public String name; // required @@ -16895,13 +17247,13 @@ public String getFieldName() { tmpMap.put(_Fields.OBJECT, new org.apache.thrift.meta_data.FieldMetaData("object", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); metaDataMap = Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(angularObjectUpdate_args.class, metaDataMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(angularObjectAdd_args.class, metaDataMap); } - public angularObjectUpdate_args() { + public angularObjectAdd_args() { } - public angularObjectUpdate_args( + public angularObjectAdd_args( String name, String noteId, String paragraphId, @@ -16917,7 +17269,7 @@ public angularObjectUpdate_args( /** * Performs a deep copy on other. */ - public angularObjectUpdate_args(angularObjectUpdate_args other) { + public angularObjectAdd_args(angularObjectAdd_args other) { if (other.isSetName()) { this.name = other.name; } @@ -16932,8 +17284,8 @@ public angularObjectUpdate_args(angularObjectUpdate_args other) { } } - public angularObjectUpdate_args deepCopy() { - return new angularObjectUpdate_args(this); + public angularObjectAdd_args deepCopy() { + return new angularObjectAdd_args(this); } @Override @@ -16948,7 +17300,7 @@ public String getName() { return this.name; } - public angularObjectUpdate_args setName(String name) { + public angularObjectAdd_args setName(String name) { this.name = name; return this; } @@ -16972,7 +17324,7 @@ public String getNoteId() { return this.noteId; } - public angularObjectUpdate_args setNoteId(String noteId) { + public angularObjectAdd_args setNoteId(String noteId) { this.noteId = noteId; return this; } @@ -16996,7 +17348,7 @@ public String getParagraphId() { return this.paragraphId; } - public angularObjectUpdate_args setParagraphId(String paragraphId) { + public angularObjectAdd_args setParagraphId(String paragraphId) { this.paragraphId = paragraphId; return this; } @@ -17020,7 +17372,7 @@ public String getObject() { return this.object; } - public angularObjectUpdate_args setObject(String object) { + public angularObjectAdd_args setObject(String object) { this.object = object; return this; } @@ -17118,12 +17470,12 @@ public boolean isSet(_Fields field) { public boolean equals(Object that) { if (that == null) return false; - if (that instanceof angularObjectUpdate_args) - return this.equals((angularObjectUpdate_args)that); + if (that instanceof angularObjectAdd_args) + return this.equals((angularObjectAdd_args)that); return false; } - public boolean equals(angularObjectUpdate_args that) { + public boolean equals(angularObjectAdd_args that) { if (that == null) return false; @@ -17194,7 +17546,7 @@ public int hashCode() { } @Override - public int compareTo(angularObjectUpdate_args other) { + public int compareTo(angularObjectAdd_args other) { if (!getClass().equals(other.getClass())) { return getClass().getName().compareTo(other.getClass().getName()); } @@ -17258,7 +17610,7 @@ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache. @Override public String toString() { - StringBuilder sb = new StringBuilder("angularObjectUpdate_args("); + StringBuilder sb = new StringBuilder("angularObjectAdd_args("); boolean first = true; sb.append("name:"); @@ -17317,15 +17669,15 @@ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException } } - private static class angularObjectUpdate_argsStandardSchemeFactory implements SchemeFactory { - public angularObjectUpdate_argsStandardScheme getScheme() { - return new angularObjectUpdate_argsStandardScheme(); + private static class angularObjectAdd_argsStandardSchemeFactory implements SchemeFactory { + public angularObjectAdd_argsStandardScheme getScheme() { + return new angularObjectAdd_argsStandardScheme(); } } - private static class angularObjectUpdate_argsStandardScheme extends StandardScheme { + private static class angularObjectAdd_argsStandardScheme extends StandardScheme { - public void read(org.apache.thrift.protocol.TProtocol iprot, angularObjectUpdate_args struct) throws org.apache.thrift.TException { + public void read(org.apache.thrift.protocol.TProtocol iprot, angularObjectAdd_args struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TField schemeField; iprot.readStructBegin(); while (true) @@ -17378,7 +17730,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, angularObjectUpdate struct.validate(); } - public void write(org.apache.thrift.protocol.TProtocol oprot, angularObjectUpdate_args struct) throws org.apache.thrift.TException { + public void write(org.apache.thrift.protocol.TProtocol oprot, angularObjectAdd_args struct) throws org.apache.thrift.TException { struct.validate(); oprot.writeStructBegin(STRUCT_DESC); @@ -17408,16 +17760,16 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, angularObjectUpdat } - private static class angularObjectUpdate_argsTupleSchemeFactory implements SchemeFactory { - public angularObjectUpdate_argsTupleScheme getScheme() { - return new angularObjectUpdate_argsTupleScheme(); + private static class angularObjectAdd_argsTupleSchemeFactory implements SchemeFactory { + public angularObjectAdd_argsTupleScheme getScheme() { + return new angularObjectAdd_argsTupleScheme(); } } - private static class angularObjectUpdate_argsTupleScheme extends TupleScheme { + private static class angularObjectAdd_argsTupleScheme extends TupleScheme { @Override - public void write(org.apache.thrift.protocol.TProtocol prot, angularObjectUpdate_args struct) throws org.apache.thrift.TException { + public void write(org.apache.thrift.protocol.TProtocol prot, angularObjectAdd_args struct) throws org.apache.thrift.TException { TTupleProtocol oprot = (TTupleProtocol) prot; BitSet optionals = new BitSet(); if (struct.isSetName()) { @@ -17448,7 +17800,7 @@ public void write(org.apache.thrift.protocol.TProtocol prot, angularObjectUpdate } @Override - public void read(org.apache.thrift.protocol.TProtocol prot, angularObjectUpdate_args struct) throws org.apache.thrift.TException { + public void read(org.apache.thrift.protocol.TProtocol prot, angularObjectAdd_args struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; BitSet incoming = iprot.readBitSet(4); if (incoming.get(0)) { @@ -17472,14 +17824,14 @@ public void read(org.apache.thrift.protocol.TProtocol prot, angularObjectUpdate_ } - public static class angularObjectUpdate_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("angularObjectUpdate_result"); + public static class angularObjectAdd_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("angularObjectAdd_result"); private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); static { - schemes.put(StandardScheme.class, new angularObjectUpdate_resultStandardSchemeFactory()); - schemes.put(TupleScheme.class, new angularObjectUpdate_resultTupleSchemeFactory()); + schemes.put(StandardScheme.class, new angularObjectAdd_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new angularObjectAdd_resultTupleSchemeFactory()); } @@ -17542,20 +17894,20 @@ public String getFieldName() { static { Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); metaDataMap = Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(angularObjectUpdate_result.class, metaDataMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(angularObjectAdd_result.class, metaDataMap); } - public angularObjectUpdate_result() { + public angularObjectAdd_result() { } /** * Performs a deep copy on other. */ - public angularObjectUpdate_result(angularObjectUpdate_result other) { + public angularObjectAdd_result(angularObjectAdd_result other) { } - public angularObjectUpdate_result deepCopy() { - return new angularObjectUpdate_result(this); + public angularObjectAdd_result deepCopy() { + return new angularObjectAdd_result(this); } @Override @@ -17588,12 +17940,12 @@ public boolean isSet(_Fields field) { public boolean equals(Object that) { if (that == null) return false; - if (that instanceof angularObjectUpdate_result) - return this.equals((angularObjectUpdate_result)that); + if (that instanceof angularObjectAdd_result) + return this.equals((angularObjectAdd_result)that); return false; } - public boolean equals(angularObjectUpdate_result that) { + public boolean equals(angularObjectAdd_result that) { if (that == null) return false; @@ -17608,7 +17960,7 @@ public int hashCode() { } @Override - public int compareTo(angularObjectUpdate_result other) { + public int compareTo(angularObjectAdd_result other) { if (!getClass().equals(other.getClass())) { return getClass().getName().compareTo(other.getClass().getName()); } @@ -17632,7 +17984,7 @@ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache. @Override public String toString() { - StringBuilder sb = new StringBuilder("angularObjectUpdate_result("); + StringBuilder sb = new StringBuilder("angularObjectAdd_result("); boolean first = true; sb.append(")"); @@ -17660,15 +18012,15 @@ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException } } - private static class angularObjectUpdate_resultStandardSchemeFactory implements SchemeFactory { - public angularObjectUpdate_resultStandardScheme getScheme() { - return new angularObjectUpdate_resultStandardScheme(); + private static class angularObjectAdd_resultStandardSchemeFactory implements SchemeFactory { + public angularObjectAdd_resultStandardScheme getScheme() { + return new angularObjectAdd_resultStandardScheme(); } } - private static class angularObjectUpdate_resultStandardScheme extends StandardScheme { + private static class angularObjectAdd_resultStandardScheme extends StandardScheme { - public void read(org.apache.thrift.protocol.TProtocol iprot, angularObjectUpdate_result struct) throws org.apache.thrift.TException { + public void read(org.apache.thrift.protocol.TProtocol iprot, angularObjectAdd_result struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TField schemeField; iprot.readStructBegin(); while (true) @@ -17689,7 +18041,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, angularObjectUpdate struct.validate(); } - public void write(org.apache.thrift.protocol.TProtocol oprot, angularObjectUpdate_result struct) throws org.apache.thrift.TException { + public void write(org.apache.thrift.protocol.TProtocol oprot, angularObjectAdd_result struct) throws org.apache.thrift.TException { struct.validate(); oprot.writeStructBegin(STRUCT_DESC); @@ -17699,52 +18051,49 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, angularObjectUpdat } - private static class angularObjectUpdate_resultTupleSchemeFactory implements SchemeFactory { - public angularObjectUpdate_resultTupleScheme getScheme() { - return new angularObjectUpdate_resultTupleScheme(); + private static class angularObjectAdd_resultTupleSchemeFactory implements SchemeFactory { + public angularObjectAdd_resultTupleScheme getScheme() { + return new angularObjectAdd_resultTupleScheme(); } } - private static class angularObjectUpdate_resultTupleScheme extends TupleScheme { + private static class angularObjectAdd_resultTupleScheme extends TupleScheme { @Override - public void write(org.apache.thrift.protocol.TProtocol prot, angularObjectUpdate_result struct) throws org.apache.thrift.TException { + public void write(org.apache.thrift.protocol.TProtocol prot, angularObjectAdd_result struct) throws org.apache.thrift.TException { TTupleProtocol oprot = (TTupleProtocol) prot; } @Override - public void read(org.apache.thrift.protocol.TProtocol prot, angularObjectUpdate_result struct) throws org.apache.thrift.TException { + public void read(org.apache.thrift.protocol.TProtocol prot, angularObjectAdd_result struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; } } } - public static class angularObjectAdd_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("angularObjectAdd_args"); + public static class angularObjectRemove_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("angularObjectRemove_args"); private static final org.apache.thrift.protocol.TField NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("name", org.apache.thrift.protocol.TType.STRING, (short)1); private static final org.apache.thrift.protocol.TField NOTE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("noteId", org.apache.thrift.protocol.TType.STRING, (short)2); private static final org.apache.thrift.protocol.TField PARAGRAPH_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("paragraphId", org.apache.thrift.protocol.TType.STRING, (short)3); - private static final org.apache.thrift.protocol.TField OBJECT_FIELD_DESC = new org.apache.thrift.protocol.TField("object", org.apache.thrift.protocol.TType.STRING, (short)4); private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); static { - schemes.put(StandardScheme.class, new angularObjectAdd_argsStandardSchemeFactory()); - schemes.put(TupleScheme.class, new angularObjectAdd_argsTupleSchemeFactory()); + schemes.put(StandardScheme.class, new angularObjectRemove_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new angularObjectRemove_argsTupleSchemeFactory()); } public String name; // required public String noteId; // required public String paragraphId; // required - public String object; // required /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { NAME((short)1, "name"), NOTE_ID((short)2, "noteId"), - PARAGRAPH_ID((short)3, "paragraphId"), - OBJECT((short)4, "object"); + PARAGRAPH_ID((short)3, "paragraphId"); private static final Map byName = new HashMap(); @@ -17765,8 +18114,6 @@ public static _Fields findByThriftId(int fieldId) { return NOTE_ID; case 3: // PARAGRAPH_ID return PARAGRAPH_ID; - case 4: // OBJECT - return OBJECT; default: return null; } @@ -17816,32 +18163,28 @@ public String getFieldName() { new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); tmpMap.put(_Fields.PARAGRAPH_ID, new org.apache.thrift.meta_data.FieldMetaData("paragraphId", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); - tmpMap.put(_Fields.OBJECT, new org.apache.thrift.meta_data.FieldMetaData("object", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); metaDataMap = Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(angularObjectAdd_args.class, metaDataMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(angularObjectRemove_args.class, metaDataMap); } - public angularObjectAdd_args() { + public angularObjectRemove_args() { } - public angularObjectAdd_args( + public angularObjectRemove_args( String name, String noteId, - String paragraphId, - String object) + String paragraphId) { this(); this.name = name; this.noteId = noteId; this.paragraphId = paragraphId; - this.object = object; } /** * Performs a deep copy on other. */ - public angularObjectAdd_args(angularObjectAdd_args other) { + public angularObjectRemove_args(angularObjectRemove_args other) { if (other.isSetName()) { this.name = other.name; } @@ -17851,13 +18194,10 @@ public angularObjectAdd_args(angularObjectAdd_args other) { if (other.isSetParagraphId()) { this.paragraphId = other.paragraphId; } - if (other.isSetObject()) { - this.object = other.object; - } } - public angularObjectAdd_args deepCopy() { - return new angularObjectAdd_args(this); + public angularObjectRemove_args deepCopy() { + return new angularObjectRemove_args(this); } @Override @@ -17865,14 +18205,13 @@ public void clear() { this.name = null; this.noteId = null; this.paragraphId = null; - this.object = null; } public String getName() { return this.name; } - public angularObjectAdd_args setName(String name) { + public angularObjectRemove_args setName(String name) { this.name = name; return this; } @@ -17896,7 +18235,7 @@ public String getNoteId() { return this.noteId; } - public angularObjectAdd_args setNoteId(String noteId) { + public angularObjectRemove_args setNoteId(String noteId) { this.noteId = noteId; return this; } @@ -17920,7 +18259,7 @@ public String getParagraphId() { return this.paragraphId; } - public angularObjectAdd_args setParagraphId(String paragraphId) { + public angularObjectRemove_args setParagraphId(String paragraphId) { this.paragraphId = paragraphId; return this; } @@ -17940,30 +18279,6 @@ public void setParagraphIdIsSet(boolean value) { } } - public String getObject() { - return this.object; - } - - public angularObjectAdd_args setObject(String object) { - this.object = object; - return this; - } - - public void unsetObject() { - this.object = null; - } - - /** Returns true if field object is set (has been assigned a value) and false otherwise */ - public boolean isSetObject() { - return this.object != null; - } - - public void setObjectIsSet(boolean value) { - if (!value) { - this.object = null; - } - } - public void setFieldValue(_Fields field, Object value) { switch (field) { case NAME: @@ -17990,14 +18305,6 @@ public void setFieldValue(_Fields field, Object value) { } break; - case OBJECT: - if (value == null) { - unsetObject(); - } else { - setObject((String)value); - } - break; - } } @@ -18012,9 +18319,6 @@ public Object getFieldValue(_Fields field) { case PARAGRAPH_ID: return getParagraphId(); - case OBJECT: - return getObject(); - } throw new IllegalStateException(); } @@ -18032,8 +18336,6 @@ public boolean isSet(_Fields field) { return isSetNoteId(); case PARAGRAPH_ID: return isSetParagraphId(); - case OBJECT: - return isSetObject(); } throw new IllegalStateException(); } @@ -18042,12 +18344,12 @@ public boolean isSet(_Fields field) { public boolean equals(Object that) { if (that == null) return false; - if (that instanceof angularObjectAdd_args) - return this.equals((angularObjectAdd_args)that); + if (that instanceof angularObjectRemove_args) + return this.equals((angularObjectRemove_args)that); return false; } - public boolean equals(angularObjectAdd_args that) { + public boolean equals(angularObjectRemove_args that) { if (that == null) return false; @@ -18078,15 +18380,6 @@ public boolean equals(angularObjectAdd_args that) { return false; } - boolean this_present_object = true && this.isSetObject(); - boolean that_present_object = true && that.isSetObject(); - if (this_present_object || that_present_object) { - if (!(this_present_object && that_present_object)) - return false; - if (!this.object.equals(that.object)) - return false; - } - return true; } @@ -18109,16 +18402,11 @@ public int hashCode() { if (present_paragraphId) list.add(paragraphId); - boolean present_object = true && (isSetObject()); - list.add(present_object); - if (present_object) - list.add(object); - return list.hashCode(); } @Override - public int compareTo(angularObjectAdd_args other) { + public int compareTo(angularObjectRemove_args other) { if (!getClass().equals(other.getClass())) { return getClass().getName().compareTo(other.getClass().getName()); } @@ -18155,16 +18443,6 @@ public int compareTo(angularObjectAdd_args other) { return lastComparison; } } - lastComparison = Boolean.valueOf(isSetObject()).compareTo(other.isSetObject()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetObject()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.object, other.object); - if (lastComparison != 0) { - return lastComparison; - } - } return 0; } @@ -18182,7 +18460,7 @@ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache. @Override public String toString() { - StringBuilder sb = new StringBuilder("angularObjectAdd_args("); + StringBuilder sb = new StringBuilder("angularObjectRemove_args("); boolean first = true; sb.append("name:"); @@ -18208,14 +18486,6 @@ public String toString() { sb.append(this.paragraphId); } first = false; - if (!first) sb.append(", "); - sb.append("object:"); - if (this.object == null) { - sb.append("null"); - } else { - sb.append(this.object); - } - first = false; sb.append(")"); return sb.toString(); } @@ -18241,15 +18511,15 @@ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException } } - private static class angularObjectAdd_argsStandardSchemeFactory implements SchemeFactory { - public angularObjectAdd_argsStandardScheme getScheme() { - return new angularObjectAdd_argsStandardScheme(); + private static class angularObjectRemove_argsStandardSchemeFactory implements SchemeFactory { + public angularObjectRemove_argsStandardScheme getScheme() { + return new angularObjectRemove_argsStandardScheme(); } } - private static class angularObjectAdd_argsStandardScheme extends StandardScheme { + private static class angularObjectRemove_argsStandardScheme extends StandardScheme { - public void read(org.apache.thrift.protocol.TProtocol iprot, angularObjectAdd_args struct) throws org.apache.thrift.TException { + public void read(org.apache.thrift.protocol.TProtocol iprot, angularObjectRemove_args struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TField schemeField; iprot.readStructBegin(); while (true) @@ -18283,14 +18553,6 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, angularObjectAdd_ar org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; - case 4: // OBJECT - if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.object = iprot.readString(); - struct.setObjectIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -18302,7 +18564,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, angularObjectAdd_ar struct.validate(); } - public void write(org.apache.thrift.protocol.TProtocol oprot, angularObjectAdd_args struct) throws org.apache.thrift.TException { + public void write(org.apache.thrift.protocol.TProtocol oprot, angularObjectRemove_args struct) throws org.apache.thrift.TException { struct.validate(); oprot.writeStructBegin(STRUCT_DESC); @@ -18321,27 +18583,22 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, angularObjectAdd_a oprot.writeString(struct.paragraphId); oprot.writeFieldEnd(); } - if (struct.object != null) { - oprot.writeFieldBegin(OBJECT_FIELD_DESC); - oprot.writeString(struct.object); - oprot.writeFieldEnd(); - } oprot.writeFieldStop(); oprot.writeStructEnd(); } } - private static class angularObjectAdd_argsTupleSchemeFactory implements SchemeFactory { - public angularObjectAdd_argsTupleScheme getScheme() { - return new angularObjectAdd_argsTupleScheme(); + private static class angularObjectRemove_argsTupleSchemeFactory implements SchemeFactory { + public angularObjectRemove_argsTupleScheme getScheme() { + return new angularObjectRemove_argsTupleScheme(); } } - private static class angularObjectAdd_argsTupleScheme extends TupleScheme { + private static class angularObjectRemove_argsTupleScheme extends TupleScheme { @Override - public void write(org.apache.thrift.protocol.TProtocol prot, angularObjectAdd_args struct) throws org.apache.thrift.TException { + public void write(org.apache.thrift.protocol.TProtocol prot, angularObjectRemove_args struct) throws org.apache.thrift.TException { TTupleProtocol oprot = (TTupleProtocol) prot; BitSet optionals = new BitSet(); if (struct.isSetName()) { @@ -18353,10 +18610,7 @@ public void write(org.apache.thrift.protocol.TProtocol prot, angularObjectAdd_ar if (struct.isSetParagraphId()) { optionals.set(2); } - if (struct.isSetObject()) { - optionals.set(3); - } - oprot.writeBitSet(optionals, 4); + oprot.writeBitSet(optionals, 3); if (struct.isSetName()) { oprot.writeString(struct.name); } @@ -18366,15 +18620,12 @@ public void write(org.apache.thrift.protocol.TProtocol prot, angularObjectAdd_ar if (struct.isSetParagraphId()) { oprot.writeString(struct.paragraphId); } - if (struct.isSetObject()) { - oprot.writeString(struct.object); - } } @Override - public void read(org.apache.thrift.protocol.TProtocol prot, angularObjectAdd_args struct) throws org.apache.thrift.TException { + public void read(org.apache.thrift.protocol.TProtocol prot, angularObjectRemove_args struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; - BitSet incoming = iprot.readBitSet(4); + BitSet incoming = iprot.readBitSet(3); if (incoming.get(0)) { struct.name = iprot.readString(); struct.setNameIsSet(true); @@ -18387,23 +18638,19 @@ public void read(org.apache.thrift.protocol.TProtocol prot, angularObjectAdd_arg struct.paragraphId = iprot.readString(); struct.setParagraphIdIsSet(true); } - if (incoming.get(3)) { - struct.object = iprot.readString(); - struct.setObjectIsSet(true); - } } } } - public static class angularObjectAdd_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("angularObjectAdd_result"); + public static class angularObjectRemove_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("angularObjectRemove_result"); private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); static { - schemes.put(StandardScheme.class, new angularObjectAdd_resultStandardSchemeFactory()); - schemes.put(TupleScheme.class, new angularObjectAdd_resultTupleSchemeFactory()); + schemes.put(StandardScheme.class, new angularObjectRemove_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new angularObjectRemove_resultTupleSchemeFactory()); } @@ -18466,20 +18713,20 @@ public String getFieldName() { static { Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); metaDataMap = Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(angularObjectAdd_result.class, metaDataMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(angularObjectRemove_result.class, metaDataMap); } - public angularObjectAdd_result() { + public angularObjectRemove_result() { } /** * Performs a deep copy on other. */ - public angularObjectAdd_result(angularObjectAdd_result other) { + public angularObjectRemove_result(angularObjectRemove_result other) { } - public angularObjectAdd_result deepCopy() { - return new angularObjectAdd_result(this); + public angularObjectRemove_result deepCopy() { + return new angularObjectRemove_result(this); } @Override @@ -18512,12 +18759,12 @@ public boolean isSet(_Fields field) { public boolean equals(Object that) { if (that == null) return false; - if (that instanceof angularObjectAdd_result) - return this.equals((angularObjectAdd_result)that); + if (that instanceof angularObjectRemove_result) + return this.equals((angularObjectRemove_result)that); return false; } - public boolean equals(angularObjectAdd_result that) { + public boolean equals(angularObjectRemove_result that) { if (that == null) return false; @@ -18532,7 +18779,7 @@ public int hashCode() { } @Override - public int compareTo(angularObjectAdd_result other) { + public int compareTo(angularObjectRemove_result other) { if (!getClass().equals(other.getClass())) { return getClass().getName().compareTo(other.getClass().getName()); } @@ -18556,7 +18803,7 @@ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache. @Override public String toString() { - StringBuilder sb = new StringBuilder("angularObjectAdd_result("); + StringBuilder sb = new StringBuilder("angularObjectRemove_result("); boolean first = true; sb.append(")"); @@ -18584,15 +18831,15 @@ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException } } - private static class angularObjectAdd_resultStandardSchemeFactory implements SchemeFactory { - public angularObjectAdd_resultStandardScheme getScheme() { - return new angularObjectAdd_resultStandardScheme(); + private static class angularObjectRemove_resultStandardSchemeFactory implements SchemeFactory { + public angularObjectRemove_resultStandardScheme getScheme() { + return new angularObjectRemove_resultStandardScheme(); } } - private static class angularObjectAdd_resultStandardScheme extends StandardScheme { + private static class angularObjectRemove_resultStandardScheme extends StandardScheme { - public void read(org.apache.thrift.protocol.TProtocol iprot, angularObjectAdd_result struct) throws org.apache.thrift.TException { + public void read(org.apache.thrift.protocol.TProtocol iprot, angularObjectRemove_result struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TField schemeField; iprot.readStructBegin(); while (true) @@ -18613,7 +18860,7 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, angularObjectAdd_re struct.validate(); } - public void write(org.apache.thrift.protocol.TProtocol oprot, angularObjectAdd_result struct) throws org.apache.thrift.TException { + public void write(org.apache.thrift.protocol.TProtocol oprot, angularObjectRemove_result struct) throws org.apache.thrift.TException { struct.validate(); oprot.writeStructBegin(STRUCT_DESC); @@ -18623,49 +18870,43 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, angularObjectAdd_r } - private static class angularObjectAdd_resultTupleSchemeFactory implements SchemeFactory { - public angularObjectAdd_resultTupleScheme getScheme() { - return new angularObjectAdd_resultTupleScheme(); + private static class angularObjectRemove_resultTupleSchemeFactory implements SchemeFactory { + public angularObjectRemove_resultTupleScheme getScheme() { + return new angularObjectRemove_resultTupleScheme(); } } - private static class angularObjectAdd_resultTupleScheme extends TupleScheme { + private static class angularObjectRemove_resultTupleScheme extends TupleScheme { @Override - public void write(org.apache.thrift.protocol.TProtocol prot, angularObjectAdd_result struct) throws org.apache.thrift.TException { + public void write(org.apache.thrift.protocol.TProtocol prot, angularObjectRemove_result struct) throws org.apache.thrift.TException { TTupleProtocol oprot = (TTupleProtocol) prot; } @Override - public void read(org.apache.thrift.protocol.TProtocol prot, angularObjectAdd_result struct) throws org.apache.thrift.TException { + public void read(org.apache.thrift.protocol.TProtocol prot, angularObjectRemove_result struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; } } } - public static class angularObjectRemove_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("angularObjectRemove_args"); + public static class angularRegistryPush_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("angularRegistryPush_args"); - private static final org.apache.thrift.protocol.TField NAME_FIELD_DESC = new org.apache.thrift.protocol.TField("name", org.apache.thrift.protocol.TType.STRING, (short)1); - private static final org.apache.thrift.protocol.TField NOTE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("noteId", org.apache.thrift.protocol.TType.STRING, (short)2); - private static final org.apache.thrift.protocol.TField PARAGRAPH_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("paragraphId", org.apache.thrift.protocol.TType.STRING, (short)3); + private static final org.apache.thrift.protocol.TField REGISTRY_FIELD_DESC = new org.apache.thrift.protocol.TField("registry", org.apache.thrift.protocol.TType.STRING, (short)1); private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); static { - schemes.put(StandardScheme.class, new angularObjectRemove_argsStandardSchemeFactory()); - schemes.put(TupleScheme.class, new angularObjectRemove_argsTupleSchemeFactory()); + schemes.put(StandardScheme.class, new angularRegistryPush_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new angularRegistryPush_argsTupleSchemeFactory()); } - public String name; // required - public String noteId; // required - public String paragraphId; // required + public String registry; // required /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { - NAME((short)1, "name"), - NOTE_ID((short)2, "noteId"), - PARAGRAPH_ID((short)3, "paragraphId"); + REGISTRY((short)1, "registry"); private static final Map byName = new HashMap(); @@ -18680,12 +18921,8 @@ public enum _Fields implements org.apache.thrift.TFieldIdEnum { */ public static _Fields findByThriftId(int fieldId) { switch(fieldId) { - case 1: // NAME - return NAME; - case 2: // NOTE_ID - return NOTE_ID; - case 3: // PARAGRAPH_ID - return PARAGRAPH_ID; + case 1: // REGISTRY + return REGISTRY; default: return null; } @@ -18729,181 +18966,850 @@ public String getFieldName() { public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); - tmpMap.put(_Fields.NAME, new org.apache.thrift.meta_data.FieldMetaData("name", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); - tmpMap.put(_Fields.NOTE_ID, new org.apache.thrift.meta_data.FieldMetaData("noteId", org.apache.thrift.TFieldRequirementType.DEFAULT, - new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); - tmpMap.put(_Fields.PARAGRAPH_ID, new org.apache.thrift.meta_data.FieldMetaData("paragraphId", org.apache.thrift.TFieldRequirementType.DEFAULT, + tmpMap.put(_Fields.REGISTRY, new org.apache.thrift.meta_data.FieldMetaData("registry", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); metaDataMap = Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(angularObjectRemove_args.class, metaDataMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(angularRegistryPush_args.class, metaDataMap); } - public angularObjectRemove_args() { + public angularRegistryPush_args() { } - public angularObjectRemove_args( - String name, - String noteId, - String paragraphId) + public angularRegistryPush_args( + String registry) { this(); - this.name = name; - this.noteId = noteId; - this.paragraphId = paragraphId; + this.registry = registry; } /** * Performs a deep copy on other. */ - public angularObjectRemove_args(angularObjectRemove_args other) { - if (other.isSetName()) { - this.name = other.name; - } - if (other.isSetNoteId()) { - this.noteId = other.noteId; - } - if (other.isSetParagraphId()) { - this.paragraphId = other.paragraphId; + public angularRegistryPush_args(angularRegistryPush_args other) { + if (other.isSetRegistry()) { + this.registry = other.registry; } } - public angularObjectRemove_args deepCopy() { - return new angularObjectRemove_args(this); + public angularRegistryPush_args deepCopy() { + return new angularRegistryPush_args(this); } @Override public void clear() { - this.name = null; - this.noteId = null; - this.paragraphId = null; + this.registry = null; } - public String getName() { - return this.name; + public String getRegistry() { + return this.registry; } - public angularObjectRemove_args setName(String name) { - this.name = name; + public angularRegistryPush_args setRegistry(String registry) { + this.registry = registry; return this; } - public void unsetName() { - this.name = null; + public void unsetRegistry() { + this.registry = null; } - /** Returns true if field name is set (has been assigned a value) and false otherwise */ - public boolean isSetName() { - return this.name != null; + /** Returns true if field registry is set (has been assigned a value) and false otherwise */ + public boolean isSetRegistry() { + return this.registry != null; } - public void setNameIsSet(boolean value) { + public void setRegistryIsSet(boolean value) { if (!value) { - this.name = null; + this.registry = null; } } - public String getNoteId() { - return this.noteId; - } + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case REGISTRY: + if (value == null) { + unsetRegistry(); + } else { + setRegistry((String)value); + } + break; - public angularObjectRemove_args setNoteId(String noteId) { - this.noteId = noteId; - return this; + } } - public void unsetNoteId() { - this.noteId = null; - } + public Object getFieldValue(_Fields field) { + switch (field) { + case REGISTRY: + return getRegistry(); - /** Returns true if field noteId is set (has been assigned a value) and false otherwise */ - public boolean isSetNoteId() { - return this.noteId != null; + } + throw new IllegalStateException(); } - public void setNoteIdIsSet(boolean value) { - if (!value) { - this.noteId = null; + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); } - } - public String getParagraphId() { - return this.paragraphId; + switch (field) { + case REGISTRY: + return isSetRegistry(); + } + throw new IllegalStateException(); } - public angularObjectRemove_args setParagraphId(String paragraphId) { - this.paragraphId = paragraphId; - return this; + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof angularRegistryPush_args) + return this.equals((angularRegistryPush_args)that); + return false; } - public void unsetParagraphId() { - this.paragraphId = null; - } + public boolean equals(angularRegistryPush_args that) { + if (that == null) + return false; - /** Returns true if field paragraphId is set (has been assigned a value) and false otherwise */ - public boolean isSetParagraphId() { - return this.paragraphId != null; + boolean this_present_registry = true && this.isSetRegistry(); + boolean that_present_registry = true && that.isSetRegistry(); + if (this_present_registry || that_present_registry) { + if (!(this_present_registry && that_present_registry)) + return false; + if (!this.registry.equals(that.registry)) + return false; + } + + return true; } - public void setParagraphIdIsSet(boolean value) { - if (!value) { - this.paragraphId = null; - } + @Override + public int hashCode() { + List list = new ArrayList(); + + boolean present_registry = true && (isSetRegistry()); + list.add(present_registry); + if (present_registry) + list.add(registry); + + return list.hashCode(); } - public void setFieldValue(_Fields field, Object value) { - switch (field) { - case NAME: - if (value == null) { - unsetName(); - } else { - setName((String)value); - } - break; + @Override + public int compareTo(angularRegistryPush_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } - case NOTE_ID: - if (value == null) { - unsetNoteId(); - } else { - setNoteId((String)value); - } - break; + int lastComparison = 0; - case PARAGRAPH_ID: - if (value == null) { - unsetParagraphId(); - } else { - setParagraphId((String)value); + lastComparison = Boolean.valueOf(isSetRegistry()).compareTo(other.isSetRegistry()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetRegistry()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.registry, other.registry); + if (lastComparison != 0) { + return lastComparison; } - break; - } + return 0; } - public Object getFieldValue(_Fields field) { - switch (field) { - case NAME: - return getName(); + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } - case NOTE_ID: - return getNoteId(); + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } - case PARAGRAPH_ID: - return getParagraphId(); + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + @Override + public String toString() { + StringBuilder sb = new StringBuilder("angularRegistryPush_args("); + boolean first = true; + + sb.append("registry:"); + if (this.registry == null) { + sb.append("null"); + } else { + sb.append(this.registry); } - throw new IllegalStateException(); + first = false; + sb.append(")"); + return sb.toString(); } - /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ - public boolean isSet(_Fields field) { - if (field == null) { - throw new IllegalArgumentException(); + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); } + } - switch (field) { - case NAME: - return isSetName(); + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class angularRegistryPush_argsStandardSchemeFactory implements SchemeFactory { + public angularRegistryPush_argsStandardScheme getScheme() { + return new angularRegistryPush_argsStandardScheme(); + } + } + + private static class angularRegistryPush_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, angularRegistryPush_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // REGISTRY + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.registry = iprot.readString(); + struct.setRegistryIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, angularRegistryPush_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.registry != null) { + oprot.writeFieldBegin(REGISTRY_FIELD_DESC); + oprot.writeString(struct.registry); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class angularRegistryPush_argsTupleSchemeFactory implements SchemeFactory { + public angularRegistryPush_argsTupleScheme getScheme() { + return new angularRegistryPush_argsTupleScheme(); + } + } + + private static class angularRegistryPush_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, angularRegistryPush_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetRegistry()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetRegistry()) { + oprot.writeString(struct.registry); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, angularRegistryPush_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.registry = iprot.readString(); + struct.setRegistryIsSet(true); + } + } + } + + } + + public static class angularRegistryPush_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("angularRegistryPush_result"); + + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new angularRegistryPush_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new angularRegistryPush_resultTupleSchemeFactory()); + } + + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { +; + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(angularRegistryPush_result.class, metaDataMap); + } + + public angularRegistryPush_result() { + } + + /** + * Performs a deep copy on other. + */ + public angularRegistryPush_result(angularRegistryPush_result other) { + } + + public angularRegistryPush_result deepCopy() { + return new angularRegistryPush_result(this); + } + + @Override + public void clear() { + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof angularRegistryPush_result) + return this.equals((angularRegistryPush_result)that); + return false; + } + + public boolean equals(angularRegistryPush_result that) { + if (that == null) + return false; + + return true; + } + + @Override + public int hashCode() { + List list = new ArrayList(); + + return list.hashCode(); + } + + @Override + public int compareTo(angularRegistryPush_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("angularRegistryPush_result("); + boolean first = true; + + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class angularRegistryPush_resultStandardSchemeFactory implements SchemeFactory { + public angularRegistryPush_resultStandardScheme getScheme() { + return new angularRegistryPush_resultStandardScheme(); + } + } + + private static class angularRegistryPush_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, angularRegistryPush_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, angularRegistryPush_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class angularRegistryPush_resultTupleSchemeFactory implements SchemeFactory { + public angularRegistryPush_resultTupleScheme getScheme() { + return new angularRegistryPush_resultTupleScheme(); + } + } + + private static class angularRegistryPush_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, angularRegistryPush_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, angularRegistryPush_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + } + } + + } + + public static class loadApplication_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("loadApplication_args"); + + private static final org.apache.thrift.protocol.TField APPLICATION_INSTANCE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("applicationInstanceId", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField PACKAGE_INFO_FIELD_DESC = new org.apache.thrift.protocol.TField("packageInfo", org.apache.thrift.protocol.TType.STRING, (short)2); + private static final org.apache.thrift.protocol.TField NOTE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("noteId", org.apache.thrift.protocol.TType.STRING, (short)3); + private static final org.apache.thrift.protocol.TField PARAGRAPH_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("paragraphId", org.apache.thrift.protocol.TType.STRING, (short)4); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new loadApplication_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new loadApplication_argsTupleSchemeFactory()); + } + + public String applicationInstanceId; // required + public String packageInfo; // required + public String noteId; // required + public String paragraphId; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + APPLICATION_INSTANCE_ID((short)1, "applicationInstanceId"), + PACKAGE_INFO((short)2, "packageInfo"), + NOTE_ID((short)3, "noteId"), + PARAGRAPH_ID((short)4, "paragraphId"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // APPLICATION_INSTANCE_ID + return APPLICATION_INSTANCE_ID; + case 2: // PACKAGE_INFO + return PACKAGE_INFO; + case 3: // NOTE_ID + return NOTE_ID; + case 4: // PARAGRAPH_ID + return PARAGRAPH_ID; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.APPLICATION_INSTANCE_ID, new org.apache.thrift.meta_data.FieldMetaData("applicationInstanceId", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.PACKAGE_INFO, new org.apache.thrift.meta_data.FieldMetaData("packageInfo", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.NOTE_ID, new org.apache.thrift.meta_data.FieldMetaData("noteId", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.PARAGRAPH_ID, new org.apache.thrift.meta_data.FieldMetaData("paragraphId", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(loadApplication_args.class, metaDataMap); + } + + public loadApplication_args() { + } + + public loadApplication_args( + String applicationInstanceId, + String packageInfo, + String noteId, + String paragraphId) + { + this(); + this.applicationInstanceId = applicationInstanceId; + this.packageInfo = packageInfo; + this.noteId = noteId; + this.paragraphId = paragraphId; + } + + /** + * Performs a deep copy on other. + */ + public loadApplication_args(loadApplication_args other) { + if (other.isSetApplicationInstanceId()) { + this.applicationInstanceId = other.applicationInstanceId; + } + if (other.isSetPackageInfo()) { + this.packageInfo = other.packageInfo; + } + if (other.isSetNoteId()) { + this.noteId = other.noteId; + } + if (other.isSetParagraphId()) { + this.paragraphId = other.paragraphId; + } + } + + public loadApplication_args deepCopy() { + return new loadApplication_args(this); + } + + @Override + public void clear() { + this.applicationInstanceId = null; + this.packageInfo = null; + this.noteId = null; + this.paragraphId = null; + } + + public String getApplicationInstanceId() { + return this.applicationInstanceId; + } + + public loadApplication_args setApplicationInstanceId(String applicationInstanceId) { + this.applicationInstanceId = applicationInstanceId; + return this; + } + + public void unsetApplicationInstanceId() { + this.applicationInstanceId = null; + } + + /** Returns true if field applicationInstanceId is set (has been assigned a value) and false otherwise */ + public boolean isSetApplicationInstanceId() { + return this.applicationInstanceId != null; + } + + public void setApplicationInstanceIdIsSet(boolean value) { + if (!value) { + this.applicationInstanceId = null; + } + } + + public String getPackageInfo() { + return this.packageInfo; + } + + public loadApplication_args setPackageInfo(String packageInfo) { + this.packageInfo = packageInfo; + return this; + } + + public void unsetPackageInfo() { + this.packageInfo = null; + } + + /** Returns true if field packageInfo is set (has been assigned a value) and false otherwise */ + public boolean isSetPackageInfo() { + return this.packageInfo != null; + } + + public void setPackageInfoIsSet(boolean value) { + if (!value) { + this.packageInfo = null; + } + } + + public String getNoteId() { + return this.noteId; + } + + public loadApplication_args setNoteId(String noteId) { + this.noteId = noteId; + return this; + } + + public void unsetNoteId() { + this.noteId = null; + } + + /** Returns true if field noteId is set (has been assigned a value) and false otherwise */ + public boolean isSetNoteId() { + return this.noteId != null; + } + + public void setNoteIdIsSet(boolean value) { + if (!value) { + this.noteId = null; + } + } + + public String getParagraphId() { + return this.paragraphId; + } + + public loadApplication_args setParagraphId(String paragraphId) { + this.paragraphId = paragraphId; + return this; + } + + public void unsetParagraphId() { + this.paragraphId = null; + } + + /** Returns true if field paragraphId is set (has been assigned a value) and false otherwise */ + public boolean isSetParagraphId() { + return this.paragraphId != null; + } + + public void setParagraphIdIsSet(boolean value) { + if (!value) { + this.paragraphId = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case APPLICATION_INSTANCE_ID: + if (value == null) { + unsetApplicationInstanceId(); + } else { + setApplicationInstanceId((String)value); + } + break; + + case PACKAGE_INFO: + if (value == null) { + unsetPackageInfo(); + } else { + setPackageInfo((String)value); + } + break; + + case NOTE_ID: + if (value == null) { + unsetNoteId(); + } else { + setNoteId((String)value); + } + break; + + case PARAGRAPH_ID: + if (value == null) { + unsetParagraphId(); + } else { + setParagraphId((String)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case APPLICATION_INSTANCE_ID: + return getApplicationInstanceId(); + + case PACKAGE_INFO: + return getPackageInfo(); + + case NOTE_ID: + return getNoteId(); + + case PARAGRAPH_ID: + return getParagraphId(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case APPLICATION_INSTANCE_ID: + return isSetApplicationInstanceId(); + case PACKAGE_INFO: + return isSetPackageInfo(); case NOTE_ID: return isSetNoteId(); case PARAGRAPH_ID: @@ -18916,39 +19822,913 @@ public boolean isSet(_Fields field) { public boolean equals(Object that) { if (that == null) return false; - if (that instanceof angularObjectRemove_args) - return this.equals((angularObjectRemove_args)that); + if (that instanceof loadApplication_args) + return this.equals((loadApplication_args)that); + return false; + } + + public boolean equals(loadApplication_args that) { + if (that == null) + return false; + + boolean this_present_applicationInstanceId = true && this.isSetApplicationInstanceId(); + boolean that_present_applicationInstanceId = true && that.isSetApplicationInstanceId(); + if (this_present_applicationInstanceId || that_present_applicationInstanceId) { + if (!(this_present_applicationInstanceId && that_present_applicationInstanceId)) + return false; + if (!this.applicationInstanceId.equals(that.applicationInstanceId)) + return false; + } + + boolean this_present_packageInfo = true && this.isSetPackageInfo(); + boolean that_present_packageInfo = true && that.isSetPackageInfo(); + if (this_present_packageInfo || that_present_packageInfo) { + if (!(this_present_packageInfo && that_present_packageInfo)) + return false; + if (!this.packageInfo.equals(that.packageInfo)) + return false; + } + + boolean this_present_noteId = true && this.isSetNoteId(); + boolean that_present_noteId = true && that.isSetNoteId(); + if (this_present_noteId || that_present_noteId) { + if (!(this_present_noteId && that_present_noteId)) + return false; + if (!this.noteId.equals(that.noteId)) + return false; + } + + boolean this_present_paragraphId = true && this.isSetParagraphId(); + boolean that_present_paragraphId = true && that.isSetParagraphId(); + if (this_present_paragraphId || that_present_paragraphId) { + if (!(this_present_paragraphId && that_present_paragraphId)) + return false; + if (!this.paragraphId.equals(that.paragraphId)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + List list = new ArrayList(); + + boolean present_applicationInstanceId = true && (isSetApplicationInstanceId()); + list.add(present_applicationInstanceId); + if (present_applicationInstanceId) + list.add(applicationInstanceId); + + boolean present_packageInfo = true && (isSetPackageInfo()); + list.add(present_packageInfo); + if (present_packageInfo) + list.add(packageInfo); + + boolean present_noteId = true && (isSetNoteId()); + list.add(present_noteId); + if (present_noteId) + list.add(noteId); + + boolean present_paragraphId = true && (isSetParagraphId()); + list.add(present_paragraphId); + if (present_paragraphId) + list.add(paragraphId); + + return list.hashCode(); + } + + @Override + public int compareTo(loadApplication_args other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = Boolean.valueOf(isSetApplicationInstanceId()).compareTo(other.isSetApplicationInstanceId()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetApplicationInstanceId()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.applicationInstanceId, other.applicationInstanceId); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetPackageInfo()).compareTo(other.isSetPackageInfo()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetPackageInfo()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.packageInfo, other.packageInfo); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetNoteId()).compareTo(other.isSetNoteId()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetNoteId()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.noteId, other.noteId); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetParagraphId()).compareTo(other.isSetParagraphId()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetParagraphId()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.paragraphId, other.paragraphId); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("loadApplication_args("); + boolean first = true; + + sb.append("applicationInstanceId:"); + if (this.applicationInstanceId == null) { + sb.append("null"); + } else { + sb.append(this.applicationInstanceId); + } + first = false; + if (!first) sb.append(", "); + sb.append("packageInfo:"); + if (this.packageInfo == null) { + sb.append("null"); + } else { + sb.append(this.packageInfo); + } + first = false; + if (!first) sb.append(", "); + sb.append("noteId:"); + if (this.noteId == null) { + sb.append("null"); + } else { + sb.append(this.noteId); + } + first = false; + if (!first) sb.append(", "); + sb.append("paragraphId:"); + if (this.paragraphId == null) { + sb.append("null"); + } else { + sb.append(this.paragraphId); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class loadApplication_argsStandardSchemeFactory implements SchemeFactory { + public loadApplication_argsStandardScheme getScheme() { + return new loadApplication_argsStandardScheme(); + } + } + + private static class loadApplication_argsStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, loadApplication_args struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 1: // APPLICATION_INSTANCE_ID + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.applicationInstanceId = iprot.readString(); + struct.setApplicationInstanceIdIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 2: // PACKAGE_INFO + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.packageInfo = iprot.readString(); + struct.setPackageInfoIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 3: // NOTE_ID + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.noteId = iprot.readString(); + struct.setNoteIdIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 4: // PARAGRAPH_ID + if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { + struct.paragraphId = iprot.readString(); + struct.setParagraphIdIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, loadApplication_args struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.applicationInstanceId != null) { + oprot.writeFieldBegin(APPLICATION_INSTANCE_ID_FIELD_DESC); + oprot.writeString(struct.applicationInstanceId); + oprot.writeFieldEnd(); + } + if (struct.packageInfo != null) { + oprot.writeFieldBegin(PACKAGE_INFO_FIELD_DESC); + oprot.writeString(struct.packageInfo); + oprot.writeFieldEnd(); + } + if (struct.noteId != null) { + oprot.writeFieldBegin(NOTE_ID_FIELD_DESC); + oprot.writeString(struct.noteId); + oprot.writeFieldEnd(); + } + if (struct.paragraphId != null) { + oprot.writeFieldBegin(PARAGRAPH_ID_FIELD_DESC); + oprot.writeString(struct.paragraphId); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class loadApplication_argsTupleSchemeFactory implements SchemeFactory { + public loadApplication_argsTupleScheme getScheme() { + return new loadApplication_argsTupleScheme(); + } + } + + private static class loadApplication_argsTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, loadApplication_args struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetApplicationInstanceId()) { + optionals.set(0); + } + if (struct.isSetPackageInfo()) { + optionals.set(1); + } + if (struct.isSetNoteId()) { + optionals.set(2); + } + if (struct.isSetParagraphId()) { + optionals.set(3); + } + oprot.writeBitSet(optionals, 4); + if (struct.isSetApplicationInstanceId()) { + oprot.writeString(struct.applicationInstanceId); + } + if (struct.isSetPackageInfo()) { + oprot.writeString(struct.packageInfo); + } + if (struct.isSetNoteId()) { + oprot.writeString(struct.noteId); + } + if (struct.isSetParagraphId()) { + oprot.writeString(struct.paragraphId); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, loadApplication_args struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(4); + if (incoming.get(0)) { + struct.applicationInstanceId = iprot.readString(); + struct.setApplicationInstanceIdIsSet(true); + } + if (incoming.get(1)) { + struct.packageInfo = iprot.readString(); + struct.setPackageInfoIsSet(true); + } + if (incoming.get(2)) { + struct.noteId = iprot.readString(); + struct.setNoteIdIsSet(true); + } + if (incoming.get(3)) { + struct.paragraphId = iprot.readString(); + struct.setParagraphIdIsSet(true); + } + } + } + + } + + public static class loadApplication_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("loadApplication_result"); + + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new loadApplication_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new loadApplication_resultTupleSchemeFactory()); + } + + public RemoteApplicationResult success; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + SUCCESS((short)0, "success"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, RemoteApplicationResult.class))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(loadApplication_result.class, metaDataMap); + } + + public loadApplication_result() { + } + + public loadApplication_result( + RemoteApplicationResult success) + { + this(); + this.success = success; + } + + /** + * Performs a deep copy on other. + */ + public loadApplication_result(loadApplication_result other) { + if (other.isSetSuccess()) { + this.success = new RemoteApplicationResult(other.success); + } + } + + public loadApplication_result deepCopy() { + return new loadApplication_result(this); + } + + @Override + public void clear() { + this.success = null; + } + + public RemoteApplicationResult getSuccess() { + return this.success; + } + + public loadApplication_result setSuccess(RemoteApplicationResult success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((RemoteApplicationResult)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case SUCCESS: + return getSuccess(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case SUCCESS: + return isSetSuccess(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof loadApplication_result) + return this.equals((loadApplication_result)that); + return false; + } + + public boolean equals(loadApplication_result that) { + if (that == null) + return false; + + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + + return true; + } + + @Override + public int hashCode() { + List list = new ArrayList(); + + boolean present_success = true && (isSetSuccess()); + list.add(present_success); + if (present_success) + list.add(success); + + return list.hashCode(); + } + + @Override + public int compareTo(loadApplication_result other) { + if (!getClass().equals(other.getClass())) { + return getClass().getName().compareTo(other.getClass().getName()); + } + + int lastComparison = 0; + + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success); + if (lastComparison != 0) { + return lastComparison; + } + } + return 0; + } + + public _Fields fieldForId(int fieldId) { + return _Fields.findByThriftId(fieldId); + } + + public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("loadApplication_result("); + boolean first = true; + + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; + sb.append(")"); + return sb.toString(); + } + + public void validate() throws org.apache.thrift.TException { + // check for required fields + // check for sub-struct validity + if (success != null) { + success.validate(); + } + } + + private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { + try { + write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException { + try { + read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in))); + } catch (org.apache.thrift.TException te) { + throw new java.io.IOException(te); + } + } + + private static class loadApplication_resultStandardSchemeFactory implements SchemeFactory { + public loadApplication_resultStandardScheme getScheme() { + return new loadApplication_resultStandardScheme(); + } + } + + private static class loadApplication_resultStandardScheme extends StandardScheme { + + public void read(org.apache.thrift.protocol.TProtocol iprot, loadApplication_result struct) throws org.apache.thrift.TException { + org.apache.thrift.protocol.TField schemeField; + iprot.readStructBegin(); + while (true) + { + schemeField = iprot.readFieldBegin(); + if (schemeField.type == org.apache.thrift.protocol.TType.STOP) { + break; + } + switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.success = new RemoteApplicationResult(); + struct.success.read(iprot); + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + default: + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + + // check for required fields of primitive type, which can't be checked in the validate method + struct.validate(); + } + + public void write(org.apache.thrift.protocol.TProtocol oprot, loadApplication_result struct) throws org.apache.thrift.TException { + struct.validate(); + + oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + struct.success.write(oprot); + oprot.writeFieldEnd(); + } + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + + } + + private static class loadApplication_resultTupleSchemeFactory implements SchemeFactory { + public loadApplication_resultTupleScheme getScheme() { + return new loadApplication_resultTupleScheme(); + } + } + + private static class loadApplication_resultTupleScheme extends TupleScheme { + + @Override + public void write(org.apache.thrift.protocol.TProtocol prot, loadApplication_result struct) throws org.apache.thrift.TException { + TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetSuccess()) { + struct.success.write(oprot); + } + } + + @Override + public void read(org.apache.thrift.protocol.TProtocol prot, loadApplication_result struct) throws org.apache.thrift.TException { + TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.success = new RemoteApplicationResult(); + struct.success.read(iprot); + struct.setSuccessIsSet(true); + } + } + } + + } + + public static class unloadApplication_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("unloadApplication_args"); + + private static final org.apache.thrift.protocol.TField APPLICATION_INSTANCE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("applicationInstanceId", org.apache.thrift.protocol.TType.STRING, (short)1); + + private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); + static { + schemes.put(StandardScheme.class, new unloadApplication_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new unloadApplication_argsTupleSchemeFactory()); + } + + public String applicationInstanceId; // required + + /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ + public enum _Fields implements org.apache.thrift.TFieldIdEnum { + APPLICATION_INSTANCE_ID((short)1, "applicationInstanceId"); + + private static final Map byName = new HashMap(); + + static { + for (_Fields field : EnumSet.allOf(_Fields.class)) { + byName.put(field.getFieldName(), field); + } + } + + /** + * Find the _Fields constant that matches fieldId, or null if its not found. + */ + public static _Fields findByThriftId(int fieldId) { + switch(fieldId) { + case 1: // APPLICATION_INSTANCE_ID + return APPLICATION_INSTANCE_ID; + default: + return null; + } + } + + /** + * Find the _Fields constant that matches fieldId, throwing an exception + * if it is not found. + */ + public static _Fields findByThriftIdOrThrow(int fieldId) { + _Fields fields = findByThriftId(fieldId); + if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!"); + return fields; + } + + /** + * Find the _Fields constant that matches name, or null if its not found. + */ + public static _Fields findByName(String name) { + return byName.get(name); + } + + private final short _thriftId; + private final String _fieldName; + + _Fields(short thriftId, String fieldName) { + _thriftId = thriftId; + _fieldName = fieldName; + } + + public short getThriftFieldId() { + return _thriftId; + } + + public String getFieldName() { + return _fieldName; + } + } + + // isset id assignments + public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; + static { + Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.APPLICATION_INSTANCE_ID, new org.apache.thrift.meta_data.FieldMetaData("applicationInstanceId", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + metaDataMap = Collections.unmodifiableMap(tmpMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(unloadApplication_args.class, metaDataMap); + } + + public unloadApplication_args() { + } + + public unloadApplication_args( + String applicationInstanceId) + { + this(); + this.applicationInstanceId = applicationInstanceId; + } + + /** + * Performs a deep copy on other. + */ + public unloadApplication_args(unloadApplication_args other) { + if (other.isSetApplicationInstanceId()) { + this.applicationInstanceId = other.applicationInstanceId; + } + } + + public unloadApplication_args deepCopy() { + return new unloadApplication_args(this); + } + + @Override + public void clear() { + this.applicationInstanceId = null; + } + + public String getApplicationInstanceId() { + return this.applicationInstanceId; + } + + public unloadApplication_args setApplicationInstanceId(String applicationInstanceId) { + this.applicationInstanceId = applicationInstanceId; + return this; + } + + public void unsetApplicationInstanceId() { + this.applicationInstanceId = null; + } + + /** Returns true if field applicationInstanceId is set (has been assigned a value) and false otherwise */ + public boolean isSetApplicationInstanceId() { + return this.applicationInstanceId != null; + } + + public void setApplicationInstanceIdIsSet(boolean value) { + if (!value) { + this.applicationInstanceId = null; + } + } + + public void setFieldValue(_Fields field, Object value) { + switch (field) { + case APPLICATION_INSTANCE_ID: + if (value == null) { + unsetApplicationInstanceId(); + } else { + setApplicationInstanceId((String)value); + } + break; + + } + } + + public Object getFieldValue(_Fields field) { + switch (field) { + case APPLICATION_INSTANCE_ID: + return getApplicationInstanceId(); + + } + throw new IllegalStateException(); + } + + /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ + public boolean isSet(_Fields field) { + if (field == null) { + throw new IllegalArgumentException(); + } + + switch (field) { + case APPLICATION_INSTANCE_ID: + return isSetApplicationInstanceId(); + } + throw new IllegalStateException(); + } + + @Override + public boolean equals(Object that) { + if (that == null) + return false; + if (that instanceof unloadApplication_args) + return this.equals((unloadApplication_args)that); return false; } - public boolean equals(angularObjectRemove_args that) { + public boolean equals(unloadApplication_args that) { if (that == null) return false; - boolean this_present_name = true && this.isSetName(); - boolean that_present_name = true && that.isSetName(); - if (this_present_name || that_present_name) { - if (!(this_present_name && that_present_name)) - return false; - if (!this.name.equals(that.name)) - return false; - } - - boolean this_present_noteId = true && this.isSetNoteId(); - boolean that_present_noteId = true && that.isSetNoteId(); - if (this_present_noteId || that_present_noteId) { - if (!(this_present_noteId && that_present_noteId)) - return false; - if (!this.noteId.equals(that.noteId)) - return false; - } - - boolean this_present_paragraphId = true && this.isSetParagraphId(); - boolean that_present_paragraphId = true && that.isSetParagraphId(); - if (this_present_paragraphId || that_present_paragraphId) { - if (!(this_present_paragraphId && that_present_paragraphId)) + boolean this_present_applicationInstanceId = true && this.isSetApplicationInstanceId(); + boolean that_present_applicationInstanceId = true && that.isSetApplicationInstanceId(); + if (this_present_applicationInstanceId || that_present_applicationInstanceId) { + if (!(this_present_applicationInstanceId && that_present_applicationInstanceId)) return false; - if (!this.paragraphId.equals(that.paragraphId)) + if (!this.applicationInstanceId.equals(that.applicationInstanceId)) return false; } @@ -18959,58 +20739,28 @@ public boolean equals(angularObjectRemove_args that) { public int hashCode() { List list = new ArrayList(); - boolean present_name = true && (isSetName()); - list.add(present_name); - if (present_name) - list.add(name); - - boolean present_noteId = true && (isSetNoteId()); - list.add(present_noteId); - if (present_noteId) - list.add(noteId); - - boolean present_paragraphId = true && (isSetParagraphId()); - list.add(present_paragraphId); - if (present_paragraphId) - list.add(paragraphId); + boolean present_applicationInstanceId = true && (isSetApplicationInstanceId()); + list.add(present_applicationInstanceId); + if (present_applicationInstanceId) + list.add(applicationInstanceId); return list.hashCode(); } @Override - public int compareTo(angularObjectRemove_args other) { + public int compareTo(unloadApplication_args other) { if (!getClass().equals(other.getClass())) { return getClass().getName().compareTo(other.getClass().getName()); } int lastComparison = 0; - lastComparison = Boolean.valueOf(isSetName()).compareTo(other.isSetName()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetName()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.name, other.name); - if (lastComparison != 0) { - return lastComparison; - } - } - lastComparison = Boolean.valueOf(isSetNoteId()).compareTo(other.isSetNoteId()); - if (lastComparison != 0) { - return lastComparison; - } - if (isSetNoteId()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.noteId, other.noteId); - if (lastComparison != 0) { - return lastComparison; - } - } - lastComparison = Boolean.valueOf(isSetParagraphId()).compareTo(other.isSetParagraphId()); + lastComparison = Boolean.valueOf(isSetApplicationInstanceId()).compareTo(other.isSetApplicationInstanceId()); if (lastComparison != 0) { return lastComparison; } - if (isSetParagraphId()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.paragraphId, other.paragraphId); + if (isSetApplicationInstanceId()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.applicationInstanceId, other.applicationInstanceId); if (lastComparison != 0) { return lastComparison; } @@ -19032,30 +20782,14 @@ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache. @Override public String toString() { - StringBuilder sb = new StringBuilder("angularObjectRemove_args("); + StringBuilder sb = new StringBuilder("unloadApplication_args("); boolean first = true; - sb.append("name:"); - if (this.name == null) { - sb.append("null"); - } else { - sb.append(this.name); - } - first = false; - if (!first) sb.append(", "); - sb.append("noteId:"); - if (this.noteId == null) { - sb.append("null"); - } else { - sb.append(this.noteId); - } - first = false; - if (!first) sb.append(", "); - sb.append("paragraphId:"); - if (this.paragraphId == null) { + sb.append("applicationInstanceId:"); + if (this.applicationInstanceId == null) { sb.append("null"); } else { - sb.append(this.paragraphId); + sb.append(this.applicationInstanceId); } first = false; sb.append(")"); @@ -19083,15 +20817,15 @@ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException } } - private static class angularObjectRemove_argsStandardSchemeFactory implements SchemeFactory { - public angularObjectRemove_argsStandardScheme getScheme() { - return new angularObjectRemove_argsStandardScheme(); + private static class unloadApplication_argsStandardSchemeFactory implements SchemeFactory { + public unloadApplication_argsStandardScheme getScheme() { + return new unloadApplication_argsStandardScheme(); } } - private static class angularObjectRemove_argsStandardScheme extends StandardScheme { + private static class unloadApplication_argsStandardScheme extends StandardScheme { - public void read(org.apache.thrift.protocol.TProtocol iprot, angularObjectRemove_args struct) throws org.apache.thrift.TException { + public void read(org.apache.thrift.protocol.TProtocol iprot, unloadApplication_args struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TField schemeField; iprot.readStructBegin(); while (true) @@ -19101,26 +20835,10 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, angularObjectRemove break; } switch (schemeField.id) { - case 1: // NAME - if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.name = iprot.readString(); - struct.setNameIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - case 2: // NOTE_ID - if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.noteId = iprot.readString(); - struct.setNoteIdIsSet(true); - } else { - org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); - } - break; - case 3: // PARAGRAPH_ID + case 1: // APPLICATION_INSTANCE_ID if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.paragraphId = iprot.readString(); - struct.setParagraphIdIsSet(true); + struct.applicationInstanceId = iprot.readString(); + struct.setApplicationInstanceIdIsSet(true); } else { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -19136,23 +20854,13 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, angularObjectRemove struct.validate(); } - public void write(org.apache.thrift.protocol.TProtocol oprot, angularObjectRemove_args struct) throws org.apache.thrift.TException { + public void write(org.apache.thrift.protocol.TProtocol oprot, unloadApplication_args struct) throws org.apache.thrift.TException { struct.validate(); oprot.writeStructBegin(STRUCT_DESC); - if (struct.name != null) { - oprot.writeFieldBegin(NAME_FIELD_DESC); - oprot.writeString(struct.name); - oprot.writeFieldEnd(); - } - if (struct.noteId != null) { - oprot.writeFieldBegin(NOTE_ID_FIELD_DESC); - oprot.writeString(struct.noteId); - oprot.writeFieldEnd(); - } - if (struct.paragraphId != null) { - oprot.writeFieldBegin(PARAGRAPH_ID_FIELD_DESC); - oprot.writeString(struct.paragraphId); + if (struct.applicationInstanceId != null) { + oprot.writeFieldBegin(APPLICATION_INSTANCE_ID_FIELD_DESC); + oprot.writeString(struct.applicationInstanceId); oprot.writeFieldEnd(); } oprot.writeFieldStop(); @@ -19161,74 +20869,56 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, angularObjectRemov } - private static class angularObjectRemove_argsTupleSchemeFactory implements SchemeFactory { - public angularObjectRemove_argsTupleScheme getScheme() { - return new angularObjectRemove_argsTupleScheme(); + private static class unloadApplication_argsTupleSchemeFactory implements SchemeFactory { + public unloadApplication_argsTupleScheme getScheme() { + return new unloadApplication_argsTupleScheme(); } } - private static class angularObjectRemove_argsTupleScheme extends TupleScheme { + private static class unloadApplication_argsTupleScheme extends TupleScheme { @Override - public void write(org.apache.thrift.protocol.TProtocol prot, angularObjectRemove_args struct) throws org.apache.thrift.TException { + public void write(org.apache.thrift.protocol.TProtocol prot, unloadApplication_args struct) throws org.apache.thrift.TException { TTupleProtocol oprot = (TTupleProtocol) prot; BitSet optionals = new BitSet(); - if (struct.isSetName()) { + if (struct.isSetApplicationInstanceId()) { optionals.set(0); } - if (struct.isSetNoteId()) { - optionals.set(1); - } - if (struct.isSetParagraphId()) { - optionals.set(2); - } - oprot.writeBitSet(optionals, 3); - if (struct.isSetName()) { - oprot.writeString(struct.name); - } - if (struct.isSetNoteId()) { - oprot.writeString(struct.noteId); - } - if (struct.isSetParagraphId()) { - oprot.writeString(struct.paragraphId); + oprot.writeBitSet(optionals, 1); + if (struct.isSetApplicationInstanceId()) { + oprot.writeString(struct.applicationInstanceId); } } @Override - public void read(org.apache.thrift.protocol.TProtocol prot, angularObjectRemove_args struct) throws org.apache.thrift.TException { + public void read(org.apache.thrift.protocol.TProtocol prot, unloadApplication_args struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; - BitSet incoming = iprot.readBitSet(3); + BitSet incoming = iprot.readBitSet(1); if (incoming.get(0)) { - struct.name = iprot.readString(); - struct.setNameIsSet(true); - } - if (incoming.get(1)) { - struct.noteId = iprot.readString(); - struct.setNoteIdIsSet(true); - } - if (incoming.get(2)) { - struct.paragraphId = iprot.readString(); - struct.setParagraphIdIsSet(true); + struct.applicationInstanceId = iprot.readString(); + struct.setApplicationInstanceIdIsSet(true); } } } } - public static class angularObjectRemove_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("angularObjectRemove_result"); + public static class unloadApplication_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("unloadApplication_result"); + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0); private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); static { - schemes.put(StandardScheme.class, new angularObjectRemove_resultStandardSchemeFactory()); - schemes.put(TupleScheme.class, new angularObjectRemove_resultTupleSchemeFactory()); + schemes.put(StandardScheme.class, new unloadApplication_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new unloadApplication_resultTupleSchemeFactory()); } + public RemoteApplicationResult success; // required /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { -; + SUCCESS((short)0, "success"); private static final Map byName = new HashMap(); @@ -19243,6 +20933,8 @@ public enum _Fields implements org.apache.thrift.TFieldIdEnum { */ public static _Fields findByThriftId(int fieldId) { switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; default: return null; } @@ -19281,37 +20973,87 @@ public String getFieldName() { return _fieldName; } } + + // isset id assignments public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, RemoteApplicationResult.class))); metaDataMap = Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(angularObjectRemove_result.class, metaDataMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(unloadApplication_result.class, metaDataMap); } - public angularObjectRemove_result() { + public unloadApplication_result() { + } + + public unloadApplication_result( + RemoteApplicationResult success) + { + this(); + this.success = success; } /** * Performs a deep copy on other. */ - public angularObjectRemove_result(angularObjectRemove_result other) { + public unloadApplication_result(unloadApplication_result other) { + if (other.isSetSuccess()) { + this.success = new RemoteApplicationResult(other.success); + } } - public angularObjectRemove_result deepCopy() { - return new angularObjectRemove_result(this); + public unloadApplication_result deepCopy() { + return new unloadApplication_result(this); } @Override public void clear() { + this.success = null; + } + + public RemoteApplicationResult getSuccess() { + return this.success; + } + + public unloadApplication_result setSuccess(RemoteApplicationResult success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } } public void setFieldValue(_Fields field, Object value) { switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((RemoteApplicationResult)value); + } + break; + } } public Object getFieldValue(_Fields field) { switch (field) { + case SUCCESS: + return getSuccess(); + } throw new IllegalStateException(); } @@ -19323,6 +21065,8 @@ public boolean isSet(_Fields field) { } switch (field) { + case SUCCESS: + return isSetSuccess(); } throw new IllegalStateException(); } @@ -19331,15 +21075,24 @@ public boolean isSet(_Fields field) { public boolean equals(Object that) { if (that == null) return false; - if (that instanceof angularObjectRemove_result) - return this.equals((angularObjectRemove_result)that); + if (that instanceof unloadApplication_result) + return this.equals((unloadApplication_result)that); return false; } - public boolean equals(angularObjectRemove_result that) { + public boolean equals(unloadApplication_result that) { if (that == null) return false; + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + return true; } @@ -19347,17 +21100,32 @@ public boolean equals(angularObjectRemove_result that) { public int hashCode() { List list = new ArrayList(); + boolean present_success = true && (isSetSuccess()); + list.add(present_success); + if (present_success) + list.add(success); + return list.hashCode(); } @Override - public int compareTo(angularObjectRemove_result other) { + public int compareTo(unloadApplication_result other) { if (!getClass().equals(other.getClass())) { return getClass().getName().compareTo(other.getClass().getName()); } int lastComparison = 0; + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -19375,9 +21143,16 @@ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache. @Override public String toString() { - StringBuilder sb = new StringBuilder("angularObjectRemove_result("); + StringBuilder sb = new StringBuilder("unloadApplication_result("); boolean first = true; + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; sb.append(")"); return sb.toString(); } @@ -19385,6 +21160,9 @@ public String toString() { public void validate() throws org.apache.thrift.TException { // check for required fields // check for sub-struct validity + if (success != null) { + success.validate(); + } } private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { @@ -19403,15 +21181,15 @@ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException } } - private static class angularObjectRemove_resultStandardSchemeFactory implements SchemeFactory { - public angularObjectRemove_resultStandardScheme getScheme() { - return new angularObjectRemove_resultStandardScheme(); + private static class unloadApplication_resultStandardSchemeFactory implements SchemeFactory { + public unloadApplication_resultStandardScheme getScheme() { + return new unloadApplication_resultStandardScheme(); } } - private static class angularObjectRemove_resultStandardScheme extends StandardScheme { + private static class unloadApplication_resultStandardScheme extends StandardScheme { - public void read(org.apache.thrift.protocol.TProtocol iprot, angularObjectRemove_result struct) throws org.apache.thrift.TException { + public void read(org.apache.thrift.protocol.TProtocol iprot, unloadApplication_result struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TField schemeField; iprot.readStructBegin(); while (true) @@ -19421,6 +21199,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, angularObjectRemove break; } switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.success = new RemoteApplicationResult(); + struct.success.read(iprot); + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -19432,53 +21219,72 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, angularObjectRemove struct.validate(); } - public void write(org.apache.thrift.protocol.TProtocol oprot, angularObjectRemove_result struct) throws org.apache.thrift.TException { + public void write(org.apache.thrift.protocol.TProtocol oprot, unloadApplication_result struct) throws org.apache.thrift.TException { struct.validate(); oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + struct.success.write(oprot); + oprot.writeFieldEnd(); + } oprot.writeFieldStop(); oprot.writeStructEnd(); } } - private static class angularObjectRemove_resultTupleSchemeFactory implements SchemeFactory { - public angularObjectRemove_resultTupleScheme getScheme() { - return new angularObjectRemove_resultTupleScheme(); + private static class unloadApplication_resultTupleSchemeFactory implements SchemeFactory { + public unloadApplication_resultTupleScheme getScheme() { + return new unloadApplication_resultTupleScheme(); } } - private static class angularObjectRemove_resultTupleScheme extends TupleScheme { + private static class unloadApplication_resultTupleScheme extends TupleScheme { @Override - public void write(org.apache.thrift.protocol.TProtocol prot, angularObjectRemove_result struct) throws org.apache.thrift.TException { + public void write(org.apache.thrift.protocol.TProtocol prot, unloadApplication_result struct) throws org.apache.thrift.TException { TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetSuccess()) { + struct.success.write(oprot); + } } @Override - public void read(org.apache.thrift.protocol.TProtocol prot, angularObjectRemove_result struct) throws org.apache.thrift.TException { + public void read(org.apache.thrift.protocol.TProtocol prot, unloadApplication_result struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.success = new RemoteApplicationResult(); + struct.success.read(iprot); + struct.setSuccessIsSet(true); + } } } } - public static class angularRegistryPush_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("angularRegistryPush_args"); + public static class runApplication_args implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("runApplication_args"); - private static final org.apache.thrift.protocol.TField REGISTRY_FIELD_DESC = new org.apache.thrift.protocol.TField("registry", org.apache.thrift.protocol.TType.STRING, (short)1); + private static final org.apache.thrift.protocol.TField APPLICATION_INSTANCE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("applicationInstanceId", org.apache.thrift.protocol.TType.STRING, (short)1); private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); static { - schemes.put(StandardScheme.class, new angularRegistryPush_argsStandardSchemeFactory()); - schemes.put(TupleScheme.class, new angularRegistryPush_argsTupleSchemeFactory()); + schemes.put(StandardScheme.class, new runApplication_argsStandardSchemeFactory()); + schemes.put(TupleScheme.class, new runApplication_argsTupleSchemeFactory()); } - public String registry; // required + public String applicationInstanceId; // required /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { - REGISTRY((short)1, "registry"); + APPLICATION_INSTANCE_ID((short)1, "applicationInstanceId"); private static final Map byName = new HashMap(); @@ -19493,8 +21299,8 @@ public enum _Fields implements org.apache.thrift.TFieldIdEnum { */ public static _Fields findByThriftId(int fieldId) { switch(fieldId) { - case 1: // REGISTRY - return REGISTRY; + case 1: // APPLICATION_INSTANCE_ID + return APPLICATION_INSTANCE_ID; default: return null; } @@ -19538,71 +21344,71 @@ public String getFieldName() { public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); - tmpMap.put(_Fields.REGISTRY, new org.apache.thrift.meta_data.FieldMetaData("registry", org.apache.thrift.TFieldRequirementType.DEFAULT, + tmpMap.put(_Fields.APPLICATION_INSTANCE_ID, new org.apache.thrift.meta_data.FieldMetaData("applicationInstanceId", org.apache.thrift.TFieldRequirementType.DEFAULT, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); metaDataMap = Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(angularRegistryPush_args.class, metaDataMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(runApplication_args.class, metaDataMap); } - public angularRegistryPush_args() { + public runApplication_args() { } - public angularRegistryPush_args( - String registry) + public runApplication_args( + String applicationInstanceId) { this(); - this.registry = registry; + this.applicationInstanceId = applicationInstanceId; } /** * Performs a deep copy on other. */ - public angularRegistryPush_args(angularRegistryPush_args other) { - if (other.isSetRegistry()) { - this.registry = other.registry; + public runApplication_args(runApplication_args other) { + if (other.isSetApplicationInstanceId()) { + this.applicationInstanceId = other.applicationInstanceId; } } - public angularRegistryPush_args deepCopy() { - return new angularRegistryPush_args(this); + public runApplication_args deepCopy() { + return new runApplication_args(this); } @Override public void clear() { - this.registry = null; + this.applicationInstanceId = null; } - public String getRegistry() { - return this.registry; + public String getApplicationInstanceId() { + return this.applicationInstanceId; } - public angularRegistryPush_args setRegistry(String registry) { - this.registry = registry; + public runApplication_args setApplicationInstanceId(String applicationInstanceId) { + this.applicationInstanceId = applicationInstanceId; return this; } - public void unsetRegistry() { - this.registry = null; + public void unsetApplicationInstanceId() { + this.applicationInstanceId = null; } - /** Returns true if field registry is set (has been assigned a value) and false otherwise */ - public boolean isSetRegistry() { - return this.registry != null; + /** Returns true if field applicationInstanceId is set (has been assigned a value) and false otherwise */ + public boolean isSetApplicationInstanceId() { + return this.applicationInstanceId != null; } - public void setRegistryIsSet(boolean value) { + public void setApplicationInstanceIdIsSet(boolean value) { if (!value) { - this.registry = null; + this.applicationInstanceId = null; } } public void setFieldValue(_Fields field, Object value) { switch (field) { - case REGISTRY: + case APPLICATION_INSTANCE_ID: if (value == null) { - unsetRegistry(); + unsetApplicationInstanceId(); } else { - setRegistry((String)value); + setApplicationInstanceId((String)value); } break; @@ -19611,8 +21417,8 @@ public void setFieldValue(_Fields field, Object value) { public Object getFieldValue(_Fields field) { switch (field) { - case REGISTRY: - return getRegistry(); + case APPLICATION_INSTANCE_ID: + return getApplicationInstanceId(); } throw new IllegalStateException(); @@ -19625,8 +21431,8 @@ public boolean isSet(_Fields field) { } switch (field) { - case REGISTRY: - return isSetRegistry(); + case APPLICATION_INSTANCE_ID: + return isSetApplicationInstanceId(); } throw new IllegalStateException(); } @@ -19635,21 +21441,21 @@ public boolean isSet(_Fields field) { public boolean equals(Object that) { if (that == null) return false; - if (that instanceof angularRegistryPush_args) - return this.equals((angularRegistryPush_args)that); + if (that instanceof runApplication_args) + return this.equals((runApplication_args)that); return false; } - public boolean equals(angularRegistryPush_args that) { + public boolean equals(runApplication_args that) { if (that == null) return false; - boolean this_present_registry = true && this.isSetRegistry(); - boolean that_present_registry = true && that.isSetRegistry(); - if (this_present_registry || that_present_registry) { - if (!(this_present_registry && that_present_registry)) + boolean this_present_applicationInstanceId = true && this.isSetApplicationInstanceId(); + boolean that_present_applicationInstanceId = true && that.isSetApplicationInstanceId(); + if (this_present_applicationInstanceId || that_present_applicationInstanceId) { + if (!(this_present_applicationInstanceId && that_present_applicationInstanceId)) return false; - if (!this.registry.equals(that.registry)) + if (!this.applicationInstanceId.equals(that.applicationInstanceId)) return false; } @@ -19660,28 +21466,28 @@ public boolean equals(angularRegistryPush_args that) { public int hashCode() { List list = new ArrayList(); - boolean present_registry = true && (isSetRegistry()); - list.add(present_registry); - if (present_registry) - list.add(registry); + boolean present_applicationInstanceId = true && (isSetApplicationInstanceId()); + list.add(present_applicationInstanceId); + if (present_applicationInstanceId) + list.add(applicationInstanceId); return list.hashCode(); } @Override - public int compareTo(angularRegistryPush_args other) { + public int compareTo(runApplication_args other) { if (!getClass().equals(other.getClass())) { return getClass().getName().compareTo(other.getClass().getName()); } int lastComparison = 0; - lastComparison = Boolean.valueOf(isSetRegistry()).compareTo(other.isSetRegistry()); + lastComparison = Boolean.valueOf(isSetApplicationInstanceId()).compareTo(other.isSetApplicationInstanceId()); if (lastComparison != 0) { return lastComparison; } - if (isSetRegistry()) { - lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.registry, other.registry); + if (isSetApplicationInstanceId()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.applicationInstanceId, other.applicationInstanceId); if (lastComparison != 0) { return lastComparison; } @@ -19703,14 +21509,14 @@ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache. @Override public String toString() { - StringBuilder sb = new StringBuilder("angularRegistryPush_args("); + StringBuilder sb = new StringBuilder("runApplication_args("); boolean first = true; - sb.append("registry:"); - if (this.registry == null) { + sb.append("applicationInstanceId:"); + if (this.applicationInstanceId == null) { sb.append("null"); } else { - sb.append(this.registry); + sb.append(this.applicationInstanceId); } first = false; sb.append(")"); @@ -19738,15 +21544,15 @@ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException } } - private static class angularRegistryPush_argsStandardSchemeFactory implements SchemeFactory { - public angularRegistryPush_argsStandardScheme getScheme() { - return new angularRegistryPush_argsStandardScheme(); + private static class runApplication_argsStandardSchemeFactory implements SchemeFactory { + public runApplication_argsStandardScheme getScheme() { + return new runApplication_argsStandardScheme(); } } - private static class angularRegistryPush_argsStandardScheme extends StandardScheme { + private static class runApplication_argsStandardScheme extends StandardScheme { - public void read(org.apache.thrift.protocol.TProtocol iprot, angularRegistryPush_args struct) throws org.apache.thrift.TException { + public void read(org.apache.thrift.protocol.TProtocol iprot, runApplication_args struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TField schemeField; iprot.readStructBegin(); while (true) @@ -19756,10 +21562,10 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, angularRegistryPush break; } switch (schemeField.id) { - case 1: // REGISTRY + case 1: // APPLICATION_INSTANCE_ID if (schemeField.type == org.apache.thrift.protocol.TType.STRING) { - struct.registry = iprot.readString(); - struct.setRegistryIsSet(true); + struct.applicationInstanceId = iprot.readString(); + struct.setApplicationInstanceIdIsSet(true); } else { org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -19775,13 +21581,13 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, angularRegistryPush struct.validate(); } - public void write(org.apache.thrift.protocol.TProtocol oprot, angularRegistryPush_args struct) throws org.apache.thrift.TException { + public void write(org.apache.thrift.protocol.TProtocol oprot, runApplication_args struct) throws org.apache.thrift.TException { struct.validate(); oprot.writeStructBegin(STRUCT_DESC); - if (struct.registry != null) { - oprot.writeFieldBegin(REGISTRY_FIELD_DESC); - oprot.writeString(struct.registry); + if (struct.applicationInstanceId != null) { + oprot.writeFieldBegin(APPLICATION_INSTANCE_ID_FIELD_DESC); + oprot.writeString(struct.applicationInstanceId); oprot.writeFieldEnd(); } oprot.writeFieldStop(); @@ -19790,54 +21596,56 @@ public void write(org.apache.thrift.protocol.TProtocol oprot, angularRegistryPus } - private static class angularRegistryPush_argsTupleSchemeFactory implements SchemeFactory { - public angularRegistryPush_argsTupleScheme getScheme() { - return new angularRegistryPush_argsTupleScheme(); + private static class runApplication_argsTupleSchemeFactory implements SchemeFactory { + public runApplication_argsTupleScheme getScheme() { + return new runApplication_argsTupleScheme(); } } - private static class angularRegistryPush_argsTupleScheme extends TupleScheme { + private static class runApplication_argsTupleScheme extends TupleScheme { @Override - public void write(org.apache.thrift.protocol.TProtocol prot, angularRegistryPush_args struct) throws org.apache.thrift.TException { + public void write(org.apache.thrift.protocol.TProtocol prot, runApplication_args struct) throws org.apache.thrift.TException { TTupleProtocol oprot = (TTupleProtocol) prot; BitSet optionals = new BitSet(); - if (struct.isSetRegistry()) { + if (struct.isSetApplicationInstanceId()) { optionals.set(0); } oprot.writeBitSet(optionals, 1); - if (struct.isSetRegistry()) { - oprot.writeString(struct.registry); + if (struct.isSetApplicationInstanceId()) { + oprot.writeString(struct.applicationInstanceId); } } @Override - public void read(org.apache.thrift.protocol.TProtocol prot, angularRegistryPush_args struct) throws org.apache.thrift.TException { + public void read(org.apache.thrift.protocol.TProtocol prot, runApplication_args struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; BitSet incoming = iprot.readBitSet(1); if (incoming.get(0)) { - struct.registry = iprot.readString(); - struct.setRegistryIsSet(true); + struct.applicationInstanceId = iprot.readString(); + struct.setApplicationInstanceIdIsSet(true); } } } } - public static class angularRegistryPush_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { - private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("angularRegistryPush_result"); + public static class runApplication_result implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { + private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("runApplication_result"); + private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRUCT, (short)0); private static final Map, SchemeFactory> schemes = new HashMap, SchemeFactory>(); static { - schemes.put(StandardScheme.class, new angularRegistryPush_resultStandardSchemeFactory()); - schemes.put(TupleScheme.class, new angularRegistryPush_resultTupleSchemeFactory()); + schemes.put(StandardScheme.class, new runApplication_resultStandardSchemeFactory()); + schemes.put(TupleScheme.class, new runApplication_resultTupleSchemeFactory()); } + public RemoteApplicationResult success; // required /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { -; + SUCCESS((short)0, "success"); private static final Map byName = new HashMap(); @@ -19852,6 +21660,8 @@ public enum _Fields implements org.apache.thrift.TFieldIdEnum { */ public static _Fields findByThriftId(int fieldId) { switch(fieldId) { + case 0: // SUCCESS + return SUCCESS; default: return null; } @@ -19890,37 +21700,87 @@ public String getFieldName() { return _fieldName; } } + + // isset id assignments public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); + tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT, + new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, RemoteApplicationResult.class))); metaDataMap = Collections.unmodifiableMap(tmpMap); - org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(angularRegistryPush_result.class, metaDataMap); + org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(runApplication_result.class, metaDataMap); } - public angularRegistryPush_result() { + public runApplication_result() { + } + + public runApplication_result( + RemoteApplicationResult success) + { + this(); + this.success = success; } /** * Performs a deep copy on other. */ - public angularRegistryPush_result(angularRegistryPush_result other) { + public runApplication_result(runApplication_result other) { + if (other.isSetSuccess()) { + this.success = new RemoteApplicationResult(other.success); + } } - public angularRegistryPush_result deepCopy() { - return new angularRegistryPush_result(this); + public runApplication_result deepCopy() { + return new runApplication_result(this); } @Override public void clear() { + this.success = null; + } + + public RemoteApplicationResult getSuccess() { + return this.success; + } + + public runApplication_result setSuccess(RemoteApplicationResult success) { + this.success = success; + return this; + } + + public void unsetSuccess() { + this.success = null; + } + + /** Returns true if field success is set (has been assigned a value) and false otherwise */ + public boolean isSetSuccess() { + return this.success != null; + } + + public void setSuccessIsSet(boolean value) { + if (!value) { + this.success = null; + } } public void setFieldValue(_Fields field, Object value) { switch (field) { + case SUCCESS: + if (value == null) { + unsetSuccess(); + } else { + setSuccess((RemoteApplicationResult)value); + } + break; + } } public Object getFieldValue(_Fields field) { switch (field) { + case SUCCESS: + return getSuccess(); + } throw new IllegalStateException(); } @@ -19932,6 +21792,8 @@ public boolean isSet(_Fields field) { } switch (field) { + case SUCCESS: + return isSetSuccess(); } throw new IllegalStateException(); } @@ -19940,15 +21802,24 @@ public boolean isSet(_Fields field) { public boolean equals(Object that) { if (that == null) return false; - if (that instanceof angularRegistryPush_result) - return this.equals((angularRegistryPush_result)that); + if (that instanceof runApplication_result) + return this.equals((runApplication_result)that); return false; } - public boolean equals(angularRegistryPush_result that) { + public boolean equals(runApplication_result that) { if (that == null) return false; + boolean this_present_success = true && this.isSetSuccess(); + boolean that_present_success = true && that.isSetSuccess(); + if (this_present_success || that_present_success) { + if (!(this_present_success && that_present_success)) + return false; + if (!this.success.equals(that.success)) + return false; + } + return true; } @@ -19956,17 +21827,32 @@ public boolean equals(angularRegistryPush_result that) { public int hashCode() { List list = new ArrayList(); + boolean present_success = true && (isSetSuccess()); + list.add(present_success); + if (present_success) + list.add(success); + return list.hashCode(); } @Override - public int compareTo(angularRegistryPush_result other) { + public int compareTo(runApplication_result other) { if (!getClass().equals(other.getClass())) { return getClass().getName().compareTo(other.getClass().getName()); } int lastComparison = 0; + lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(other.isSetSuccess()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetSuccess()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, other.success); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -19984,9 +21870,16 @@ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache. @Override public String toString() { - StringBuilder sb = new StringBuilder("angularRegistryPush_result("); + StringBuilder sb = new StringBuilder("runApplication_result("); boolean first = true; + sb.append("success:"); + if (this.success == null) { + sb.append("null"); + } else { + sb.append(this.success); + } + first = false; sb.append(")"); return sb.toString(); } @@ -19994,6 +21887,9 @@ public String toString() { public void validate() throws org.apache.thrift.TException { // check for required fields // check for sub-struct validity + if (success != null) { + success.validate(); + } } private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException { @@ -20012,15 +21908,15 @@ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException } } - private static class angularRegistryPush_resultStandardSchemeFactory implements SchemeFactory { - public angularRegistryPush_resultStandardScheme getScheme() { - return new angularRegistryPush_resultStandardScheme(); + private static class runApplication_resultStandardSchemeFactory implements SchemeFactory { + public runApplication_resultStandardScheme getScheme() { + return new runApplication_resultStandardScheme(); } } - private static class angularRegistryPush_resultStandardScheme extends StandardScheme { + private static class runApplication_resultStandardScheme extends StandardScheme { - public void read(org.apache.thrift.protocol.TProtocol iprot, angularRegistryPush_result struct) throws org.apache.thrift.TException { + public void read(org.apache.thrift.protocol.TProtocol iprot, runApplication_result struct) throws org.apache.thrift.TException { org.apache.thrift.protocol.TField schemeField; iprot.readStructBegin(); while (true) @@ -20030,6 +21926,15 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, angularRegistryPush break; } switch (schemeField.id) { + case 0: // SUCCESS + if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) { + struct.success = new RemoteApplicationResult(); + struct.success.read(iprot); + struct.setSuccessIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -20041,32 +21946,51 @@ public void read(org.apache.thrift.protocol.TProtocol iprot, angularRegistryPush struct.validate(); } - public void write(org.apache.thrift.protocol.TProtocol oprot, angularRegistryPush_result struct) throws org.apache.thrift.TException { + public void write(org.apache.thrift.protocol.TProtocol oprot, runApplication_result struct) throws org.apache.thrift.TException { struct.validate(); oprot.writeStructBegin(STRUCT_DESC); + if (struct.success != null) { + oprot.writeFieldBegin(SUCCESS_FIELD_DESC); + struct.success.write(oprot); + oprot.writeFieldEnd(); + } oprot.writeFieldStop(); oprot.writeStructEnd(); } } - private static class angularRegistryPush_resultTupleSchemeFactory implements SchemeFactory { - public angularRegistryPush_resultTupleScheme getScheme() { - return new angularRegistryPush_resultTupleScheme(); + private static class runApplication_resultTupleSchemeFactory implements SchemeFactory { + public runApplication_resultTupleScheme getScheme() { + return new runApplication_resultTupleScheme(); } } - private static class angularRegistryPush_resultTupleScheme extends TupleScheme { + private static class runApplication_resultTupleScheme extends TupleScheme { @Override - public void write(org.apache.thrift.protocol.TProtocol prot, angularRegistryPush_result struct) throws org.apache.thrift.TException { + public void write(org.apache.thrift.protocol.TProtocol prot, runApplication_result struct) throws org.apache.thrift.TException { TTupleProtocol oprot = (TTupleProtocol) prot; + BitSet optionals = new BitSet(); + if (struct.isSetSuccess()) { + optionals.set(0); + } + oprot.writeBitSet(optionals, 1); + if (struct.isSetSuccess()) { + struct.success.write(oprot); + } } @Override - public void read(org.apache.thrift.protocol.TProtocol prot, angularRegistryPush_result struct) throws org.apache.thrift.TException { + public void read(org.apache.thrift.protocol.TProtocol prot, runApplication_result struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; + BitSet incoming = iprot.readBitSet(1); + if (incoming.get(0)) { + struct.success = new RemoteApplicationResult(); + struct.success.read(iprot); + struct.setSuccessIsSet(true); + } } } diff --git a/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift b/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift index 80212e71cc7..dd3bbf93d1b 100644 --- a/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift +++ b/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift @@ -56,6 +56,11 @@ struct RemoteInterpreterEvent { 2: string data // json serialized data } +struct RemoteApplicationResult { + 1: bool success, + 2: string msg +} + service RemoteInterpreterService { void createInterpreter(1: string intpGroupId, 2: string noteId, 3: string className, 4: map properties); @@ -88,4 +93,8 @@ service RemoteInterpreterService { void angularObjectAdd(1: string name, 2: string noteId, 3: string paragraphId, 4: string object); void angularObjectRemove(1: string name, 2: string noteId, 3: string paragraphId); void angularRegistryPush(1: string registry); + + RemoteApplicationResult loadApplication(1: string applicationInstanceId, 2: string packageInfo, 3: string noteId, 4: string paragraphId); + RemoteApplicationResult unloadApplication(1: string applicationInstanceId); + RemoteApplicationResult runApplication(1: string applicationInstanceId); } diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/ApplicationLoaderTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/ApplicationLoaderTest.java index eac4f23f187..1df4d0587b7 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/ApplicationLoaderTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/ApplicationLoaderTest.java @@ -19,7 +19,6 @@ import org.apache.commons.io.FileUtils; import org.apache.zeppelin.dep.DependencyResolver; -import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.resource.LocalResourcePool; import org.junit.After; import org.junit.Before; @@ -46,36 +45,33 @@ public void tearDown() throws IOException { @Test public void loadUnloadApplication() throws Exception { + // given LocalResourcePool resourcePool = new LocalResourcePool("pool1"); DependencyResolver dep = new DependencyResolver(tmpDir.getAbsolutePath()); ApplicationLoader appLoader = new ApplicationLoader(resourcePool, dep); - HeliumPackageInfo pkg1 = createPackageInfo(MockApplication1.class.getName(), "artifact1"); + HeliumPackage pkg1 = createPackageInfo(MockApplication1.class.getName(), "artifact1"); ApplicationContext context1 = createContext("note1", "paragraph1"); - // app not loaded yet - assertEquals(null, appLoader.get(pkg1, context1)); - - // load application + // when load application MockApplication1 app = (MockApplication1) ((ClassLoaderApplication) appLoader.load(pkg1, context1)).getInnerApplication(); // then assertFalse(app.isUnloaded()); assertEquals(0, app.getNumRun()); - assertNotNull(appLoader.get(pkg1, context1)); - // unload application - appLoader.unload(pkg1, context1); + // when unload + app.unload(); // then assertTrue(app.isUnloaded()); assertEquals(0, app.getNumRun()); } - public HeliumPackageInfo createPackageInfo(String className, String artifact) { - HeliumPackageInfo app1 = new HeliumPackageInfo( - HeliumPackageInfo.Type.APPLICATION, + public HeliumPackage createPackageInfo(String className, String artifact) { + HeliumPackage app1 = new HeliumPackage( + HeliumPackage.Type.APPLICATION, "name1", "desc1", artifact, diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java new file mode 100644 index 00000000000..bcd1777b55a --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * Manages helium packages + */ +public class Helium { + Logger logger = LoggerFactory.getLogger(Helium.class); + private List registry = new LinkedList(); + + private final HeliumConf heliumConf; + private final String heliumConfPath; + private final Gson gson; + + public Helium(String heliumConfPath) throws IOException { + this.heliumConfPath = heliumConfPath; + + GsonBuilder builder = new GsonBuilder(); + builder.setPrettyPrinting(); + builder.registerTypeAdapter( + HeliumRegistry.class, new HeliumRegistrySerializer()); + gson = builder.create(); + + heliumConf = loadConf(heliumConfPath); + } + + /** + * Add HeliumRegistry + * @param registry + */ + public void addRegistry(HeliumRegistry registry) { + synchronized (this.registry) { + this.registry.add(registry); + } + } + + public List getAllRegistry() { + synchronized (this.registry) { + List list = new LinkedList(); + for (HeliumRegistry r : registry) { + list.add(r); + } + return list; + } + } + + private synchronized HeliumConf loadConf(String path) throws IOException { + File heliumConfFile = new File(path); + if (!heliumConfFile.isFile()) { + logger.warn("{} does not exists", path); + return new HeliumConf(); + } else { + String jsonString = FileUtils.readFileToString(heliumConfFile); + HeliumConf conf = gson.fromJson(jsonString, HeliumConf.class); + this.registry = conf.getRegistry(); + return conf; + } + } + + public synchronized void save() throws IOException { + String jsonString; + synchronized (registry) { + heliumConf.setRegistry(registry); + jsonString = gson.toJson(heliumConf); + } + + File heliumConfFile = new File(heliumConfPath); + if (!heliumConfFile.exists()) { + heliumConfFile.createNewFile(); + } + + FileUtils.writeStringToFile(heliumConfFile, jsonString); + } + + public List getAllPackageInfo() { + List list = new LinkedList(); + synchronized (registry) { + for (HeliumRegistry r : registry) { + try { + for (HeliumPackage pkg : r.getAll()) { + list.add(new HeliumPackageSearchResult(r.name(), pkg)); + } + } catch (IOException e) { + logger.error(e.getMessage(), e); + } + } + } + return list; + } + +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumConf.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumConf.java new file mode 100644 index 00000000000..2a93caa0b57 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumConf.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import java.util.LinkedList; +import java.util.List; + +/** + * Helium config. This object will be persisted to conf/heliumc.conf + */ +public class HeliumConf { + List registry = new LinkedList(); + + public List getRegistry() { + return registry; + } + + public void setRegistry(List registry) { + this.registry = registry; + } +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumLocalRegistry.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumLocalRegistry.java new file mode 100644 index 00000000000..50d35175b44 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumLocalRegistry.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import com.google.gson.Gson; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.util.LinkedList; +import java.util.List; + +/** + * Simple Helium registry on local filesystem + */ +public class HeliumLocalRegistry extends HeliumRegistry { + Logger logger = LoggerFactory.getLogger(HeliumLocalRegistry.class); + + private final Gson gson; + + public HeliumLocalRegistry(String name, URI uri) { + super(name, uri); + gson = new Gson(); + } + + + @Override + public synchronized List getAll() throws IOException { + List result = new LinkedList(); + + File file = new File(uri()); + File [] files = file.listFiles(); + if (files == null) { + return result; + } + + for (File f : files) { + if (f.getName().startsWith(".")) { + continue; + } + + HeliumPackage pkgInfo = readPackageInfo(f); + if (pkgInfo != null) { + result.add(pkgInfo); + } + } + return result; + } + + private HeliumPackage readPackageInfo(File f) { + try { + return gson.fromJson(FileUtils.readFileToString(f), HeliumPackage.class); + } catch (IOException e) { + logger.error(e.getMessage(), e); + return null; + } + } + +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumPackageSearchResult.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumPackageSearchResult.java new file mode 100644 index 00000000000..57a9d4512f4 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumPackageSearchResult.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +/** + * search result + */ +public class HeliumPackageSearchResult { + private final String registry; + private final HeliumPackage pkg; + + /** + * Create search result item + * @param registry registry name + * @param pkg package information + */ + public HeliumPackageSearchResult(String registry, HeliumPackage pkg) { + this.registry = registry; + this.pkg = pkg; + } + + public String getRegistry() { + return registry; + } + + public HeliumPackage getPkg() { + return pkg; + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumRegistry.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumRegistry.java similarity index 77% rename from zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumRegistry.java rename to zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumRegistry.java index a893df6be95..48626e0adce 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumRegistry.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumRegistry.java @@ -16,20 +16,26 @@ */ package org.apache.zeppelin.helium; +import java.io.IOException; import java.net.URI; +import java.util.List; /** - * Helium registry + * Helium package registry */ public abstract class HeliumRegistry { + private final String name; private final URI uri; - public HeliumRegistry(URI uri) { + public HeliumRegistry(String name, URI uri) { + this.name = name; this.uri = uri; } - + public String name() { + return name; + } public URI uri() { return uri; } - + public abstract List getAll() throws IOException; } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumRegistrySerializer.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumRegistrySerializer.java new file mode 100644 index 00000000000..3940c9d32e7 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumRegistrySerializer.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import com.google.gson.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Type; +import java.net.URI; +import java.net.URISyntaxException; + +/** + * HeliumRegistrySerializer (and deserializer) for gson + */ +public class HeliumRegistrySerializer + implements JsonSerializer, JsonDeserializer { + Logger logger = LoggerFactory.getLogger(HeliumRegistrySerializer.class); + + @Override + public HeliumRegistry deserialize(JsonElement json, + Type type, + JsonDeserializationContext jsonDeserializationContext) + throws JsonParseException { + JsonObject jsonObject = json.getAsJsonObject(); + String className = jsonObject.get("class").getAsString(); + URI uri = null; + try { + uri = new URI(jsonObject.get("uri").getAsString()); + } catch (URISyntaxException e) { + new JsonParseException(e); + } + String name = jsonObject.get("name").getAsString(); + + try { + logger.info("Restore helium registry {} {} {}", name, className, uri); + Class cls = + (Class) getClass().getClassLoader().loadClass(className); + Constructor constructor = cls.getConstructor(String.class, URI.class); + HeliumRegistry registry = constructor.newInstance(name, uri); + return registry; + } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | + InstantiationException | InvocationTargetException e) { + logger.error(e.getMessage(), e); + return null; + } + } + + @Override + public JsonElement serialize(HeliumRegistry heliumRegistry, + Type type, + JsonSerializationContext jsonSerializationContext) { + JsonObject json = new JsonObject(); + json.addProperty("class", heliumRegistry.getClass().getName()); + json.addProperty("uri", heliumRegistry.uri().toString()); + json.addProperty("name", heliumRegistry.name()); + return json; + } +} diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumLocalRegistryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumLocalRegistryTest.java new file mode 100644 index 00000000000..78ecec6d7cf --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumLocalRegistryTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import com.google.gson.Gson; +import org.apache.commons.io.FileUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; + +import static org.junit.Assert.assertEquals; + +public class HeliumLocalRegistryTest { + private File tmpDir; + + @Before + public void setUp() throws Exception { + tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis()); + tmpDir.mkdirs(); + } + + @After + public void tearDown() throws IOException { + FileUtils.deleteDirectory(tmpDir); + } + + @Test + public void testGetAllPackage() throws IOException { + // given + File r1Path = new File(tmpDir, "r1"); + HeliumLocalRegistry r1 = new HeliumLocalRegistry("r1", r1Path.toURI()); + assertEquals(0, r1.getAll().size()); + + // when + Gson gson = new Gson(); + HeliumPackage pkg1 = new HeliumPackage(HeliumPackage.Type.APPLICATION, + "app1", + "desc1", + "artifact1", + "classname1", + new String[][]{}); + FileUtils.writeStringToFile(new File(r1Path, "pkg1.json"), gson.toJson(pkg1)); + + // then + assertEquals(1, r1.getAll().size()); + } +} diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java new file mode 100644 index 00000000000..fac93b0de4e --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import org.apache.commons.io.FileUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class HeliumTest { + private File tmpDir; + + @Before + public void setUp() throws Exception { + tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis()); + tmpDir.mkdirs(); + } + + @After + public void tearDown() throws IOException { + FileUtils.deleteDirectory(tmpDir); + } + + @Test + public void testSaveLoadConf() throws IOException, URISyntaxException { + // given + File heliumConf = new File(tmpDir, "helium.conf"); + Helium helium = new Helium(heliumConf.getAbsolutePath()); + assertFalse(heliumConf.exists()); + HeliumTestRegistry registry1 = new HeliumTestRegistry("r1", new URI("file:///r1")); + helium.addRegistry(registry1); + assertEquals(1, helium.getAllRegistry().size()); + assertEquals(0, helium.getAllPackageInfo().size()); + + // when + helium.save(); + + // then + assertTrue(heliumConf.exists()); + + // then + Helium heliumRestored = new Helium(heliumConf.getAbsolutePath()); + assertEquals(1, heliumRestored.getAllRegistry().size()); + } + + @Test + public void testRestoreRegistryInstances() throws IOException, URISyntaxException { + File heliumConf = new File(tmpDir, "helium.conf"); + Helium helium = new Helium(heliumConf.getAbsolutePath()); + HeliumTestRegistry registry1 = new HeliumTestRegistry("r1", new URI("file:///r1")); + HeliumTestRegistry registry2 = new HeliumTestRegistry("r2", new URI("file:///r2")); + helium.addRegistry(registry1); + helium.addRegistry(registry2); + + // when + registry1.add(new HeliumPackage( + HeliumPackage.Type.APPLICATION, + "name1", + "desc1", + "artifact1", + "className1", + new String[][]{})); + + registry2.add(new HeliumPackage( + HeliumPackage.Type.APPLICATION, + "name2", + "desc2", + "artifact2", + "className2", + new String[][]{})); + + // then + assertEquals(2, helium.getAllPackageInfo().size()); + } +} diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTestRegistry.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTestRegistry.java new file mode 100644 index 00000000000..494d0367fd0 --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTestRegistry.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import java.io.IOException; +import java.net.URI; +import java.util.LinkedList; +import java.util.List; + +public class HeliumTestRegistry extends HeliumRegistry { + List infos = new LinkedList(); + + public HeliumTestRegistry(String name, URI uri) { + super(name, uri); + } + + @Override + public List getAll() throws IOException { + return infos; + } + + public void add(HeliumPackage info) { + infos.add(info); + } +} From b239f1b96b01264b4f92aef0d5d05d5173794c3b Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 19 Mar 2016 08:25:30 -0700 Subject: [PATCH 04/69] Helium application factory --- .../zeppelin/helium/ApplicationLoader.java | 2 +- .../apache/zeppelin/helium/HeliumPackage.java | 3 + .../zeppelin/server/ZeppelinServer.java | 4 + .../zeppelin/conf/ZeppelinConfiguration.java | 4 + .../org/apache/zeppelin/helium/Helium.java | 1 - .../helium/HeliumApplicationFactory.java | 228 ++++++++++++++++++ .../zeppelin/notebook/ApplicationState.java | 23 ++ .../apache/zeppelin/notebook/Paragraph.java | 2 + 8 files changed, 265 insertions(+), 2 deletions(-) create mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java create mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java index ae8889c6c9a..0a37babdba8 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java @@ -101,7 +101,7 @@ public boolean equals(Object o) { * @throws Exception */ public Application load(HeliumPackage packageInfo, ApplicationContext context) - throws Exception { + throws Exception { if (packageInfo.getType() != HeliumPackage.Type.APPLICATION) { throw new ApplicationException( "Can't instantiate " + packageInfo.getType() + " package using ApplicationLoader"); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackage.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackage.java index abd63f4c221..98c4d42e93f 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackage.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackage.java @@ -28,6 +28,9 @@ public class HeliumPackage { private String [][] resources; // resource classnames that requires // [[ .. and .. and .. ] or [ .. and .. and ..] ..] + /** + * Type of package + */ public static enum Type { INTERPRETER, NOTEBOOK_REPO, diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index 08b179bf635..952f112afbb 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -31,6 +31,7 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.dep.DependencyResolver; +import org.apache.zeppelin.helium.Helium; import org.apache.zeppelin.interpreter.InterpreterFactory; import org.apache.zeppelin.notebook.Notebook; import org.apache.zeppelin.notebook.NotebookAuthorization; @@ -74,12 +75,15 @@ public class ZeppelinServer extends Application { private SearchService notebookIndex; private NotebookAuthorization notebookAuthorization; private DependencyResolver depResolver; + private Helium helium; public ZeppelinServer() throws Exception { ZeppelinConfiguration conf = ZeppelinConfiguration.create(); this.depResolver = new DependencyResolver( conf.getString(ConfVars.ZEPPELIN_INTERPRETER_LOCALREPO)); + + this.helium = new Helium(conf.getHeliumConfPath()); this.schedulerFactory = new SchedulerFactory(); this.replFactory = new InterpreterFactory(conf, notebookWsServer, notebookWsServer, depResolver); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 25a9b121a76..70706fc44ef 100755 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -342,6 +342,10 @@ public String getInterpreterSettingPath() { return getRelativeDir(String.format("%s/interpreter.json", getConfDir())); } + public String getHeliumConfPath() { + return getRelativeDir(String.format("%s/helium.json", getConfDir())); + } + public String getNotebookAuthorizationPath() { return getRelativeDir(String.format("%s/notebook-authorization.json", getConfDir())); } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java index bcd1777b55a..b01ea671f7d 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java @@ -115,5 +115,4 @@ public List getAllPackageInfo() { } return list; } - } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java new file mode 100644 index 00000000000..848b0c3258e --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import com.google.gson.Gson; +import org.apache.thrift.TException; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; +import org.apache.zeppelin.interpreter.thrift.RemoteApplicationResult; +import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.Paragraph; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; + +/** + * HeliumApplicationFactory + */ +public class HeliumApplicationFactory { + Logger logger = LoggerFactory.getLogger(HeliumApplicationFactory.class); + + private final Gson gson; + Map apps = + Collections.synchronizedMap(new HashMap()); + + + public HeliumApplicationFactory() { + gson = new Gson(); + } + + private static class RunningApplicationInfo { + HeliumPackage pkg; + Paragraph paragraph; + RemoteInterpreterProcess process; + + public RunningApplicationInfo(HeliumPackage pkg, + RemoteInterpreterProcess process, + Paragraph paragraph) { + this.pkg = pkg; + this.paragraph = paragraph; + this.process = process; + } + + public HeliumPackage getPkg() { + return pkg; + } + + public Paragraph getParagraph() { + return paragraph; + } + + public RemoteInterpreterProcess getProcess() { + return process; + } + } + + + + private static String generateApplicationId() { + return "app_" + System.currentTimeMillis() + "_" + + new Random(System.currentTimeMillis()).nextInt(); + } + + + /** + * Load pkg + */ + public void load(HeliumPackage pkg, Paragraph paragraph) throws ApplicationException { + Interpreter intp = paragraph.getRepl(paragraph.getRequiredReplName()); + RemoteInterpreterProcess intpProcess = intp.getInterpreterGroup().getRemoteInterpreterProcess(); + + RemoteInterpreterService.Client client; + try { + client = intpProcess.getClient(); + } catch (Exception e) { + throw new ApplicationException(e); + } + + String appId = generateApplicationId(); + String pkgInfo = gson.toJson(pkg); + try { + RemoteApplicationResult ret = client.loadApplication( + appId, + pkgInfo, + paragraph.getNote().getId(), + paragraph.getId()); + + if (ret.isSuccess()) { + RunningApplicationInfo appInfo = new RunningApplicationInfo(pkg, intpProcess, paragraph); + apps.put(appId, appInfo); + } else { + throw new ApplicationException(ret.getMsg()); + } + } catch (TException e) { + intpProcess.releaseBrokenClient(client); + throw new ApplicationException(e); + } finally { + intpProcess.releaseClient(client); + } + } + + + /** + * Unload pkg + */ + public void unload(String appId) throws ApplicationException { + RunningApplicationInfo appInfo = apps.get(appId); + if (appInfo == null) { + return; + } + + RemoteInterpreterProcess intpProcess = appInfo.getProcess(); + + RemoteInterpreterService.Client client; + try { + client = intpProcess.getClient(); + } catch (Exception e) { + throw new ApplicationException(e); + } + + try { + RemoteApplicationResult ret = client.unloadApplication(appId); + + if (ret.isSuccess()) { + apps.remove(appId); + } else { + throw new ApplicationException(ret.getMsg()); + } + } catch (TException e) { + intpProcess.releaseBrokenClient(client); + throw new ApplicationException(e); + } finally { + intpProcess.releaseClient(client); + } + } + + /** + * run pkg + */ + public void run(String appId) throws ApplicationException { + RunningApplicationInfo appInfo = apps.get(appId); + if (appInfo == null) { + return; + } + + RemoteInterpreterProcess intpProcess = appInfo.getProcess(); + + RemoteInterpreterService.Client client; + try { + client = intpProcess.getClient(); + } catch (Exception e) { + throw new ApplicationException(e); + } + + try { + RemoteApplicationResult ret = client.runApplication(appId); + + if (ret.isSuccess()) { + // success + } else { + throw new ApplicationException(ret.getMsg()); + } + } catch (TException e) { + intpProcess.releaseBrokenClient(client); + throw new ApplicationException(e); + } finally { + intpProcess.releaseClient(client); + } + } + + public void unloadAllInTheInteprreterProcess(RemoteInterpreterProcess process) { + for (String appId : apps.keySet()) { + RunningApplicationInfo app = apps.get(appId); + if (app.getProcess() == process) { + try { + unload(appId); + } catch (ApplicationException e) { + logger.error(e.getMessage(), e); + } + } + } + } + + public void unloadAllInTheNote(Note note) { + for (String appId : apps.keySet()) { + RunningApplicationInfo app = apps.get(appId); + if (app.getParagraph().getNote().getId().equals(note.getId())) { + try { + unload(appId); + } catch (ApplicationException e) { + logger.error(e.getMessage(), e); + } + } + } + } + + public void unloadAllInTheParagraph(Paragraph paragraph) { + for (String appId : apps.keySet()) { + RunningApplicationInfo app = apps.get(appId); + if (app.getParagraph().getId().equals(paragraph.getId())) { + try { + unload(appId); + } catch (ApplicationException e) { + logger.error(e.getMessage(), e); + } + } + } + } +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java new file mode 100644 index 00000000000..e1f8584bc2a --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.notebook; + +/** + * Running ApplicationState + */ +public class ApplicationState { +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java index bb4d69b717f..337e32d3ac9 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java @@ -53,6 +53,7 @@ public class Paragraph extends Job implements Serializable, Cloneable { Date dateUpdated; private Map config; // paragraph configs like isOpen, colWidth, etc public final GUI settings; // form and parameter settings + private Map apps; // helium applications running @VisibleForTesting Paragraph() { @@ -71,6 +72,7 @@ public Paragraph(Note note, JobListener listener, NoteInterpreterLoader replLoad dateUpdated = null; settings = new GUI(); config = new HashMap(); + apps = new HashMap(); } private static String generateId() { From 9f5c493e414e7c240abb08f2fa8600761514d63f Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Fri, 1 Apr 2016 11:41:57 -0700 Subject: [PATCH 05/69] Application output --- .../zeppelin/helium/ApplicationContext.java | 6 +- .../helium/ApplicationEventListener.java | 26 ++ .../interpreter/InterpreterGroup.java | 3 + .../interpreter/remote/RemoteInterpreter.java | 48 ++- .../remote/RemoteInterpreterEventClient.java | 24 ++ .../remote/RemoteInterpreterEventPoller.java | 21 +- .../remote/RemoteInterpreterProcess.java | 9 +- .../remote/RemoteInterpreterServer.java | 28 +- .../helium/ApplicationLoaderTest.java | 15 +- .../remote/RemoteAngularObjectTest.java | 1 + .../RemoteInterpreterOutputTestStream.java | 3 +- .../remote/RemoteInterpreterProcessTest.java | 2 +- .../remote/RemoteInterpreterTest.java | 6 +- .../resource/DistributedResourcePoolTest.java | 6 +- .../scheduler/RemoteSchedulerTest.java | 6 +- .../zeppelin/server/ZeppelinServer.java | 10 +- .../org/apache/zeppelin/socket/Message.java | 4 +- .../zeppelin/socket/NotebookServer.java | 39 +- .../helium/HeliumApplicationFactory.java | 359 +++++++++++------- .../interpreter/InterpreterFactory.java | 23 +- .../zeppelin/notebook/ApplicationState.java | 78 ++++ .../apache/zeppelin/notebook/Notebook.java | 1 + .../apache/zeppelin/notebook/Paragraph.java | 3 +- .../helium/HeliumApplicationFactoryTest.java | 168 ++++++++ .../helium/HeliumTestApplication.java | 43 +++ .../interpreter/InterpreterFactoryTest.java | 2 +- .../notebook/NoteInterpreterLoaderTest.java | 2 +- .../zeppelin/notebook/NotebookTest.java | 2 +- .../notebook/repo/NotebookRepoSyncTest.java | 2 +- .../notebook/repo/VFSNotebookRepoTest.java | 2 +- 30 files changed, 761 insertions(+), 181 deletions(-) create mode 100644 zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationEventListener.java create mode 100644 zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java create mode 100644 zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTestApplication.java diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationContext.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationContext.java index 58a2d49fe2e..303356a30d5 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationContext.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationContext.java @@ -16,16 +16,20 @@ */ package org.apache.zeppelin.helium; +import org.apache.zeppelin.interpreter.InterpreterOutput; + /** * ApplicationContext */ public class ApplicationContext { private final String noteId; private final String paragraphId; + public final InterpreterOutput out; - public ApplicationContext(String noteId, String paragraphId) { + public ApplicationContext(String noteId, String paragraphId, InterpreterOutput out) { this.noteId = noteId; this.paragraphId = paragraphId; + this.out = out; } public String getNoteId() { diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationEventListener.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationEventListener.java new file mode 100644 index 00000000000..6ab4ea49f7e --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationEventListener.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +/** + * Event from HeliumApplication running on remote interpreter process + */ +public interface ApplicationEventListener { + public void onOutputAppend(String noteId, String paragraphId, String appId, String output); + public void onOutputUpdated(String noteId, String paragraphId, String appId, String output); + +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java index b5d5863ab5c..462526e5803 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java @@ -22,6 +22,9 @@ import org.apache.log4j.Logger; import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.helium.Application; +import org.apache.zeppelin.helium.ApplicationException; +import org.apache.zeppelin.helium.HeliumPackage; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService; import org.apache.zeppelin.resource.ResourcePool; diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java index 137b6053632..3314111a103 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java @@ -23,6 +23,7 @@ import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.GUI; +import org.apache.zeppelin.helium.ApplicationEventListener; import org.apache.zeppelin.interpreter.*; import org.apache.zeppelin.interpreter.InterpreterResult.Type; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterContext; @@ -41,6 +42,7 @@ */ public class RemoteInterpreter extends Interpreter { private final RemoteInterpreterProcessListener remoteInterpreterProcessListener; + private final ApplicationEventListener applicationEventListener; Logger logger = LoggerFactory.getLogger(RemoteInterpreter.class); Gson gson = new Gson(); private String interpreterRunner; @@ -56,14 +58,15 @@ public class RemoteInterpreter extends Interpreter { private static String schedulerName; public RemoteInterpreter(Properties property, - String noteId, - String className, - String interpreterRunner, - String interpreterPath, - String localRepoPath, - int connectTimeout, - int maxPoolSize, - RemoteInterpreterProcessListener remoteInterpreterProcessListener) { + String noteId, + String className, + String interpreterRunner, + String interpreterPath, + String localRepoPath, + int connectTimeout, + int maxPoolSize, + RemoteInterpreterProcessListener remoteInterpreterProcessListener, + ApplicationEventListener appListener) { super(property); this.noteId = noteId; this.className = className; @@ -75,17 +78,19 @@ public RemoteInterpreter(Properties property, this.connectTimeout = connectTimeout; this.maxPoolSize = maxPoolSize; this.remoteInterpreterProcessListener = remoteInterpreterProcessListener; + this.applicationEventListener = appListener; } public RemoteInterpreter(Properties property, - String noteId, - String className, - String interpreterRunner, - String interpreterPath, - String localRepoPath, - Map env, - int connectTimeout, - RemoteInterpreterProcessListener remoteInterpreterProcessListener) { + String noteId, + String className, + String interpreterRunner, + String interpreterPath, + String localRepoPath, + Map env, + int connectTimeout, + RemoteInterpreterProcessListener remoteInterpreterProcessListener, + ApplicationEventListener appListener) { super(property); this.className = className; this.noteId = noteId; @@ -96,6 +101,7 @@ public RemoteInterpreter(Properties property, this.connectTimeout = connectTimeout; this.maxPoolSize = 10; this.remoteInterpreterProcessListener = remoteInterpreterProcessListener; + this.applicationEventListener = appListener; } @Override @@ -114,7 +120,7 @@ public RemoteInterpreterProcess getInterpreterProcess() { // create new remote process RemoteInterpreterProcess remoteProcess = new RemoteInterpreterProcess( interpreterRunner, interpreterPath, localRepoPath, env, connectTimeout, - remoteInterpreterProcessListener); + remoteInterpreterProcessListener, applicationEventListener); intpGroup.setRemoteInterpreterProcess(remoteProcess); } @@ -423,4 +429,12 @@ void pushAngularObjectRegistryToRemote(Client client) throws TException { client.angularRegistryPush(gson.toJson(registry, registryType)); } } + + public Map getEnv() { + return env; + } + + public void setEnv(Map env) { + this.env = env; + } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java index 158f145de45..b9b6b0a8222 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java @@ -242,4 +242,28 @@ private void sendEvent(RemoteInterpreterEvent event) { } } + public void onAppOutputAppend(String noteId, String paragraphId, String appId, String output) { + Map appendOutput = new HashMap(); + appendOutput.put("noteId", noteId); + appendOutput.put("paragraphId", paragraphId); + appendOutput.put("appId", appId); + appendOutput.put("data", output); + + sendEvent(new RemoteInterpreterEvent( + RemoteInterpreterEventType.OUTPUT_APPEND, + gson.toJson(appendOutput))); + } + + + public void onAppOutputUpdate(String noteId, String paragraphId, String appId, String output) { + Map appendOutput = new HashMap(); + appendOutput.put("noteId", noteId); + appendOutput.put("paragraphId", paragraphId); + appendOutput.put("appId", appId); + appendOutput.put("data", output); + + sendEvent(new RemoteInterpreterEvent( + RemoteInterpreterEventType.OUTPUT_UPDATE, + gson.toJson(appendOutput))); + } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java index 48e79de8109..6dead0eb93e 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java @@ -22,6 +22,7 @@ import org.apache.thrift.TException; import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.helium.ApplicationEventListener; import org.apache.zeppelin.interpreter.InterpreterContextRunner; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterOutputListener; @@ -47,14 +48,18 @@ public class RemoteInterpreterEventPoller extends Thread { private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreterEventPoller.class); private final RemoteInterpreterProcessListener listener; + private final ApplicationEventListener appListener; private volatile boolean shutdown; private RemoteInterpreterProcess interpreterProcess; private InterpreterGroup interpreterGroup; - public RemoteInterpreterEventPoller(RemoteInterpreterProcessListener listener) { + public RemoteInterpreterEventPoller( + RemoteInterpreterProcessListener listener, + ApplicationEventListener appListener) { this.listener = listener; + this.appListener = appListener; shutdown = false; } @@ -141,8 +146,13 @@ public void run() { String noteId = outputAppend.get("noteId"); String paragraphId = outputAppend.get("paragraphId"); String outputToAppend = outputAppend.get("data"); + String appId = outputAppend.get("appId"); - listener.onOutputAppend(noteId, paragraphId, outputToAppend); + if (appId == null) { + listener.onOutputAppend(noteId, paragraphId, outputToAppend); + } else { + appListener.onOutputAppend(noteId, paragraphId, appId, outputToAppend); + } } else if (event.getType() == RemoteInterpreterEventType.OUTPUT_UPDATE) { // on output update Map outputAppend = gson.fromJson( @@ -150,8 +160,13 @@ public void run() { String noteId = outputAppend.get("noteId"); String paragraphId = outputAppend.get("paragraphId"); String outputToUpdate = outputAppend.get("data"); + String appId = outputAppend.get("appId"); - listener.onOutputUpdated(noteId, paragraphId, outputToUpdate); + if (appId == null) { + listener.onOutputUpdated(noteId, paragraphId, outputToUpdate); + } else { + appListener.onOutputUpdated(noteId, paragraphId, appId, outputToUpdate); + } } logger.debug("Event from remoteproceess {}", event.getType()); } catch (Exception e) { diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java index 67a048ba9cc..43b94918748 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java @@ -22,6 +22,7 @@ import org.apache.commons.exec.environment.EnvironmentUtils; import org.apache.commons.pool2.impl.GenericObjectPool; import org.apache.thrift.TException; +import org.apache.zeppelin.helium.ApplicationEventListener; import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; @@ -53,17 +54,19 @@ public class RemoteInterpreterProcess implements ExecuteResultHandler { private final InterpreterContextRunnerPool interpreterContextRunnerPool; private int connectTimeout; - public RemoteInterpreterProcess(String intpRunner, + public RemoteInterpreterProcess( + String intpRunner, String intpDir, String localRepoDir, Map env, int connectTimeout, - RemoteInterpreterProcessListener listener) { + RemoteInterpreterProcessListener listener, + ApplicationEventListener appListener) { this(intpRunner, intpDir, localRepoDir, env, - new RemoteInterpreterEventPoller(listener), + new RemoteInterpreterEventPoller(listener, appListener), connectTimeout); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java index 7cd378b76b4..19826ac1bd8 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java @@ -706,9 +706,28 @@ public void angularRegistryPush(String registryAsString) throws TException { } } + private InterpreterOutput createAppOutput(final String noteId, + final String paragraphId, + final String appId) { + return new InterpreterOutput(new InterpreterOutputListener() { + @Override + public void onAppend(InterpreterOutput out, byte[] line) { + logger.info("Append output ----------------" + new String(line)); + eventClient.onAppOutputAppend(noteId, paragraphId, appId, new String(line)); + } + + @Override + public void onUpdate(InterpreterOutput out, byte[] output) { + eventClient.onAppOutputUpdate(noteId, paragraphId, appId, new String(output)); + } + }); + } + private ApplicationContext getApplicationContext( - HeliumPackage packageInfo, String noteId, String paragraphId) { - return new ApplicationContext(noteId, paragraphId); + HeliumPackage packageInfo, String noteId, String paragraphId, String applicationInstanceId) { + System.err.println("get app context ************"); + InterpreterOutput out = createAppOutput(noteId, paragraphId, applicationInstanceId); + return new ApplicationContext(noteId, paragraphId, out); } @Override @@ -720,11 +739,12 @@ public RemoteApplicationResult loadApplication( return new RemoteApplicationResult(true, ""); } HeliumPackage pkgInfo = gson.fromJson(packageInfo, HeliumPackage.class); - ApplicationContext context = getApplicationContext(pkgInfo, noteId, paragraphId); + ApplicationContext context = getApplicationContext( + pkgInfo, noteId, paragraphId, applicationInstanceId); try { Application app = null; logger.info( - "Loading application {}({}). artifact={}, className={} into note={}, paragraph={}", + "Loading application {}({}), artifact={}, className={} into note={}, paragraph={}", pkgInfo.getName(), applicationInstanceId, pkgInfo.getArtifact(), diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/ApplicationLoaderTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/ApplicationLoaderTest.java index 1df4d0587b7..fa1915fef45 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/ApplicationLoaderTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/ApplicationLoaderTest.java @@ -19,6 +19,8 @@ import org.apache.commons.io.FileUtils; import org.apache.zeppelin.dep.DependencyResolver; +import org.apache.zeppelin.interpreter.InterpreterOutput; +import org.apache.zeppelin.interpreter.InterpreterOutputListener; import org.apache.zeppelin.resource.LocalResourcePool; import org.junit.After; import org.junit.Before; @@ -83,7 +85,18 @@ public HeliumPackage createPackageInfo(String className, String artifact) { public ApplicationContext createContext(String noteId, String paragraphId) { ApplicationContext context1 = new ApplicationContext( noteId, - paragraphId); + paragraphId, + new InterpreterOutput(new InterpreterOutputListener() { + @Override + public void onAppend(InterpreterOutput out, byte[] line) { + + } + + @Override + public void onUpdate(InterpreterOutput out, byte[] output) { + + } + })); return context1; } } diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java index 84327dd6c45..262a1b4c342 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java @@ -68,6 +68,7 @@ public void setUp() throws Exception { "fakeRepo", env, 10 * 1000, + null, null ); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java index 3fbf5bcc8b2..cbb9322ae5f 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java @@ -66,7 +66,8 @@ private RemoteInterpreter createMockInterpreter() { "fakeRepo", env, 10 * 1000, - this); + this, + null); intpGroup.get("note").add(intp); intp.setInterpreterGroup(intpGroup); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java index 7beaee111f6..0526c057740 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java @@ -34,7 +34,7 @@ public void testStartStop() { InterpreterGroup intpGroup = new InterpreterGroup(); RemoteInterpreterProcess rip = new RemoteInterpreterProcess( "../bin/interpreter.sh", "nonexists", "fakeRepo", new HashMap(), - 10 * 1000, null); + 10 * 1000, null, null); assertFalse(rip.isRunning()); assertEquals(0, rip.referenceCount()); assertEquals(1, rip.reference(intpGroup)); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java index f3b936b6852..66e25dd7894 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java @@ -83,6 +83,7 @@ private RemoteInterpreter createMockInterpreterA(Properties p, String noteId) { "fakeRepo", env, 10 * 1000, + null, null); } @@ -100,6 +101,7 @@ private RemoteInterpreter createMockInterpreterB(Properties p, String noteId) { "fakeRepo", env, 10 * 1000, + null, null); } @@ -198,6 +200,7 @@ public void testRemoteSchedulerSharing() throws TTransportException, IOException "fakeRepo", env, 10 * 1000, + null, null); @@ -213,6 +216,7 @@ public void testRemoteSchedulerSharing() throws TTransportException, IOException "fakeRepo", env, 10 * 1000, + null, null); intpGroup.get("note").add(intpB); @@ -677,7 +681,7 @@ public void should_push_local_angular_repo_to_remote() throws Exception { //Given final Client client = Mockito.mock(Client.class); final RemoteInterpreter intr = new RemoteInterpreter(new Properties(), "noteId", - MockInterpreterA.class.getName(), "runner", "path","localRepo", env, 10 * 1000, null); + MockInterpreterA.class.getName(), "runner", "path","localRepo", env, 10 * 1000, null, null); final AngularObjectRegistry registry = new AngularObjectRegistry("spark", null); registry.add("name", "DuyHai DOAN", "nodeId", "paragraphId"); final InterpreterGroup interpreterGroup = new InterpreterGroup("groupId"); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java index e49b437f3ea..605be549048 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java @@ -65,6 +65,7 @@ public void setUp() throws Exception { "fakeRepo", env, 10 * 1000, + null, null ); @@ -82,6 +83,7 @@ public void setUp() throws Exception { "fakeRepo", env, 10 * 1000, + null, null ); @@ -106,11 +108,11 @@ public void setUp() throws Exception { intp1.open(); intp2.open(); - eventPoller1 = new RemoteInterpreterEventPoller(null); + eventPoller1 = new RemoteInterpreterEventPoller(null, null); eventPoller1.setInterpreterGroup(intpGroup1); eventPoller1.setInterpreterProcess(intpGroup1.getRemoteInterpreterProcess()); - eventPoller2 = new RemoteInterpreterEventPoller(null); + eventPoller2 = new RemoteInterpreterEventPoller(null, null); eventPoller2.setInterpreterGroup(intpGroup2); eventPoller2.setInterpreterProcess(intpGroup2.getRemoteInterpreterProcess()); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java index 9ce7a65a6a4..8c526141216 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java @@ -76,7 +76,8 @@ public void test() throws Exception { "fakeRepo", env, 10 * 1000, - this); + this, + null); intpGroup.put("note", new LinkedList()); intpGroup.get("note").add(intpA); @@ -164,7 +165,8 @@ public void testAbortOnPending() throws Exception { "fakeRepo", env, 10 * 1000, - this); + this, + null); intpGroup.put("note", new LinkedList()); intpGroup.get("note").add(intpA); diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index 952f112afbb..70f44006031 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -32,6 +32,7 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.dep.DependencyResolver; import org.apache.zeppelin.helium.Helium; +import org.apache.zeppelin.helium.HeliumApplicationFactory; import org.apache.zeppelin.interpreter.InterpreterFactory; import org.apache.zeppelin.notebook.Notebook; import org.apache.zeppelin.notebook.NotebookAuthorization; @@ -76,6 +77,7 @@ public class ZeppelinServer extends Application { private NotebookAuthorization notebookAuthorization; private DependencyResolver depResolver; private Helium helium; + private HeliumApplicationFactory heliumApplicationFactory; public ZeppelinServer() throws Exception { ZeppelinConfiguration conf = ZeppelinConfiguration.create(); @@ -84,15 +86,21 @@ public ZeppelinServer() throws Exception { conf.getString(ConfVars.ZEPPELIN_INTERPRETER_LOCALREPO)); this.helium = new Helium(conf.getHeliumConfPath()); + this.heliumApplicationFactory = new HeliumApplicationFactory(); this.schedulerFactory = new SchedulerFactory(); this.replFactory = new InterpreterFactory(conf, notebookWsServer, - notebookWsServer, depResolver); + notebookWsServer, heliumApplicationFactory, depResolver); this.notebookRepo = new NotebookRepoSync(conf); this.notebookIndex = new LuceneSearch(); this.notebookAuthorization = new NotebookAuthorization(conf); notebook = new Notebook(conf, notebookRepo, schedulerFactory, replFactory, notebookWsServer, notebookIndex, notebookAuthorization); + + // to update notebook from application event from remote process. + heliumApplicationFactory.setNotebook(notebook); + // to update fire websocket event on application event. + heliumApplicationFactory.setApplicationEventListener(notebookWsServer); } public static void main(String[] args) throws InterruptedException { diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/Message.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/Message.java index f0913648251..7c71b4cdd75 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/Message.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/Message.java @@ -109,10 +109,12 @@ public static enum OP { CONFIGURATIONS_INFO, // [s-c] all key/value pairs of configurations // @param settings serialized Map object - CHECKPOINT_NOTEBOOK // [c-s] checkpoint notebook to storage repository + CHECKPOINT_NOTEBOOK, // [c-s] checkpoint notebook to storage repository // @param noteId // @param checkpointName + APP_APPEND_OUTPUT, // [s-c] append output + APP_UPDATE_OUTPUT // [s-c] update (replace) output } public OP op; diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 98a1aaaec13..7fe105de398 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -34,6 +34,7 @@ import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.AngularObjectRegistryListener; +import org.apache.zeppelin.helium.ApplicationEventListener; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; import org.apache.zeppelin.user.AuthenticationInfo; @@ -60,7 +61,7 @@ */ public class NotebookServer extends WebSocketServlet implements NotebookSocketListener, JobListenerFactory, AngularObjectRegistryListener, - RemoteInterpreterProcessListener { + RemoteInterpreterProcessListener, ApplicationEventListener { private static final Logger LOG = LoggerFactory.getLogger(NotebookServer.class); Gson gson = new Gson(); final Map> noteSocketMap = new HashMap<>(); @@ -965,7 +966,6 @@ public void onOutputAppend(String noteId, String paragraphId, String output) { .put("noteId", noteId) .put("paragraphId", paragraphId) .put("data", output); - Paragraph paragraph = notebook().getNote(noteId).getParagraph(paragraphId); broadcast(noteId, msg); } @@ -981,7 +981,40 @@ public void onOutputUpdated(String noteId, String paragraphId, String output) { .put("noteId", noteId) .put("paragraphId", paragraphId) .put("data", output); - Paragraph paragraph = notebook().getNote(noteId).getParagraph(paragraphId); + broadcast(noteId, msg); + } + + /** + * When application append output + * @param noteId + * @param paragraphId + * @param appId + * @param output + */ + @Override + public void onOutputAppend(String noteId, String paragraphId, String appId, String output) { + Message msg = new Message(OP.APP_APPEND_OUTPUT) + .put("noteId", noteId) + .put("paragraphId", paragraphId) + .put("appId", appId) + .put("data", output); + broadcast(noteId, msg); + } + + /** + * When application update output + * @param noteId + * @param paragraphId + * @param appId + * @param output + */ + @Override + public void onOutputUpdated(String noteId, String paragraphId, String appId, String output) { + Message msg = new Message(OP.APP_UPDATE_OUTPUT) + .put("noteId", noteId) + .put("paragraphId", paragraphId) + .put("appId", appId) + .put("data", output); broadcast(noteId, msg); } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java index 848b0c3258e..5317825997c 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java @@ -19,74 +19,60 @@ import com.google.gson.Gson; import org.apache.thrift.TException; import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; import org.apache.zeppelin.interpreter.thrift.RemoteApplicationResult; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService; +import org.apache.zeppelin.notebook.ApplicationState; import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.Notebook; import org.apache.zeppelin.notebook.Paragraph; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Random; +import java.util.*; /** * HeliumApplicationFactory + * + * 1. sync api -> async api + * 2. unload app when paragraph / note / interpreter remove + * 3. front-end job + * 4. example app + * 5. dev mode + * 6. app launcher */ -public class HeliumApplicationFactory { +public class HeliumApplicationFactory implements ApplicationEventListener { Logger logger = LoggerFactory.getLogger(HeliumApplicationFactory.class); private final Gson gson; - Map apps = - Collections.synchronizedMap(new HashMap()); - + private Notebook notebook; + private ApplicationEventListener applicationEventListener; public HeliumApplicationFactory() { gson = new Gson(); } - private static class RunningApplicationInfo { - HeliumPackage pkg; - Paragraph paragraph; - RemoteInterpreterProcess process; - - public RunningApplicationInfo(HeliumPackage pkg, - RemoteInterpreterProcess process, - Paragraph paragraph) { - this.pkg = pkg; - this.paragraph = paragraph; - this.process = process; - } - - public HeliumPackage getPkg() { - return pkg; - } - - public Paragraph getParagraph() { - return paragraph; - } - public RemoteInterpreterProcess getProcess() { - return process; - } + private static String generateApplicationId(HeliumPackage pkg, Paragraph paragraph) { + return "app_" + paragraph.getNote().getId() + "-" + paragraph.getId() + pkg.getName(); } - - - private static String generateApplicationId() { - return "app_" + System.currentTimeMillis() + "_" - + new Random(System.currentTimeMillis()).nextInt(); + private boolean isRemote(InterpreterGroup group) { + return group.getAngularObjectRegistry() instanceof RemoteAngularObjectRegistry; } - /** * Load pkg */ - public void load(HeliumPackage pkg, Paragraph paragraph) throws ApplicationException { + public String load(HeliumPackage pkg, Paragraph paragraph) throws ApplicationException { Interpreter intp = paragraph.getRepl(paragraph.getRequiredReplName()); - RemoteInterpreterProcess intpProcess = intp.getInterpreterGroup().getRemoteInterpreterProcess(); + InterpreterGroup intpGroup = intp.getInterpreterGroup(); + RemoteInterpreterProcess intpProcess = intpGroup.getRemoteInterpreterProcess(); + if (intpProcess == null) { + throw new ApplicationException("Target interpreter process is not running"); + } RemoteInterpreterService.Client client; try { @@ -95,134 +81,247 @@ public void load(HeliumPackage pkg, Paragraph paragraph) throws ApplicationExcep throw new ApplicationException(e); } - String appId = generateApplicationId(); + String appId = generateApplicationId(pkg, paragraph); String pkgInfo = gson.toJson(pkg); - try { - RemoteApplicationResult ret = client.loadApplication( - appId, - pkgInfo, - paragraph.getNote().getId(), - paragraph.getId()); - - if (ret.isSuccess()) { - RunningApplicationInfo appInfo = new RunningApplicationInfo(pkg, intpProcess, paragraph); - apps.put(appId, appInfo); - } else { - throw new ApplicationException(ret.getMsg()); + + ApplicationState appState = null; + synchronized (paragraph.apps) { + for (ApplicationState as : paragraph.apps) { + if (as.getName().equals(pkg.getName())) { + appState = as; + break; + } + } + + if (appState == null) { + appState = new ApplicationState(appId, pkg.getName()); + paragraph.apps.add(appState); } - } catch (TException e) { - intpProcess.releaseBrokenClient(client); - throw new ApplicationException(e); - } finally { - intpProcess.releaseClient(client); } - } + synchronized (appState) { + if (appState.getStatus() == ApplicationState.ApplicationStatus.LOADED) { + logger.info("Application {} already loaded on paragraph {}", + pkg.getName(), paragraph.getId()); + return appId; + } + appState.setStatus(ApplicationState.ApplicationStatus.LOADING); + try { + RemoteApplicationResult ret = client.loadApplication( + appId, + pkgInfo, + paragraph.getNote().getId(), + paragraph.getId()); + + if (ret.isSuccess()) { + appState.setStatus(ApplicationState.ApplicationStatus.LOADED); + } else { + appState.setStatus(ApplicationState.ApplicationStatus.UNLOADED); + throw new ApplicationException(ret.getMsg()); + } + return appId; + } catch (TException e) { + intpProcess.releaseBrokenClient(client); + appState.setStatus(ApplicationState.ApplicationStatus.UNLOADED); + throw new ApplicationException(e); + } finally { + intpProcess.releaseClient(client); + } + } + } /** * Unload pkg */ - public void unload(String appId) throws ApplicationException { - RunningApplicationInfo appInfo = apps.get(appId); - if (appInfo == null) { - return; - } + public void unload(Paragraph paragraph, String appName) throws ApplicationException { - RemoteInterpreterProcess intpProcess = appInfo.getProcess(); + ApplicationState appsToUnload = null; - RemoteInterpreterService.Client client; - try { - client = intpProcess.getClient(); - } catch (Exception e) { - throw new ApplicationException(e); + synchronized (paragraph.apps) { + for (ApplicationState as : paragraph.apps) { + if (as.getName().equals(appName)) { + appsToUnload = as; + break; + } + } } - try { - RemoteApplicationResult ret = client.unloadApplication(appId); + if (appsToUnload == null) { + logger.warn("Can not find {} to unload from {}", appName, paragraph.getId()); + return; + } - if (ret.isSuccess()) { - apps.remove(appId); - } else { - throw new ApplicationException(ret.getMsg()); + synchronized (appsToUnload) { + if (appsToUnload.getStatus() != ApplicationState.ApplicationStatus.LOADED) { + throw new ApplicationException( + "Can't unload application status " + appsToUnload.getStatus()); + } + appsToUnload.setStatus(ApplicationState.ApplicationStatus.UNLOADING); + RemoteInterpreterProcess intpProcess = + paragraph.getCurrentRepl().getInterpreterGroup().getRemoteInterpreterProcess(); + + RemoteInterpreterService.Client client; + try { + client = intpProcess.getClient(); + } catch (Exception e) { + throw new ApplicationException(e); + } + + try { + RemoteApplicationResult ret = client.unloadApplication(appsToUnload.getId()); + + if (ret.isSuccess()) { + appsToUnload.setStatus(ApplicationState.ApplicationStatus.UNLOADED); + } else { + appsToUnload.setStatus(ApplicationState.ApplicationStatus.LOADED); + throw new ApplicationException(ret.getMsg()); + } + } catch (TException e) { + intpProcess.releaseBrokenClient(client); + appsToUnload.setStatus(ApplicationState.ApplicationStatus.LOADED); + throw new ApplicationException(e); + } finally { + intpProcess.releaseClient(client); } - } catch (TException e) { - intpProcess.releaseBrokenClient(client); - throw new ApplicationException(e); - } finally { - intpProcess.releaseClient(client); } } /** * run pkg */ - public void run(String appId) throws ApplicationException { - RunningApplicationInfo appInfo = apps.get(appId); - if (appInfo == null) { - return; + public void run(Paragraph paragraph, String appName) throws ApplicationException { + ApplicationState app = null; + synchronized (paragraph.apps) { + for (ApplicationState as : paragraph.apps) { + if (as.getName().equals(appName)) { + app = as; + break; + } + } } - RemoteInterpreterProcess intpProcess = appInfo.getProcess(); - - RemoteInterpreterService.Client client; - try { - client = intpProcess.getClient(); - } catch (Exception e) { - throw new ApplicationException(e); + if (app == null) { + logger.warn("Can not find app {} from {}", appName, paragraph.getId()); + return; } - try { - RemoteApplicationResult ret = client.runApplication(appId); - - if (ret.isSuccess()) { - // success - } else { - throw new ApplicationException(ret.getMsg()); + synchronized (app) { + if (app.getStatus() != ApplicationState.ApplicationStatus.LOADED) { + throw new ApplicationException( + "Can't run application status " + app.getStatus()); + } + RemoteInterpreterProcess intpProcess = + paragraph.getCurrentRepl().getInterpreterGroup().getRemoteInterpreterProcess(); + + RemoteInterpreterService.Client client; + try { + client = intpProcess.getClient(); + } catch (Exception e) { + throw new ApplicationException(e); } - } catch (TException e) { - intpProcess.releaseBrokenClient(client); - throw new ApplicationException(e); - } finally { - intpProcess.releaseClient(client); - } - } - public void unloadAllInTheInteprreterProcess(RemoteInterpreterProcess process) { - for (String appId : apps.keySet()) { - RunningApplicationInfo app = apps.get(appId); - if (app.getProcess() == process) { - try { - unload(appId); - } catch (ApplicationException e) { - logger.error(e.getMessage(), e); + try { + RemoteApplicationResult ret = client.runApplication(app.getId()); + + if (ret.isSuccess()) { + // success + } else { + throw new ApplicationException(ret.getMsg()); } + } catch (TException e) { + intpProcess.releaseBrokenClient(client); + throw new ApplicationException(e); + } finally { + intpProcess.releaseClient(client); } } } + public void unloadAllInTheInterpreterProcess(RemoteInterpreterProcess process) { + // TODO + } + public void unloadAllInTheNote(Note note) { - for (String appId : apps.keySet()) { - RunningApplicationInfo app = apps.get(appId); - if (app.getParagraph().getNote().getId().equals(note.getId())) { - try { - unload(appId); - } catch (ApplicationException e) { - logger.error(e.getMessage(), e); - } - } - } + // TODO } public void unloadAllInTheParagraph(Paragraph paragraph) { - for (String appId : apps.keySet()) { - RunningApplicationInfo app = apps.get(appId); - if (app.getParagraph().getId().equals(paragraph.getId())) { - try { - unload(appId); - } catch (ApplicationException e) { - logger.error(e.getMessage(), e); + // TODO + } + + + @Override + public void onOutputAppend(String noteId, String paragraphId, String appId, String output) { + ApplicationState appToUpdate = getAppState(noteId, paragraphId, appId); + + if (appToUpdate != null) { + appToUpdate.appendOutput(output); + } else { + logger.error("Can't find app {}", appId); + } + + if (applicationEventListener != null) { + applicationEventListener.onOutputAppend(noteId, paragraphId, appId, output); + } + } + + @Override + public void onOutputUpdated(String noteId, String paragraphId, String appId, String output) { + ApplicationState appToUpdate = getAppState(noteId, paragraphId, appId); + + if (appToUpdate != null) { + appToUpdate.setOutput(output); + } else { + logger.error("Can't find app {}", appId); + } + + if (applicationEventListener != null) { + applicationEventListener.onOutputUpdated(noteId, paragraphId, appId, output); + } + } + + private ApplicationState getAppState(String noteId, String paragraphId, String appId) { + if (notebook == null) { + return null; + } + + Note note = notebook.getNote(noteId); + if (note == null) { + logger.error("Can't get note {}", noteId); + return null; + } + Paragraph paragraph = note.getParagraph(paragraphId); + if (paragraph == null) { + logger.error("Can't get paragraph {}", paragraphId); + return null; + } + + ApplicationState appFound = null; + synchronized (paragraph.apps) { + for (ApplicationState app : paragraph.apps) { + if (app.getId().equals(appId)) { + appFound = app; + break; } } } + + return appFound; + } + + public Notebook getNotebook() { + return notebook; + } + + public void setNotebook(Notebook notebook) { + this.notebook = notebook; + } + + public ApplicationEventListener getApplicationEventListener() { + return applicationEventListener; + } + + public void setApplicationEventListener(ApplicationEventListener applicationEventListener) { + this.applicationEventListener = applicationEventListener; } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index f0fa385aa12..08ddba1ec58 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -29,6 +29,7 @@ import org.apache.zeppelin.dep.DependencyResolver; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.AngularObjectRegistryListener; +import org.apache.zeppelin.helium.ApplicationEventListener; import org.apache.zeppelin.interpreter.Interpreter.RegisteredInterpreter; import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; import org.apache.zeppelin.interpreter.remote.RemoteInterpreter; @@ -73,22 +74,27 @@ public class InterpreterFactory { AngularObjectRegistryListener angularObjectRegistryListener; private final RemoteInterpreterProcessListener remoteInterpreterProcessListener; + private final ApplicationEventListener appEventListener; private DependencyResolver depResolver; + private Map env; + public InterpreterFactory(ZeppelinConfiguration conf, AngularObjectRegistryListener angularObjectRegistryListener, RemoteInterpreterProcessListener remoteInterpreterProcessListener, + ApplicationEventListener appEventListener, DependencyResolver depResolver) throws InterpreterException, IOException, RepositoryException { this(conf, new InterpreterOption(true), angularObjectRegistryListener, - remoteInterpreterProcessListener, depResolver); + remoteInterpreterProcessListener, appEventListener, depResolver); } public InterpreterFactory(ZeppelinConfiguration conf, InterpreterOption defaultOption, AngularObjectRegistryListener angularObjectRegistryListener, RemoteInterpreterProcessListener remoteInterpreterProcessListener, + ApplicationEventListener appEventListener, DependencyResolver depResolver) throws InterpreterException, IOException, RepositoryException { this.conf = conf; @@ -97,6 +103,7 @@ public InterpreterFactory(ZeppelinConfiguration conf, InterpreterOption defaultO this.depResolver = depResolver; this.interpreterRepositories = depResolver.getRepos(); this.remoteInterpreterProcessListener = remoteInterpreterProcessListener; + this.appEventListener = appEventListener; String replsConf = conf.getString(ConfVars.ZEPPELIN_INTERPRETERS); interpreterClassList = replsConf.split(","); @@ -812,10 +819,12 @@ private Interpreter createRemoteRepl(String interpreterPath, String noteId, Stri int connectTimeout = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT); String localRepoPath = conf.getInterpreterLocalRepoPath() + "/" + interpreterId; int maxPoolSize = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE); - LazyOpenInterpreter intp = new LazyOpenInterpreter(new RemoteInterpreter( + RemoteInterpreter remoteInterpreter = new RemoteInterpreter( property, noteId, className, conf.getInterpreterRemoteRunnerPath(), interpreterPath, localRepoPath, connectTimeout, - maxPoolSize, remoteInterpreterProcessListener)); + maxPoolSize, remoteInterpreterProcessListener, appEventListener); + remoteInterpreter.setEnv(env); + LazyOpenInterpreter intp = new LazyOpenInterpreter(remoteInterpreter); return intp; } @@ -853,4 +862,12 @@ public void removeRepository(String id) throws IOException { depResolver.delRepo(id); saveToFile(); } + + public Map getEnv() { + return env; + } + + public void setEnv(Map env) { + this.env = env; + } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java index e1f8584bc2a..5f93bb6c4c0 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java @@ -16,8 +16,86 @@ */ package org.apache.zeppelin.notebook; +import org.apache.zeppelin.helium.Application; +import org.apache.zeppelin.helium.HeliumPackage; +import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; + /** * Running ApplicationState */ public class ApplicationState { + + /** + * Status of Application + */ + public static enum ApplicationStatus { + LOADING, + LOADED, + UNLOADING, + UNLOADED + }; + + + String id; // unique id for this instance similar to note id or paragraph id + String name; // name of app + ApplicationStatus status; + String output; + + public ApplicationState(String id, String name) { + this.id = id; + this.name = name; + status = ApplicationStatus.UNLOADED; + } + + @Override + public boolean equals(Object o) { + String compareName; + if (o instanceof ApplicationState) { + compareName = ((ApplicationState) o).name; + } else if (o instanceof String) { + compareName = (String) o; + } else { + return false; + } + + return name.equals(compareName); + } + + @Override + public int hashCode() { + return name.hashCode(); + } + + public String getId() { + return id; + } + + public void setStatus(ApplicationStatus status) { + this.status = status; + } + + public ApplicationStatus getStatus() { + return status; + } + + public String getOutput() { + return output; + } + + public void setOutput(String output) { + this.output = output; + } + + public synchronized void appendOutput(String output) { + if (this.output == null) { + this.output = output; + } else { + this.output += output; + } + } + + public String getName() { + return name; + } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java index 4827bfff414..606973450c1 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java @@ -34,6 +34,7 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.helium.ApplicationEventListener; import org.apache.zeppelin.interpreter.InterpreterFactory; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterSetting; diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java index 337e32d3ac9..b7f354504cc 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java @@ -53,7 +53,7 @@ public class Paragraph extends Job implements Serializable, Cloneable { Date dateUpdated; private Map config; // paragraph configs like isOpen, colWidth, etc public final GUI settings; // form and parameter settings - private Map apps; // helium applications running + public final List apps = new LinkedList(); @VisibleForTesting Paragraph() { @@ -72,7 +72,6 @@ public Paragraph(Note note, JobListener listener, NoteInterpreterLoader replLoad dateUpdated = null; settings = new GUI(); config = new HashMap(); - apps = new HashMap(); } private static String generateId() { diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java new file mode 100644 index 00000000000..a7c4e0a2825 --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import org.apache.commons.io.FileUtils; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.dep.DependencyResolver; +import org.apache.zeppelin.interpreter.InterpreterFactory; +import org.apache.zeppelin.interpreter.InterpreterOption; +import org.apache.zeppelin.interpreter.InterpreterOutput; +import org.apache.zeppelin.interpreter.mock.MockInterpreter1; +import org.apache.zeppelin.interpreter.mock.MockInterpreter2; +import org.apache.zeppelin.notebook.*; +import org.apache.zeppelin.notebook.repo.VFSNotebookRepo; +import org.apache.zeppelin.scheduler.Job; +import org.apache.zeppelin.scheduler.SchedulerFactory; +import org.apache.zeppelin.search.SearchService; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; + +public class HeliumApplicationFactoryTest implements JobListenerFactory { + private File tmpDir; + private File notebookDir; + private ZeppelinConfiguration conf; + private SchedulerFactory schedulerFactory; + private DependencyResolver depResolver; + private InterpreterFactory factory; + private VFSNotebookRepo notebookRepo; + private Notebook notebook; + private HeliumApplicationFactory heliumAppFactory; + + @Before + public void setUp() throws Exception { + tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis()); + tmpDir.mkdirs(); + File confDir = new File(tmpDir, "conf"); + confDir.mkdirs(); + notebookDir = new File(tmpDir + "/notebook"); + notebookDir.mkdirs(); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_CONF_DIR.getVarName(), confDir.getAbsolutePath()); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath()); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETERS.getVarName(), "org.apache.zeppelin.interpreter.mock.MockInterpreter1,org.apache.zeppelin.interpreter.mock.MockInterpreter2"); + + conf = ZeppelinConfiguration.create(); + + this.schedulerFactory = new SchedulerFactory(); + + MockInterpreter1.register("mock1", "org.apache.zeppelin.interpreter.mock.MockInterpreter1"); + MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2"); + + heliumAppFactory = new HeliumApplicationFactory(); + + depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); + factory = new InterpreterFactory(conf, + new InterpreterOption(true), null, null, heliumAppFactory, depResolver); + HashMap env = new HashMap(); + env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath()); + factory.setEnv(env); + + SearchService search = mock(SearchService.class); + notebookRepo = new VFSNotebookRepo(conf); + NotebookAuthorization notebookAuthorization = new NotebookAuthorization(conf); + notebook = new Notebook( + conf, + notebookRepo, + schedulerFactory, + factory, + this, + search, + notebookAuthorization); + + heliumAppFactory.setNotebook(notebook); + } + + @After + public void tearDown() throws Exception { + FileUtils.deleteDirectory(tmpDir); + } + + + @Test + public void testLoadRunUnloadApplication() + throws IOException, ApplicationException, InterruptedException { + // given + HeliumPackage pkg1 = new HeliumPackage(HeliumPackage.Type.APPLICATION, + "name1", + "desc1", + "", + HeliumTestApplication.class.getName(), + new String[][]{}); + + Note note1 = notebook.createNote(); + note1.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); + + Paragraph p1 = note1.addParagraph(); + + // make sure interpreter process running + p1.setText("job"); + note1.run(p1.getId()); + while(p1.isTerminated()==false || p1.getResult()==null) Thread.yield(); + + assertEquals("repl1: job", p1.getResult().message()); + + // when + String appId = heliumAppFactory.load(pkg1, p1); + heliumAppFactory.run(p1, pkg1.getName()); + + // then + Thread.sleep(1000); + assertEquals("Hello world", p1.apps.get(0).getOutput()); + + // clean + heliumAppFactory.unload(p1, pkg1.getName()); + notebook.removeNote(note1.getId()); + } + + + @Override + public ParagraphJobListener getParagraphJobListener(Note note) { + return new ParagraphJobListener() { + @Override + public void onOutputAppend(Paragraph paragraph, InterpreterOutput out, String output) { + } + + @Override + public void onOutputUpdate(Paragraph paragraph, InterpreterOutput out, String output) { + + } + + @Override + public void onProgressUpdate(Job job, int progress) { + + } + + @Override + public void beforeStatusChange(Job job, Job.Status before, Job.Status after) { + + } + + @Override + public void afterStatusChange(Job job, Job.Status before, Job.Status after) { + + } + }; + } +} diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTestApplication.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTestApplication.java new file mode 100644 index 00000000000..c28be012a07 --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTestApplication.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import org.apache.zeppelin.resource.ResourceSet; + +import java.io.IOException; + +public class HeliumTestApplication extends Application { + public HeliumTestApplication(ResourceSet args, ApplicationContext context) + throws ApplicationException { + super(args, context); + } + + @Override + public void run() throws ApplicationException { + try { + context().out.write("Hello world"); + context().out.flush(); + } catch (IOException e) { + throw new ApplicationException(e); + } + } + + @Override + public void unload() throws ApplicationException { + + } +} diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java index d215e7c41cd..49c49337aec 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java @@ -60,7 +60,7 @@ public void setUp() throws Exception { System.setProperty(ConfVars.ZEPPELIN_INTERPRETERS.getVarName(), "org.apache.zeppelin.interpreter.mock.MockInterpreter1,org.apache.zeppelin.interpreter.mock.MockInterpreter2"); conf = new ZeppelinConfiguration(); depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); - factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, depResolver); + factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver); context = new InterpreterContext("note", "id", "title", "text", null, null, null, null, null, null, null); } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java index aa98afd6db1..3c56a8a5e24 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java @@ -61,7 +61,7 @@ public void setUp() throws Exception { MockInterpreter2.register("mock2", "group2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2"); depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); - factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, depResolver); + factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver); } @After diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index 3c89c35a72d..92471879419 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -85,7 +85,7 @@ public void setUp() throws Exception { MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2"); depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); - factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, depResolver); + factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver); SearchService search = mock(SearchService.class); notebookRepo = new VFSNotebookRepo(conf); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java index 69b50b43933..de8790ce350 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/NotebookRepoSyncTest.java @@ -92,7 +92,7 @@ public void setUp() throws Exception { MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2"); depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo"); - factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, depResolver); + factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver); search = mock(SearchService.class); notebookRepoSync = new NotebookRepoSync(conf); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java index 4932ae2fde2..64608e146ce 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java @@ -76,7 +76,7 @@ public void setUp() throws Exception { this.schedulerFactory = new SchedulerFactory(); depResolver = new DependencyResolver(mainZepDir.getAbsolutePath() + "/local-repo"); - factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, depResolver); + factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver); SearchService search = mock(SearchService.class); notebookRepo = new VFSNotebookRepo(conf); From 4eaeea720561962766f25438b19d7a334a8cb0bc Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sun, 3 Apr 2016 05:18:54 +0900 Subject: [PATCH 06/69] sync -> async api --- .../helium/ApplicationEventListener.java | 1 - .../remote/RemoteInterpreterServer.java | 2 - .../helium/HeliumApplicationFactory.java | 366 +++++++++++------- .../zeppelin/notebook/ApplicationState.java | 33 +- .../apache/zeppelin/notebook/Paragraph.java | 46 ++- .../helium/HeliumApplicationFactoryTest.java | 19 +- 6 files changed, 302 insertions(+), 165 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationEventListener.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationEventListener.java index 6ab4ea49f7e..49899fa3028 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationEventListener.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationEventListener.java @@ -22,5 +22,4 @@ public interface ApplicationEventListener { public void onOutputAppend(String noteId, String paragraphId, String appId, String output); public void onOutputUpdated(String noteId, String paragraphId, String appId, String output); - } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java index 19826ac1bd8..129d7b94981 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java @@ -712,7 +712,6 @@ private InterpreterOutput createAppOutput(final String noteId, return new InterpreterOutput(new InterpreterOutputListener() { @Override public void onAppend(InterpreterOutput out, byte[] line) { - logger.info("Append output ----------------" + new String(line)); eventClient.onAppOutputAppend(noteId, paragraphId, appId, new String(line)); } @@ -725,7 +724,6 @@ public void onUpdate(InterpreterOutput out, byte[] output) { private ApplicationContext getApplicationContext( HeliumPackage packageInfo, String noteId, String paragraphId, String applicationInstanceId) { - System.err.println("get app context ************"); InterpreterOutput out = createAppOutput(noteId, paragraphId, applicationInstanceId); return new ApplicationContext(noteId, paragraphId, out); } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java index 5317825997c..8c9a4a304dd 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java @@ -30,8 +30,12 @@ import org.apache.zeppelin.notebook.Paragraph; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import sun.reflect.annotation.ExceptionProxy; -import java.util.*; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; /** * HeliumApplicationFactory @@ -42,201 +46,287 @@ * 4. example app * 5. dev mode * 6. app launcher + * 7. offline mode. front-end table data / pivot panel access */ public class HeliumApplicationFactory implements ApplicationEventListener { - Logger logger = LoggerFactory.getLogger(HeliumApplicationFactory.class); - + private final Logger logger = LoggerFactory.getLogger(HeliumApplicationFactory.class); + private final ExecutorService executor; private final Gson gson; private Notebook notebook; private ApplicationEventListener applicationEventListener; - public HeliumApplicationFactory() { + public HeliumApplicationFactory(ExecutorService executor) { gson = new Gson(); + this.executor = executor; } - private static String generateApplicationId(HeliumPackage pkg, Paragraph paragraph) { - return "app_" + paragraph.getNote().getId() + "-" + paragraph.getId() + pkg.getName(); - } private boolean isRemote(InterpreterGroup group) { return group.getAngularObjectRegistry() instanceof RemoteAngularObjectRegistry; } + /** - * Load pkg + * Load pkg and run task */ - public String load(HeliumPackage pkg, Paragraph paragraph) throws ApplicationException { - Interpreter intp = paragraph.getRepl(paragraph.getRequiredReplName()); - InterpreterGroup intpGroup = intp.getInterpreterGroup(); - RemoteInterpreterProcess intpProcess = intpGroup.getRemoteInterpreterProcess(); - if (intpProcess == null) { - throw new ApplicationException("Target interpreter process is not running"); - } + public String loadAndRun(HeliumPackage pkg, Paragraph paragraph) { + ApplicationState appState = paragraph.createOrGetApplicationState(pkg); + executor.submit(new LoadApplication(appState, pkg, paragraph)); + return appState.getId(); + } - RemoteInterpreterService.Client client; - try { - client = intpProcess.getClient(); - } catch (Exception e) { - throw new ApplicationException(e); + /** + * Load application and run in the remote process + */ + private class LoadApplication implements Runnable { + private final HeliumPackage pkg; + private final Paragraph paragraph; + private final ApplicationState appState; + + public LoadApplication(ApplicationState appState, HeliumPackage pkg, Paragraph paragraph) { + this.appState = appState; + this.pkg = pkg; + this.paragraph = paragraph; } - String appId = generateApplicationId(pkg, paragraph); - String pkgInfo = gson.toJson(pkg); - - ApplicationState appState = null; - synchronized (paragraph.apps) { - for (ApplicationState as : paragraph.apps) { - if (as.getName().equals(pkg.getName())) { - appState = as; - break; + @Override + public void run() { + try { + // get interpreter process + Interpreter intp = paragraph.getRepl(paragraph.getRequiredReplName()); + InterpreterGroup intpGroup = intp.getInterpreterGroup(); + RemoteInterpreterProcess intpProcess = intpGroup.getRemoteInterpreterProcess(); + if (intpProcess == null) { + throw new ApplicationException("Target interpreter process is not running"); } - } - if (appState == null) { - appState = new ApplicationState(appId, pkg.getName()); - paragraph.apps.add(appState); + // load application + load(intpProcess, appState); + + // run application + RunApplication runTask = new RunApplication(paragraph, appState.getId()); + runTask.run(); + } catch (Exception e) { + logger.error(e.getMessage(), e); + + if (appState != null) { + appState.setStatus(ApplicationState.Status.ERROR); + appState.setOutput(e.getMessage()); + } } } - synchronized (appState) { - if (appState.getStatus() == ApplicationState.ApplicationStatus.LOADED) { - logger.info("Application {} already loaded on paragraph {}", - pkg.getName(), paragraph.getId()); - return appId; - } - appState.setStatus(ApplicationState.ApplicationStatus.LOADING); + private void load(RemoteInterpreterProcess intpProcess, ApplicationState appState) + throws Exception { + + RemoteInterpreterService.Client client; try { - RemoteApplicationResult ret = client.loadApplication( - appId, - pkgInfo, - paragraph.getNote().getId(), - paragraph.getId()); - - if (ret.isSuccess()) { - appState.setStatus(ApplicationState.ApplicationStatus.LOADED); - } else { - appState.setStatus(ApplicationState.ApplicationStatus.UNLOADED); - throw new ApplicationException(ret.getMsg()); - } - return appId; - } catch (TException e) { - intpProcess.releaseBrokenClient(client); - appState.setStatus(ApplicationState.ApplicationStatus.UNLOADED); + client = intpProcess.getClient(); + } catch (Exception e) { throw new ApplicationException(e); - } finally { - intpProcess.releaseClient(client); + } + + synchronized (appState) { + if (appState.getStatus() == ApplicationState.Status.LOADED) { + // already loaded + return; + } + appState.setStatus(ApplicationState.Status.LOADING); + try { + String pkgInfo = gson.toJson(pkg); + String appId = appState.getId(); + + RemoteApplicationResult ret = client.loadApplication( + appId, + pkgInfo, + paragraph.getNote().getId(), + paragraph.getId()); + + if (ret.isSuccess()) { + appState.setStatus(ApplicationState.Status.LOADED); + } else { + throw new ApplicationException(ret.getMsg()); + } + } catch (TException e) { + intpProcess.releaseBrokenClient(client); + throw e; + } finally { + intpProcess.releaseClient(client); + } } } } /** - * Unload pkg + * Get ApplicationState + * @param paragraph + * @param appId + * @return */ - public void unload(Paragraph paragraph, String appName) throws ApplicationException { + public ApplicationState get(Paragraph paragraph, String appId) { + return paragraph.getApplicationState(appId); + } - ApplicationState appsToUnload = null; + /** + * Unload application + * It does not remove ApplicationState + * + * @param paragraph + * @param appId + */ + public void unload(Paragraph paragraph, String appId) { + executor.execute(new UnloadApplication(paragraph, appId)); + } - synchronized (paragraph.apps) { - for (ApplicationState as : paragraph.apps) { - if (as.getName().equals(appName)) { - appsToUnload = as; - break; - } - } - } + /** + * Unload application task + */ + private class UnloadApplication implements Runnable { + private final Paragraph paragraph; + private final String appId; - if (appsToUnload == null) { - logger.warn("Can not find {} to unload from {}", appName, paragraph.getId()); - return; + public UnloadApplication(Paragraph paragraph, String appId) { + this.paragraph = paragraph; + this.appId = appId; } - synchronized (appsToUnload) { - if (appsToUnload.getStatus() != ApplicationState.ApplicationStatus.LOADED) { - throw new ApplicationException( - "Can't unload application status " + appsToUnload.getStatus()); - } - appsToUnload.setStatus(ApplicationState.ApplicationStatus.UNLOADING); - RemoteInterpreterProcess intpProcess = - paragraph.getCurrentRepl().getInterpreterGroup().getRemoteInterpreterProcess(); - - RemoteInterpreterService.Client client; + @Override + public void run() { + ApplicationState appState = null; try { - client = intpProcess.getClient(); + appState = paragraph.getApplicationState(appId); + + if (appState == null) { + logger.warn("Can not find {} to unload from {}", appId, paragraph.getId()); + return; + } + + unload(appState); } catch (Exception e) { - throw new ApplicationException(e); + logger.error(e.getMessage(), e); + if (appState != null) { + appState.setStatus(ApplicationState.Status.ERROR); + appState.setOutput(e.getMessage()); + } } + } - try { - RemoteApplicationResult ret = client.unloadApplication(appsToUnload.getId()); + private void unload(ApplicationState appsToUnload) throws ApplicationException { + synchronized (appsToUnload) { + if (appsToUnload.getStatus() != ApplicationState.Status.LOADED) { + throw new ApplicationException( + "Can't unload application status " + appsToUnload.getStatus()); + } + appsToUnload.setStatus(ApplicationState.Status.UNLOADING); + RemoteInterpreterProcess intpProcess = + paragraph.getCurrentRepl().getInterpreterGroup().getRemoteInterpreterProcess(); + + RemoteInterpreterService.Client client; + try { + client = intpProcess.getClient(); + } catch (Exception e) { + throw new ApplicationException(e); + } - if (ret.isSuccess()) { - appsToUnload.setStatus(ApplicationState.ApplicationStatus.UNLOADED); - } else { - appsToUnload.setStatus(ApplicationState.ApplicationStatus.LOADED); - throw new ApplicationException(ret.getMsg()); + try { + RemoteApplicationResult ret = client.unloadApplication(appsToUnload.getId()); + + if (ret.isSuccess()) { + appsToUnload.setStatus(ApplicationState.Status.UNLOADED); + } else { + throw new ApplicationException(ret.getMsg()); + } + } catch (TException e) { + intpProcess.releaseBrokenClient(client); + throw new ApplicationException(e); + } finally { + intpProcess.releaseClient(client); } - } catch (TException e) { - intpProcess.releaseBrokenClient(client); - appsToUnload.setStatus(ApplicationState.ApplicationStatus.LOADED); - throw new ApplicationException(e); - } finally { - intpProcess.releaseClient(client); } } } /** - * run pkg + * Run application + * It does not remove ApplicationState + * + * @param paragraph + * @param appId */ - public void run(Paragraph paragraph, String appName) throws ApplicationException { - ApplicationState app = null; - synchronized (paragraph.apps) { - for (ApplicationState as : paragraph.apps) { - if (as.getName().equals(appName)) { - app = as; - break; - } - } - } + public void run(Paragraph paragraph, String appId) { + executor.execute(new RunApplication(paragraph, appId)); + } - if (app == null) { - logger.warn("Can not find app {} from {}", appName, paragraph.getId()); - return; - } + /** + * Run application task + */ + private class RunApplication implements Runnable { + private final Paragraph paragraph; + private final String appId; - synchronized (app) { - if (app.getStatus() != ApplicationState.ApplicationStatus.LOADED) { - throw new ApplicationException( - "Can't run application status " + app.getStatus()); - } - RemoteInterpreterProcess intpProcess = - paragraph.getCurrentRepl().getInterpreterGroup().getRemoteInterpreterProcess(); + public RunApplication(Paragraph paragraph, String appId) { + this.paragraph = paragraph; + this.appId = appId; + } - RemoteInterpreterService.Client client; + @Override + public void run() { + ApplicationState appState = null; try { - client = intpProcess.getClient(); + appState = paragraph.getApplicationState(appId); + + if (appState == null) { + logger.warn("Can not find {} to unload from {}", appId, paragraph.getId()); + return; + } + + run(appState); } catch (Exception e) { - throw new ApplicationException(e); + logger.error(e.getMessage(), e); + if (appState != null) { + appState.setStatus(ApplicationState.Status.ERROR); + appState.setOutput(e.getMessage()); + } } + } - try { - RemoteApplicationResult ret = client.runApplication(app.getId()); + private void run(ApplicationState app) throws ApplicationException { + synchronized (app) { + if (app.getStatus() != ApplicationState.Status.LOADED) { + throw new ApplicationException( + "Can't run application status " + app.getStatus()); + } + RemoteInterpreterProcess intpProcess = + paragraph.getCurrentRepl().getInterpreterGroup().getRemoteInterpreterProcess(); + + RemoteInterpreterService.Client client; + try { + client = intpProcess.getClient(); + } catch (Exception e) { + throw new ApplicationException(e); + } - if (ret.isSuccess()) { - // success - } else { - throw new ApplicationException(ret.getMsg()); + try { + RemoteApplicationResult ret = client.runApplication(app.getId()); + + if (ret.isSuccess()) { + // success + } else { + throw new ApplicationException(ret.getMsg()); + } + } catch (TException e) { + intpProcess.releaseBrokenClient(client); + throw new ApplicationException(e); + } finally { + intpProcess.releaseClient(client); } - } catch (TException e) { - intpProcess.releaseBrokenClient(client); - throw new ApplicationException(e); - } finally { - intpProcess.releaseClient(client); } } } + + public void unloadAllInTheInterpreterProcess(RemoteInterpreterProcess process) { // TODO } @@ -296,15 +386,7 @@ private ApplicationState getAppState(String noteId, String paragraphId, String a return null; } - ApplicationState appFound = null; - synchronized (paragraph.apps) { - for (ApplicationState app : paragraph.apps) { - if (app.getId().equals(appId)) { - appFound = app; - break; - } - } - } + ApplicationState appFound = paragraph.getApplicationState(appId); return appFound; } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java index 5f93bb6c4c0..0ff75fe62d0 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java @@ -16,38 +16,45 @@ */ package org.apache.zeppelin.notebook; -import org.apache.zeppelin.helium.Application; -import org.apache.zeppelin.helium.HeliumPackage; -import org.apache.zeppelin.interpreter.InterpreterGroup; -import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; - /** - * Running ApplicationState + * Current state of application */ public class ApplicationState { /** * Status of Application */ - public static enum ApplicationStatus { + public static enum Status { LOADING, LOADED, UNLOADING, - UNLOADED + UNLOADED, + ERROR }; + Status status = Status.UNLOADED; - String id; // unique id for this instance similar to note id or paragraph id + String id; // unique id for this instance. Similar to note id or paragraph id String name; // name of app - ApplicationStatus status; String output; public ApplicationState(String id, String name) { this.id = id; this.name = name; - status = ApplicationStatus.UNLOADED; } + /** + * After ApplicationState is restored from NotebookRepo, + * such as after Zeppelin daemon starts or Notebook import, + * Application status need to be reset. + */ + public void resetStatus() { + if (status != Status.ERROR) { + status = Status.UNLOADED; + } + } + + @Override public boolean equals(Object o) { String compareName; @@ -71,11 +78,11 @@ public String getId() { return id; } - public void setStatus(ApplicationStatus status) { + public void setStatus(Status status) { this.status = status; } - public ApplicationStatus getStatus() { + public Status getStatus() { return status; } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java index b7f354504cc..4a162949952 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java @@ -18,6 +18,7 @@ package org.apache.zeppelin.notebook; import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.helium.HeliumPackage; import org.apache.zeppelin.user.AuthenticationInfo; import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.display.Input; @@ -53,7 +54,11 @@ public class Paragraph extends Job implements Serializable, Cloneable { Date dateUpdated; private Map config; // paragraph configs like isOpen, colWidth, etc public final GUI settings; // form and parameter settings - public final List apps = new LinkedList(); + + /** + * Applicaiton states in this paragraph + */ + private final List apps = new LinkedList(); @VisibleForTesting Paragraph() { @@ -391,4 +396,43 @@ public Object clone() throws CloneNotSupportedException { Paragraph paraClone = (Paragraph) this.clone(); return paraClone; } + + + private String getApplicationId(HeliumPackage pkg) { + return "app_" + getNote().getId() + "-" + getId() + pkg.getName(); + } + + public ApplicationState createOrGetApplicationState(HeliumPackage pkg) { + synchronized (apps) { + for (ApplicationState as : apps) { + if (as.getName().equals(pkg.getName())) { + return as; + } + } + + String appId = getApplicationId(pkg); + ApplicationState appState = new ApplicationState(appId, pkg.getName()); + apps.add(appState); + return appState; + } + } + + + public ApplicationState getApplicationState(String appId) { + synchronized (apps) { + for (ApplicationState as : apps) { + if (as.getId().equals(appId)) { + return as; + } + } + } + + return null; + } + + public List getAllApplicationStates() { + synchronized (apps) { + return new LinkedList(apps); + } + } } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java index a7c4e0a2825..77a22ab310e 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java @@ -26,6 +26,7 @@ import org.apache.zeppelin.interpreter.mock.MockInterpreter2; import org.apache.zeppelin.notebook.*; import org.apache.zeppelin.notebook.repo.VFSNotebookRepo; +import org.apache.zeppelin.scheduler.ExecutorFactory; import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.SchedulerFactory; import org.apache.zeppelin.search.SearchService; @@ -36,6 +37,8 @@ import java.io.File; import java.io.IOException; import java.util.HashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; @@ -70,7 +73,9 @@ public void setUp() throws Exception { MockInterpreter1.register("mock1", "org.apache.zeppelin.interpreter.mock.MockInterpreter1"); MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2"); - heliumAppFactory = new HeliumApplicationFactory(); + + ExecutorService executor = ExecutorFactory.singleton().createOrGet("schedulerFactory", 100); + heliumAppFactory = new HeliumApplicationFactory(executor); depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); factory = new InterpreterFactory(conf, @@ -124,15 +129,17 @@ public void testLoadRunUnloadApplication() assertEquals("repl1: job", p1.getResult().message()); // when - String appId = heliumAppFactory.load(pkg1, p1); - heliumAppFactory.run(p1, pkg1.getName()); + assertEquals(0, p1.getAllApplicationStates().size()); + String appId = heliumAppFactory.loadAndRun(pkg1, p1); + assertEquals(1, p1.getAllApplicationStates().size()); + ApplicationState app = p1.getApplicationState(appId); + Thread.sleep(1000); // wait for enough time // then - Thread.sleep(1000); - assertEquals("Hello world", p1.apps.get(0).getOutput()); + assertEquals("Hello world", app.getOutput()); // clean - heliumAppFactory.unload(p1, pkg1.getName()); + heliumAppFactory.unload(p1, appId); notebook.removeNote(note1.getId()); } From 7aeb64addf55e0712aa270e131f4af39d06f470c Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sun, 3 Apr 2016 12:14:23 +0900 Subject: [PATCH 07/69] Unload app on paragraph / note removal as well as interpreter unbind --- .../interpreter/InterpreterOutput.java | 3 +- .../zeppelin/server/ZeppelinServer.java | 3 + .../helium/HeliumApplicationFactory.java | 77 ++++++++------- .../org/apache/zeppelin/notebook/Note.java | 28 +++++- .../zeppelin/notebook/NoteEventListener.java | 9 ++ .../apache/zeppelin/notebook/Notebook.java | 61 ++++++++++-- .../notebook/NotebookEventListener.java | 13 +++ .../helium/HeliumApplicationFactoryTest.java | 94 ++++++++++++++++++- .../helium/HeliumTestApplication.java | 5 +- .../zeppelin/notebook/NotebookTest.java | 57 +++++++++++ .../zeppelin/search/LuceneSearchTest.java | 2 +- 11 files changed, 304 insertions(+), 48 deletions(-) create mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteEventListener.java create mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookEventListener.java diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutput.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutput.java index 42ebe485e1b..b28386c7726 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutput.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutput.java @@ -61,7 +61,6 @@ public InterpreterResult.Type getType() { public void setType(InterpreterResult.Type type) { if (this.type != type) { clear(); - flushListener.onUpdate(this, new byte[]{}); this.type = type; } } @@ -74,6 +73,8 @@ public void clear() { if (watcher != null) { watcher.clear(); } + + flushListener.onUpdate(this, new byte[]{}); } } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index 70f44006031..c5b06f09bb7 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -22,6 +22,7 @@ import java.util.EnumSet; import java.util.HashSet; import java.util.Set; +import java.util.concurrent.ExecutorService; import javax.net.ssl.SSLContext; import javax.servlet.DispatcherType; @@ -101,6 +102,8 @@ public ZeppelinServer() throws Exception { heliumApplicationFactory.setNotebook(notebook); // to update fire websocket event on application event. heliumApplicationFactory.setApplicationEventListener(notebookWsServer); + + notebook.addNotebookEventListener(heliumApplicationFactory); } public static void main(String[] args) throws InterruptedException { diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java index 8c9a4a304dd..08704c5c9b8 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java @@ -20,48 +20,40 @@ import org.apache.thrift.TException; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; import org.apache.zeppelin.interpreter.thrift.RemoteApplicationResult; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService; -import org.apache.zeppelin.notebook.ApplicationState; -import org.apache.zeppelin.notebook.Note; -import org.apache.zeppelin.notebook.Notebook; -import org.apache.zeppelin.notebook.Paragraph; +import org.apache.zeppelin.notebook.*; +import org.apache.zeppelin.scheduler.ExecutorFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import sun.reflect.annotation.ExceptionProxy; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; +import java.util.List; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; /** * HeliumApplicationFactory * - * 1. sync api -> async api - * 2. unload app when paragraph / note / interpreter remove * 3. front-end job * 4. example app * 5. dev mode * 6. app launcher * 7. offline mode. front-end table data / pivot panel access */ -public class HeliumApplicationFactory implements ApplicationEventListener { +public class HeliumApplicationFactory implements ApplicationEventListener, NotebookEventListener { private final Logger logger = LoggerFactory.getLogger(HeliumApplicationFactory.class); private final ExecutorService executor; - private final Gson gson; + private final Gson gson = new Gson(); private Notebook notebook; private ApplicationEventListener applicationEventListener; - public HeliumApplicationFactory(ExecutorService executor) { - gson = new Gson(); - this.executor = executor; + public HeliumApplicationFactory() { + executor = ExecutorFactory.singleton().createOrGet( + HeliumApplicationFactory.class.getName(), 10); } - - private boolean isRemote(InterpreterGroup group) { return group.getAngularObjectRegistry() instanceof RemoteAngularObjectRegistry; } @@ -325,21 +317,6 @@ private void run(ApplicationState app) throws ApplicationException { } } - - - public void unloadAllInTheInterpreterProcess(RemoteInterpreterProcess process) { - // TODO - } - - public void unloadAllInTheNote(Note note) { - // TODO - } - - public void unloadAllInTheParagraph(Paragraph paragraph) { - // TODO - } - - @Override public void onOutputAppend(String noteId, String paragraphId, String appId, String output) { ApplicationState appToUpdate = getAppState(noteId, paragraphId, appId); @@ -406,4 +383,40 @@ public ApplicationEventListener getApplicationEventListener() { public void setApplicationEventListener(ApplicationEventListener applicationEventListener) { this.applicationEventListener = applicationEventListener; } + + @Override + public void onNoteRemove(Note note) { + } + + @Override + public void onNoteCreate(Note note) { + + } + + @Override + public void onUnbindInterpreter(Note note, InterpreterSetting setting) { + for (Paragraph p : note.getParagraphs()) { + Interpreter currentInterpreter = p.getCurrentRepl(); + List infos = setting.getInterpreterInfos(); + for (InterpreterSetting.InterpreterInfo info : infos) { + if (info.getClassName().equals(currentInterpreter.getClassName())) { + onParagraphRemove(p); + break; + } + } + } + } + + @Override + public void onParagraphRemove(Paragraph paragraph) { + List appStates = paragraph.getAllApplicationStates(); + for (ApplicationState app : appStates) { + unload(paragraph, app.getId()); + } + } + + @Override + public void onParagraphCreate(Paragraph p) { + + } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java index 6a097355386..2e97d26b2d5 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java @@ -69,6 +69,7 @@ public class Note implements Serializable, JobListener { private transient NotebookRepo repo; private transient SearchService index; private transient ScheduledFuture delayedPersist; + private transient NoteEventListener noteEventListener; /** * note configurations. @@ -88,11 +89,12 @@ public class Note implements Serializable, JobListener { public Note() {} public Note(NotebookRepo repo, NoteInterpreterLoader replLoader, - JobListenerFactory jlFactory, SearchService noteIndex) { + JobListenerFactory jlFactory, SearchService noteIndex, NoteEventListener noteEventListener) { this.repo = repo; this.replLoader = replLoader; this.jobListenerFactory = jlFactory; this.index = noteIndex; + this.noteEventListener = noteEventListener; generateId(); } @@ -158,6 +160,9 @@ public Paragraph addParagraph() { synchronized (paragraphs) { paragraphs.add(p); } + if (noteEventListener != null) { + noteEventListener.onParagraphCreate(p); + } return p; } @@ -187,6 +192,9 @@ public void addCloneParagraph(Paragraph srcParagraph) { synchronized (paragraphs) { paragraphs.add(newParagraph); } + if (noteEventListener != null) { + noteEventListener.onParagraphCreate(newParagraph); + } } /** @@ -199,6 +207,9 @@ public Paragraph insertParagraph(int index) { synchronized (paragraphs) { paragraphs.add(index, p); } + if (noteEventListener != null) { + noteEventListener.onParagraphCreate(p); + } return p; } @@ -218,12 +229,14 @@ public Paragraph removeParagraph(String paragraphId) { if (p.getId().equals(paragraphId)) { index.deleteIndexDoc(this, p); i.remove(); + + if (noteEventListener != null) { + noteEventListener.onParagraphRemove(p); + } return p; } } } - - return null; } @@ -513,4 +526,13 @@ public void afterStatusChange(Job job, Status before, Status after) { @Override public void onProgressUpdate(Job job, int progress) {} + + + public NoteEventListener getNoteEventListener() { + return noteEventListener; + } + + public void setNoteEventListener(NoteEventListener noteEventListener) { + this.noteEventListener = noteEventListener; + } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteEventListener.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteEventListener.java new file mode 100644 index 00000000000..4b55eb18a19 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteEventListener.java @@ -0,0 +1,9 @@ +package org.apache.zeppelin.notebook; + +/** + * NoteEventListener + */ +public interface NoteEventListener { + public void onParagraphRemove(Paragraph p); + public void onParagraphCreate(Paragraph p); +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java index 606973450c1..843d10520db 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java @@ -34,7 +34,6 @@ import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; -import org.apache.zeppelin.helium.ApplicationEventListener; import org.apache.zeppelin.interpreter.InterpreterFactory; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterSetting; @@ -63,7 +62,7 @@ /** * Collection of Notes. */ -public class Notebook { +public class Notebook implements NoteEventListener { static Logger logger = LoggerFactory.getLogger(Notebook.class); @SuppressWarnings("unused") @Deprecated //TODO(bzz): remove unused @@ -79,6 +78,8 @@ public class Notebook { private NotebookRepo notebookRepo; private SearchService notebookIndex; private NotebookAuthorization notebookAuthorization; + private final List notebookEventListeners = + Collections.synchronizedList(new LinkedList()); /** * Main constructor \w manual Dependency Injection @@ -147,7 +148,7 @@ public Note createNote() throws IOException { */ public Note createNote(List interpreterIds) throws IOException { NoteInterpreterLoader intpLoader = new NoteInterpreterLoader(replFactory); - Note note = new Note(notebookRepo, intpLoader, jobListenerFactory, notebookIndex); + Note note = new Note(notebookRepo, intpLoader, jobListenerFactory, notebookIndex, this); intpLoader.setNoteId(note.id()); synchronized (notes) { notes.put(note.id(), note); @@ -158,6 +159,7 @@ public Note createNote(List interpreterIds) throws IOException { notebookIndex.addIndexDoc(note); note.persist(); + fireNoteCreateEvent(note); return note; } @@ -209,7 +211,7 @@ public Note importNote(String sourceJson, String noteName) throws IOException { logger.error(e.toString(), e); throw e; } - + return newNote; } @@ -246,10 +248,17 @@ public Note cloneNote(String sourceNoteID, String newNoteName) throws } public void bindInterpretersToNote(String id, - List interpreterSettingIds) throws IOException { + List newBindings) throws IOException { Note note = getNote(id); if (note != null) { - note.getNoteReplLoader().setInterpreters(interpreterSettingIds); + List currentBindings = note.getNoteReplLoader().getInterpreterSettings(); + for (InterpreterSetting setting : currentBindings) { + if (!newBindings.contains(setting.id())) { + fireUnbindInterpreter(note, setting); + } + } + + note.getNoteReplLoader().setInterpreters(newBindings); // comment out while note.getNoteReplLoader().setInterpreters(...) do the same // replFactory.putNoteInterpreterSettingBinding(id, interpreterSettingIds); } @@ -311,6 +320,8 @@ public void removeNote(String id) { ResourcePoolUtils.removeResourcesBelongsToNote(id); + fireNoteRemoveEvent(note); + try { note.unpersist(); } catch (IOException e) { @@ -372,6 +383,8 @@ private Note loadNoteFromRepo(String id) { } } + note.setNoteEventListener(this); + synchronized (notes) { notes.put(note.id(), note); refreshCron(note.id()); @@ -395,6 +408,7 @@ private Note loadNoteFromRepo(String id) { } } } + return note; } @@ -597,4 +611,39 @@ public void close() { this.notebookIndex.close(); } + public void addNotebookEventListener(NotebookEventListener listener) { + notebookEventListeners.add(listener); + } + + private void fireNoteCreateEvent(Note note) { + for (NotebookEventListener listener : notebookEventListeners) { + listener.onNoteCreate(note); + } + } + + private void fireNoteRemoveEvent(Note note) { + for (NotebookEventListener listener : notebookEventListeners) { + listener.onNoteRemove(note); + } + } + + private void fireUnbindInterpreter(Note note, InterpreterSetting setting) { + for (NotebookEventListener listener : notebookEventListeners) { + listener.onUnbindInterpreter(note, setting); + } + } + + @Override + public void onParagraphRemove(Paragraph p) { + for (NotebookEventListener listener : notebookEventListeners) { + listener.onParagraphRemove(p); + } + } + + @Override + public void onParagraphCreate(Paragraph p) { + for (NotebookEventListener listener : notebookEventListeners) { + listener.onParagraphCreate(p); + } + } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookEventListener.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookEventListener.java new file mode 100644 index 00000000000..d1c3020e0b2 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookEventListener.java @@ -0,0 +1,13 @@ +package org.apache.zeppelin.notebook; + +import org.apache.zeppelin.interpreter.InterpreterSetting; + +/** + * Notebook event + */ +public interface NotebookEventListener extends NoteEventListener { + public void onNoteRemove(Note note); + public void onNoteCreate(Note note); + + public void onUnbindInterpreter(Note note, InterpreterSetting setting); +} diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java index 77a22ab310e..9ac1ce9f927 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java @@ -37,6 +37,7 @@ import java.io.File; import java.io.IOException; import java.util.HashMap; +import java.util.LinkedList; import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicInteger; @@ -74,8 +75,7 @@ public void setUp() throws Exception { MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2"); - ExecutorService executor = ExecutorFactory.singleton().createOrGet("schedulerFactory", 100); - heliumAppFactory = new HeliumApplicationFactory(executor); + heliumAppFactory = new HeliumApplicationFactory(); depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); factory = new InterpreterFactory(conf, @@ -97,6 +97,8 @@ public void setUp() throws Exception { notebookAuthorization); heliumAppFactory.setNotebook(notebook); + + notebook.addNotebookEventListener(heliumAppFactory); } @After @@ -133,16 +135,100 @@ public void testLoadRunUnloadApplication() String appId = heliumAppFactory.loadAndRun(pkg1, p1); assertEquals(1, p1.getAllApplicationStates().size()); ApplicationState app = p1.getApplicationState(appId); - Thread.sleep(1000); // wait for enough time + Thread.sleep(500); // wait for enough time + + // then + assertEquals("Hello world 1", app.getOutput()); + + // when + heliumAppFactory.run(p1, appId); + Thread.sleep(500); // wait for enough time // then - assertEquals("Hello world", app.getOutput()); + assertEquals("Hello world 2", app.getOutput()); // clean heliumAppFactory.unload(p1, appId); notebook.removeNote(note1.getId()); } + @Test + public void testUnloadOnParagraphRemove() throws IOException { + // given + HeliumPackage pkg1 = new HeliumPackage(HeliumPackage.Type.APPLICATION, + "name1", + "desc1", + "", + HeliumTestApplication.class.getName(), + new String[][]{}); + + Note note1 = notebook.createNote(); + note1.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); + + Paragraph p1 = note1.addParagraph(); + + // make sure interpreter process running + p1.setText("job"); + note1.run(p1.getId()); + while(p1.isTerminated()==false || p1.getResult()==null) Thread.yield(); + + assertEquals(0, p1.getAllApplicationStates().size()); + String appId = heliumAppFactory.loadAndRun(pkg1, p1); + ApplicationState app = p1.getApplicationState(appId); + while (app.getStatus() != ApplicationState.Status.LOADED) { + Thread.yield(); + } + + // when remove paragraph + note1.removeParagraph(p1.getId()); + while (app.getStatus() != ApplicationState.Status.UNLOADED) { + Thread.yield(); + } + + // then + assertEquals(ApplicationState.Status.UNLOADED, app.getStatus()); + + // clean + notebook.removeNote(note1.getId()); + } + + + @Test + public void testUnloadOnInterpreterUnbind() throws IOException { + // given + HeliumPackage pkg1 = new HeliumPackage(HeliumPackage.Type.APPLICATION, + "name1", + "desc1", + "", + HeliumTestApplication.class.getName(), + new String[][]{}); + + Note note1 = notebook.createNote(); + note1.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); + + Paragraph p1 = note1.addParagraph(); + + // make sure interpreter process running + p1.setText("job"); + note1.run(p1.getId()); + while(p1.isTerminated()==false || p1.getResult()==null) Thread.yield(); + + assertEquals(0, p1.getAllApplicationStates().size()); + String appId = heliumAppFactory.loadAndRun(pkg1, p1); + ApplicationState app = p1.getApplicationState(appId); + while (app.getStatus() != ApplicationState.Status.LOADED) { + Thread.yield(); + } + + // when unbind interpreter + note1.getNoteReplLoader().setInterpreters(new LinkedList()); + + // then + assertEquals(ApplicationState.Status.UNLOADED, app.getStatus()); + + // clean + notebook.removeNote(note1.getId()); + } @Override public ParagraphJobListener getParagraphJobListener(Note note) { diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTestApplication.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTestApplication.java index c28be012a07..e41c85aa385 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTestApplication.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTestApplication.java @@ -19,8 +19,10 @@ import org.apache.zeppelin.resource.ResourceSet; import java.io.IOException; +import java.util.concurrent.atomic.AtomicInteger; public class HeliumTestApplication extends Application { + AtomicInteger numRun = new AtomicInteger(0); public HeliumTestApplication(ResourceSet args, ApplicationContext context) throws ApplicationException { super(args, context); @@ -29,7 +31,8 @@ public HeliumTestApplication(ResourceSet args, ApplicationContext context) @Override public void run() throws ApplicationException { try { - context().out.write("Hello world"); + context().out.clear(); + context().out.write("Hello world " + numRun.incrementAndGet()); context().out.flush(); } catch (IOException e) { throw new ApplicationException(e); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index 92471879419..c3497a820f4 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -27,6 +27,7 @@ import java.io.File; import java.io.IOException; import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.io.FileUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; @@ -625,6 +626,62 @@ public void testPerSessionInterpreterCloseOnUnbindInterpreterSetting() throws IO notebook.removeNote(note1.getId()); } + @Test + public void testNotebookEventListener() throws IOException { + final AtomicInteger onNoteRemove = new AtomicInteger(0); + final AtomicInteger onNoteCreate = new AtomicInteger(0); + final AtomicInteger onParagraphRemove = new AtomicInteger(0); + final AtomicInteger onParagraphCreate = new AtomicInteger(0); + final AtomicInteger unbindInterpreter = new AtomicInteger(0); + + notebook.addNotebookEventListener(new NotebookEventListener() { + @Override + public void onNoteRemove(Note note) { + onNoteRemove.incrementAndGet(); + } + + @Override + public void onNoteCreate(Note note) { + onNoteCreate.incrementAndGet(); + } + + @Override + public void onUnbindInterpreter(Note note, InterpreterSetting setting) { + unbindInterpreter.incrementAndGet(); + } + + @Override + public void onParagraphRemove(Paragraph p) { + onParagraphRemove.incrementAndGet(); + } + + @Override + public void onParagraphCreate(Paragraph p) { + onParagraphCreate.incrementAndGet(); + } + }); + + Note note1 = notebook.createNote(); + assertEquals(1, onNoteCreate.get()); + + Paragraph p1 = note1.addParagraph(); + assertEquals(1, onParagraphCreate.get()); + + note1.addCloneParagraph(p1); + assertEquals(2, onParagraphCreate.get()); + + note1.removeParagraph(p1.getId()); + assertEquals(1, onParagraphRemove.get()); + + List settings = notebook.getBindedInterpreterSettingsIds(note1.id()); + notebook.bindInterpretersToNote(note1.id(), new LinkedList()); + assertEquals(settings.size(), unbindInterpreter.get()); + + notebook.removeNote(note1.getId()); + assertEquals(1, onNoteRemove.get()); + assertEquals(1, onParagraphRemove.get()); + } + private void delete(File file){ if(file.isFile()) file.delete(); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java index f74d95eb375..5de1f8ada98 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java @@ -251,7 +251,7 @@ private Paragraph addParagraphWithText(Note note, String text) { } private Note newNote(String name) { - Note note = new Note(notebookRepoMock, replLoaderMock, null, notebookIndex); + Note note = new Note(notebookRepoMock, replLoaderMock, null, notebookIndex, null); note.setName(name); return note; } From 134bbe640ae2636bb847d9844b28f87409027d08 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Wed, 6 Apr 2016 10:32:47 +0900 Subject: [PATCH 08/69] Change HeliumRegistry constructor argument type --- .../apache/zeppelin/helium/HeliumLocalRegistry.java | 2 +- .../org/apache/zeppelin/helium/HeliumRegistry.java | 6 +++--- .../zeppelin/helium/HeliumRegistrySerializer.java | 11 +++-------- .../zeppelin/helium/HeliumLocalRegistryTest.java | 2 +- .../java/org/apache/zeppelin/helium/HeliumTest.java | 10 +++++----- .../apache/zeppelin/helium/HeliumTestRegistry.java | 2 +- 6 files changed, 14 insertions(+), 19 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumLocalRegistry.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumLocalRegistry.java index 50d35175b44..7e79d85896a 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumLocalRegistry.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumLocalRegistry.java @@ -35,7 +35,7 @@ public class HeliumLocalRegistry extends HeliumRegistry { private final Gson gson; - public HeliumLocalRegistry(String name, URI uri) { + public HeliumLocalRegistry(String name, String uri) { super(name, uri); gson = new Gson(); } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumRegistry.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumRegistry.java index 48626e0adce..125ad9298c2 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumRegistry.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumRegistry.java @@ -25,16 +25,16 @@ */ public abstract class HeliumRegistry { private final String name; - private final URI uri; + private final String uri; - public HeliumRegistry(String name, URI uri) { + public HeliumRegistry(String name, String uri) { this.name = name; this.uri = uri; } public String name() { return name; } - public URI uri() { + public String uri() { return uri; } public abstract List getAll() throws IOException; diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumRegistrySerializer.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumRegistrySerializer.java index 3940c9d32e7..3abcb9ffe1a 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumRegistrySerializer.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumRegistrySerializer.java @@ -40,19 +40,14 @@ public HeliumRegistry deserialize(JsonElement json, throws JsonParseException { JsonObject jsonObject = json.getAsJsonObject(); String className = jsonObject.get("class").getAsString(); - URI uri = null; - try { - uri = new URI(jsonObject.get("uri").getAsString()); - } catch (URISyntaxException e) { - new JsonParseException(e); - } + String uri = jsonObject.get("uri").getAsString(); String name = jsonObject.get("name").getAsString(); try { logger.info("Restore helium registry {} {} {}", name, className, uri); Class cls = (Class) getClass().getClassLoader().loadClass(className); - Constructor constructor = cls.getConstructor(String.class, URI.class); + Constructor constructor = cls.getConstructor(String.class, String.class); HeliumRegistry registry = constructor.newInstance(name, uri); return registry; } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | @@ -68,7 +63,7 @@ public JsonElement serialize(HeliumRegistry heliumRegistry, JsonSerializationContext jsonSerializationContext) { JsonObject json = new JsonObject(); json.addProperty("class", heliumRegistry.getClass().getName()); - json.addProperty("uri", heliumRegistry.uri().toString()); + json.addProperty("uri", heliumRegistry.uri()); json.addProperty("name", heliumRegistry.name()); return json; } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumLocalRegistryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumLocalRegistryTest.java index 78ecec6d7cf..e90cbc30a73 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumLocalRegistryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumLocalRegistryTest.java @@ -45,7 +45,7 @@ public void tearDown() throws IOException { public void testGetAllPackage() throws IOException { // given File r1Path = new File(tmpDir, "r1"); - HeliumLocalRegistry r1 = new HeliumLocalRegistry("r1", r1Path.toURI()); + HeliumLocalRegistry r1 = new HeliumLocalRegistry("r1", r1Path.getAbsolutePath()); assertEquals(0, r1.getAll().size()); // when diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java index fac93b0de4e..084729adffa 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java @@ -50,9 +50,9 @@ public void testSaveLoadConf() throws IOException, URISyntaxException { File heliumConf = new File(tmpDir, "helium.conf"); Helium helium = new Helium(heliumConf.getAbsolutePath()); assertFalse(heliumConf.exists()); - HeliumTestRegistry registry1 = new HeliumTestRegistry("r1", new URI("file:///r1")); + HeliumTestRegistry registry1 = new HeliumTestRegistry("r1", "r1"); helium.addRegistry(registry1); - assertEquals(1, helium.getAllRegistry().size()); + assertEquals(2, helium.getAllRegistry().size()); assertEquals(0, helium.getAllPackageInfo().size()); // when @@ -63,15 +63,15 @@ public void testSaveLoadConf() throws IOException, URISyntaxException { // then Helium heliumRestored = new Helium(heliumConf.getAbsolutePath()); - assertEquals(1, heliumRestored.getAllRegistry().size()); + assertEquals(2, heliumRestored.getAllRegistry().size()); } @Test public void testRestoreRegistryInstances() throws IOException, URISyntaxException { File heliumConf = new File(tmpDir, "helium.conf"); Helium helium = new Helium(heliumConf.getAbsolutePath()); - HeliumTestRegistry registry1 = new HeliumTestRegistry("r1", new URI("file:///r1")); - HeliumTestRegistry registry2 = new HeliumTestRegistry("r2", new URI("file:///r2")); + HeliumTestRegistry registry1 = new HeliumTestRegistry("r1", "r1"); + HeliumTestRegistry registry2 = new HeliumTestRegistry("r2", "r2"); helium.addRegistry(registry1); helium.addRegistry(registry2); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTestRegistry.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTestRegistry.java index 494d0367fd0..d99f73f2817 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTestRegistry.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTestRegistry.java @@ -24,7 +24,7 @@ public class HeliumTestRegistry extends HeliumRegistry { List infos = new LinkedList(); - public HeliumTestRegistry(String name, URI uri) { + public HeliumTestRegistry(String name, String uri) { super(name, uri); } From 94b490d76c7875a8d2f1a30232a762ec374664b4 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Wed, 6 Apr 2016 10:35:37 +0900 Subject: [PATCH 09/69] initial rest api impl --- .../zeppelin/helium/ApplicationLoader.java | 21 +++- .../interpreter/remote/RemoteInterpreter.java | 1 + .../apache/zeppelin/rest/HeliumRestApi.java | 105 ++++++++++++++++++ .../zeppelin/server/ZeppelinServer.java | 7 +- .../org/apache/zeppelin/helium/Helium.java | 51 ++++++++- .../helium/HeliumApplicationFactory.java | 26 ++++- .../helium/HeliumPackageSuggestion.java | 53 +++++++++ .../interpreter/InterpreterFactory.java | 2 +- .../apache/zeppelin/notebook/Paragraph.java | 3 + .../helium/HeliumApplicationFactoryTest.java | 7 +- 10 files changed, 257 insertions(+), 19 deletions(-) create mode 100644 zeppelin-server/src/main/java/org/apache/zeppelin/rest/HeliumRestApi.java create mode 100644 zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumPackageSuggestion.java diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java index 0a37babdba8..40065373c2f 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java @@ -136,8 +136,7 @@ public Application load(HeliumPackage packageInfo, ApplicationContext context) } private ResourceSet findRequiredResourceSet( - String [][] requiredResources, String noteId, String paragraphId) - throws ApplicationException { + String [][] requiredResources, String noteId, String paragraphId) { if (requiredResources == null || requiredResources.length == 0) { return new ResourceSet(); } @@ -151,7 +150,19 @@ private ResourceSet findRequiredResourceSet( allResources = resourcePool.getAll(); } - allResources = allResources.filterByNoteId(noteId).filterByParagraphId(paragraphId); + return findRequiredResourceSet(requiredResources, noteId, paragraphId, allResources); + } + + static ResourceSet findRequiredResourceSet(String [][] requiredResources, + String noteId, + String paragraphId, + ResourceSet resources) { + ResourceSet args = new ResourceSet(); + if (requiredResources == null || requiredResources.length == 0) { + return args; + } + + resources = resources.filterByNoteId(noteId).filterByParagraphId(paragraphId); for (String [] requires : requiredResources) { args.clear(); @@ -159,7 +170,7 @@ private ResourceSet findRequiredResourceSet( for (String require : requires) { boolean found = false; - for (Resource r : allResources) { + for (Resource r : resources) { if (r.getClassName().equals(require)) { args.add(r); found = true; @@ -177,7 +188,7 @@ private ResourceSet findRequiredResourceSet( } } - throw new ApplicationException("Can not find available resources"); + return null; } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java index 3314111a103..9df3db014b9 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java @@ -81,6 +81,7 @@ public RemoteInterpreter(Properties property, this.applicationEventListener = appListener; } + // VisibleForTesting public RemoteInterpreter(Properties property, String noteId, String className, diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/HeliumRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/HeliumRestApi.java new file mode 100644 index 00000000000..b53c7e2ccbf --- /dev/null +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/HeliumRestApi.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.rest; + +import com.google.gson.Gson; +import org.apache.zeppelin.helium.Helium; +import org.apache.zeppelin.helium.HeliumApplicationFactory; +import org.apache.zeppelin.helium.HeliumPackage; +import org.apache.zeppelin.notebook.Note; +import org.apache.zeppelin.notebook.Notebook; +import org.apache.zeppelin.notebook.Paragraph; +import org.apache.zeppelin.server.JsonResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.*; +import javax.ws.rs.core.Response; + +/** + * Helium Rest Api + */ +@Path("/helium") +@Produces("application/json") +public class HeliumRestApi { + Logger logger = LoggerFactory.getLogger(HeliumRestApi.class); + + private Helium helium; + private HeliumApplicationFactory applicationFactory; + private Notebook notebook; + private Gson gson = new Gson(); + + public HeliumRestApi(Helium helium, + HeliumApplicationFactory heliumApplicationFactory, + Notebook notebook) { + this.helium = helium; + this.applicationFactory = heliumApplicationFactory; + this.notebook = notebook; + } + + /** + * Get all packages + * @return + */ + @GET + @Path("all") + public Response getAll() { + return new JsonResponse(Response.Status.OK, "", helium.getAllPackageInfo()).build(); + } + + @GET + @Path("suggest/{noteId}/{paragraphId}") + public Response suggest(@PathParam("noteId") String noteId, + @PathParam("paragraphId") String paragraphId) { + Note note = notebook.getNote(noteId); + if (note == null) { + return new JsonResponse(Response.Status.NOT_FOUND, "Note " + noteId + " not found").build(); + } + + Paragraph paragraph = note.getParagraph(paragraphId); + if (paragraph == null) { + return new JsonResponse(Response.Status.NOT_FOUND, "Paragraph " + paragraphId + " not found") + .build(); + } + + return new JsonResponse(Response.Status.OK, "", helium.suggestApp(paragraph)).build(); + } + + @POST + @Path("load/{noteId}/{paragraphId}") + public Response suggest(@PathParam("noteId") String noteId, + @PathParam("paragraphId") String paragraphId, + String heliumPackage) { + + Note note = notebook.getNote(noteId); + if (note == null) { + return new JsonResponse(Response.Status.NOT_FOUND, "Note " + noteId + " not found").build(); + } + + Paragraph paragraph = note.getParagraph(paragraphId); + if (paragraph == null) { + return new JsonResponse(Response.Status.NOT_FOUND, "Paragraph " + paragraphId + " not found") + .build(); + } + HeliumPackage pkg = gson.fromJson(heliumPackage, HeliumPackage.class); + + String appId = applicationFactory.loadAndRun(pkg, paragraph); + return new JsonResponse(Response.Status.OK, "", appId).build(); + } + +} diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index c5b06f09bb7..717877d65b9 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -70,6 +70,8 @@ public class ZeppelinServer extends Application { public static Notebook notebook; public static Server jettyWebServer; public static NotebookServer notebookWsServer; + public static Helium helium; + public static HeliumApplicationFactory heliumApplicationFactory; private SchedulerFactory schedulerFactory; private InterpreterFactory replFactory; @@ -77,8 +79,6 @@ public class ZeppelinServer extends Application { private SearchService notebookIndex; private NotebookAuthorization notebookAuthorization; private DependencyResolver depResolver; - private Helium helium; - private HeliumApplicationFactory heliumApplicationFactory; public ZeppelinServer() throws Exception { ZeppelinConfiguration conf = ZeppelinConfiguration.create(); @@ -298,6 +298,9 @@ public Set getSingletons() { NotebookRestApi notebookApi = new NotebookRestApi(notebook, notebookWsServer, notebookIndex); singletons.add(notebookApi); + HeliumRestApi heliumApi = new HeliumRestApi(helium, heliumApplicationFactory, notebook); + singletons.add(heliumApi); + InterpreterRestApi interpreterApi = new InterpreterRestApi(replFactory); singletons.add(interpreterApi); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java index b01ea671f7d..8324d16e23f 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java @@ -19,11 +19,19 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import org.apache.commons.io.FileUtils; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.notebook.Paragraph; +import org.apache.zeppelin.resource.DistributedResourcePool; +import org.apache.zeppelin.resource.ResourcePool; +import org.apache.zeppelin.resource.ResourcePoolUtils; +import org.apache.zeppelin.resource.ResourceSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.util.HashMap; import java.util.LinkedList; import java.util.List; @@ -76,7 +84,12 @@ private synchronized HeliumConf loadConf(String path) throws IOException { File heliumConfFile = new File(path); if (!heliumConfFile.isFile()) { logger.warn("{} does not exists", path); - return new HeliumConf(); + HeliumConf conf = new HeliumConf(); + LinkedList defaultRegistry = new LinkedList(); + defaultRegistry.add(new HeliumLocalRegistry("local", "../helium")); + conf.setRegistry(defaultRegistry); + this.registry = conf.getRegistry(); + return conf; } else { String jsonString = FileUtils.readFileToString(heliumConfFile); HeliumConf conf = gson.fromJson(jsonString, HeliumConf.class); @@ -115,4 +128,40 @@ public List getAllPackageInfo() { } return list; } + + public HeliumPackageSuggestion suggestApp(Paragraph paragraph) { + HeliumPackageSuggestion suggestion = new HeliumPackageSuggestion(); + + Interpreter intp = paragraph.getCurrentRepl(); + if (intp == null) { + return suggestion; + } + + ResourcePool resourcePool = intp.getInterpreterGroup().getResourcePool(); + ResourceSet allResources; + + if (resourcePool == null) { + allResources = new ResourceSet(); + } else if (resourcePool instanceof DistributedResourcePool) { + allResources = ((DistributedResourcePool) resourcePool).getAll(false); + } else { + allResources = resourcePool.getAll(); + } + + for (HeliumPackageSearchResult pkg : getAllPackageInfo()) { + ResourceSet resources = ApplicationLoader.findRequiredResourceSet( + pkg.getPkg().getResources(), + paragraph.getNote().getId(), + paragraph.getId(), + allResources); + if (resources == null) { + continue; + } else { + suggestion.addAvailablePackage(pkg); + } + } + + suggestion.sort(); + return suggestion; + } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java index 08704c5c9b8..c7d8f86b5fd 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java @@ -36,6 +36,7 @@ /** * HeliumApplicationFactory * + * 2. unload on interpreter restart * 3. front-end job * 4. example app * 5. dev mode @@ -211,8 +212,15 @@ private void unload(ApplicationState appsToUnload) throws ApplicationException { "Can't unload application status " + appsToUnload.getStatus()); } appsToUnload.setStatus(ApplicationState.Status.UNLOADING); - RemoteInterpreterProcess intpProcess = - paragraph.getCurrentRepl().getInterpreterGroup().getRemoteInterpreterProcess(); + Interpreter intp = paragraph.getCurrentRepl(); + if (intp == null) { + throw new ApplicationException("No interpreter found"); + } + + RemoteInterpreterProcess intpProcess = intp.getInterpreterGroup().getRemoteInterpreterProcess(); + if (intpProcess == null) { + throw new ApplicationException("Target interpreter process is not running"); + } RemoteInterpreterService.Client client; try { @@ -289,9 +297,16 @@ private void run(ApplicationState app) throws ApplicationException { throw new ApplicationException( "Can't run application status " + app.getStatus()); } - RemoteInterpreterProcess intpProcess = - paragraph.getCurrentRepl().getInterpreterGroup().getRemoteInterpreterProcess(); + Interpreter intp = paragraph.getCurrentRepl(); + if (intp == null) { + throw new ApplicationException("No interpreter found"); + } + + RemoteInterpreterProcess intpProcess = intp.getInterpreterGroup().getRemoteInterpreterProcess(); + if (intpProcess == null) { + throw new ApplicationException("Target interpreter process is not running"); + } RemoteInterpreterService.Client client; try { client = intpProcess.getClient(); @@ -411,7 +426,8 @@ public void onUnbindInterpreter(Note note, InterpreterSetting setting) { public void onParagraphRemove(Paragraph paragraph) { List appStates = paragraph.getAllApplicationStates(); for (ApplicationState app : appStates) { - unload(paragraph, app.getId()); + UnloadApplication unloadJob = new UnloadApplication(paragraph, app.getId()); + unloadJob.run(); } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumPackageSuggestion.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumPackageSuggestion.java new file mode 100644 index 00000000000..45c16403bd4 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumPackageSuggestion.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedList; +import java.util.List; + +/** + * Suggested apps + */ +public class HeliumPackageSuggestion { + private final List available = + new LinkedList(); + + /* + * possible future improvement + * provides n - 'favorite' list, based on occurrence of apps in notebook + */ + + public HeliumPackageSuggestion() { + + } + + public void addAvailablePackage(HeliumPackageSearchResult r) { + available.add(r); + + } + + public void sort() { + Collections.sort(available, new Comparator() { + @Override + public int compare(HeliumPackageSearchResult o1, HeliumPackageSearchResult o2) { + return o1.getPkg().getName().compareTo(o2.getPkg().getName()); + } + }); + } +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index 08ddba1ec58..024dd9e3f0c 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -78,7 +78,7 @@ public class InterpreterFactory { private DependencyResolver depResolver; - private Map env; + private Map env = new HashMap(); public InterpreterFactory(ZeppelinConfiguration conf, AngularObjectRegistryListener angularObjectRegistryListener, diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java index 4a162949952..b71f87854c6 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java @@ -175,6 +175,9 @@ public NoteInterpreterLoader getNoteReplLoader() { } public Interpreter getRepl(String name) { + if (replLoader == null) { + return null; + } return replLoader.get(name); } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java index 9ac1ce9f927..61495dc9070 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java @@ -181,9 +181,6 @@ public void testUnloadOnParagraphRemove() throws IOException { // when remove paragraph note1.removeParagraph(p1.getId()); - while (app.getStatus() != ApplicationState.Status.UNLOADED) { - Thread.yield(); - } // then assertEquals(ApplicationState.Status.UNLOADED, app.getStatus()); @@ -204,7 +201,7 @@ public void testUnloadOnInterpreterUnbind() throws IOException { new String[][]{}); Note note1 = notebook.createNote(); - note1.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); + notebook.bindInterpretersToNote(note1.id(), factory.getDefaultInterpreterSettingList()); Paragraph p1 = note1.addParagraph(); @@ -221,7 +218,7 @@ public void testUnloadOnInterpreterUnbind() throws IOException { } // when unbind interpreter - note1.getNoteReplLoader().setInterpreters(new LinkedList()); + notebook.bindInterpretersToNote(note1.id(), new LinkedList()); // then assertEquals(ApplicationState.Status.UNLOADED, app.getStatus()); From bd0f467d23aecd32adf0152b939d8ed8eea6f56f Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 9 Apr 2016 09:38:24 +0900 Subject: [PATCH 10/69] Style --- .../apache/zeppelin/helium/HeliumApplicationFactory.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java index c7d8f86b5fd..489257dd83c 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java @@ -217,7 +217,8 @@ private void unload(ApplicationState appsToUnload) throws ApplicationException { throw new ApplicationException("No interpreter found"); } - RemoteInterpreterProcess intpProcess = intp.getInterpreterGroup().getRemoteInterpreterProcess(); + RemoteInterpreterProcess intpProcess = + intp.getInterpreterGroup().getRemoteInterpreterProcess(); if (intpProcess == null) { throw new ApplicationException("Target interpreter process is not running"); } @@ -303,7 +304,8 @@ private void run(ApplicationState app) throws ApplicationException { throw new ApplicationException("No interpreter found"); } - RemoteInterpreterProcess intpProcess = intp.getInterpreterGroup().getRemoteInterpreterProcess(); + RemoteInterpreterProcess intpProcess = + intp.getInterpreterGroup().getRemoteInterpreterProcess(); if (intpProcess == null) { throw new ApplicationException("Target interpreter process is not running"); } From 6223cd44d9fb3b7190c2cf7e817a031e38b5eb6f Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 12 Apr 2016 10:10:58 +0900 Subject: [PATCH 11/69] App output display --- .../helium/ApplicationEventListener.java | 2 + .../apache/zeppelin/helium/HeliumPackage.java | 6 +- .../remote/RemoteInterpreterEventPoller.java | 4 +- .../apache/zeppelin/rest/HeliumRestApi.java | 3 + .../org/apache/zeppelin/socket/Message.java | 4 +- .../zeppelin/socket/NotebookServer.java | 21 ++ .../paragraph/paragraph-chart-selector.html | 45 +++- .../notebook/paragraph/paragraph-results.html | 9 + .../paragraph/paragraph.controller.js | 203 +++++++++++++++++- .../websocketEvents.factory.js | 8 + .../test/spec/controllers/paragraph.js | 11 +- .../org/apache/zeppelin/helium/Helium.java | 1 + .../helium/HeliumApplicationFactory.java | 83 +++++-- .../zeppelin/notebook/ApplicationState.java | 22 +- .../org/apache/zeppelin/notebook/Note.java | 41 +++- .../zeppelin/notebook/NoteEventListener.java | 3 + .../apache/zeppelin/notebook/Notebook.java | 8 + .../apache/zeppelin/notebook/Paragraph.java | 4 +- .../notebook/repo/VFSNotebookRepo.java | 10 + .../zeppelin/notebook/NotebookTest.java | 4 + 20 files changed, 441 insertions(+), 51 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationEventListener.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationEventListener.java index 49899fa3028..f89bb211265 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationEventListener.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationEventListener.java @@ -22,4 +22,6 @@ public interface ApplicationEventListener { public void onOutputAppend(String noteId, String paragraphId, String appId, String output); public void onOutputUpdated(String noteId, String paragraphId, String appId, String output); + public void onLoad(String noteId, String paragraphId, String appId, HeliumPackage pkg); + public void onStatusChange(String noteId, String paragraphId, String appId, String status); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackage.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackage.java index 98c4d42e93f..6fb971df826 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackage.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumPackage.java @@ -27,7 +27,7 @@ public class HeliumPackage { private String className; // entry point private String [][] resources; // resource classnames that requires // [[ .. and .. and .. ] or [ .. and .. and ..] ..] - + private String icon; /** * Type of package */ @@ -89,4 +89,8 @@ public String getClassName() { public String[][] getResources() { return resources; } + + public String getIcon() { + return icon; + } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java index 6dead0eb93e..aab74b60033 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java @@ -147,7 +147,7 @@ public void run() { String paragraphId = outputAppend.get("paragraphId"); String outputToAppend = outputAppend.get("data"); String appId = outputAppend.get("appId"); - + logger.info("Append " + outputToAppend + ", appId = " + appId); if (appId == null) { listener.onOutputAppend(noteId, paragraphId, outputToAppend); } else { @@ -161,7 +161,7 @@ public void run() { String paragraphId = outputAppend.get("paragraphId"); String outputToUpdate = outputAppend.get("data"); String appId = outputAppend.get("appId"); - + logger.info("Update " + outputToUpdate + ", appId = " + appId); if (appId == null) { listener.onOutputUpdated(noteId, paragraphId, outputToUpdate); } else { diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/HeliumRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/HeliumRestApi.java index b53c7e2ccbf..062f5b93978 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/HeliumRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/HeliumRestApi.java @@ -44,6 +44,9 @@ public class HeliumRestApi { private Notebook notebook; private Gson gson = new Gson(); + public HeliumRestApi() { + } + public HeliumRestApi(Helium helium, HeliumApplicationFactory heliumApplicationFactory, Notebook notebook) { diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/Message.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/Message.java index 7c71b4cdd75..1e7dc0b4aab 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/Message.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/Message.java @@ -114,7 +114,9 @@ public static enum OP { // @param checkpointName APP_APPEND_OUTPUT, // [s-c] append output - APP_UPDATE_OUTPUT // [s-c] update (replace) output + APP_UPDATE_OUTPUT, // [s-c] update (replace) output + APP_LOAD, // [s-c] on app load + APP_STATUS_CHANGE // [s-c] on app status change } public OP op; diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 7fe105de398..0f0e83bae46 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -35,6 +35,7 @@ import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.AngularObjectRegistryListener; import org.apache.zeppelin.helium.ApplicationEventListener; +import org.apache.zeppelin.helium.HeliumPackage; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; import org.apache.zeppelin.user.AuthenticationInfo; @@ -1018,6 +1019,26 @@ public void onOutputUpdated(String noteId, String paragraphId, String appId, Str broadcast(noteId, msg); } + @Override + public void onLoad(String noteId, String paragraphId, String appId, HeliumPackage pkg) { + Message msg = new Message(OP.APP_LOAD) + .put("noteId", noteId) + .put("paragraphId", paragraphId) + .put("appId", appId) + .put("pkg", pkg); + broadcast(noteId, msg); + } + + @Override + public void onStatusChange(String noteId, String paragraphId, String appId, String status) { + Message msg = new Message(OP.APP_STATUS_CHANGE) + .put("noteId", noteId) + .put("paragraphId", paragraphId) + .put("appId", appId) + .put("status", status); + broadcast(noteId, msg); + } + /** * Need description here. * diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph-chart-selector.html b/zeppelin-web/src/app/notebook/paragraph/paragraph-chart-selector.html index b7ebbedc2d0..55721c12474 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph-chart-selector.html +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph-chart-selector.html @@ -13,35 +13,76 @@ -->
+ +
- + + + + + + settings diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph-results.html b/zeppelin-web/src/app/notebook/paragraph/paragraph-results.html index 77d74519795..6c274b7a1ce 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph-results.html +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph-results.html @@ -13,6 +13,7 @@ -->
@@ -59,3 +60,11 @@ ng-bind="paragraph.errorMessage">
+ +
+
+
+
+ + diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index 097ee84fbb6..39dbc3c7f3f 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -16,7 +16,7 @@ angular.module('zeppelinWebApp') .controller('ParagraphCtrl', function($scope,$rootScope, $route, $window, $element, $routeParams, $location, - $timeout, $compile, websocketMsgSrv) { + $timeout, $compile, $http, websocketMsgSrv, baseUrlSrv) { var ANGULAR_FUNCTION_OBJECT_NAME_PREFIX = '_Z_ANGULAR_FUNC_'; $scope.paragraph = null; $scope.originalText = ''; @@ -75,10 +75,21 @@ angular.module('zeppelinWebApp') } else if ($scope.getResultType() === 'TEXT') { $scope.renderText(); } + + getApplicationStates(); + getSuggestions(); + + var activeApp = _.get($scope.paragraph.config, 'helium.activeApp'); + if (activeApp) { + var app = _.find($scope.apps, {id: activeApp}); + renderApp(app); + } }; - $scope.renderHtml = function() { - var retryRenderer = function() { + + + $scope.renderHtml = function() { + var retryRenderer = function() { if (angular.element('#p' + $scope.paragraph.id + '_html').length) { try { angular.element('#p' + $scope.paragraph.id + '_html').html($scope.paragraph.result.msg); @@ -328,6 +339,9 @@ angular.module('zeppelinWebApp') var statusChanged = (data.paragraph.status !== $scope.paragraph.status); + var oldActiveApp = _.get($scope.paragraph.config, 'helium.activeApp'); + var newActiveApp = _.get(data.paragraph.config, 'helium.activeApp'); + //console.log("updateParagraph oldData %o, newData %o. type %o -> %o, mode %o -> %o", $scope.paragraph, data, oldType, newType, oldGraphMode, newGraphMode); if ($scope.paragraph.text !== data.paragraph.text) { @@ -388,6 +402,14 @@ angular.module('zeppelinWebApp') $scope.renderText(); } + getApplicationStates(); + getSuggestions(); + + if (newActiveApp && newActiveApp !== oldActiveApp) { + var app = _.find($scope.apps, { id : newActiveApp }); + renderApp(app); + } + if (statusChanged || resultRefreshed) { // when last paragraph runs, zeppelin automatically appends new paragraph. // this broadcast will focus to the newly inserted paragraph @@ -1160,6 +1182,9 @@ angular.module('zeppelinWebApp') // graph options newConfig.graph.mode = newMode; + // see switchApp() + _.set(newConfig, 'helium.activeApp', undefined); + commitParagraph($scope.paragraph.title, $scope.paragraph.text, newConfig, newParams); }; @@ -1417,7 +1442,8 @@ angular.module('zeppelinWebApp') }; $scope.isGraphMode = function(graphName) { - if ($scope.getResultType() === 'TABLE' && $scope.getGraphMode()===graphName) { + var activeAppId = _.get($scope.paragraph.config, 'helium.activeApp'); + if ($scope.getResultType() === 'TABLE' && $scope.getGraphMode()===graphName && !activeAppId) { return true; } else { return false; @@ -2142,4 +2168,173 @@ angular.module('zeppelinWebApp') $scope.keepScrollDown = false; }; + // Helium --------------------------------------------- + + // app states + $scope.apps = []; + + // suggested apps + $scope.suggestion = {}; + + $scope.switchApp = function(appId) { + var app = _.find($scope.apps, { id : appId }); + var config = $scope.paragraph.config; + var settings = $scope.paragraph.settings; + + var newConfig = angular.copy(config); + var newParams = angular.copy(settings.params); + + // 'helium.activeApp' can be cleared by setGraphMode() + _.set(newConfig, 'helium.activeApp', appId); + + commitConfig(newConfig, newParams); + }; + + $scope.loadApp = function(heliumPackage) { + var noteId = $route.current.pathParams.noteId; + $http.post(baseUrlSrv.getRestApiBase() + '/helium/load/' + noteId + '/' + $scope.paragraph.id, + heliumPackage) + .success(function(data, status, headers, config) { + console.log('Load app %o', data); + }) + .error(function(err, status, headers, config) { + console.log('Error %o', err); + }); + }; + + var commitConfig = function(config, params) { + var paragraph = $scope.paragraph; + commitParagraph(paragraph.title, paragraph.text, config, params); + }; + + var getApplicationStates = function() { + var appStates = []; + var paragraph = $scope.paragraph; + + // Display ApplicationState + if (paragraph.apps) { + _.forEach(paragraph.apps, function (app) { + appStates.push({ + id: app.id, + pkg: app.pkg, + status: app.status, + output: app.output + }); + }); + } + + // update or remove app states no longer exists + _.forEach($scope.apps, function(currentAppState, idx) { + var newAppState = _.find(appStates, { id : currentAppState.id }); + if (newAppState) { + angular.extend($scope.apps[idx], newAppState); + } else { + $scope.apps.splice(idx, 1); + } + }); + + // add new app states + _.forEach(appStates, function(app, idx) { + if ($scope.apps.length <= idx || $scope.apps[idx].id !== app.id) { + $scope.apps.splice(idx, 0, app); + } + }); + }; + + var getSuggestions = function() { + // Get suggested apps + var noteId = $route.current.pathParams.noteId; + $http.get(baseUrlSrv.getRestApiBase() + '/helium/suggest/' + noteId + '/' + $scope.paragraph.id) + .success(function(data, status, headers, config) { + console.log('Suggested apps %o', data); + $scope.suggestion = data.body; + }) + .error(function(err, status, headers, config) { + console.log('Error %o', err); + }); + }; + + + var renderApp = function(appState) { + var retryRenderer = function() { + var targetEl = angular.element(document.getElementById('p' + appState.id)); + console.log('retry renderApp %o', targetEl); + if (targetEl.length) { + try { + console.log('renderApp %o', appState); + targetEl.html(appState.output); + $compile(targetEl.contents())(paragraphScope); + } catch(err) { + console.log('App rendering error %o', err); + } + } else { + $timeout(retryRenderer, 1000); + } + }; + $timeout(retryRenderer); + }; + + $scope.$on('appendAppOutput', function(event, data) { + if ($scope.paragraph.id === data.paragraphId) { + var app = _.find($scope.apps, { id : data.appId }); + if (app) { + app.output += data.data; + + var paragraphAppState = _.find($scope.paragraph.apps, { id : data.appId }); + paragraphAppState.output = app.output; + + var targetEl = angular.element(document.getElementById('p' + app.id)); + targetEl.html(app.output); + $compile(targetEl.contents())(paragraphScope); + console.log('append app output %o', $scope.apps); + } + } + }); + + $scope.$on('updateAppOutput', function(event, data) { + if ($scope.paragraph.id === data.paragraphId) { + var app = _.find($scope.apps, { id : data.appId }); + if (app) { + app.output = data.data; + + var paragraphAppState = _.find($scope.paragraph.apps, { id : data.appId }); + paragraphAppState.output = app.output; + + var targetEl = angular.element(document.getElementById('p' + app.id)); + targetEl.html(app.output); + $compile(targetEl.contents())(paragraphScope); + console.log('append app output'); + } + } + }); + + $scope.$on('appLoad', function(event, data) { + if ($scope.paragraph.id === data.paragraphId) { + var app = _.find($scope.apps, {id: data.appId}); + if (!app) { + app = { + id: data.appId, + pkg: data.pkg, + status: 'UNLOADED', + output: '' + }; + + $scope.apps.push(app); + $scope.paragraph.apps.push(app); + } + + $scope.switchApp(app.id); + } + }); + + $scope.$on('appStatusChange', function(event, data) { + if ($scope.paragraph.id === data.paragraphId) { + var app = _.find($scope.apps, {id: data.appId}); + if (app) { + app.status = data.status; + var paragraphAppState = _.find($scope.paragraph.apps, { id : data.appId }); + paragraphAppState.status = app.status; + } + } + }); }); diff --git a/zeppelin-web/src/components/websocketEvents/websocketEvents.factory.js b/zeppelin-web/src/components/websocketEvents/websocketEvents.factory.js index f40c47f3079..7c4b25b78cb 100644 --- a/zeppelin-web/src/components/websocketEvents/websocketEvents.factory.js +++ b/zeppelin-web/src/components/websocketEvents/websocketEvents.factory.js @@ -69,6 +69,14 @@ angular.module('zeppelinWebApp').factory('websocketEvents', function($rootScope, $rootScope.$broadcast('angularObjectUpdate', data); } else if (op === 'ANGULAR_OBJECT_REMOVE') { $rootScope.$broadcast('angularObjectRemove', data); + } else if (op === 'APP_APPEND_OUTPUT') { + $rootScope.$broadcast('appendAppOutput', data); + } else if (op === 'APP_UPDATE_OUTPUT') { + $rootScope.$broadcast('updateAppOutput', data); + } else if (op === 'APP_LOAD') { + $rootScope.$broadcast('appLoad', data); + } else if (op === 'APP_STATUS_CHANGE') { + $rootScope.$broadcast('appStatusChange', data); } }); diff --git a/zeppelin-web/test/spec/controllers/paragraph.js b/zeppelin-web/test/spec/controllers/paragraph.js index 7cdf74876f1..77fc495cc6d 100644 --- a/zeppelin-web/test/spec/controllers/paragraph.js +++ b/zeppelin-web/test/spec/controllers/paragraph.js @@ -10,6 +10,13 @@ describe('Controller: ParagraphCtrl', function() { var paragraphMock = { config: {} }; + var route = { + current : { + pathParams : { + noteId : 'noteId' + } + } + }; beforeEach(inject(function($controller, $rootScope) { scope = $rootScope.$new(); @@ -18,8 +25,10 @@ describe('Controller: ParagraphCtrl', function() { ParagraphCtrl = $controller('ParagraphCtrl', { $scope: scope, websocketMsgSrv: websocketMsgSrvMock, - $element: {} + $element: {}, + $route: route }); + scope.init(paragraphMock); })); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java index 8324d16e23f..7cdcdd149ee 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java @@ -62,6 +62,7 @@ public Helium(String heliumConfPath) throws IOException { /** * Add HeliumRegistry + * * @param registry */ public void addRegistry(HeliumRegistry registry) { diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java index 489257dd83c..b370c1e399b 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java @@ -27,6 +27,7 @@ import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService; import org.apache.zeppelin.notebook.*; import org.apache.zeppelin.scheduler.ExecutorFactory; +import org.apache.zeppelin.scheduler.Job; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,12 +37,7 @@ /** * HeliumApplicationFactory * - * 2. unload on interpreter restart - * 3. front-end job - * 4. example app - * 5. dev mode - * 6. app launcher - * 7. offline mode. front-end table data / pivot panel access + * TODO(moon): unload apps on interpreter restart */ public class HeliumApplicationFactory implements ApplicationEventListener, NotebookEventListener { private final Logger logger = LoggerFactory.getLogger(HeliumApplicationFactory.class); @@ -65,6 +61,8 @@ private boolean isRemote(InterpreterGroup group) { */ public String loadAndRun(HeliumPackage pkg, Paragraph paragraph) { ApplicationState appState = paragraph.createOrGetApplicationState(pkg); + onLoad(paragraph.getNote().getId(), paragraph.getId(), appState.getId(), + appState.getHeliumPackage()); executor.submit(new LoadApplication(appState, pkg, paragraph)); return appState.getId(); } @@ -104,7 +102,7 @@ public void run() { logger.error(e.getMessage(), e); if (appState != null) { - appState.setStatus(ApplicationState.Status.ERROR); + appStatusChange(paragraph, appState.getId(), ApplicationState.Status.ERROR); appState.setOutput(e.getMessage()); } } @@ -113,23 +111,20 @@ public void run() { private void load(RemoteInterpreterProcess intpProcess, ApplicationState appState) throws Exception { - RemoteInterpreterService.Client client; - try { - client = intpProcess.getClient(); - } catch (Exception e) { - throw new ApplicationException(e); - } + RemoteInterpreterService.Client client = null; synchronized (appState) { if (appState.getStatus() == ApplicationState.Status.LOADED) { // already loaded return; } - appState.setStatus(ApplicationState.Status.LOADING); + try { + appStatusChange(paragraph, appState.getId(), ApplicationState.Status.LOADING); String pkgInfo = gson.toJson(pkg); String appId = appState.getId(); + client = intpProcess.getClient(); RemoteApplicationResult ret = client.loadApplication( appId, pkgInfo, @@ -137,7 +132,7 @@ private void load(RemoteInterpreterProcess intpProcess, ApplicationState appStat paragraph.getId()); if (ret.isSuccess()) { - appState.setStatus(ApplicationState.Status.LOADED); + appStatusChange(paragraph, appState.getId(), ApplicationState.Status.LOADED); } else { throw new ApplicationException(ret.getMsg()); } @@ -145,7 +140,9 @@ private void load(RemoteInterpreterProcess intpProcess, ApplicationState appStat intpProcess.releaseBrokenClient(client); throw e; } finally { - intpProcess.releaseClient(client); + if (client != null) { + intpProcess.releaseClient(client); + } } } } @@ -194,12 +191,15 @@ public void run() { logger.warn("Can not find {} to unload from {}", appId, paragraph.getId()); return; } - + if (appState.getStatus() == ApplicationState.Status.UNLOADED) { + // not loaded + return; + } unload(appState); } catch (Exception e) { logger.error(e.getMessage(), e); if (appState != null) { - appState.setStatus(ApplicationState.Status.ERROR); + appStatusChange(paragraph, appId, ApplicationState.Status.ERROR); appState.setOutput(e.getMessage()); } } @@ -211,7 +211,7 @@ private void unload(ApplicationState appsToUnload) throws ApplicationException { throw new ApplicationException( "Can't unload application status " + appsToUnload.getStatus()); } - appsToUnload.setStatus(ApplicationState.Status.UNLOADING); + appStatusChange(paragraph, appsToUnload.getId(), ApplicationState.Status.UNLOADING); Interpreter intp = paragraph.getCurrentRepl(); if (intp == null) { throw new ApplicationException("No interpreter found"); @@ -234,7 +234,7 @@ private void unload(ApplicationState appsToUnload) throws ApplicationException { RemoteApplicationResult ret = client.unloadApplication(appsToUnload.getId()); if (ret.isSuccess()) { - appsToUnload.setStatus(ApplicationState.Status.UNLOADED); + appStatusChange(paragraph, appsToUnload.getId(), ApplicationState.Status.UNLOADED); } else { throw new ApplicationException(ret.getMsg()); } @@ -286,7 +286,7 @@ public void run() { } catch (Exception e) { logger.error(e.getMessage(), e); if (appState != null) { - appState.setStatus(ApplicationState.Status.ERROR); + appStatusChange(paragraph, appId, ApplicationState.Status.UNLOADED); appState.setOutput(e.getMessage()); } } @@ -309,7 +309,7 @@ private void run(ApplicationState app) throws ApplicationException { if (intpProcess == null) { throw new ApplicationException("Target interpreter process is not running"); } - RemoteInterpreterService.Client client; + RemoteInterpreterService.Client client = null; try { client = intpProcess.getClient(); } catch (Exception e) { @@ -326,9 +326,12 @@ private void run(ApplicationState app) throws ApplicationException { } } catch (TException e) { intpProcess.releaseBrokenClient(client); + client = null; throw new ApplicationException(e); } finally { - intpProcess.releaseClient(client); + if (client != null) { + intpProcess.releaseClient(client); + } } } } @@ -364,6 +367,28 @@ public void onOutputUpdated(String noteId, String paragraphId, String appId, Str } } + @Override + public void onLoad(String noteId, String paragraphId, String appId, HeliumPackage pkg) { + if (applicationEventListener != null) { + applicationEventListener.onLoad(noteId, paragraphId, appId, pkg); + } + } + + @Override + public void onStatusChange(String noteId, String paragraphId, String appId, String status) { + if (applicationEventListener != null) { + applicationEventListener.onStatusChange(noteId, paragraphId, appId, status); + } + } + + private void appStatusChange(Paragraph paragraph, + String appId, + ApplicationState.Status status) { + ApplicationState app = paragraph.getApplicationState(appId); + app.setStatus(status); + onStatusChange(paragraph.getNote().getId(), paragraph.getId(), appId, status.toString()); + } + private ApplicationState getAppState(String noteId, String paragraphId, String appId) { if (notebook == null) { return null; @@ -437,4 +462,16 @@ public void onParagraphRemove(Paragraph paragraph) { public void onParagraphCreate(Paragraph p) { } + + @Override + public void onParagraphStatusChange(Paragraph p, Job.Status status) { + if (status == Job.Status.FINISHED) { + // refresh application + List appStates = p.getAllApplicationStates(); + + for (ApplicationState app : appStates) { + loadAndRun(app.getHeliumPackage(), p); + } + } + } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java index 0ff75fe62d0..bc71d893221 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java @@ -16,6 +16,8 @@ */ package org.apache.zeppelin.notebook; +import org.apache.zeppelin.helium.HeliumPackage; + /** * Current state of application */ @@ -35,12 +37,12 @@ public static enum Status { Status status = Status.UNLOADED; String id; // unique id for this instance. Similar to note id or paragraph id - String name; // name of app + HeliumPackage pkg; String output; - public ApplicationState(String id, String name) { + public ApplicationState(String id, HeliumPackage pkg) { this.id = id; - this.name = name; + this.pkg = pkg; } /** @@ -59,19 +61,17 @@ public void resetStatus() { public boolean equals(Object o) { String compareName; if (o instanceof ApplicationState) { - compareName = ((ApplicationState) o).name; - } else if (o instanceof String) { - compareName = (String) o; + return pkg.equals(((ApplicationState) o).getHeliumPackage()); + } else if (o instanceof HeliumPackage) { + return pkg.equals((HeliumPackage) o); } else { return false; } - - return name.equals(compareName); } @Override public int hashCode() { - return name.hashCode(); + return pkg.hashCode(); } public String getId() { @@ -102,7 +102,7 @@ public synchronized void appendOutput(String output) { } } - public String getName() { - return name; + public HeliumPackage getHeliumPackage() { + return pkg; } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java index 2e97d26b2d5..f800e8340c0 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java @@ -28,6 +28,7 @@ import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.Input; +import org.apache.zeppelin.helium.HeliumApplicationFactory; import org.apache.zeppelin.interpreter.*; import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; import org.apache.zeppelin.notebook.repo.NotebookRepo; @@ -45,7 +46,7 @@ /** * Binded interpreters for a note */ -public class Note implements Serializable, JobListener { +public class Note implements Serializable, ParagraphJobListener { static Logger logger = LoggerFactory.getLogger(Note.class); private static final long serialVersionUID = 7920699076577612429L; @@ -369,7 +370,7 @@ public void runAll() { continue; } p.setNoteReplLoader(replLoader); - p.setListener(jobListenerFactory.getParagraphJobListener(this)); + p.setListener(this); Interpreter intp = replLoader.get(p.getRequiredReplName()); intp.getScheduler().submit(p); } @@ -384,7 +385,7 @@ public void runAll() { public void run(String paragraphId) { Paragraph p = getParagraph(paragraphId); p.setNoteReplLoader(replLoader); - p.setListener(jobListenerFactory.getParagraphJobListener(this)); + p.setListener(this); Interpreter intp = replLoader.get(p.getRequiredReplName()); if (intp == null) { throw new InterpreterException("Interpreter " + p.getRequiredReplName() + " not found"); @@ -517,14 +518,45 @@ public void setInfo(Map info) { @Override public void beforeStatusChange(Job job, Status before, Status after) { + if (jobListenerFactory != null) { + ParagraphJobListener listener = jobListenerFactory.getParagraphJobListener(this); + listener.beforeStatusChange(job, before, after); + } } @Override public void afterStatusChange(Job job, Status before, Status after) { + if (jobListenerFactory != null) { + ParagraphJobListener listener = jobListenerFactory.getParagraphJobListener(this); + listener.afterStatusChange(job, before, after); + } + noteEventListener.onParagraphStatusChange((Paragraph) job, after); } @Override - public void onProgressUpdate(Job job, int progress) {} + public void onProgressUpdate(Job job, int progress) { + if (jobListenerFactory != null) { + ParagraphJobListener listener = jobListenerFactory.getParagraphJobListener(this); + listener.onProgressUpdate(job, progress); + } + } + + + @Override + public void onOutputAppend(Paragraph paragraph, InterpreterOutput out, String output) { + if (jobListenerFactory != null) { + ParagraphJobListener listener = jobListenerFactory.getParagraphJobListener(this); + listener.onOutputAppend(paragraph, out, output); + } + } + + @Override + public void onOutputUpdate(Paragraph paragraph, InterpreterOutput out, String output) { + if (jobListenerFactory != null) { + ParagraphJobListener listener = jobListenerFactory.getParagraphJobListener(this); + listener.onOutputUpdate(paragraph, out, output); + } + } @@ -535,4 +567,5 @@ public NoteEventListener getNoteEventListener() { public void setNoteEventListener(NoteEventListener noteEventListener) { this.noteEventListener = noteEventListener; } + } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteEventListener.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteEventListener.java index 4b55eb18a19..c89901f51e0 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteEventListener.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteEventListener.java @@ -1,9 +1,12 @@ package org.apache.zeppelin.notebook; +import org.apache.zeppelin.scheduler.Job; + /** * NoteEventListener */ public interface NoteEventListener { public void onParagraphRemove(Paragraph p); public void onParagraphCreate(Paragraph p); + public void onParagraphStatusChange(Paragraph p, Job.Status status); } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java index 843d10520db..15e7dff2b07 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java @@ -41,6 +41,7 @@ import org.apache.zeppelin.notebook.repo.NotebookRepo; import org.apache.zeppelin.notebook.repo.NotebookRepoSync; import org.apache.zeppelin.resource.ResourcePoolUtils; +import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.SchedulerFactory; import org.apache.zeppelin.search.SearchService; import org.quartz.CronScheduleBuilder; @@ -646,4 +647,11 @@ public void onParagraphCreate(Paragraph p) { listener.onParagraphCreate(p); } } + + @Override + public void onParagraphStatusChange(Paragraph p, Job.Status status) { + for (NotebookEventListener listener : notebookEventListeners) { + listener.onParagraphStatusChange(p, status); + } + } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java index b71f87854c6..0ef4b4c5e0d 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java @@ -408,13 +408,13 @@ private String getApplicationId(HeliumPackage pkg) { public ApplicationState createOrGetApplicationState(HeliumPackage pkg) { synchronized (apps) { for (ApplicationState as : apps) { - if (as.getName().equals(pkg.getName())) { + if (as.equals(pkg)) { return as; } } String appId = getApplicationId(pkg); - ApplicationState appState = new ApplicationState(appId, pkg.getName()); + ApplicationState appState = new ApplicationState(appId, pkg); apps.add(appState); return appState; } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java index 1f9308f6700..13007919f31 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java @@ -36,6 +36,7 @@ import org.apache.commons.vfs2.VFS; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.notebook.ApplicationState; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.NoteInfo; import org.apache.zeppelin.notebook.Paragraph; @@ -169,6 +170,15 @@ private Note getNote(FileObject noteDir) throws IOException { if (p.getStatus() == Status.PENDING || p.getStatus() == Status.RUNNING) { p.setStatus(Status.ABORT); } + + List appStates = p.getAllApplicationStates(); + if (appStates != null) { + for (ApplicationState app : appStates) { + if (app.getStatus() != ApplicationState.Status.ERROR) { + app.setStatus(ApplicationState.Status.UNLOADED); + } + } + } } return note; diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index c3497a820f4..c05be770f31 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -659,6 +659,10 @@ public void onParagraphRemove(Paragraph p) { public void onParagraphCreate(Paragraph p) { onParagraphCreate.incrementAndGet(); } + + @Override + public void onParagraphStatusChange(Paragraph p, Status status) { + } }); Note note1 = notebook.createNote(); From 16f68871daaf98b4d3d4ca759b3f750e597bf5a4 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 12 Apr 2016 13:53:34 +0900 Subject: [PATCH 12/69] Angular object update for helium app --- .../zeppelin/helium/ApplicationContext.java | 12 ++- .../HeliumAppAngularObjectRegistry.java | 55 +++++++++++ .../remote/RemoteInterpreterServer.java | 6 +- .../helium/ApplicationLoaderTest.java | 1 + .../paragraph/paragraph.controller.js | 92 ++++++++++++++----- .../org/apache/zeppelin/notebook/Note.java | 17 ++++ .../apache/zeppelin/notebook/Notebook.java | 17 ++++ 7 files changed, 175 insertions(+), 25 deletions(-) create mode 100644 zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumAppAngularObjectRegistry.java diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationContext.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationContext.java index 303356a30d5..33d7f40f229 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationContext.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationContext.java @@ -16,6 +16,7 @@ */ package org.apache.zeppelin.helium; +import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.interpreter.InterpreterOutput; /** @@ -24,11 +25,16 @@ public class ApplicationContext { private final String noteId; private final String paragraphId; + private final HeliumAppAngularObjectRegistry angularObjectRegistry; public final InterpreterOutput out; - public ApplicationContext(String noteId, String paragraphId, InterpreterOutput out) { + public ApplicationContext(String noteId, + String paragraphId, + HeliumAppAngularObjectRegistry angularObjectRegistry, + InterpreterOutput out) { this.noteId = noteId; this.paragraphId = paragraphId; + this.angularObjectRegistry = angularObjectRegistry; this.out = out; } @@ -39,4 +45,8 @@ public String getNoteId() { public String getParagraphId() { return paragraphId; } + + public HeliumAppAngularObjectRegistry getAngularObjectRegistry() { + return angularObjectRegistry; + } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumAppAngularObjectRegistry.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumAppAngularObjectRegistry.java new file mode 100644 index 00000000000..dedb603add8 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/HeliumAppAngularObjectRegistry.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.helium; + +import org.apache.zeppelin.display.AngularObject; +import org.apache.zeppelin.display.AngularObjectRegistry; + +import java.util.List; + +/** + * Angular Registry for helium app + */ +public class HeliumAppAngularObjectRegistry { + private final String noteId; + private final String appId; + private final AngularObjectRegistry angularObjectRegistry; + + public HeliumAppAngularObjectRegistry(AngularObjectRegistry angularObjectRegistry, + String noteId, + String appId) { + this.angularObjectRegistry = angularObjectRegistry; + this.noteId = noteId; + this.appId = appId; + } + + public AngularObject add(String name, Object o) { + return angularObjectRegistry.add(name, o, noteId, appId); + } + + public AngularObject remove(String name) { + return angularObjectRegistry.remove(name, noteId, appId); + } + + public AngularObject get(String name) { + return angularObjectRegistry.get(name, noteId, appId); + } + + public List getAll() { + return angularObjectRegistry.getAll(noteId, appId); + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java index 129d7b94981..c83afd83ac3 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java @@ -725,7 +725,11 @@ public void onUpdate(InterpreterOutput out, byte[] output) { private ApplicationContext getApplicationContext( HeliumPackage packageInfo, String noteId, String paragraphId, String applicationInstanceId) { InterpreterOutput out = createAppOutput(noteId, paragraphId, applicationInstanceId); - return new ApplicationContext(noteId, paragraphId, out); + return new ApplicationContext( + noteId, + paragraphId, + new HeliumAppAngularObjectRegistry(angularObjectRegistry, noteId, applicationInstanceId), + out); } @Override diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/ApplicationLoaderTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/ApplicationLoaderTest.java index fa1915fef45..72e15053a5e 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/ApplicationLoaderTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/ApplicationLoaderTest.java @@ -86,6 +86,7 @@ public ApplicationContext createContext(String noteId, String paragraphId) { ApplicationContext context1 = new ApplicationContext( noteId, paragraphId, + null, new InterpreterOutput(new InterpreterOutputListener() { @Override public void onAppend(InterpreterOutput out, byte[] line) { diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index 39dbc3c7f3f..be62e4693d5 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -172,8 +172,23 @@ angular.module('zeppelinWebApp') $scope.$on('angularObjectUpdate', function(event, data) { var noteId = $route.current.pathParams.noteId; - if (!data.noteId || (data.noteId === noteId && (!data.paragraphId || data.paragraphId === $scope.paragraph.id))) { - var scope = paragraphScope; + if (!data.noteId || data.noteId === noteId) { + var scope; + var registry; + + if (!data.paragraphId || data.paragraphId === $scope.paragraph.id) { + scope = paragraphScope; + registry = angularObjectRegistry; + } else { + var app = _.find($scope.apps, { id: data.paragraphId}) + if (app) { + scope = getAppScope(app); + registry = getAppRegistry(app); + } else { + // no matching app in this paragraph + return; + } + } var varName = data.angularObject.name; if (angular.equals(data.angularObject.object, scope[varName])) { @@ -181,32 +196,32 @@ angular.module('zeppelinWebApp') return; } - if (!angularObjectRegistry[varName]) { - angularObjectRegistry[varName] = { + if (!registry[varName]) { + registry[varName] = { interpreterGroupId : data.interpreterGroupId, noteId : data.noteId, paragraphId : data.paragraphId }; } else { - angularObjectRegistry[varName].noteId = angularObjectRegistry[varName].noteId || data.noteId; - angularObjectRegistry[varName].paragraphId = angularObjectRegistry[varName].paragraphId || data.paragraphId; + registry[varName].noteId = registry[varName].noteId || data.noteId; + registry[varName].paragraphId = registry[varName].paragraphId || data.paragraphId; } - angularObjectRegistry[varName].skipEmit = true; + registry[varName].skipEmit = true; - if (!angularObjectRegistry[varName].clearWatcher) { - angularObjectRegistry[varName].clearWatcher = scope.$watch(varName, function(newValue, oldValue) { - console.log('angular object (paragraph) updated %o %o', varName, angularObjectRegistry[varName]); - if (angularObjectRegistry[varName].skipEmit) { - angularObjectRegistry[varName].skipEmit = false; + if (!registry[varName].clearWatcher) { + registry[varName].clearWatcher = scope.$watch(varName, function(newValue, oldValue) { + console.log('angular object (paragraph) updated %o %o', varName, registry[varName]); + if (registry[varName].skipEmit) { + registry[varName].skipEmit = false; return; } websocketMsgSrv.updateAngularObject( - angularObjectRegistry[varName].noteId, - angularObjectRegistry[varName].paragraphId, + registry[varName].noteId, + registry[varName].paragraphId, varName, newValue, - angularObjectRegistry[varName].interpreterGroupId); + registry[varName].interpreterGroupId); }); } console.log('angular object (paragraph) created %o', varName); @@ -228,14 +243,30 @@ angular.module('zeppelinWebApp') $scope.$on('angularObjectRemove', function(event, data) { var noteId = $route.current.pathParams.noteId; - if (!data.noteId || (data.noteId === noteId && (!data.paragraphId || data.paragraphId === $scope.paragraph.id))) { - var scope = paragraphScope; + if (!data.noteId || data.noteId === noteId) { + var scope; + var registry; + + if (!data.paragraphId || data.paragraphId === $scope.paragraph.id) { + scope = paragraphScope; + registry = angularObjectRegistry; + } else { + var app = _.find($scope.apps, { id: data.paragraphId}) + if (app) { + scope = getAppScope(app); + registry = getAppRegistry(app); + } else { + // no matching app in this paragraph + return; + } + } + var varName = data.name; // clear watcher - if (angularObjectRegistry[varName]) { - angularObjectRegistry[varName].clearWatcher(); - angularObjectRegistry[varName] = undefined; + if (registry[varName]) { + registry[varName].clearWatcher(); + registry[varName] = undefined; } // remove scope variable @@ -2254,6 +2285,21 @@ angular.module('zeppelinWebApp') }); }; + var getAppScope = function(appState) { + if (!appState.scope) { + appState.scope = $rootScope.$new(true, $rootScope); + } + + return appState.scope; + }; + + var getAppRegistry = function(appState) { + if (!appState.registry) { + appState.registry = {}; + } + + return appState.registry; + }; var renderApp = function(appState) { var retryRenderer = function() { @@ -2263,7 +2309,7 @@ angular.module('zeppelinWebApp') try { console.log('renderApp %o', appState); targetEl.html(appState.output); - $compile(targetEl.contents())(paragraphScope); + $compile(targetEl.contents())(getAppScope(appState)); } catch(err) { console.log('App rendering error %o', err); } @@ -2285,7 +2331,7 @@ angular.module('zeppelinWebApp') var targetEl = angular.element(document.getElementById('p' + app.id)); targetEl.html(app.output); - $compile(targetEl.contents())(paragraphScope); + $compile(targetEl.contents())(getAppScope(app)); console.log('append app output %o', $scope.apps); } } @@ -2302,7 +2348,7 @@ angular.module('zeppelinWebApp') var targetEl = angular.element(document.getElementById('p' + app.id)); targetEl.html(app.output); - $compile(targetEl.contents())(paragraphScope); + $compile(targetEl.contents())(getAppScope(app)); console.log('append app output'); } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java index f800e8340c0..f535b07aee0 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java @@ -438,8 +438,25 @@ private void removeAllAngularObjectInParagraph(String paragraphId) { if (registry instanceof RemoteAngularObjectRegistry) { // remove paragraph scope object ((RemoteAngularObjectRegistry) registry).removeAllAndNotifyRemoteProcess(id, paragraphId); + + // remove app scope object + List appStates = getParagraph(paragraphId).getAllApplicationStates(); + if (appStates != null) { + for (ApplicationState app : appStates) { + ((RemoteAngularObjectRegistry) registry).removeAllAndNotifyRemoteProcess( + id, app.getId()); + } + } } else { registry.removeAll(id, paragraphId); + + // remove app scope object + List appStates = getParagraph(paragraphId).getAllApplicationStates(); + if (appStates != null) { + for (ApplicationState app : appStates) { + registry.removeAll(id, app.getId()); + } + } } } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java index 15e7dff2b07..5bfd748ba63 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java @@ -306,6 +306,15 @@ public void removeNote(String id) { // remove paragraph scope object for (Paragraph p : note.getParagraphs()) { ((RemoteAngularObjectRegistry) registry).removeAllAndNotifyRemoteProcess(id, p.getId()); + + // remove app scope object + List appStates = p.getAllApplicationStates(); + if (appStates != null) { + for (ApplicationState app : appStates) { + ((RemoteAngularObjectRegistry) registry).removeAllAndNotifyRemoteProcess( + id, app.getId()); + } + } } // remove notebook scope object ((RemoteAngularObjectRegistry) registry).removeAllAndNotifyRemoteProcess(id, null); @@ -313,6 +322,14 @@ public void removeNote(String id) { // remove paragraph scope object for (Paragraph p : note.getParagraphs()) { registry.removeAll(id, p.getId()); + + // remove app scope object + List appStates = p.getAllApplicationStates(); + if (appStates != null) { + for (ApplicationState app : appStates) { + registry.removeAll(id, app.getId()); + } + } } // remove notebook scope object registry.removeAll(id, null); From 412480a2d222e31c7bd74198c5a4249dfea7ba8b Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 12 Apr 2016 07:33:45 +0200 Subject: [PATCH 13/69] Fix style --- .../src/app/notebook/paragraph/paragraph.controller.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index be62e4693d5..1841c56af17 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -180,7 +180,7 @@ angular.module('zeppelinWebApp') scope = paragraphScope; registry = angularObjectRegistry; } else { - var app = _.find($scope.apps, { id: data.paragraphId}) + var app = _.find($scope.apps, { id: data.paragraphId}); if (app) { scope = getAppScope(app); registry = getAppRegistry(app); @@ -251,7 +251,7 @@ angular.module('zeppelinWebApp') scope = paragraphScope; registry = angularObjectRegistry; } else { - var app = _.find($scope.apps, { id: data.paragraphId}) + var app = _.find($scope.apps, { id: data.paragraphId}); if (app) { scope = getAppScope(app); registry = getAppRegistry(app); From be3a1fa7b3da908051605945926790e1342ffd0a Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 12 Apr 2016 07:52:15 +0200 Subject: [PATCH 14/69] Add license header --- .../helium/ClassLoaderApplication.java | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ClassLoaderApplication.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ClassLoaderApplication.java index 7e3f61310d7..04cb6e6e1cb 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ClassLoaderApplication.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ClassLoaderApplication.java @@ -1,7 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.zeppelin.helium; -import org.apache.zeppelin.resource.ResourceSet; - /** * Application wrapper */ From 5503f9c100bfde93f5c3eb2f12bf88e6e18ab65b Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 12 Apr 2016 14:13:16 +0200 Subject: [PATCH 15/69] Improved --- .../zeppelin/helium/ApplicationLoader.java | 11 ++++++---- .../remote/RemoteAngularObjectRegistry.java | 2 +- .../remote/RemoteInterpreterServer.java | 12 ++++++----- .../resource/WellKnownResourceName.java | 2 +- .../paragraph/paragraph-chart-selector.html | 21 ++++++++++++++----- .../paragraph/paragraph.controller.js | 12 ++++++----- .../src/app/notebook/paragraph/paragraph.css | 5 +++++ .../org/apache/zeppelin/helium/Helium.java | 12 ++++++----- 8 files changed, 51 insertions(+), 26 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java index 40065373c2f..407e099488e 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java @@ -141,8 +141,6 @@ private ResourceSet findRequiredResourceSet( return new ResourceSet(); } - String localResourcePoolId = resourcePool.id(); - ResourceSet args = new ResourceSet(); ResourceSet allResources; if (resourcePool instanceof DistributedResourcePool) { allResources = ((DistributedResourcePool) resourcePool).getAll(false); @@ -171,9 +169,14 @@ static ResourceSet findRequiredResourceSet(String [][] requiredResources, boolean found = false; for (Resource r : resources) { - if (r.getClassName().equals(require)) { - args.add(r); + if (require.startsWith(":") && r.getClassName().equals(require)) { + found = true; + } else if (r.getResourceId().getName().equals(require)) { found = true; + } + + if (found) { + args.add(r); break; } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java index b80a2526e92..583d8049fdf 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java @@ -96,7 +96,7 @@ public AngularObject addAndNotifyRemoteProcess(String name, Object o, String not public AngularObject removeAndNotifyRemoteProcess(String name, String noteId, String paragraphId) { RemoteInterpreterProcess remoteInterpreterProcess = getRemoteInterpreterProcess(); - if (!remoteInterpreterProcess.isRunning()) { + if (remoteInterpreterProcess == null || !remoteInterpreterProcess.isRunning()) { return super.remove(name, noteId, paragraphId); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java index c83afd83ac3..254c0364d97 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java @@ -362,11 +362,13 @@ protected Object jobRun() throws Throwable { } // put result into resource pool - context.getResourcePool().put( - context.getNoteId(), - context.getParagraphId(), - WellKnownResourceName.ParagraphResult.toString(), - combinedResult); + if (combinedResult.type() == InterpreterResult.Type.TABLE) { + context.getResourcePool().put( + context.getNoteId(), + context.getParagraphId(), + WellKnownResourceName.ZeppelinTableResult.toString(), + combinedResult); + } return combinedResult; } finally { InterpreterContext.remove(); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/WellKnownResourceName.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/WellKnownResourceName.java index 2d14fd4bc97..646df2f49f1 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/WellKnownResourceName.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/WellKnownResourceName.java @@ -20,7 +20,7 @@ * Well known resource names in ResourcePool */ public enum WellKnownResourceName { - ParagraphResult("zeppelin.paragraph.result"); // paragraph run result + ZeppelinTableResult("zeppelin.paragraph.result.table"); // paragraph run result String name; WellKnownResourceName(String name) { diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph-chart-selector.html b/zeppelin-web/src/app/notebook/paragraph/paragraph-chart-selector.html index 55721c12474..edbc62e3046 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph-chart-selector.html +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph-chart-selector.html @@ -46,6 +46,13 @@ ng-class="{'active': isGraphMode('scatterChart')}" ng-click="setGraphMode('scatterChart', true)"> + + diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index 1841c56af17..72443faa0eb 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -366,12 +366,15 @@ angular.module('zeppelinWebApp') var newType = $scope.getResultType(data.paragraph); var oldGraphMode = $scope.getGraphMode(); var newGraphMode = $scope.getGraphMode(data.paragraph); - var resultRefreshed = (data.paragraph.dateFinished !== $scope.paragraph.dateFinished) || isEmpty(data.paragraph.result) !== isEmpty($scope.paragraph.result); + var oldActiveApp = _.get($scope.paragraph.config, 'helium.activeApp'); + var newActiveApp = _.get(data.paragraph.config, 'helium.activeApp'); + + var resultRefreshed = (data.paragraph.dateFinished !== $scope.paragraph.dateFinished) || + isEmpty(data.paragraph.result) !== isEmpty($scope.paragraph.result) || + (!newActiveApp && oldActiveApp !== newActiveApp); var statusChanged = (data.paragraph.status !== $scope.paragraph.status); - var oldActiveApp = _.get($scope.paragraph.config, 'helium.activeApp'); - var newActiveApp = _.get(data.paragraph.config, 'helium.activeApp'); //console.log("updateParagraph oldData %o, newData %o. type %o -> %o, mode %o -> %o", $scope.paragraph, data, oldType, newType, oldGraphMode, newGraphMode); @@ -2367,9 +2370,8 @@ angular.module('zeppelinWebApp') $scope.apps.push(app); $scope.paragraph.apps.push(app); + $scope.switchApp(app.id); } - - $scope.switchApp(app.id); } }); diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.css b/zeppelin-web/src/app/notebook/paragraph/paragraph.css index f170ca08078..3d885a4eb94 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.css +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.css @@ -434,3 +434,8 @@ table.table-striped { position: absolute; right: 15px; } + +.appSuggestion { + width: 200px; + padding: 5px 10px 5px 10px; +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java index 7cdcdd149ee..0507eb21a1b 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java @@ -141,12 +141,14 @@ public HeliumPackageSuggestion suggestApp(Paragraph paragraph) { ResourcePool resourcePool = intp.getInterpreterGroup().getResourcePool(); ResourceSet allResources; - if (resourcePool == null) { - allResources = new ResourceSet(); - } else if (resourcePool instanceof DistributedResourcePool) { - allResources = ((DistributedResourcePool) resourcePool).getAll(false); + if (resourcePool != null) { + if (resourcePool instanceof DistributedResourcePool) { + allResources = ((DistributedResourcePool) resourcePool).getAll(true); + } else { + allResources = resourcePool.getAll(); + } } else { - allResources = resourcePool.getAll(); + allResources = ResourcePoolUtils.getAllResources(); } for (HeliumPackageSearchResult pkg : getAllPackageInfo()) { From b47ca744b989d8644e7cafe4ec3f2d336b890f92 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Wed, 13 Apr 2016 00:46:45 +0200 Subject: [PATCH 16/69] Fix tests --- .../resource/DistributedResourcePoolTest.java | 12 +++++------- .../org/apache/zeppelin/server/ZeppelinServer.java | 2 +- .../apache/zeppelin/conf/ZeppelinConfiguration.java | 5 +++++ .../main/java/org/apache/zeppelin/helium/Helium.java | 6 ++++-- .../main/java/org/apache/zeppelin/notebook/Note.java | 5 ++++- .../java/org/apache/zeppelin/helium/HeliumTest.java | 9 ++++++--- 6 files changed, 25 insertions(+), 14 deletions(-) diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java index 605be549048..4e8fee2e54f 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java @@ -138,13 +138,12 @@ public void testRemoteDistributedResourcePool() { InterpreterResult ret; intp1.interpret("put key1 value1", context); intp2.interpret("put key2 value2", context); - int numInterpreterResult = 2; ret = intp1.interpret("getAll", context); - assertEquals(numInterpreterResult + 2, gson.fromJson(ret.message(), ResourceSet.class).size()); + assertEquals(2, gson.fromJson(ret.message(), ResourceSet.class).size()); ret = intp2.interpret("getAll", context); - assertEquals(numInterpreterResult + 2, gson.fromJson(ret.message(), ResourceSet.class).size()); + assertEquals(2, gson.fromJson(ret.message(), ResourceSet.class).size()); ret = intp1.interpret("get key1", context); assertEquals("value1", gson.fromJson(ret.message(), String.class)); @@ -216,16 +215,15 @@ public void testResourcePoolUtils() { intp2.interpret("put note2:paragraph1:key1 value1", context); intp2.interpret("put note2:paragraph2:key2 value2", context); - int numInterpreterResult = 2; // then get all resources. - assertEquals(numInterpreterResult + 4, ResourcePoolUtils.getAllResources().size()); + assertEquals(4, ResourcePoolUtils.getAllResources().size()); // when remove all resources from note1 ResourcePoolUtils.removeResourcesBelongsToNote("note1"); // then resources should be removed. - assertEquals(numInterpreterResult + 2, ResourcePoolUtils.getAllResources().size()); + assertEquals(2, ResourcePoolUtils.getAllResources().size()); assertEquals("", gson.fromJson( intp1.interpret("get note1:paragraph1:key1", context).message(), String.class)); @@ -238,7 +236,7 @@ public void testResourcePoolUtils() { ResourcePoolUtils.removeResourcesBelongsToParagraph("note2", "paragraph1"); // then 1 - assertEquals(numInterpreterResult + 1, ResourcePoolUtils.getAllResources().size()); + assertEquals(1, ResourcePoolUtils.getAllResources().size()); assertEquals("value2", gson.fromJson( intp1.interpret("get note2:paragraph2:key2", context).message(), String.class)); diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java index 717877d65b9..36a555d7563 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/server/ZeppelinServer.java @@ -86,7 +86,7 @@ public ZeppelinServer() throws Exception { this.depResolver = new DependencyResolver( conf.getString(ConfVars.ZEPPELIN_INTERPRETER_LOCALREPO)); - this.helium = new Helium(conf.getHeliumConfPath()); + this.helium = new Helium(conf.getHeliumConfPath(), conf.getHeliumDefaultLocalRegistryPath()); this.heliumApplicationFactory = new HeliumApplicationFactory(); this.schedulerFactory = new SchedulerFactory(); this.replFactory = new InterpreterFactory(conf, notebookWsServer, diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 70706fc44ef..eeb82b9828f 100755 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -346,6 +346,10 @@ public String getHeliumConfPath() { return getRelativeDir(String.format("%s/helium.json", getConfDir())); } + public String getHeliumDefaultLocalRegistryPath() { + return getRelativeDir(ConfVars.ZEPPELIN_HELIUM_LOCALREGISTRY_DEFAULT); + } + public String getNotebookAuthorizationPath() { return getRelativeDir(String.format("%s/notebook-authorization.json", getConfDir())); } @@ -490,6 +494,7 @@ public static enum ConfVars { ZEPPELIN_NOTEBOOK_AUTO_INTERPRETER_BINDING("zeppelin.notebook.autoInterpreterBinding", true), ZEPPELIN_CONF_DIR("zeppelin.conf.dir", "conf"), ZEPPELIN_DEP_LOCALREPO("zeppelin.dep.localrepo", "local-repo"), + ZEPPELIN_HELIUM_LOCALREGISTRY_DEFAULT("zeppelin.helium.localregistry.default", "helium"), // Allows a way to specify a ',' separated list of allowed origins for rest and websockets // i.e. http://localhost:8080 ZEPPELIN_ALLOWED_ORIGINS("zeppelin.server.allowed.origins", "*"), diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java index 0507eb21a1b..a07f5f0fde5 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/Helium.java @@ -46,10 +46,12 @@ public class Helium { private final HeliumConf heliumConf; private final String heliumConfPath; + private final String defaultLocalRegistryPath; private final Gson gson; - public Helium(String heliumConfPath) throws IOException { + public Helium(String heliumConfPath, String defaultLocalRegistryPath) throws IOException { this.heliumConfPath = heliumConfPath; + this.defaultLocalRegistryPath = defaultLocalRegistryPath; GsonBuilder builder = new GsonBuilder(); builder.setPrettyPrinting(); @@ -87,7 +89,7 @@ private synchronized HeliumConf loadConf(String path) throws IOException { logger.warn("{} does not exists", path); HeliumConf conf = new HeliumConf(); LinkedList defaultRegistry = new LinkedList(); - defaultRegistry.add(new HeliumLocalRegistry("local", "../helium")); + defaultRegistry.add(new HeliumLocalRegistry("local", defaultLocalRegistryPath)); conf.setRegistry(defaultRegistry); this.registry = conf.getRegistry(); return conf; diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java index f535b07aee0..ea62e04dc01 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java @@ -547,7 +547,10 @@ public void afterStatusChange(Job job, Status before, Status after) { ParagraphJobListener listener = jobListenerFactory.getParagraphJobListener(this); listener.afterStatusChange(job, before, after); } - noteEventListener.onParagraphStatusChange((Paragraph) job, after); + + if (noteEventListener != null) { + noteEventListener.onParagraphStatusChange((Paragraph) job, after); + } } @Override diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java index 084729adffa..1ed31c5bb95 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTest.java @@ -32,11 +32,14 @@ public class HeliumTest { private File tmpDir; + private File localRegistryPath; @Before public void setUp() throws Exception { tmpDir = new File(System.getProperty("java.io.tmpdir") + "/ZeppelinLTest_" + System.currentTimeMillis()); tmpDir.mkdirs(); + localRegistryPath = new File(tmpDir, "helium"); + localRegistryPath.mkdirs(); } @After @@ -48,7 +51,7 @@ public void tearDown() throws IOException { public void testSaveLoadConf() throws IOException, URISyntaxException { // given File heliumConf = new File(tmpDir, "helium.conf"); - Helium helium = new Helium(heliumConf.getAbsolutePath()); + Helium helium = new Helium(heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath()); assertFalse(heliumConf.exists()); HeliumTestRegistry registry1 = new HeliumTestRegistry("r1", "r1"); helium.addRegistry(registry1); @@ -62,14 +65,14 @@ public void testSaveLoadConf() throws IOException, URISyntaxException { assertTrue(heliumConf.exists()); // then - Helium heliumRestored = new Helium(heliumConf.getAbsolutePath()); + Helium heliumRestored = new Helium(heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath()); assertEquals(2, heliumRestored.getAllRegistry().size()); } @Test public void testRestoreRegistryInstances() throws IOException, URISyntaxException { File heliumConf = new File(tmpDir, "helium.conf"); - Helium helium = new Helium(heliumConf.getAbsolutePath()); + Helium helium = new Helium(heliumConf.getAbsolutePath(), localRegistryPath.getAbsolutePath()); HeliumTestRegistry registry1 = new HeliumTestRegistry("r1", "r1"); HeliumTestRegistry registry2 = new HeliumTestRegistry("r2", "r2"); helium.addRegistry(registry1); From 98f3872c6a27e8699717b9fc20b353b8fe9a1d2d Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Wed, 13 Apr 2016 02:13:49 +0200 Subject: [PATCH 17/69] Managed interpreter process and Running interpreter process --- .../interpreter/InterpreterGroup.java | 4 - .../remote/RemoteAngularObjectRegistry.java | 2 - .../interpreter/remote/RemoteInterpreter.java | 54 +++++- .../remote/RemoteInterpreterEventPoller.java | 2 - .../RemoteInterpreterManagedProcess.java | 164 ++++++++++++++++++ .../remote/RemoteInterpreterProcess.java | 129 ++------------ .../RemoteInterpreterRunningProcess.java | 67 +++++++ .../zeppelin/resource/ResourcePoolUtils.java | 1 + .../zeppelin/scheduler/RemoteScheduler.java | 5 +- .../remote/RemoteInterpreterProcessTest.java | 4 +- .../remote/RemoteInterpreterTest.java | 1 - .../interpreter/InterpreterFactory.java | 37 +++- .../interpreter/InterpreterOption.java | 17 +- 13 files changed, 352 insertions(+), 135 deletions(-) create mode 100644 zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java create mode 100644 zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterRunningProcess.java diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java index 462526e5803..bc56784b15b 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterGroup.java @@ -22,11 +22,7 @@ import org.apache.log4j.Logger; import org.apache.zeppelin.display.AngularObjectRegistry; -import org.apache.zeppelin.helium.Application; -import org.apache.zeppelin.helium.ApplicationException; -import org.apache.zeppelin.helium.HeliumPackage; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; -import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService; import org.apache.zeppelin.resource.ResourcePool; import org.apache.zeppelin.scheduler.Scheduler; import org.apache.zeppelin.scheduler.SchedulerFactory; diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java index 583d8049fdf..0ac71165348 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectRegistry.java @@ -23,9 +23,7 @@ import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.AngularObjectRegistryListener; -import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterGroup; -import org.apache.zeppelin.interpreter.WrappedInterpreter; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java index 9df3db014b9..c5b146debef 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java @@ -55,8 +55,12 @@ public class RemoteInterpreter extends Interpreter { private Map env; private int connectTimeout; private int maxPoolSize; - private static String schedulerName; + private String host; + private int port; + /** + * Remote interpreter and manage interpreter process + */ public RemoteInterpreter(Properties property, String noteId, String className, @@ -81,6 +85,32 @@ public RemoteInterpreter(Properties property, this.applicationEventListener = appListener; } + + /** + * Connect to existing process + */ + public RemoteInterpreter(Properties property, + String noteId, + String className, + String host, + int port, + int connectTimeout, + int maxPoolSize, + RemoteInterpreterProcessListener remoteInterpreterProcessListener, + ApplicationEventListener appListener) { + super(property); + this.noteId = noteId; + this.className = className; + initialized = false; + this.host = host; + this.port = port; + this.connectTimeout = connectTimeout; + this.maxPoolSize = maxPoolSize; + this.remoteInterpreterProcessListener = remoteInterpreterProcessListener; + this.applicationEventListener = appListener; + } + + // VisibleForTesting public RemoteInterpreter(Properties property, String noteId, @@ -110,6 +140,10 @@ public String getClassName() { return className; } + private boolean connectToExistingProcess() { + return host != null && port > 0; + } + public RemoteInterpreterProcess getInterpreterProcess() { InterpreterGroup intpGroup = getInterpreterGroup(); if (intpGroup == null) { @@ -118,10 +152,20 @@ public RemoteInterpreterProcess getInterpreterProcess() { synchronized (intpGroup) { if (intpGroup.getRemoteInterpreterProcess() == null) { - // create new remote process - RemoteInterpreterProcess remoteProcess = new RemoteInterpreterProcess( - interpreterRunner, interpreterPath, localRepoPath, env, connectTimeout, - remoteInterpreterProcessListener, applicationEventListener); + RemoteInterpreterProcess remoteProcess; + if (connectToExistingProcess()) { + remoteProcess = new RemoteInterpreterRunningProcess( + connectTimeout, + remoteInterpreterProcessListener, + applicationEventListener, + host, + port); + } else { + // create new remote process + remoteProcess = new RemoteInterpreterManagedProcess( + interpreterRunner, interpreterPath, localRepoPath, env, connectTimeout, + remoteInterpreterProcessListener, applicationEventListener); + } intpGroup.setRemoteInterpreterProcess(remoteProcess); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java index aab74b60033..86b4f690133 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java @@ -25,7 +25,6 @@ import org.apache.zeppelin.helium.ApplicationEventListener; import org.apache.zeppelin.interpreter.InterpreterContextRunner; import org.apache.zeppelin.interpreter.InterpreterGroup; -import org.apache.zeppelin.interpreter.InterpreterOutputListener; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEvent; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEventType; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; @@ -37,7 +36,6 @@ import org.slf4j.LoggerFactory; import java.nio.ByteBuffer; -import java.util.Collections; import java.util.LinkedList; import java.util.List; import java.util.Map; diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java new file mode 100644 index 00000000000..648932bea38 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.interpreter.remote; + +import org.apache.commons.exec.*; +import org.apache.commons.exec.environment.EnvironmentUtils; +import org.apache.zeppelin.helium.ApplicationEventListener; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Map; + +/** + * This class manages start / stop of remote interpreter process + */ +public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess implements ExecuteResultHandler { + private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreterManagedProcess.class); + private final String interpreterRunner; + + private DefaultExecutor executor; + private ExecuteWatchdog watchdog; + boolean running = false; + private int port = -1; + private final String interpreterDir; + private final String localRepoDir; + + private Map env; + + public RemoteInterpreterManagedProcess( + String intpRunner, + String intpDir, + String localRepoDir, + Map env, + int connectTimeout, + RemoteInterpreterProcessListener listener, + ApplicationEventListener appListener) { + super(new RemoteInterpreterEventPoller(listener, appListener), + connectTimeout); + this.interpreterRunner = intpRunner; + this.env = env; + this.interpreterDir = intpDir; + this.localRepoDir = localRepoDir; + } + + RemoteInterpreterManagedProcess(String intpRunner, + String intpDir, + String localRepoDir, + Map env, + RemoteInterpreterEventPoller remoteInterpreterEventPoller, + int connectTimeout) { + super(remoteInterpreterEventPoller, + connectTimeout); + this.interpreterRunner = intpRunner; + this.env = env; + this.interpreterDir = intpDir; + this.localRepoDir = localRepoDir; + } + + @Override + public String getHost() { + return "localhost"; + } + + @Override + public int getPort() { + return port; + } + + @Override + public void start() { + // start server process + try { + port = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(); + } catch (IOException e1) { + throw new InterpreterException(e1); + } + + CommandLine cmdLine = CommandLine.parse(interpreterRunner); + cmdLine.addArgument("-d", false); + cmdLine.addArgument(interpreterDir, false); + cmdLine.addArgument("-p", false); + cmdLine.addArgument(Integer.toString(port), false); + cmdLine.addArgument("-l", false); + cmdLine.addArgument(localRepoDir, false); + + executor = new DefaultExecutor(); + + watchdog = new ExecuteWatchdog(ExecuteWatchdog.INFINITE_TIMEOUT); + executor.setWatchdog(watchdog); + + try { + Map procEnv = EnvironmentUtils.getProcEnvironment(); + procEnv.putAll(env); + + logger.info("Run interpreter process {}", cmdLine); + executor.execute(cmdLine, procEnv, this); + running = true; + } catch (IOException e) { + running = false; + throw new InterpreterException(e); + } + + + long startTime = System.currentTimeMillis(); + while (System.currentTimeMillis() - startTime < getConnectTimeout()) { + if (RemoteInterpreterUtils.checkIfRemoteEndpointAccessible("localhost", port)) { + break; + } else { + try { + Thread.sleep(500); + } catch (InterruptedException e) { + logger.error("Exception in RemoteInterpreterProcess while synchronized reference " + + "Thread.sleep", e); + } + } + } + } + + public void stop() { + if (isRunning()) { + logger.info("kill interpreter process"); + watchdog.destroyProcess(); + } + + executor = null; + watchdog = null; + running = false; + logger.info("Remote process terminated"); + } + + @Override + public void onProcessComplete(int exitValue) { + logger.info("Interpreter process exited {}", exitValue); + running = false; + + } + + @Override + public void onProcessFailed(ExecuteException e) { + logger.info("Interpreter process failed {}", e); + running = false; + } + + public boolean isRunning() { + return running; + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java index 43b94918748..6755cdaa6df 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java @@ -14,134 +14,68 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.zeppelin.interpreter.remote; import com.google.gson.Gson; import org.apache.commons.exec.*; -import org.apache.commons.exec.environment.EnvironmentUtils; import org.apache.commons.pool2.impl.GenericObjectPool; import org.apache.thrift.TException; import org.apache.zeppelin.helium.ApplicationEventListener; -import org.apache.zeppelin.interpreter.InterpreterException; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; /** - * + * Abstract class for interpreter process */ -public class RemoteInterpreterProcess implements ExecuteResultHandler { +public abstract class RemoteInterpreterProcess { private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreterProcess.class); private final AtomicInteger referenceCount; - private DefaultExecutor executor; private ExecuteWatchdog watchdog; - boolean running = false; - private int port = -1; - private final String interpreterRunner; - private final String interpreterDir; - private final String localRepoDir; private GenericObjectPool clientPool; - private Map env; private final RemoteInterpreterEventPoller remoteInterpreterEventPoller; private final InterpreterContextRunnerPool interpreterContextRunnerPool; private int connectTimeout; public RemoteInterpreterProcess( - String intpRunner, - String intpDir, - String localRepoDir, - Map env, int connectTimeout, RemoteInterpreterProcessListener listener, ApplicationEventListener appListener) { - this(intpRunner, - intpDir, - localRepoDir, - env, - new RemoteInterpreterEventPoller(listener, appListener), + this(new RemoteInterpreterEventPoller(listener, appListener), connectTimeout); } - RemoteInterpreterProcess(String intpRunner, - String intpDir, - String localRepoDir, - Map env, - RemoteInterpreterEventPoller remoteInterpreterEventPoller, - int connectTimeout) { - this.interpreterRunner = intpRunner; - this.interpreterDir = intpDir; - this.localRepoDir = localRepoDir; - this.env = env; + RemoteInterpreterProcess(RemoteInterpreterEventPoller remoteInterpreterEventPoller, + int connectTimeout) { this.interpreterContextRunnerPool = new InterpreterContextRunnerPool(); referenceCount = new AtomicInteger(0); this.remoteInterpreterEventPoller = remoteInterpreterEventPoller; this.connectTimeout = connectTimeout; } + public abstract String getHost(); + public abstract int getPort(); + public abstract void start(); + public abstract void stop(); + public abstract boolean isRunning(); - public int getPort() { - return port; + public int getConnectTimeout() { + return connectTimeout; } public int reference(InterpreterGroup interpreterGroup) { synchronized (referenceCount) { - if (executor == null) { - // start server process - try { - port = RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces(); - } catch (IOException e1) { - throw new InterpreterException(e1); - } - - CommandLine cmdLine = CommandLine.parse(interpreterRunner); - cmdLine.addArgument("-d", false); - cmdLine.addArgument(interpreterDir, false); - cmdLine.addArgument("-p", false); - cmdLine.addArgument(Integer.toString(port), false); - cmdLine.addArgument("-l", false); - cmdLine.addArgument(localRepoDir, false); - - executor = new DefaultExecutor(); - - watchdog = new ExecuteWatchdog(ExecuteWatchdog.INFINITE_TIMEOUT); - executor.setWatchdog(watchdog); - - running = true; - try { - Map procEnv = EnvironmentUtils.getProcEnvironment(); - procEnv.putAll(env); - - logger.info("Run interpreter process {}", cmdLine); - executor.execute(cmdLine, procEnv, this); - } catch (IOException e) { - running = false; - throw new InterpreterException(e); - } - - - long startTime = System.currentTimeMillis(); - while (System.currentTimeMillis() - startTime < connectTimeout) { - if (RemoteInterpreterUtils.checkIfRemoteEndpointAccessible("localhost", port)) { - break; - } else { - try { - Thread.sleep(500); - } catch (InterruptedException e) { - logger.error("Exception in RemoteInterpreterProcess while synchronized reference " + - "Thread.sleep", e); - } - } - } + if (!isRunning()) { + start(); + } - clientPool = new GenericObjectPool(new ClientFactory("localhost", port)); + if (clientPool == null) { + clientPool = new GenericObjectPool(new ClientFactory(getHost(), getPort())); + clientPool.setTestOnReturn(true); remoteInterpreterEventPoller.setInterpreterGroup(interpreterGroup); remoteInterpreterEventPoller.setInterpreterProcess(this); @@ -224,16 +158,6 @@ public int dereference() { break; } } - - if (isRunning()) { - logger.info("kill interpreter process"); - watchdog.destroyProcess(); - } - - executor = null; - watchdog = null; - running = false; - logger.info("Remote process terminated"); } return r; } @@ -245,23 +169,6 @@ public int referenceCount() { } } - @Override - public void onProcessComplete(int exitValue) { - logger.info("Interpreter process exited {}", exitValue); - running = false; - - } - - @Override - public void onProcessFailed(ExecuteException e) { - logger.info("Interpreter process failed {}", e); - running = false; - } - - public boolean isRunning() { - return running; - } - public int getNumActiveClient() { if (clientPool == null) { return 0; diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterRunningProcess.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterRunningProcess.java new file mode 100644 index 00000000000..42e6250e3ca --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterRunningProcess.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.interpreter.remote; + +import org.apache.zeppelin.helium.ApplicationEventListener; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class connects to existing process + */ +public class RemoteInterpreterRunningProcess extends RemoteInterpreterProcess { + private final Logger logger = LoggerFactory.getLogger(RemoteInterpreterRunningProcess.class); + private final String host; + private final int port; + + public RemoteInterpreterRunningProcess( + int connectTimeout, + RemoteInterpreterProcessListener listener, + ApplicationEventListener appListener, + String host, + int port + ) { + super(connectTimeout, listener, appListener); + this.host = host; + this.port = port; + } + + @Override + public String getHost() { + return host; + } + + @Override + public int getPort() { + return port; + } + + @Override + public void start() { + // assume process is externally managed. nothing to do + } + + @Override + public void stop() { + // assume process is externally managed. nothing to do + } + + @Override + public boolean isRunning() { + return RemoteInterpreterUtils.checkIfRemoteEndpointAccessible(getHost(), getPort()); + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java index 3ad8ff95624..9878d7e766c 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java @@ -19,6 +19,7 @@ import com.google.gson.Gson; import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterManagedProcess; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService; import org.slf4j.Logger; diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/RemoteScheduler.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/RemoteScheduler.java index 33a3ca63c75..28c743740ed 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/RemoteScheduler.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/scheduler/RemoteScheduler.java @@ -20,6 +20,7 @@ import org.apache.thrift.TException; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.InterpreterResult.Code; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterManagedProcess; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; import org.apache.zeppelin.scheduler.Job.Status; @@ -49,8 +50,8 @@ public class RemoteScheduler implements Scheduler { private RemoteInterpreterProcess interpreterProcess; public RemoteScheduler(String name, ExecutorService executor, String noteId, - RemoteInterpreterProcess interpreterProcess, SchedulerListener listener, - int maxConcurrency) { + RemoteInterpreterProcess interpreterProcess, SchedulerListener listener, + int maxConcurrency) { this.name = name; this.executor = executor; this.listener = listener; diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java index 0526c057740..e601a23a281 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java @@ -32,7 +32,7 @@ public class RemoteInterpreterProcessTest { @Test public void testStartStop() { InterpreterGroup intpGroup = new InterpreterGroup(); - RemoteInterpreterProcess rip = new RemoteInterpreterProcess( + RemoteInterpreterManagedProcess rip = new RemoteInterpreterManagedProcess( "../bin/interpreter.sh", "nonexists", "fakeRepo", new HashMap(), 10 * 1000, null, null); assertFalse(rip.isRunning()); @@ -49,7 +49,7 @@ public void testStartStop() { @Test public void testClientFactory() throws Exception { InterpreterGroup intpGroup = new InterpreterGroup(); - RemoteInterpreterProcess rip = new RemoteInterpreterProcess( + RemoteInterpreterManagedProcess rip = new RemoteInterpreterManagedProcess( "../bin/interpreter.sh", "nonexists", "fakeRepo", new HashMap(), mock(RemoteInterpreterEventPoller.class), 10 * 1000); rip.reference(intpGroup); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java index 66e25dd7894..d6fb10b334b 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java @@ -30,7 +30,6 @@ import org.apache.thrift.transport.TTransportException; import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; -import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; import org.apache.zeppelin.user.AuthenticationInfo; import org.apache.zeppelin.display.GUI; diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index 024dd9e3f0c..18ad5ce072c 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -506,11 +506,18 @@ public void createInterpretersForNote( Interpreter intp; if (option.isRemote()) { - intp = createRemoteRepl(info.getPath(), - noteId, - info.getClassName(), - properties, - interpreterGroup.id); + if (option.isConnectExistingProcess()) { + intp = connectToRemoteRepl( + noteId, + info.getClassName(), + option.getHost(), option.getPort(), properties); + } else { + intp = createRemoteRepl(info.getPath(), + noteId, + info.getClassName(), + properties, + interpreterGroup.id); + } } else { intp = createRepl(info.getPath(), info.getClassName(), @@ -813,6 +820,26 @@ private Interpreter createRepl(String dirName, String className, } } + private Interpreter connectToRemoteRepl(String noteId, + String className, + String host, + int port, + Properties property) { + int connectTimeout = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT); + int maxPoolSize = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE); + LazyOpenInterpreter intp = new LazyOpenInterpreter( + new RemoteInterpreter( + property, + noteId, + className, + host, + port, + connectTimeout, + maxPoolSize, + remoteInterpreterProcessListener, + appEventListener)); + return intp; + } private Interpreter createRemoteRepl(String interpreterPath, String noteId, String className, Properties property, String interpreterId) { diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java index 29a4748213f..b916eb88379 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java @@ -22,6 +22,8 @@ */ public class InterpreterOption { boolean remote; + String host = null; + int port = -1; boolean perNoteSession; public InterpreterOption() { @@ -47,4 +49,17 @@ public boolean isPerNoteSession() { public void setPerNoteSession(boolean perNoteSession) { this.perNoteSession = perNoteSession; } -} + + + public boolean isConnectExistingProcess() { + return (host != null && port != -1); + } + + public String getHost() { + return host; + } + + public int getPort() { + return port; + } +} \ No newline at end of file From 024d7fc2c52c54a6b5925598330d6326194fa8b4 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Wed, 13 Apr 2016 09:37:40 +0100 Subject: [PATCH 18/69] Dev mode --- .../helium/ClassLoaderApplication.java | 12 ++ .../interpreter/dev/DevInterpreter.java | 117 ++++++++++++++++ .../dev/ZeppelinApplicationDevServer.java | 131 ++++++++++++++++++ .../interpreter/dev/ZeppelinDevServer.java | 128 +++++++++++++++++ .../interpreter/remote/ClientFactory.java | 5 + .../interpreter/remote/RemoteInterpreter.java | 4 +- .../remote/RemoteInterpreterEventPoller.java | 16 ++- .../RemoteInterpreterManagedProcess.java | 6 +- .../remote/RemoteInterpreterProcess.java | 2 +- .../remote/RemoteInterpreterServer.java | 38 +++-- .../zeppelin/socket/NotebookServer.java | 46 +++--- .../paragraph/paragraph.controller.js | 2 + .../interpreter/InterpreterFactory.java | 26 ++++ .../interpreter/InterpreterOption.java | 2 +- .../notebook/NoteInterpreterLoader.java | 6 + 15 files changed, 496 insertions(+), 45 deletions(-) create mode 100644 zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/DevInterpreter.java create mode 100644 zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinApplicationDevServer.java create mode 100644 zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinDevServer.java diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ClassLoaderApplication.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ClassLoaderApplication.java index 04cb6e6e1cb..1603ced5e3d 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ClassLoaderApplication.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ClassLoaderApplication.java @@ -16,6 +16,8 @@ */ package org.apache.zeppelin.helium; +import org.apache.zeppelin.resource.ResourceSet; + /** * Application wrapper */ @@ -67,4 +69,14 @@ public ClassLoader getClassLoader() { public Application getInnerApplication() { return app; } + + @Override + public ResourceSet args() { + return app.args(); + } + + @Override + public ApplicationContext context() { + return app.context(); + } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/DevInterpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/DevInterpreter.java new file mode 100644 index 00000000000..db6a18257b8 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/DevInterpreter.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.interpreter.dev; + +import java.util.LinkedList; +import java.util.List; +import java.util.Properties; + +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContext; +import org.apache.zeppelin.interpreter.InterpreterContextRunner; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder; +import org.apache.zeppelin.interpreter.InterpreterResult; + +/** + * Dummy interpreter to support development mode for Zeppelin app + */ +public class DevInterpreter extends Interpreter { + static { + Interpreter.register( + "dev", + "dev", + DevInterpreter.class.getName(), + new InterpreterPropertyBuilder().build()); + } + + private InterpreterEvent interpreterEvent; + private InterpreterContext context; + + public static boolean isInterpreterName(String replName) { + return replName.equals("dev"); + } + + /** + * event handler for ZeppelinApplicationDevServer + */ + public static interface InterpreterEvent { + public InterpreterResult interpret(String st, InterpreterContext context); + } + + public DevInterpreter(Properties property) { + super(property); + } + + public DevInterpreter(Properties property, InterpreterEvent interpreterEvent) { + super(property); + this.interpreterEvent = interpreterEvent; + } + + @Override + public void open() { + } + + @Override + public void close() { + } + + public void rerun() { + for (InterpreterContextRunner r : context.getRunners()) { + if (context.getParagraphId().equals(r.getParagraphId())) { + r.run(); + } + } + } + + @Override + public InterpreterResult interpret(String st, InterpreterContext context) { + this.context = context; + try { + return interpreterEvent.interpret(st, context); + } catch (Exception e) { + throw new InterpreterException(e); + } + } + + @Override + public void cancel(InterpreterContext context) { + } + + @Override + public FormType getFormType() { + return FormType.NATIVE; + } + + @Override + public int getProgress(InterpreterContext context) { + return 0; + } + + @Override + public List completion(String buf, int cursor) { + return new LinkedList(); + } + + public InterpreterContext getLastInterpretContext() { + return context; + } + + public void setInterpreterEvent(InterpreterEvent event) { + this.interpreterEvent = event; + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinApplicationDevServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinApplicationDevServer.java new file mode 100644 index 00000000000..d749fc42955 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinApplicationDevServer.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.interpreter.dev; + +import java.io.IOException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; + +import org.apache.log4j.ConsoleAppender; +import org.apache.log4j.Level; +import org.apache.log4j.PatternLayout; +import org.apache.zeppelin.helium.*; +import org.apache.zeppelin.interpreter.*; +import org.apache.zeppelin.interpreter.InterpreterResult.Code; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterEventClient; +import org.apache.zeppelin.resource.ResourceSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Run this server for development mode. + */ +public class ZeppelinApplicationDevServer extends ZeppelinDevServer { + final Logger logger = LoggerFactory.getLogger(ZeppelinApplicationDevServer.class); + + private final String className; + private final ResourceSet resourceSet; + private Application app; + private InterpreterOutput out; + + public ZeppelinApplicationDevServer(final String className, ResourceSet resourceSet) throws + Exception { + this(ZeppelinDevServer.DEFAULT_TEST_INTERPRETER_PORT, className, resourceSet); + } + + public ZeppelinApplicationDevServer(int port, String className, ResourceSet resourceSet) throws + Exception { + super(port); + this.className = className; + this.resourceSet = resourceSet; + setLogger(); + }; + + void setLogger() { + ConsoleAppender console = new ConsoleAppender(); //create appender + //configure the appender + String PATTERN = "%d [%p|%c|%C{1}] %m%n"; + console.setLayout(new PatternLayout(PATTERN)); + console.setThreshold(Level.DEBUG); + console.activateOptions(); + //add appender to any Logger (here is root) + org.apache.log4j.Logger.getRootLogger().addAppender(console); + } + + + @Override + public InterpreterResult interpret(String st, InterpreterContext context) { + if (app == null) { + logger.info("Create instance " + className); + try { + Class appClass = ClassLoader.getSystemClassLoader().loadClass(className); + Constructor constructor = appClass.getConstructor( + ResourceSet.class, ApplicationContext.class); + app = (Application) constructor.newInstance(resourceSet, getApplicationContext(context)); + } catch (Exception e) { + logger.error(e.getMessage(), e); + return new InterpreterResult(Code.ERROR, e.getMessage()); + } + } + + try { + logger.info("Run " + className); + app.context().out.setType(InterpreterResult.Type.ANGULAR); + app.run(); + } catch (ApplicationException e) { + logger.error(e.getMessage(), e); + return new InterpreterResult(Code.ERROR, e.getMessage()); + } + return new InterpreterResult(Code.SUCCESS, ""); + } + + ApplicationContext getApplicationContext(InterpreterContext interpreterContext) { + return new ApplicationContext( + interpreterContext.getNoteId(), + interpreterContext.getParagraphId(), + new HeliumAppAngularObjectRegistry( + interpreterContext.getAngularObjectRegistry(), + interpreterContext.getNoteId(), + interpreterContext.getParagraphId()), + interpreterContext.out); + } + + @Override + protected InterpreterOutput createInterpreterOutput( + final String noteId, final String paragraphId) { + if (out == null) { + final RemoteInterpreterEventClient eventClient = getEventClient(); + try { + out = new InterpreterOutput(new InterpreterOutputListener() { + @Override + public void onAppend(InterpreterOutput out, byte[] line) { + eventClient.onInterpreterOutputAppend(noteId, paragraphId, new String(line)); + } + + @Override + public void onUpdate(InterpreterOutput out, byte[] output) { + eventClient.onInterpreterOutputUpdate(noteId, paragraphId, new String(output)); + } + }, this); + } catch (IOException e) { + return null; + } + } + + return out; + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinDevServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinDevServer.java new file mode 100644 index 00000000000..51010b1018b --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinDevServer.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.interpreter.dev; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.Properties; + +import org.apache.thrift.TException; +import org.apache.zeppelin.interpreter.*; +import org.apache.zeppelin.interpreter.dev.DevInterpreter.InterpreterEvent; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterEventClient; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Interpreter development server + */ +public class ZeppelinDevServer extends + RemoteInterpreterServer implements InterpreterEvent, InterpreterOutputChangeListener { + final Logger logger = LoggerFactory.getLogger(ZeppelinDevServer.class); + public static final int DEFAULT_TEST_INTERPRETER_PORT = 29914; + + DevInterpreter interpreter = null; + private InterpreterEvent listener; + InterpreterOutput out; + public ZeppelinDevServer(int port) throws TException { + super(port); + } + + @Override + protected Interpreter getInterpreter(String noteId, String className) throws TException { + synchronized (this) { + InterpreterGroup interpreterGroup = getInterpreterGroup(); + if (interpreterGroup == null) { + createInterpreter( + "dev", + noteId, + DevInterpreter.class.getName(), + new HashMap()); + + Interpreter intp = super.getInterpreter(noteId, className); + interpreter = (DevInterpreter) ( + (ClassloaderInterpreter) ((LazyOpenInterpreter) intp).getInnerInterpreter()) + .getInnerInterpreter(); + interpreter.setInterpreterEvent(this); + notify(); + } + } + return super.getInterpreter(noteId, className); + } + + @Override + protected InterpreterOutput createInterpreterOutput( + final String noteId, final String paragraphId) { + if (out == null) { + final RemoteInterpreterEventClient eventClient = getEventClient(); + try { + out = new InterpreterOutput(new InterpreterOutputListener() { + @Override + public void onAppend(InterpreterOutput out, byte[] line) { + eventClient.onInterpreterOutputAppend(noteId, paragraphId, new String(line)); + } + + @Override + public void onUpdate(InterpreterOutput out, byte[] output) { + eventClient.onInterpreterOutputUpdate(noteId, paragraphId, new String(output)); + } + }, this); + } catch (IOException e) { + return null; + } + } + + out.clear(); + return out; + } + + @Override + public void fileChanged(File file) { + refresh(); + } + + @Override + public InterpreterResult interpret(String st, InterpreterContext context) { + waitForConnected(); + return new InterpreterResult(InterpreterResult.Code.SUCCESS, ""); + } + + public void refresh() { + interpreter.rerun(); + } + + /** + * Wait until %dev paragraph is executed and connected to this process + */ + public void waitForConnected() { + synchronized (this) { + while (!isConnected()) { + try { + this.wait(10 * 1000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + } + + public boolean isConnected() { + return !(interpreter == null || interpreter.getLastInterpretContext() == null); + } +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/ClientFactory.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/ClientFactory.java index 6126f75d3db..809c574cbc8 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/ClientFactory.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/ClientFactory.java @@ -76,4 +76,9 @@ public void destroyObject(PooledObject p) { } } } + + @Override + public boolean validateObject(PooledObject p) { + return p.getObject().getOutputProtocol().getTransport().isOpen(); + } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java index c5b146debef..97fffca31eb 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java @@ -198,7 +198,9 @@ public synchronized void init() { boolean broken = false; try { logger.info("Create remote interpreter {}", getClassName()); - property.put("zeppelin.interpreter.localRepo", localRepoPath); + if (localRepoPath != null) { + property.put("zeppelin.interpreter.localRepo", localRepoPath); + } client.createInterpreter(groupId, noteId, getClassName(), (Map) property); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java index 86b4f690133..9d94d97a8a5 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java @@ -73,7 +73,17 @@ public void setInterpreterGroup(InterpreterGroup interpreterGroup) { public void run() { Client client = null; - while (!shutdown && interpreterProcess.isRunning()) { + while (!shutdown) { + // wait and retry + if (!interpreterProcess.isRunning()) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + // nothing to do + } + continue; + } + try { client = interpreterProcess.getClient(); } catch (Exception e1) { @@ -145,7 +155,7 @@ public void run() { String paragraphId = outputAppend.get("paragraphId"); String outputToAppend = outputAppend.get("data"); String appId = outputAppend.get("appId"); - logger.info("Append " + outputToAppend + ", appId = " + appId); + if (appId == null) { listener.onOutputAppend(noteId, paragraphId, outputToAppend); } else { @@ -159,7 +169,7 @@ public void run() { String paragraphId = outputAppend.get("paragraphId"); String outputToUpdate = outputAppend.get("data"); String appId = outputAppend.get("appId"); - logger.info("Update " + outputToUpdate + ", appId = " + appId); + if (appId == null) { listener.onOutputUpdated(noteId, paragraphId, outputToUpdate); } else { diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java index 648932bea38..098a9d48431 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterManagedProcess.java @@ -30,8 +30,10 @@ /** * This class manages start / stop of remote interpreter process */ -public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess implements ExecuteResultHandler { - private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreterManagedProcess.class); +public class RemoteInterpreterManagedProcess extends RemoteInterpreterProcess + implements ExecuteResultHandler { + private static final Logger logger = LoggerFactory.getLogger( + RemoteInterpreterManagedProcess.class); private final String interpreterRunner; private DefaultExecutor executor; diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java index 6755cdaa6df..024c8adb15e 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcess.java @@ -75,7 +75,7 @@ public int reference(InterpreterGroup interpreterGroup) { if (clientPool == null) { clientPool = new GenericObjectPool(new ClientFactory(getHost(), getPort())); - clientPool.setTestOnReturn(true); + clientPool.setTestOnBorrow(true); remoteInterpreterEventPoller.setInterpreterGroup(interpreterGroup); remoteInterpreterEventPoller.setInterpreterProcess(this); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java index 254c0364d97..41697786e97 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java @@ -75,7 +75,6 @@ public class RemoteInterpreterServer private final Map runningApplications = Collections.synchronizedMap(new HashMap()); - public RemoteInterpreterServer(int port) throws TTransportException { this.port = port; @@ -188,7 +187,19 @@ public void createInterpreter(String interpreterGroupId, String noteId, String } } - private Interpreter getInterpreter(String noteId, String className) throws TException { + protected InterpreterGroup getInterpreterGroup() { + return interpreterGroup; + } + + protected ResourcePool getResourcePool() { + return resourcePool; + } + + protected RemoteInterpreterEventClient getEventClient() { + return eventClient; + } + + protected Interpreter getInterpreter(String noteId, String className) throws TException { if (interpreterGroup == null) { throw new TException( new InterpreterException("Interpreter instance " + className + " not created")); @@ -393,7 +404,7 @@ public void cancel(String noteId, String className, RemoteInterpreterContext int if (job != null) { job.setStatus(Status.ABORT); } else { - intp.cancel(convert(interpreterContext)); + intp.cancel(convert(interpreterContext, null)); } } @@ -402,7 +413,7 @@ public int getProgress(String noteId, String className, RemoteInterpreterContext interpreterContext) throws TException { Interpreter intp = getInterpreter(noteId, className); - return intp.getProgress(convert(interpreterContext)); + return intp.getProgress(convert(interpreterContext, null)); } @@ -420,6 +431,10 @@ public List completion(String noteId, String className, String buf, int } private InterpreterContext convert(RemoteInterpreterContext ric) { + return convert(ric, createInterpreterOutput(ric.getNoteId(), ric.getParagraphId())); + } + + private InterpreterContext convert(RemoteInterpreterContext ric, InterpreterOutput output) { List contextRunners = new LinkedList(); List runners = gson.fromJson(ric.getRunners(), new TypeToken>() { @@ -440,11 +455,12 @@ private InterpreterContext convert(RemoteInterpreterContext ric) { gson.fromJson(ric.getGui(), GUI.class), interpreterGroup.getAngularObjectRegistry(), interpreterGroup.getResourcePool(), - contextRunners, createInterpreterOutput(ric.getNoteId(), ric.getParagraphId())); + contextRunners, output); } - private InterpreterOutput createInterpreterOutput(final String noteId, final String paragraphId) { + protected InterpreterOutput createInterpreterOutput(final String noteId, final String + paragraphId) { return new InterpreterOutput(new InterpreterOutputListener() { @Override public void onAppend(InterpreterOutput out, byte[] line) { @@ -659,9 +675,14 @@ public void resourceResponseGet(String resourceId, ByteBuffer object) throws TEx @Override public List resourcePoolGetAll() throws TException { logger.debug("Request getAll from ZeppelinServer"); + List result = new LinkedList(); + + if (resourcePool == null) { + return result; + } ResourceSet resourceSet = resourcePool.getAll(false); - List result = new LinkedList(); + Gson gson = new Gson(); for (Resource r : resourceSet) { @@ -708,7 +729,7 @@ public void angularRegistryPush(String registryAsString) throws TException { } } - private InterpreterOutput createAppOutput(final String noteId, + protected InterpreterOutput createAppOutput(final String noteId, final String paragraphId, final String appId) { return new InterpreterOutput(new InterpreterOutputListener() { @@ -791,6 +812,7 @@ public RemoteApplicationResult runApplication(String applicationInstanceId) return new RemoteApplicationResult(false, "Application instance does not exists"); } else { try { + app.context().out.clear(); app.run(); return new RemoteApplicationResult(true, ""); } catch (ApplicationException e) { diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 0f0e83bae46..373f3e4a56e 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -659,15 +659,10 @@ private void angularObjectUpdated(NotebookSocket conn, HashSet userAndRo // propagate change to (Remote) AngularObjectRegistry Note note = notebook.getNote(noteId); if (note != null) { - List settings = note.getNoteReplLoader() - .getInterpreterSettings(); - for (InterpreterSetting setting : settings) { - if (setting.getInterpreterGroup() == null) { - continue; - } - if (interpreterGroupId.equals(setting.getInterpreterGroup().getId())) { - AngularObjectRegistry angularObjectRegistry = setting - .getInterpreterGroup().getAngularObjectRegistry(); + Collection interpreterGroups = InterpreterGroup.getAll(); + for (InterpreterGroup interpreterGroup : interpreterGroups) { + if (interpreterGroupId.equals(interpreterGroup.getId())) { + AngularObjectRegistry angularObjectRegistry = interpreterGroup.getAngularObjectRegistry(); // first trying to get local registry ao = angularObjectRegistry.get(varName, noteId, paragraphId); if (ao == null) { @@ -1164,19 +1159,17 @@ public void onUpdate(String interpreterGroupId, AngularObject object) { List intpSettings = note.getNoteReplLoader() .getInterpreterSettings(); - if (intpSettings.isEmpty()) + if (intpSettings.isEmpty()) { continue; - for (InterpreterSetting setting : intpSettings) { - if (setting.getInterpreterGroup().getId().equals(interpreterGroupId)) { - broadcast( - note.id(), - new Message(OP.ANGULAR_OBJECT_UPDATE) - .put("angularObject", object) - .put("interpreterGroupId", interpreterGroupId) - .put("noteId", note.id()) - .put("paragraphId", object.getParagraphId())); - } } + + broadcast( + note.id(), + new Message(OP.ANGULAR_OBJECT_UPDATE) + .put("angularObject", object) + .put("interpreterGroupId", interpreterGroupId) + .put("noteId", note.id()) + .put("paragraphId", object.getParagraphId())); } } @@ -1189,15 +1182,10 @@ public void onRemove(String interpreterGroupId, String name, String noteId, Stri continue; } - List ids = note.getNoteReplLoader().getInterpreters(); - for (String id : ids) { - if (id.equals(interpreterGroupId)) { - broadcast( - note.id(), - new Message(OP.ANGULAR_OBJECT_REMOVE).put("name", name).put( - "noteId", noteId).put("paragraphId", paragraphId)); - } - } + broadcast( + note.id(), + new Message(OP.ANGULAR_OBJECT_REMOVE).put("name", name).put( + "noteId", noteId).put("paragraphId", paragraphId)); } } } diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index 72443faa0eb..c861f63da9b 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -179,11 +179,13 @@ angular.module('zeppelinWebApp') if (!data.paragraphId || data.paragraphId === $scope.paragraph.id) { scope = paragraphScope; registry = angularObjectRegistry; + console.log("paragraph scope"); } else { var app = _.find($scope.apps, { id: data.paragraphId}); if (app) { scope = getAppScope(app); registry = getAppRegistry(app); + console.log("app scope"); } else { // no matching app in this paragraph return; diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index 18ad5ce072c..98c4da526fe 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -31,6 +31,8 @@ import org.apache.zeppelin.display.AngularObjectRegistryListener; import org.apache.zeppelin.helium.ApplicationEventListener; import org.apache.zeppelin.interpreter.Interpreter.RegisteredInterpreter; +import org.apache.zeppelin.interpreter.dev.DevInterpreter; +import org.apache.zeppelin.interpreter.dev.ZeppelinDevServer; import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; import org.apache.zeppelin.interpreter.remote.RemoteInterpreter; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; @@ -80,6 +82,8 @@ public class InterpreterFactory { private Map env = new HashMap(); + private Interpreter devInterpreter; + public InterpreterFactory(ZeppelinConfiguration conf, AngularObjectRegistryListener angularObjectRegistryListener, RemoteInterpreterProcessListener remoteInterpreterProcessListener, @@ -897,4 +901,26 @@ public Map getEnv() { public void setEnv(Map env) { this.env = env; } + + + public Interpreter getDevInterpreter() { + if (devInterpreter == null) { + InterpreterOption option = new InterpreterOption(); + option.setRemote(true); + + InterpreterGroup interpreterGroup = createInterpreterGroup("dev", option); + + devInterpreter = connectToRemoteRepl("dev", DevInterpreter.class.getName(), + "localhost", + ZeppelinDevServer.DEFAULT_TEST_INTERPRETER_PORT, + new Properties()); + + LinkedList intpList = new LinkedList(); + intpList.add(devInterpreter); + interpreterGroup.put("dev", intpList); + + devInterpreter.setInterpreterGroup(interpreterGroup); + } + return devInterpreter; + } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java index b916eb88379..af88f32b5ce 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java @@ -62,4 +62,4 @@ public String getHost() { public int getPort() { return port; } -} \ No newline at end of file +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteInterpreterLoader.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteInterpreterLoader.java index 4e7626b4fd9..c4f055a98c2 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteInterpreterLoader.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteInterpreterLoader.java @@ -27,6 +27,7 @@ import org.apache.zeppelin.interpreter.InterpreterFactory; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterSetting; +import org.apache.zeppelin.interpreter.dev.DevInterpreter; /** * Interpreter loader per note. @@ -183,6 +184,11 @@ public Interpreter get(String replName) { } } + // dev interpreter + if (DevInterpreter.isInterpreterName(replName)) { + return factory.getDevInterpreter(); + } + throw new InterpreterException(replName + " interpreter not found"); } } From 71f814dce41113f71fdcfff4c7d468604cd35f4b Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Wed, 13 Apr 2016 13:29:34 +0100 Subject: [PATCH 19/69] Better way to find resource dir for InterpreterOutput watcher --- .../interpreter/InterpreterOutput.java | 41 +++++++++++-------- .../dev/ZeppelinApplicationDevServer.java | 8 ++++ .../interpreter/dev/ZeppelinDevServer.java | 1 - 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutput.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutput.java index b28386c7726..a5ec3f65387 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutput.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutput.java @@ -22,8 +22,10 @@ import java.io.*; import java.net.URL; +import java.util.Collections; import java.util.LinkedList; import java.util.List; +import java.util.concurrent.ConcurrentHashMap; /** * InterpreterOutput is OutputStream that supposed to print content on notebook @@ -32,6 +34,7 @@ public class InterpreterOutput extends OutputStream { Logger logger = LoggerFactory.getLogger(InterpreterOutput.class); private final int NEW_LINE_CHAR = '\n'; + private List resourceSearchPaths = Collections.synchronizedList(new LinkedList()); ByteArrayOutputStream buffer = new ByteArrayOutputStream(); @@ -157,26 +160,32 @@ public void write(URL url) throws IOException { } } + public void addResourceSearchPath(String path) { + resourceSearchPaths.add(path); + } + public void writeResource(String resourceName) throws IOException { - // search file under resource dir first for dev mode - File mainResource = new File("./src/main/resources/" + resourceName); - File testResource = new File("./src/test/resources/" + resourceName); - if (mainResource.isFile()) { - write(mainResource); - } else if (testResource.isFile()) { - write(testResource); - } else { - // search from classpath - ClassLoader cl = Thread.currentThread().getContextClassLoader(); - if (cl == null) { - cl = this.getClass().getClassLoader(); - } - if (cl == null) { - cl = ClassLoader.getSystemClassLoader(); + // search file under provided paths first, for dev mode + for (String path : resourceSearchPaths) { + File res = new File(path + "/" + resourceName); + logger.info("path = " + path + ", res = " + resourceName); + logger.info("Search resource " + res.getAbsolutePath()); + if (res.isFile()) { + write(res); + return; } + } - write(cl.getResource(resourceName)); + // search from classpath + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + if (cl == null) { + cl = this.getClass().getClassLoader(); + } + if (cl == null) { + cl = ClassLoader.getSystemClassLoader(); } + + write(cl.getResource(resourceName)); } public byte[] toByteArray() throws IOException { diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinApplicationDevServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinApplicationDevServer.java index d749fc42955..edb980fa546 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinApplicationDevServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinApplicationDevServer.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; +import java.net.URL; import org.apache.log4j.ConsoleAppender; import org.apache.log4j.Level; @@ -75,6 +76,13 @@ public InterpreterResult interpret(String st, InterpreterContext context) { Class appClass = ClassLoader.getSystemClassLoader().loadClass(className); Constructor constructor = appClass.getConstructor( ResourceSet.class, ApplicationContext.class); + + // classPath will be ..../target/classes in dev mode most cases + String classPath = appClass.getProtectionDomain().getCodeSource().getLocation().getPath(); + + context.out.addResourceSearchPath(classPath + "../../src/main/resources/"); + context.out.addResourceSearchPath(classPath + "../../src/test/resources/"); + app = (Application) constructor.newInstance(resourceSet, getApplicationContext(context)); } catch (Exception e) { logger.error(e.getMessage(), e); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinDevServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinDevServer.java index 51010b1018b..c887f86a626 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinDevServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinDevServer.java @@ -38,7 +38,6 @@ public class ZeppelinDevServer extends public static final int DEFAULT_TEST_INTERPRETER_PORT = 29914; DevInterpreter interpreter = null; - private InterpreterEvent listener; InterpreterOutput out; public ZeppelinDevServer(int port) throws TException { super(port); From f2ab95dcd37e2968c9c7c75f595d32bc2ef56dee Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Wed, 13 Apr 2016 23:22:45 +0100 Subject: [PATCH 20/69] Prevent unnecessary output update --- .../remote/RemoteInterpreterServer.java | 1 - .../apache/zeppelin/notebook/Paragraph.java | 66 ++++++++++--------- 2 files changed, 35 insertions(+), 32 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java index 41697786e97..538a8446ef0 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java @@ -356,7 +356,6 @@ protected Object jobRun() throws Throwable { context.out.flush(); InterpreterResult.Type outputType = context.out.getType(); byte[] interpreterOutput = context.out.toByteArray(); - context.out.clear(); if (interpreterOutput != null && interpreterOutput.length > 0) { message = new String(interpreterOutput); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java index 0ef4b4c5e0d..8976f375214 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java @@ -212,7 +212,7 @@ public int progress() { String replName = getRequiredReplName(); Interpreter repl = getRepl(replName); if (repl != null) { - return repl.getProgress(getInterpreterContext()); + return repl.getProgress(getInterpreterContext(null)); } else { return 0; } @@ -259,7 +259,6 @@ protected Object jobRun() throws Throwable { context.out.flush(); InterpreterResult.Type outputType = context.out.getType(); byte[] interpreterOutput = context.out.toByteArray(); - context.out.clear(); if (interpreterOutput != null && interpreterOutput.length > 0) { message = new String(interpreterOutput); @@ -298,12 +297,44 @@ protected boolean jobAbort() { if (job != null) { job.setStatus(Status.ABORT); } else { - repl.cancel(getInterpreterContext()); + repl.cancel(getInterpreterContext(null)); } return true; } private InterpreterContext getInterpreterContext() { + final Paragraph self = this; + + return getInterpreterContext(new InterpreterOutput(new InterpreterOutputListener() { + @Override + public void onAppend(InterpreterOutput out, byte[] line) { + updateParagraphResult(out); + ((ParagraphJobListener) getListener()).onOutputAppend(self, out, new String(line)); + } + + @Override + public void onUpdate(InterpreterOutput out, byte[] output) { + updateParagraphResult(out); + ((ParagraphJobListener) getListener()).onOutputUpdate(self, out, + new String(output)); + } + + private void updateParagraphResult(InterpreterOutput out) { + // update paragraph result + Throwable t = null; + String message = null; + try { + message = new String(out.toByteArray()); + } catch (IOException e) { + logger().error(e.getMessage(), e); + t = e; + } + setReturn(new InterpreterResult(Code.SUCCESS, out.getType(), message), t); + } + })); + } + + private InterpreterContext getInterpreterContext(InterpreterOutput output) { AngularObjectRegistry registry = null; ResourcePool resourcePool = null; @@ -318,7 +349,6 @@ private InterpreterContext getInterpreterContext() { runners.add(new ParagraphRunner(note, note.id(), p.getId())); } - final Paragraph self = this; InterpreterContext interpreterContext = new InterpreterContext( note.id(), getId(), @@ -330,33 +360,7 @@ private InterpreterContext getInterpreterContext() { registry, resourcePool, runners, - new InterpreterOutput(new InterpreterOutputListener() { - @Override - public void onAppend(InterpreterOutput out, byte[] line) { - updateParagraphResult(out); - ((ParagraphJobListener) getListener()).onOutputAppend(self, out, new String(line)); - } - - @Override - public void onUpdate(InterpreterOutput out, byte[] output) { - updateParagraphResult(out); - ((ParagraphJobListener) getListener()).onOutputUpdate(self, out, - new String(output)); - } - - private void updateParagraphResult(InterpreterOutput out) { - // update paragraph result - Throwable t = null; - String message = null; - try { - message = new String(out.toByteArray()); - } catch (IOException e) { - logger().error(e.getMessage(), e); - t = e; - } - setReturn(new InterpreterResult(Code.SUCCESS, out.getType(), message), t); - } - })); + output); return interpreterContext; } From b4ff52fbcf7f0d7dbeac3d56f8ab4f4c283cdeba Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Thu, 14 Apr 2016 01:05:54 +0100 Subject: [PATCH 21/69] Put last value of scala repl into resource pool --- .../zeppelin/spark/SparkInterpreter.java | 35 ++++++++++++++++--- .../resource/WellKnownResourceName.java | 1 + 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 5bd50cea576..a4540d3274e 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -51,6 +51,8 @@ import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.InterpreterUtils; import org.apache.zeppelin.interpreter.WrappedInterpreter; +import org.apache.zeppelin.resource.ResourcePool; +import org.apache.zeppelin.resource.WellKnownResourceName; import org.apache.zeppelin.scheduler.Scheduler; import org.apache.zeppelin.scheduler.SchedulerFactory; import org.apache.zeppelin.spark.dep.SparkDependencyContext; @@ -545,7 +547,9 @@ public void open() { binder.put("sc", sc); binder.put("sqlc", sqlc); binder.put("z", z); - + binder.put("intp", intp); + intp.interpret("@transient val intp = _binder.get(\"intp\").asInstanceOf[org.apache.spark" + + ".repl.SparkIMain]"); intp.interpret("@transient val z = " + "_binder.get(\"z\").asInstanceOf[org.apache.zeppelin.spark.ZeppelinContext]"); intp.interpret("@transient val sc = " @@ -761,13 +765,10 @@ public InterpreterResult interpret(String[] lines, InterpreterContext context) { public InterpreterResult interpretInput(String[] lines, InterpreterContext context) { SparkEnv.set(env); - // add print("") to make sure not finishing with comment - // see https://github.com/NFLabs/zeppelin/issues/151 - String[] linesToRun = new String[lines.length + 1]; + String[] linesToRun = new String[lines.length]; for (int i = 0; i < lines.length; i++) { linesToRun[i] = lines[i]; } - linesToRun[lines.length] = "print(\"\")"; Console.setOut(context.out); out.setInterpreterOutput(context.out); @@ -809,17 +810,41 @@ public InterpreterResult interpretInput(String[] lines, InterpreterContext conte } } + // make sure code does not finish with comment + if (r == Code.INCOMPLETE) { + scala.tools.nsc.interpreter.Results.Result res; + res = intp.interpret(incomplete + "\nprint(\"\")"); + r = getResultCode(res); + } + if (r == Code.INCOMPLETE) { sc.clearJobGroup(); out.setInterpreterOutput(null); return new InterpreterResult(r, "Incomplete expression"); } else { sc.clearJobGroup(); + putLatestVarInResourcePool(context); out.setInterpreterOutput(null); return new InterpreterResult(Code.SUCCESS); } } + private void putLatestVarInResourcePool(InterpreterContext context) { + String varName = intp.mostRecentVar(); + if (varName == null || varName.isEmpty()) { + return; + } + + Object lastObj = getValue(varName); + if (lastObj != null) { + logger.info("Put LastValue {} {} {} into ResourcePool", + varName, lastObj, lastObj.getClass().getName()); + ResourcePool resourcePool = context.getResourcePool(); + resourcePool.put(context.getNoteId(), context.getParagraphId(), + WellKnownResourceName.ZeppelinReplResult.toString(), lastObj); + } + }; + @Override public void cancel(InterpreterContext context) { diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/WellKnownResourceName.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/WellKnownResourceName.java index 646df2f49f1..4613c62a30a 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/WellKnownResourceName.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/WellKnownResourceName.java @@ -20,6 +20,7 @@ * Well known resource names in ResourcePool */ public enum WellKnownResourceName { + ZeppelinReplResult("zeppelin.repl.result"), // last object of repl ZeppelinTableResult("zeppelin.paragraph.result.table"); // paragraph run result String name; From ec2fdea60bcd7e10438e2cc8d888d6995a2151f8 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Thu, 14 Apr 2016 01:26:33 +0100 Subject: [PATCH 22/69] Match classname correctly --- .../main/java/org/apache/zeppelin/helium/ApplicationLoader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java index 407e099488e..92f686fc354 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java @@ -169,7 +169,7 @@ static ResourceSet findRequiredResourceSet(String [][] requiredResources, boolean found = false; for (Resource r : resources) { - if (require.startsWith(":") && r.getClassName().equals(require)) { + if (require.startsWith(":") && r.getClassName().equals(require.substring(1))) { found = true; } else if (r.getResourceId().getName().equals(require)) { found = true; From 0e4d81c841791ea4a4bd580478e57051d61a8b10 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Thu, 14 Apr 2016 01:26:51 +0100 Subject: [PATCH 23/69] Remove unnecessary log --- zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js index c861f63da9b..72443faa0eb 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph.controller.js @@ -179,13 +179,11 @@ angular.module('zeppelinWebApp') if (!data.paragraphId || data.paragraphId === $scope.paragraph.id) { scope = paragraphScope; registry = angularObjectRegistry; - console.log("paragraph scope"); } else { var app = _.find($scope.apps, { id: data.paragraphId}); if (app) { scope = getAppScope(app); registry = getAppRegistry(app); - console.log("app scope"); } else { // no matching app in this paragraph return; From fade3c1f0f04603d28ce594a672f35e9558b7499 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Thu, 14 Apr 2016 03:48:10 +0100 Subject: [PATCH 24/69] Handle output update in angular mode --- .../org/apache/zeppelin/spark/SparkInterpreter.java | 2 -- .../apache/zeppelin/interpreter/InterpreterOutput.java | 9 ++------- .../zeppelin/interpreter/dev/DevInterpreter.java | 4 ++++ .../interpreter/dev/ZeppelinApplicationDevServer.java | 1 + .../interpreter/remote/RemoteInterpreterServer.java | 10 +++++++++- 5 files changed, 16 insertions(+), 10 deletions(-) diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index a4540d3274e..91accc5ff4f 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -837,8 +837,6 @@ private void putLatestVarInResourcePool(InterpreterContext context) { Object lastObj = getValue(varName); if (lastObj != null) { - logger.info("Put LastValue {} {} {} into ResourcePool", - varName, lastObj, lastObj.getClass().getName()); ResourcePool resourcePool = context.getResourcePool(); resourcePool.put(context.getNoteId(), context.getParagraphId(), WellKnownResourceName.ZeppelinReplResult.toString(), lastObj); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutput.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutput.java index a5ec3f65387..df0c210b8d9 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutput.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOutput.java @@ -21,6 +21,7 @@ import org.slf4j.LoggerFactory; import java.io.*; +import java.net.URISyntaxException; import java.net.URL; import java.util.Collections; import java.util.LinkedList; @@ -153,11 +154,7 @@ public void write(String string) throws IOException { * @throws IOException */ public void write(URL url) throws IOException { - if ("file".equals(url.getProtocol())) { - write(new File(url.getPath())); - } else { - outList.add(url); - } + outList.add(url); } public void addResourceSearchPath(String path) { @@ -168,8 +165,6 @@ public void writeResource(String resourceName) throws IOException { // search file under provided paths first, for dev mode for (String path : resourceSearchPaths) { File res = new File(path + "/" + resourceName); - logger.info("path = " + path + ", res = " + resourceName); - logger.info("Search resource " + res.getAbsolutePath()); if (res.isFile()) { write(res); return; diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/DevInterpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/DevInterpreter.java index db6a18257b8..86fd1aff006 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/DevInterpreter.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/DevInterpreter.java @@ -114,4 +114,8 @@ public InterpreterContext getLastInterpretContext() { public void setInterpreterEvent(InterpreterEvent event) { this.interpreterEvent = event; } + + public InterpreterEvent getInterpreterEvent() { + return interpreterEvent; + } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinApplicationDevServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinApplicationDevServer.java index edb980fa546..dfb12340005 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinApplicationDevServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinApplicationDevServer.java @@ -92,6 +92,7 @@ public InterpreterResult interpret(String st, InterpreterContext context) { try { logger.info("Run " + className); + app.context().out.clear(); app.context().out.setType(InterpreterResult.Type.ANGULAR); app.run(); } catch (ApplicationException e) { diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java index 538a8446ef0..1d768ab752e 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java @@ -812,9 +812,17 @@ public RemoteApplicationResult runApplication(String applicationInstanceId) } else { try { app.context().out.clear(); + app.context().out.setType(InterpreterResult.Type.ANGULAR); app.run(); + String output = new String(app.context().out.toByteArray()); + logger.info("Update app output " + output); + eventClient.onAppOutputUpdate( + app.context().getNoteId(), + app.context().getParagraphId(), + applicationInstanceId, + output); return new RemoteApplicationResult(true, ""); - } catch (ApplicationException e) { + } catch (ApplicationException | IOException e) { return new RemoteApplicationResult(false, e.getMessage()); } } From 03be3a12056ae96e47de128d32a00860ac0786a3 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Thu, 14 Apr 2016 05:24:43 +0100 Subject: [PATCH 25/69] Pass required resource to run() method --- .../apache/zeppelin/helium/Application.java | 14 +++--- .../zeppelin/helium/ApplicationLoader.java | 7 ++- .../helium/ClassLoaderApplication.java | 16 ++---- .../dev/ZeppelinApplicationDevServer.java | 10 ++-- .../remote/RemoteInterpreterServer.java | 50 +++++++++++++------ .../zeppelin/helium/MockApplication1.java | 6 +-- .../helium/HeliumTestApplication.java | 7 ++- 7 files changed, 58 insertions(+), 52 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/Application.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/Application.java index bee1397bab8..59506b7624c 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/Application.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/Application.java @@ -19,22 +19,19 @@ import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.resource.ResourceSet; +import java.io.IOException; + /** * Zeppelin Application base */ public abstract class Application { - private final ResourceSet args; + private final ApplicationContext context; - public Application(ResourceSet args, ApplicationContext context) throws ApplicationException { - this.args = args; + public Application(ApplicationContext context) { this.context = context; } - public ResourceSet args() { - return args; - } - public ApplicationContext context() { return context; } @@ -43,7 +40,8 @@ public ApplicationContext context() { * This method can be invoked multiple times before unload(), * Either just after application selected or when paragraph re-run after application load */ - public abstract void run() throws ApplicationException; + public abstract void run(ResourceSet args) + throws ApplicationException, IOException; /** diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java index 92f686fc354..be36ad96fec 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationLoader.java @@ -123,10 +123,9 @@ public Application load(HeliumPackage packageInfo, ApplicationContext context) ClassLoader cl = appClass.getClassLoader(); Thread.currentThread().setContextClassLoader(cl); try { - Constructor constructor = - appClass.getConstructor(ResourceSet.class, ApplicationContext.class); + Constructor constructor = appClass.getConstructor(ApplicationContext.class); - Application app = new ClassLoaderApplication(constructor.newInstance(resources, context), cl); + Application app = new ClassLoaderApplication(constructor.newInstance(context), cl); return app; } catch (Exception e) { throw new ApplicationException(e); @@ -135,7 +134,7 @@ public Application load(HeliumPackage packageInfo, ApplicationContext context) } } - private ResourceSet findRequiredResourceSet( + public ResourceSet findRequiredResourceSet( String [][] requiredResources, String noteId, String paragraphId) { if (requiredResources == null || requiredResources.length == 0) { return new ResourceSet(); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ClassLoaderApplication.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ClassLoaderApplication.java index 1603ced5e3d..272a152448e 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ClassLoaderApplication.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ClassLoaderApplication.java @@ -25,18 +25,18 @@ public class ClassLoaderApplication extends Application { Application app; ClassLoader cl; public ClassLoaderApplication(Application app, ClassLoader cl) throws ApplicationException { - super(null, null); + super(app.context()); this.app = app; this.cl = cl; } @Override - public void run() throws ApplicationException { + public void run(ResourceSet args) throws ApplicationException { // instantiate ClassLoader oldcl = Thread.currentThread().getContextClassLoader(); Thread.currentThread().setContextClassLoader(cl); try { - app.run(); + app.run(args); } catch (ApplicationException e) { throw e; } catch (Exception e) { @@ -69,14 +69,4 @@ public ClassLoader getClassLoader() { public Application getInnerApplication() { return app; } - - @Override - public ResourceSet args() { - return app.args(); - } - - @Override - public ApplicationContext context() { - return app.context(); - } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinApplicationDevServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinApplicationDevServer.java index dfb12340005..e8f67d04600 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinApplicationDevServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinApplicationDevServer.java @@ -74,8 +74,7 @@ public InterpreterResult interpret(String st, InterpreterContext context) { logger.info("Create instance " + className); try { Class appClass = ClassLoader.getSystemClassLoader().loadClass(className); - Constructor constructor = appClass.getConstructor( - ResourceSet.class, ApplicationContext.class); + Constructor constructor = appClass.getConstructor(ApplicationContext.class); // classPath will be ..../target/classes in dev mode most cases String classPath = appClass.getProtectionDomain().getCodeSource().getLocation().getPath(); @@ -83,7 +82,8 @@ public InterpreterResult interpret(String st, InterpreterContext context) { context.out.addResourceSearchPath(classPath + "../../src/main/resources/"); context.out.addResourceSearchPath(classPath + "../../src/test/resources/"); - app = (Application) constructor.newInstance(resourceSet, getApplicationContext(context)); + ApplicationContext appContext = getApplicationContext(context); + app = (Application) constructor.newInstance(appContext); } catch (Exception e) { logger.error(e.getMessage(), e); return new InterpreterResult(Code.ERROR, e.getMessage()); @@ -94,8 +94,8 @@ public InterpreterResult interpret(String st, InterpreterContext context) { logger.info("Run " + className); app.context().out.clear(); app.context().out.setType(InterpreterResult.Type.ANGULAR); - app.run(); - } catch (ApplicationException e) { + app.run(resourceSet); + } catch (IOException | ApplicationException e) { logger.error(e.getMessage(), e); return new InterpreterResult(Code.ERROR, e.getMessage()); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java index 1d768ab752e..59c62436640 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java @@ -72,8 +72,8 @@ public class RemoteInterpreterServer RemoteInterpreterEventClient eventClient = new RemoteInterpreterEventClient(); private DependencyResolver depLoader; - private final Map runningApplications = - Collections.synchronizedMap(new HashMap()); + private final Map runningApplications = + Collections.synchronizedMap(new HashMap()); public RemoteInterpreterServer(int port) throws TTransportException { this.port = port; @@ -776,7 +776,7 @@ public RemoteApplicationResult loadApplication( noteId, paragraphId); app = appLoader.load(pkgInfo, context); - runningApplications.put(applicationInstanceId, app); + runningApplications.put(applicationInstanceId, new RunningApplication(pkgInfo, app)); return new RemoteApplicationResult(true, ""); } catch (Exception e) { logger.error(e.getMessage(), e); @@ -787,11 +787,11 @@ public RemoteApplicationResult loadApplication( @Override public RemoteApplicationResult unloadApplication(String applicationInstanceId) throws TException { - Application app = runningApplications.remove(applicationInstanceId); - if (app != null) { + RunningApplication runningApplication = runningApplications.remove(applicationInstanceId); + if (runningApplication != null) { try { logger.info("Unloading application {}", applicationInstanceId); - app.unload(); + runningApplication.app.unload(); } catch (ApplicationException e) { logger.error(e.getMessage(), e); return new RemoteApplicationResult(false, e.getMessage()); @@ -805,20 +805,27 @@ public RemoteApplicationResult runApplication(String applicationInstanceId) throws TException { logger.info("run application {}", applicationInstanceId); - Application app = runningApplications.get(applicationInstanceId); - if (app == null) { + RunningApplication runningApp = runningApplications.get(applicationInstanceId); + if (runningApp == null) { logger.error("Application instance {} not exists", applicationInstanceId); return new RemoteApplicationResult(false, "Application instance does not exists"); } else { + ApplicationContext context = runningApp.app.context(); try { - app.context().out.clear(); - app.context().out.setType(InterpreterResult.Type.ANGULAR); - app.run(); - String output = new String(app.context().out.toByteArray()); - logger.info("Update app output " + output); + context.out.clear(); + context.out.setType(InterpreterResult.Type.ANGULAR); + ResourceSet resource = appLoader.findRequiredResourceSet( + runningApp.pkg.getResources(), + context.getNoteId(), + context.getParagraphId()); + for (Resource res : resource) { + System.err.println("Resource " + res.get()); + } + runningApp.app.run(resource); + String output = new String(context.out.toByteArray()); eventClient.onAppOutputUpdate( - app.context().getNoteId(), - app.context().getParagraphId(), + context.getNoteId(), + context.getParagraphId(), applicationInstanceId, output); return new RemoteApplicationResult(true, ""); @@ -826,5 +833,18 @@ public RemoteApplicationResult runApplication(String applicationInstanceId) return new RemoteApplicationResult(false, e.getMessage()); } } + + + } + + private static class RunningApplication { + public final Application app; + public final HeliumPackage pkg; + + public RunningApplication(HeliumPackage pkg, Application app) { + this.app = app; + this.pkg = pkg; + } + }; } diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/MockApplication1.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/MockApplication1.java index 90cf63a7c68..df3afeffc10 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/MockApplication1.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/MockApplication1.java @@ -26,14 +26,14 @@ public class MockApplication1 extends Application { boolean unloaded; int run; - public MockApplication1(ResourceSet args, ApplicationContext context) throws ApplicationException { - super(args, context); + public MockApplication1(ApplicationContext context) { + super(context); unloaded = false; run = 0; } @Override - public void run() { + public void run(ResourceSet args) { run++; } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTestApplication.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTestApplication.java index e41c85aa385..8b1d14ff17b 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTestApplication.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumTestApplication.java @@ -23,13 +23,12 @@ public class HeliumTestApplication extends Application { AtomicInteger numRun = new AtomicInteger(0); - public HeliumTestApplication(ResourceSet args, ApplicationContext context) - throws ApplicationException { - super(args, context); + public HeliumTestApplication(ApplicationContext context) { + super(context); } @Override - public void run() throws ApplicationException { + public void run(ResourceSet args) throws ApplicationException { try { context().out.clear(); context().out.write("Hello world " + numRun.incrementAndGet()); From c30f53c7bb77c84c9fc07bf72674811bae04a064 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Thu, 14 Apr 2016 06:03:35 +0100 Subject: [PATCH 26/69] null check --- .../org/apache/zeppelin/notebook/Note.java | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java index ea62e04dc01..b4f3b7281df 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java @@ -537,7 +537,9 @@ public void setInfo(Map info) { public void beforeStatusChange(Job job, Status before, Status after) { if (jobListenerFactory != null) { ParagraphJobListener listener = jobListenerFactory.getParagraphJobListener(this); - listener.beforeStatusChange(job, before, after); + if (listener != null) { + listener.beforeStatusChange(job, before, after); + } } } @@ -545,7 +547,9 @@ public void beforeStatusChange(Job job, Status before, Status after) { public void afterStatusChange(Job job, Status before, Status after) { if (jobListenerFactory != null) { ParagraphJobListener listener = jobListenerFactory.getParagraphJobListener(this); - listener.afterStatusChange(job, before, after); + if (listener != null) { + listener.afterStatusChange(job, before, after); + } } if (noteEventListener != null) { @@ -557,7 +561,9 @@ public void afterStatusChange(Job job, Status before, Status after) { public void onProgressUpdate(Job job, int progress) { if (jobListenerFactory != null) { ParagraphJobListener listener = jobListenerFactory.getParagraphJobListener(this); - listener.onProgressUpdate(job, progress); + if (listener != null) { + listener.onProgressUpdate(job, progress); + } } } @@ -566,7 +572,9 @@ public void onProgressUpdate(Job job, int progress) { public void onOutputAppend(Paragraph paragraph, InterpreterOutput out, String output) { if (jobListenerFactory != null) { ParagraphJobListener listener = jobListenerFactory.getParagraphJobListener(this); - listener.onOutputAppend(paragraph, out, output); + if (listener != null) { + listener.onOutputAppend(paragraph, out, output); + } } } @@ -574,7 +582,9 @@ public void onOutputAppend(Paragraph paragraph, InterpreterOutput out, String ou public void onOutputUpdate(Paragraph paragraph, InterpreterOutput out, String output) { if (jobListenerFactory != null) { ParagraphJobListener listener = jobListenerFactory.getParagraphJobListener(this); - listener.onOutputUpdate(paragraph, out, output); + if (listener != null) { + listener.onOutputUpdate(paragraph, out, output); + } } } From 1f1a3b5c4848df16c90e2faa32a3ce29cf789648 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sun, 17 Apr 2016 05:40:30 +0100 Subject: [PATCH 27/69] Fix test --- .../helium/HeliumApplicationFactoryTest.java | 6 ++++-- .../interpreter/InterpreterFactoryTest.java | 16 ++-------------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java index 61495dc9070..9a882757cc7 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java @@ -63,11 +63,11 @@ public void setUp() throws Exception { confDir.mkdirs(); notebookDir = new File(tmpDir + "/notebook"); notebookDir.mkdirs(); - System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_CONF_DIR.getVarName(), confDir.getAbsolutePath()); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_CONF_DIR.getVarName(), tmpDir.getAbsolutePath() + "/conf"); System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath()); System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETERS.getVarName(), "org.apache.zeppelin.interpreter.mock.MockInterpreter1,org.apache.zeppelin.interpreter.mock.MockInterpreter2"); - conf = ZeppelinConfiguration.create(); + conf = new ZeppelinConfiguration(); this.schedulerFactory = new SchedulerFactory(); @@ -104,6 +104,8 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { FileUtils.deleteDirectory(tmpDir); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_CONF_DIR.getVarName(), + ZeppelinConfiguration.ConfVars.ZEPPELIN_CONF_DIR.getStringValue()); } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java index 49c49337aec..bdded066266 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.Properties; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang.NullArgumentException; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; @@ -67,20 +68,7 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { - delete(tmpDir); - } - - private void delete(File file){ - if(file.isFile()) file.delete(); - else if(file.isDirectory()){ - File [] files = file.listFiles(); - if(files!=null && files.length>0){ - for(File f : files){ - delete(f); - } - } - file.delete(); - } + FileUtils.deleteDirectory(tmpDir); } @Test From 01a16468614c7c52b00b4d04034caf0295e51793 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sun, 17 Apr 2016 08:33:29 +0100 Subject: [PATCH 28/69] Add example --- README.md | 8 + pom.xml | 7 + zeppelin-examples/pom.xml | 64 +++++++ .../zeppelin-example-clock/pom.xml | 170 ++++++++++++++++++ .../zeppelin/example/app/clock/Clock.java | 113 ++++++++++++ .../resources/example/app/clock/clock.html | 14 ++ .../zeppelin-example-clock.json | 9 + .../zeppelin/dep/DependencyResolver.java | 28 +-- 8 files changed, 403 insertions(+), 10 deletions(-) create mode 100644 zeppelin-examples/pom.xml create mode 100644 zeppelin-examples/zeppelin-example-clock/pom.xml create mode 100644 zeppelin-examples/zeppelin-example-clock/src/main/java/org/apache/zeppelin/example/app/clock/Clock.java create mode 100644 zeppelin-examples/zeppelin-example-clock/src/main/resources/example/app/clock/clock.html create mode 100644 zeppelin-examples/zeppelin-example-clock/zeppelin-example-clock.json diff --git a/README.md b/README.md index 20b2669094c..c59611e9d15 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,13 @@ Available profiles are -Pmapr51 ``` +#### -Pexamples (optional) + +Bulid examples under zeppelin-examples directory + + + + Here're some examples: @@ -162,6 +169,7 @@ mvn clean package -Dignite.version=1.1.0-incubating -DskipTests mvn clean package -Pscalding -DskipTests ``` + ### Configure If you wish to configure Zeppelin option (like port number), configure the following files: diff --git a/pom.xml b/pom.xml index c7d93126915..ace398973db 100755 --- a/pom.xml +++ b/pom.xml @@ -706,6 +706,13 @@ + + examples + + zeppelin-examples + + + build-distr diff --git a/zeppelin-examples/pom.xml b/zeppelin-examples/pom.xml new file mode 100644 index 00000000000..f29e2934219 --- /dev/null +++ b/zeppelin-examples/pom.xml @@ -0,0 +1,64 @@ + + + + + 4.0.0 + + + zeppelin + org.apache.zeppelin + 0.6.0-incubating-SNAPSHOT + .. + + + org.apache.zeppelin + zeppelin-examples + pom + 0.6.0-incubating-SNAPSHOT + Zeppelin: Examples + Zeppelin examples + http://zeppelin.incubator.apache.org + + + zeppelin-example-clock + + + + + + org.apache.maven.plugins + maven-deploy-plugin + 2.7 + + true + + + + + maven-enforcer-plugin + 1.3.1 + + + enforce + none + + + + + + diff --git a/zeppelin-examples/zeppelin-example-clock/pom.xml b/zeppelin-examples/zeppelin-example-clock/pom.xml new file mode 100644 index 00000000000..9a871c0b183 --- /dev/null +++ b/zeppelin-examples/zeppelin-example-clock/pom.xml @@ -0,0 +1,170 @@ + + + + + 4.0.0 + + + zeppelin-examples + org.apache.zeppelin + 0.6.0-incubating-SNAPSHOT + .. + + + org.apache.zeppelin + zeppelin-example-clock + jar + 0.6.0-incubating-SNAPSHOT + Zeppelin: Example application - Clock + http://zeppelin.incubator.apache.org + + + + ${project.groupId} + zeppelin-interpreter + ${project.version} + + + + org.slf4j + slf4j-api + + + + org.slf4j + slf4j-log4j12 + + + + junit + junit + test + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + 2.7 + + true + + + + + + + maven-clean-plugin + + + + + ${project.basedir}/../../helium + + ${project.artifactId}.json + + + + + + + + + + maven-resources-plugin + 2.7 + + + generate-resources + + copy-resources + + + + ${project.basedir}/../../helium/ + + + ${project.basedir} + + ${project.artifactId}.json + + + + + + + + + + + + diff --git a/zeppelin-examples/zeppelin-example-clock/src/main/java/org/apache/zeppelin/example/app/clock/Clock.java b/zeppelin-examples/zeppelin-example-clock/src/main/java/org/apache/zeppelin/example/app/clock/Clock.java new file mode 100644 index 00000000000..632b35ff251 --- /dev/null +++ b/zeppelin-examples/zeppelin-example-clock/src/main/java/org/apache/zeppelin/example/app/clock/Clock.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.example.app.clock; + +import org.apache.zeppelin.helium.Application; +import org.apache.zeppelin.helium.ApplicationContext; +import org.apache.zeppelin.helium.ApplicationException; +import org.apache.zeppelin.interpreter.dev.ZeppelinApplicationDevServer; +import org.apache.zeppelin.resource.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.Date; + +/** + * Basic example application. + * Get java.util.Date from resource pool and display it + */ +public class Clock extends Application { + private final Logger logger = LoggerFactory.getLogger(Clock.class); + + Date date; + boolean shutdown = false; + private Thread updateThread; + + public Clock(ApplicationContext context) { + super(context); + } + + @Override + public void run(ResourceSet resources) throws ApplicationException { + // Get data from resource args + date = (Date) resources.get(0).get(); + + // print view template + try { + context().out.writeResource("example/app/clock/clock.html"); + } catch (IOException e) { + throw new ApplicationException(e); + } + + if (updateThread == null) { + start(); + } + } + + + public void start() { + updateThread = new Thread() { + public void run() { + while (!shutdown) { + // format date + SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + + // put formatted string to angular object. + context().getAngularObjectRegistry().add("date", df.format(date)); + + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + // nothing todo + } + date = new Date(date.getTime() + 1000); + } + } + }; + + updateThread.start(); + } + + + @Override + public void unload() throws ApplicationException { + shutdown = true; + try { + updateThread.join(); + } catch (InterruptedException e) { + // nothing to do + } + context().getAngularObjectRegistry().remove("date"); + } + + /** + * Development mode + */ + public static void main(String[] args) throws Exception { + LocalResourcePool pool = new LocalResourcePool("dev"); + pool.put("date", new Date()); + + ZeppelinApplicationDevServer devServer = new ZeppelinApplicationDevServer( + Clock.class.getName(), + pool.getAll()); + + devServer.start(); + devServer.join(); + } +} diff --git a/zeppelin-examples/zeppelin-example-clock/src/main/resources/example/app/clock/clock.html b/zeppelin-examples/zeppelin-example-clock/src/main/resources/example/app/clock/clock.html new file mode 100644 index 00000000000..ff492f03e5c --- /dev/null +++ b/zeppelin-examples/zeppelin-example-clock/src/main/resources/example/app/clock/clock.html @@ -0,0 +1,14 @@ + +

{{date}}

\ No newline at end of file diff --git a/zeppelin-examples/zeppelin-example-clock/zeppelin-example-clock.json b/zeppelin-examples/zeppelin-example-clock/zeppelin-example-clock.json new file mode 100644 index 00000000000..c797adba108 --- /dev/null +++ b/zeppelin-examples/zeppelin-example-clock/zeppelin-example-clock.json @@ -0,0 +1,9 @@ +{ + type : "APPLICATION", + name : "zeppelin.clock", + description : "Clock (example)", + artifact : "zeppelin-examples/zeppelin-example-clock/target/zeppelin-example-clock-0.6.0-incubating-SNAPSHOT.jar", + className : "org.apache.zeppelin.example.app.clock.Clock", + resources : [[":java.util.Date"]], + icon : '' +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java index 60ef1f9e6e0..79e8da48e97 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java @@ -81,18 +81,15 @@ public synchronized List load(String artifact, Collection excludes return loadFromMvn(artifact, excludes); } else { LinkedList libs = new LinkedList(); - libs.add(new File(artifact)); + libs.add(new File(getPath(artifact))); return libs; } } - - public List load(String artifact, Collection excludes, String destPath) - throws RepositoryException, IOException { - List libs = new LinkedList(); - - if (StringUtils.isNotBlank(artifact)) { - libs = load(artifact, excludes); + private String getPath(String path) { + if (path.startsWith("/")) { + return path; + } else { // find home dir String home = System.getenv("ZEPPELIN_HOME"); if (home == null) { @@ -102,11 +99,22 @@ public List load(String artifact, Collection excludes, String dest home = ".."; } + return home + "/" + path; + } + } + + public List load(String artifact, Collection excludes, String destPath) + throws RepositoryException, IOException { + List libs = new LinkedList(); + + if (StringUtils.isNotBlank(artifact)) { + libs = load(artifact, excludes); + for (File srcFile : libs) { - File destFile = new File(home + "/" + destPath, srcFile.getName()); + File destFile = new File(getPath(destPath), srcFile.getName()); if (!destFile.exists() || !FileUtils.contentEquals(srcFile, destFile)) { FileUtils.copyFile(srcFile, destFile); - logger.info("copy {} to {}", srcFile.getAbsolutePath(), destPath); + logger.info("copy {} to {}", srcFile.getAbsolutePath(), getPath(destPath)); } } } From 0730ece85147fc0ed0eabe175cb875c92f6e1316 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sun, 17 Apr 2016 09:31:12 +0100 Subject: [PATCH 29/69] Add license into package json --- .../zeppelin-example-clock.json | 16 ++++++++++++++++ .../zeppelin/helium/HeliumLocalRegistry.java | 8 +++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/zeppelin-examples/zeppelin-example-clock/zeppelin-example-clock.json b/zeppelin-examples/zeppelin-example-clock/zeppelin-example-clock.json index c797adba108..1deff591b81 100644 --- a/zeppelin-examples/zeppelin-example-clock/zeppelin-example-clock.json +++ b/zeppelin-examples/zeppelin-example-clock/zeppelin-example-clock.json @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ { type : "APPLICATION", name : "zeppelin.clock", diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumLocalRegistry.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumLocalRegistry.java index 7e79d85896a..ef2883531e8 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumLocalRegistry.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumLocalRegistry.java @@ -17,12 +17,14 @@ package org.apache.zeppelin.helium; import com.google.gson.Gson; +import com.google.gson.stream.JsonReader; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; +import java.io.StringReader; import java.net.URI; import java.util.LinkedList; import java.util.List; @@ -38,6 +40,7 @@ public class HeliumLocalRegistry extends HeliumRegistry { public HeliumLocalRegistry(String name, String uri) { super(name, uri); gson = new Gson(); + } @@ -66,7 +69,10 @@ public synchronized List getAll() throws IOException { private HeliumPackage readPackageInfo(File f) { try { - return gson.fromJson(FileUtils.readFileToString(f), HeliumPackage.class); + JsonReader reader = new JsonReader(new StringReader(FileUtils.readFileToString(f))); + reader.setLenient(true); + + return gson.fromJson(reader, HeliumPackage.class); } catch (IOException e) { logger.error(e.getMessage(), e); return null; From 3b891deb588a9e9e2472a4c25fa11b1f1c3347e7 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sun, 17 Apr 2016 10:05:44 +0100 Subject: [PATCH 30/69] Provide resource pool for testing --- .../java/org/apache/zeppelin/spark/SparkInterpreterTest.java | 3 ++- .../org/apache/zeppelin/spark/SparkSqlInterpreterTest.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java b/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java index 5b132777419..421e26c0ed7 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java +++ b/spark/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java @@ -29,6 +29,7 @@ import org.apache.spark.SparkConf; import org.apache.spark.SparkContext; import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.resource.LocalResourcePool; import org.apache.zeppelin.user.AuthenticationInfo; import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.interpreter.*; @@ -85,7 +86,7 @@ public void setUp() throws Exception { new HashMap(), new GUI(), new AngularObjectRegistry(intpGroup.getId(), null), - null, + new LocalResourcePool("id"), new LinkedList(), new InterpreterOutput(new InterpreterOutputListener() { @Override diff --git a/spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java b/spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java index c2cc1e637c6..6d3fb84e775 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java +++ b/spark/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java @@ -24,6 +24,7 @@ import java.util.Properties; import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.resource.LocalResourcePool; import org.apache.zeppelin.user.AuthenticationInfo; import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.interpreter.*; @@ -73,7 +74,7 @@ public void setUp() throws Exception { context = new InterpreterContext("note", "id", "title", "text", new AuthenticationInfo(), new HashMap(), new GUI(), new AngularObjectRegistry(intpGroup.getId(), null), - null, + new LocalResourcePool("id"), new LinkedList(), new InterpreterOutput(new InterpreterOutputListener() { @Override public void onAppend(InterpreterOutput out, byte[] line) { From 83eba9826ed2d3a2ccca6e1bc3e2dfe8c0f3db35 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Mon, 18 Apr 2016 05:44:39 +0100 Subject: [PATCH 31/69] UnloadApp on interpreter restart --- .../remote/RemoteInterpreterEventClient.java | 12 ++++++ .../remote/RemoteInterpreterEventPoller.java | 13 +++++- .../remote/RemoteInterpreterServer.java | 30 +++++++++++++- .../thrift/RemoteInterpreterEventType.java | 3 +- .../helium/HeliumApplicationFactoryTest.java | 41 +++++++++++++++++++ 5 files changed, 95 insertions(+), 4 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java index b9b6b0a8222..8cec32a6f3c 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java @@ -266,4 +266,16 @@ public void onAppOutputUpdate(String noteId, String paragraphId, String appId, S RemoteInterpreterEventType.OUTPUT_UPDATE, gson.toJson(appendOutput))); } + + public void onAppStatusUpdate(String noteId, String paragraphId, String appId, String status) { + Map appendOutput = new HashMap(); + appendOutput.put("noteId", noteId); + appendOutput.put("paragraphId", paragraphId); + appendOutput.put("appId", appId); + appendOutput.put("status", status); + + sendEvent(new RemoteInterpreterEvent( + RemoteInterpreterEventType.APP_STATUS_UPDATE, + gson.toJson(appendOutput))); + } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java index 9d94d97a8a5..48c14d50bde 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java @@ -164,7 +164,7 @@ public void run() { } else if (event.getType() == RemoteInterpreterEventType.OUTPUT_UPDATE) { // on output update Map outputAppend = gson.fromJson( - event.getData(), new TypeToken>() {}.getType()); + event.getData(), new TypeToken>() {}.getType()); String noteId = outputAppend.get("noteId"); String paragraphId = outputAppend.get("paragraphId"); String outputToUpdate = outputAppend.get("data"); @@ -175,6 +175,17 @@ public void run() { } else { appListener.onOutputUpdated(noteId, paragraphId, appId, outputToUpdate); } + } else if (event.getType() == RemoteInterpreterEventType.APP_STATUS_UPDATE) { + // on output update + Map appStatusUpdate = gson.fromJson( + event.getData(), new TypeToken>() {}.getType()); + + String noteId = appStatusUpdate.get("noteId"); + String paragraphId = appStatusUpdate.get("paragraphId"); + String appId = appStatusUpdate.get("appId"); + String status = appStatusUpdate.get("status"); + + appListener.onStatusChange(noteId, paragraphId, appId, status); } logger.debug("Event from remoteproceess {}", event.getType()); } catch (Exception e) { diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java index ffb658ef2a8..6cc7e893650 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java @@ -242,6 +242,23 @@ public void open(String noteId, String className) throws TException { @Override public void close(String noteId, String className) throws TException { + // unload all applications + for (String appId : runningApplications.keySet()) { + RunningApplication appInfo = runningApplications.get(appId); + + // see NoteInterpreterLoader.SHARED_SESSION + if (appInfo.noteId.equals(noteId) || noteId.equals("shared_session")) { + try { + appInfo.app.unload(); + // see ApplicationState.Status.UNLOADED + eventClient.onAppStatusUpdate(appInfo.noteId, appInfo.paragraphId, appId, "UNLOADED"); + } catch (ApplicationException e) { + logger.error(e.getMessage(), e); + } + } + } + + // close interpreters synchronized (interpreterGroup) { List interpreters = interpreterGroup.get(noteId); if (interpreters != null) { @@ -790,7 +807,9 @@ public RemoteApplicationResult loadApplication( noteId, paragraphId); app = appLoader.load(pkgInfo, context); - runningApplications.put(applicationInstanceId, new RunningApplication(pkgInfo, app)); + runningApplications.put( + applicationInstanceId, + new RunningApplication(pkgInfo, app, noteId, paragraphId)); return new RemoteApplicationResult(true, ""); } catch (Exception e) { logger.error(e.getMessage(), e); @@ -855,10 +874,17 @@ public RemoteApplicationResult runApplication(String applicationInstanceId) private static class RunningApplication { public final Application app; public final HeliumPackage pkg; + public final String noteId; + public final String paragraphId; - public RunningApplication(HeliumPackage pkg, Application app) { + public RunningApplication(HeliumPackage pkg, + Application app, + String noteId, + String paragraphId) { this.app = app; this.pkg = pkg; + this.noteId = noteId; + this.paragraphId = paragraphId; } }; } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java index 66631d2f2a8..32dfef6c5b8 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java @@ -38,7 +38,8 @@ public enum RemoteInterpreterEventType implements org.apache.thrift.TEnum { RESOURCE_GET(7), OUTPUT_APPEND(8), OUTPUT_UPDATE(9), - ANGULAR_REGISTRY_PUSH(10); + ANGULAR_REGISTRY_PUSH(10), + APP_STATUS_UPDATE(11); private final int value; diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java index 9a882757cc7..6e499d400cd 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java @@ -229,6 +229,47 @@ public void testUnloadOnInterpreterUnbind() throws IOException { notebook.removeNote(note1.getId()); } + + @Test + public void testUnloadOnInterpreterRestart() throws IOException { + // given + HeliumPackage pkg1 = new HeliumPackage(HeliumPackage.Type.APPLICATION, + "name1", + "desc1", + "", + HeliumTestApplication.class.getName(), + new String[][]{}); + + Note note1 = notebook.createNote(); + notebook.bindInterpretersToNote(note1.id(), factory.getDefaultInterpreterSettingList()); + + Paragraph p1 = note1.addParagraph(); + + // make sure interpreter process running + p1.setText("job"); + note1.run(p1.getId()); + while(p1.isTerminated()==false || p1.getResult()==null) Thread.yield(); + + assertEquals(0, p1.getAllApplicationStates().size()); + String appId = heliumAppFactory.loadAndRun(pkg1, p1); + ApplicationState app = p1.getApplicationState(appId); + while (app.getStatus() != ApplicationState.Status.LOADED) { + Thread.yield(); + } + + // when unbind interpreter + factory.restart(factory.getDefaultInterpreterSettingList().get(0)); + while (app.getStatus() == ApplicationState.Status.LOADED) { + Thread.yield(); + } + + // then + assertEquals(ApplicationState.Status.UNLOADED, app.getStatus()); + + // clean + notebook.removeNote(note1.getId()); + } + @Override public ParagraphJobListener getParagraphJobListener(Note note) { return new ParagraphJobListener() { From 51fc113046635bdb26d58a59e66c2f29d15326a2 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Mon, 18 Apr 2016 10:06:33 +0100 Subject: [PATCH 32/69] Include examples in CI build --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f60870442ea..44901430166 100644 --- a/.travis.yml +++ b/.travis.yml @@ -34,7 +34,7 @@ matrix: include: # Test all modules - jdk: "oraclejdk7" - env: SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS="" + env: SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Pr -Phadoop-2.3 -Ppyspark -Psparkr -Pscalding -Pexamples" BUILD_FLAG="package -Pbuild-distr" TEST_FLAG="verify -Pusing-packaged-distr" TEST_PROJECTS="" # Test spark module for 1.5.2 - jdk: "oraclejdk7" @@ -58,7 +58,7 @@ matrix: # Test selenium with spark module for 1.6.1 - jdk: "oraclejdk7" - env: TEST_SELENIUM="true" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false" + env: TEST_SELENIUM="true" SPARK_VER="1.6.1" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark -Pexamples" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false" before_install: - "ls -la .spark-dist" From f07ada1d94784fec79d51b5067ae7c586c03f660 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 7 May 2016 13:48:46 -0700 Subject: [PATCH 33/69] Construct classpath correctly --- bin/interpreter.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/interpreter.sh b/bin/interpreter.sh index 17c9028b853..483840a9cf6 100755 --- a/bin/interpreter.sh +++ b/bin/interpreter.sh @@ -85,10 +85,10 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then export SPARK_SUBMIT="${SPARK_HOME}/bin/spark-submit" SPARK_APP_JAR="$(ls ${ZEPPELIN_HOME}/interpreter/spark/zeppelin-spark*.jar)" # This will evantually passes SPARK_APP_JAR to classpath of SparkIMain - ZEPPELIN_CLASSPATH=${SPARK_APP_JAR} + ZEPPELIN_CLASSPATH+=${SPARK_APP_JAR} # Need to add the R Interpreter RZEPPELINPATH="$(ls ${ZEPPELIN_HOME}/interpreter/spark/zeppelin-zr*.jar)" - ZEPPELIN_CLASSPATH="${ZEPPELIN_CLASSPATH}:${RZEPPELINPATH}" + ZEPPELIN_CLASSPATH+=":${RZEPPELINPATH}" pattern="$SPARK_HOME/python/lib/py4j-*-src.zip" py4j=($pattern) @@ -134,7 +134,7 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then fi RZEPPELINPATH="$(ls ${ZEPPELIN_HOME}/interpreter/spark/zeppelin-zr*.jar)" - ZEPPELIN_CLASSPATH="${ZEPPELIN_CLASSPATH}:${RZEPPELINPATH}" + ZEPPELIN_CLASSPATH+=":${RZEPPELINPATH}" export SPARK_CLASSPATH+=":${ZEPPELIN_CLASSPATH}" fi elif [[ "${INTERPRETER_ID}" == "hbase" ]]; then From 8186daf58e059463b7871df557d9a223d7127d9b Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 7 May 2016 22:11:35 -0700 Subject: [PATCH 34/69] Update Application Status from RemoteInterpreterProcess event --- .../remote/RemoteInterpreterEventClient.java | 15 +++++++++++++++ .../remote/RemoteInterpreterServer.java | 2 ++ .../thrift/RemoteApplicationResult.java | 2 +- .../thrift/RemoteInterpreterContext.java | 2 +- .../thrift/RemoteInterpreterEvent.java | 2 +- .../thrift/RemoteInterpreterEventType.java | 2 ++ .../thrift/RemoteInterpreterResult.java | 2 +- .../thrift/RemoteInterpreterService.java | 2 +- .../main/thrift/RemoteInterpreterService.thrift | 3 ++- .../zeppelin/helium/HeliumApplicationFactory.java | 7 +++++-- .../zeppelin/notebook/ApplicationState.java | 1 + .../helium/HeliumApplicationFactoryTest.java | 8 ++++++-- 12 files changed, 38 insertions(+), 10 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java index 8cec32a6f3c..6f26ffd959b 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java @@ -278,4 +278,19 @@ public void onAppStatusUpdate(String noteId, String paragraphId, String appId, S RemoteInterpreterEventType.APP_STATUS_UPDATE, gson.toJson(appendOutput))); } + + /** + * Wait for eventQueue becomes empty + */ + public void waitForEventQueueBecomesEmpty() { + synchronized (eventQueue) { + while (!eventQueue.isEmpty()) { + try { + eventQueue.wait(100); + } catch (InterruptedException e) { + // ignore exception + } + } + } + } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java index 6cc7e893650..b455bc75ed9 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java @@ -92,6 +92,7 @@ public void run() { @Override public void shutdown() throws TException { + eventClient.waitForEventQueueBecomesEmpty(); if (interpreterGroup != null) { interpreterGroup.close(); interpreterGroup.destroy(); @@ -249,6 +250,7 @@ public void close(String noteId, String className) throws TException { // see NoteInterpreterLoader.SHARED_SESSION if (appInfo.noteId.equals(noteId) || noteId.equals("shared_session")) { try { + logger.info("Unload App {} ", appInfo.pkg.getName()); appInfo.app.unload(); // see ApplicationState.Status.UNLOADED eventClient.onAppStatusUpdate(appInfo.noteId, appInfo.paragraphId, appId, "UNLOADED"); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteApplicationResult.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteApplicationResult.java index 2a3945fa406..a192899daa1 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteApplicationResult.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteApplicationResult.java @@ -51,7 +51,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2016-3-18") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2016-5-7") public class RemoteApplicationResult implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteApplicationResult"); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterContext.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterContext.java index 5fadb74b49d..3e0379c6e05 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterContext.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterContext.java @@ -51,7 +51,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2016-3-18") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2016-5-7") public class RemoteInterpreterContext implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteInterpreterContext"); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java index 6198ff06cd2..ed6e6090b2d 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEvent.java @@ -51,7 +51,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2016-3-18") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2016-5-7") public class RemoteInterpreterEvent implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteInterpreterEvent"); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java index 32dfef6c5b8..955461951f3 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterEventType.java @@ -80,6 +80,8 @@ public static RemoteInterpreterEventType findByValue(int value) { return OUTPUT_UPDATE; case 10: return ANGULAR_REGISTRY_PUSH; + case 11: + return APP_STATUS_UPDATE; default: return null; } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterResult.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterResult.java index 3078147ea9f..89884928a8f 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterResult.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterResult.java @@ -51,7 +51,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2016-3-18") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2016-5-7") public class RemoteInterpreterResult implements org.apache.thrift.TBase, java.io.Serializable, Cloneable, Comparable { private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("RemoteInterpreterResult"); diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterService.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterService.java index 3b5a9c1384b..bafa3bee86f 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterService.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/thrift/RemoteInterpreterService.java @@ -51,7 +51,7 @@ import org.slf4j.LoggerFactory; @SuppressWarnings({"cast", "rawtypes", "serial", "unchecked"}) -@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2016-3-18") +@Generated(value = "Autogenerated by Thrift Compiler (0.9.2)", date = "2016-5-7") public class RemoteInterpreterService { public interface Iface { diff --git a/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift b/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift index dd3bbf93d1b..7a1678d45ce 100644 --- a/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift +++ b/zeppelin-interpreter/src/main/thrift/RemoteInterpreterService.thrift @@ -48,7 +48,8 @@ enum RemoteInterpreterEventType { RESOURCE_GET = 7 OUTPUT_APPEND = 8, OUTPUT_UPDATE = 9, - ANGULAR_REGISTRY_PUSH=10 + ANGULAR_REGISTRY_PUSH = 10, + APP_STATUS_UPDATE = 11, } struct RemoteInterpreterEvent { diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java index b370c1e399b..6759f979d13 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java @@ -36,8 +36,6 @@ /** * HeliumApplicationFactory - * - * TODO(moon): unload apps on interpreter restart */ public class HeliumApplicationFactory implements ApplicationEventListener, NotebookEventListener { private final Logger logger = LoggerFactory.getLogger(HeliumApplicationFactory.class); @@ -376,6 +374,11 @@ public void onLoad(String noteId, String paragraphId, String appId, HeliumPackag @Override public void onStatusChange(String noteId, String paragraphId, String appId, String status) { + ApplicationState appToUpdate = getAppState(noteId, paragraphId, appId); + if (appToUpdate != null) { + appToUpdate.setStatus(ApplicationState.Status.valueOf(status)); + } + if (applicationEventListener != null) { applicationEventListener.onStatusChange(noteId, paragraphId, appId, status); } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java index bc71d893221..1505db9ada3 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/ApplicationState.java @@ -17,6 +17,7 @@ package org.apache.zeppelin.notebook; import org.apache.zeppelin.helium.HeliumPackage; +import org.apache.zeppelin.interpreter.InterpreterGroup; /** * Current state of application diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java index 6e499d400cd..4907ef7bf94 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java @@ -76,7 +76,6 @@ public void setUp() throws Exception { heliumAppFactory = new HeliumApplicationFactory(); - depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); factory = new InterpreterFactory(conf, new InterpreterOption(true), null, null, heliumAppFactory, depResolver); @@ -257,7 +256,12 @@ public void testUnloadOnInterpreterRestart() throws IOException { Thread.yield(); } - // when unbind interpreter + // wait until application is executed + while (!"Hello world 1".equals(app.getOutput())) { + Thread.yield(); + } + + // when restart interpreter factory.restart(factory.getDefaultInterpreterSettingList().get(0)); while (app.getStatus() == ApplicationState.Status.LOADED) { Thread.yield(); From b6e4141badb40ff473224bda3b4efb19156f23cc Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Thu, 12 May 2016 11:59:05 -0700 Subject: [PATCH 35/69] helper for printing javascript and inject $z --- .../apache/zeppelin/helium/Application.java | 44 +++++++++++++++++++ .../zeppelin/helium/ApplicationContext.java | 8 ++++ .../dev/ZeppelinApplicationDevServer.java | 22 ++++++++++ .../remote/RemoteInterpreterServer.java | 1 + .../helium/ApplicationLoaderTest.java | 5 ++- 5 files changed, 78 insertions(+), 2 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/Application.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/Application.java index 59506b7624c..02910f6fa41 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/Application.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/Application.java @@ -48,4 +48,48 @@ public abstract void run(ResourceSet args) * this method is invoked just before application is removed */ public abstract void unload() throws ApplicationException; + + public void print(String string) throws IOException { + context.out.write(string); + } + + public void println(String string) throws IOException { + print(string + "\n"); + } + + public void printResource(String resourceName) throws IOException { + context.out.writeResource(resourceName); + } + + public void printResourceAsJavascript(String resourceName) throws IOException { + beginJavascript(); + context.out.writeResource(resourceName); + endJavascript(); + } + + public void printStringAsJavascript(String js) throws IOException { + beginJavascript(); + context.out.write(js); + endJavascript(); + } + + private void beginJavascript() throws IOException { + StringBuffer js = new StringBuffer(); + js.append("\n\n"); + context.out.write(js.toString()); + } } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationContext.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationContext.java index 33d7f40f229..e0ea94cb01f 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationContext.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/ApplicationContext.java @@ -25,15 +25,19 @@ public class ApplicationContext { private final String noteId; private final String paragraphId; + private final String applicationInstanceId; private final HeliumAppAngularObjectRegistry angularObjectRegistry; public final InterpreterOutput out; + public ApplicationContext(String noteId, String paragraphId, + String applicationInstanceId, HeliumAppAngularObjectRegistry angularObjectRegistry, InterpreterOutput out) { this.noteId = noteId; this.paragraphId = paragraphId; + this.applicationInstanceId = applicationInstanceId; this.angularObjectRegistry = angularObjectRegistry; this.out = out; } @@ -46,6 +50,10 @@ public String getParagraphId() { return paragraphId; } + public String getApplicationInstanceId() { + return applicationInstanceId; + } + public HeliumAppAngularObjectRegistry getAngularObjectRegistry() { return angularObjectRegistry; } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinApplicationDevServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinApplicationDevServer.java index e8f67d04600..941fdfe1f06 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinApplicationDevServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/dev/ZeppelinApplicationDevServer.java @@ -21,6 +21,7 @@ import java.lang.reflect.InvocationTargetException; import java.net.URL; +import com.google.gson.Gson; import org.apache.log4j.ConsoleAppender; import org.apache.log4j.Level; import org.apache.log4j.PatternLayout; @@ -29,6 +30,7 @@ import org.apache.zeppelin.interpreter.InterpreterResult.Code; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterEventClient; import org.apache.zeppelin.resource.ResourceSet; +import org.apache.zeppelin.resource.WellKnownResourceName; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -94,6 +96,7 @@ public InterpreterResult interpret(String st, InterpreterContext context) { logger.info("Run " + className); app.context().out.clear(); app.context().out.setType(InterpreterResult.Type.ANGULAR); + transferTableResultDataToFrontend(); app.run(resourceSet); } catch (IOException | ApplicationException e) { logger.error(e.getMessage(), e); @@ -102,10 +105,29 @@ public InterpreterResult interpret(String st, InterpreterContext context) { return new InterpreterResult(Code.SUCCESS, ""); } + private void transferTableResultDataToFrontend() throws IOException { + ResourceSet results = resourceSet.filterByClassname(InterpreterResult.class.getName()); + if (results.size() == 0) { + return; + } + + InterpreterResult result = (InterpreterResult) results.get(0).get(); + Gson gson = new Gson(); + String resultJson = gson.toJson(result); + StringBuffer transferResult = new StringBuffer(); + transferResult.append("$z.result = " + resultJson + ";\n"); + if (result.type() == InterpreterResult.Type.TABLE) { + transferResult.append("$z.scope.loadTableData($z.result);\n"); + } + transferResult.append("$z.scope._devmodeResult = $z.result;\n"); + app.printStringAsJavascript(transferResult.toString()); + } + ApplicationContext getApplicationContext(InterpreterContext interpreterContext) { return new ApplicationContext( interpreterContext.getNoteId(), interpreterContext.getParagraphId(), + "app_" + this.hashCode(), new HeliumAppAngularObjectRegistry( interpreterContext.getAngularObjectRegistry(), interpreterContext.getNoteId(), diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java index b455bc75ed9..ae3435f2e6f 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServer.java @@ -783,6 +783,7 @@ private ApplicationContext getApplicationContext( return new ApplicationContext( noteId, paragraphId, + applicationInstanceId, new HeliumAppAngularObjectRegistry(angularObjectRegistry, noteId, applicationInstanceId), out); } diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/ApplicationLoaderTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/ApplicationLoaderTest.java index 72e15053a5e..06c4a6b8e29 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/ApplicationLoaderTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/helium/ApplicationLoaderTest.java @@ -53,7 +53,7 @@ public void loadUnloadApplication() throws Exception { ApplicationLoader appLoader = new ApplicationLoader(resourcePool, dep); HeliumPackage pkg1 = createPackageInfo(MockApplication1.class.getName(), "artifact1"); - ApplicationContext context1 = createContext("note1", "paragraph1"); + ApplicationContext context1 = createContext("note1", "paragraph1", "app1"); // when load application MockApplication1 app = (MockApplication1) ((ClassLoaderApplication) @@ -82,10 +82,11 @@ public HeliumPackage createPackageInfo(String className, String artifact) { return app1; } - public ApplicationContext createContext(String noteId, String paragraphId) { + public ApplicationContext createContext(String noteId, String paragraphId, String appInstanceId) { ApplicationContext context1 = new ApplicationContext( noteId, paragraphId, + appInstanceId, null, new InterpreterOutput(new InterpreterOutputListener() { @Override From b0680655db6d8d7c9b0b2ebd18342322d15cecd5 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 14 May 2016 20:14:35 -0700 Subject: [PATCH 36/69] Replace . to _ --- .../src/main/java/org/apache/zeppelin/notebook/Paragraph.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java index 8953074f236..935e2c9e1ff 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java @@ -425,7 +425,7 @@ public Object clone() throws CloneNotSupportedException { } private String getApplicationId(HeliumPackage pkg) { - return "app_" + getNote().getId() + "-" + getId() + pkg.getName(); + return "app_" + getNote().getId() + "-" + getId() + pkg.getName().replaceAll("\\.", "_"); } public ApplicationState createOrGetApplicationState(HeliumPackage pkg) { From 52255511bbd909fa70fbb51ebb12a3d72f254760 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Sat, 14 May 2016 20:15:01 -0700 Subject: [PATCH 37/69] let instead of var --- .../src/main/java/org/apache/zeppelin/helium/Application.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/Application.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/Application.java index 02910f6fa41..5dd58273e18 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/Application.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/helium/Application.java @@ -77,7 +77,7 @@ private void beginJavascript() throws IOException { StringBuffer js = new StringBuffer(); js.append("\n tag. + * Javascript printed using this method will be run in the un-named function. + * i.e. each method call will creates different variable scope for the javascript code. + * + * This method inject '$z' into the variable scope for convenience. + * + * $z.scope : angularjs scope object for this application + * $z.id : unique id for this application instance + * + * @param resourceName + * @throws IOException + */ @Experimental public void printResourceAsJavascript(String resourceName) throws IOException { beginJavascript(); @@ -75,6 +107,21 @@ public void printResourceAsJavascript(String resourceName) throws IOException { endJavascript(); } + /** + * Print string as a javascript + * + * Using this method does not require print javascript inside of tag. + * Javascript printed using this method will be run in the un-named function. + * i.e. each method call will creates different variable scope for the javascript code. + * + * This method inject '$z' into the variable scope for convenience. + * + * $z.scope : angularjs scope object for this application + * $z.id : unique id for this application instance + * + * @param js + * @throws IOException + */ @Experimental public void printStringAsJavascript(String js) throws IOException { beginJavascript(); From e60cd8ec1c8eedbf991e218578a4ad6bdba69954 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Wed, 8 Jun 2016 15:21:47 -0700 Subject: [PATCH 45/69] Add docs --- .../themes/zeppelin/_navigation.html | 1 + .../development/writingzeppelinapplication.md | 185 ++++++++++++++++++ 2 files changed, 186 insertions(+) create mode 100644 docs/development/writingzeppelinapplication.md diff --git a/docs/_includes/themes/zeppelin/_navigation.html b/docs/_includes/themes/zeppelin/_navigation.html index 179ede8ae1b..c674e5aa5ac 100644 --- a/docs/_includes/themes/zeppelin/_navigation.html +++ b/docs/_includes/themes/zeppelin/_navigation.html @@ -101,6 +101,7 @@
  • Writing Zeppelin Interpreter
    • +
  • Writing Zeppelin Application
  • How to contribute (code)
  • How to contribute (website)
    • diff --git a/docs/development/writingzeppelinapplication.md b/docs/development/writingzeppelinapplication.md new file mode 100644 index 00000000000..44b90e4221d --- /dev/null +++ b/docs/development/writingzeppelinapplication.md @@ -0,0 +1,185 @@ +--- +layout: page +title: "Writing Zeppelin Application" +description: "" +group: development +--- + +{% include JB/setup %} + +### What is Zeppelin Application (Experimental) + +Zeppelin Application is a package that runs on Interpreter process and display it's output inside of the notebook. While application runs on Interpreter process, it's able to access resources provided by Interpreter through ResourcePool. Output is always rendered by AngularDisplaySystem. Therefore application provides all the possiblity of making interactive graphical application that uses data and processing power of any Interpreter. + + + +### Writing your own Application + +Writing Application means extending `org.apache.zeppelin.helium.Application`. You can use your favorite IDE and language while Java class files are packaged into jar. `Application` class looks like + +```java + +/** + * Constructor. Invoked when application is loaded + */ +public Application(ApplicationContext context); + +/** + * Invoked when there're (possible) updates in required resource set. + * i.e. invoked after application load and after paragraph finishes. + */ +public abstract void run(ResourceSet args); + +/** + * Invoked before application unload. + * Application is automatically unloaded with paragraph/notebook removal + */ +public abstract void unload(); +``` + + +You can check example applications under [./zeppelin-examples](https://github.com/apache/incubator-zeppelin/tree/master/zeppelin-examples) directory. + + +### Development mode + +In the development mode, you can run your Application in your IDE as a normal java application and see the result inside of Zeppelin notebook. + +org.apache.zeppelin.interpreter.dev.ZeppelinApplicationDevServer can run Zeppelin Application in development mode. + +```java + +// entry point for development mode +public static void main(String[] args) throws Exception { + + // add resources for development mode + LocalResourcePool pool = new LocalResourcePool("dev"); + pool.put("date", new Date()); + + // run application in devlopment mode with give resource + // in this case, Clock.class.getName() will be the application class name + ZeppelinApplicationDevServer devServer = new ZeppelinApplicationDevServer( + Clock.class.getName(), + pool.getAll()); + + // start development mode + devServer.start(); + devServer.join(); +} +``` + + +In the Zeppelin notebook, run `%dev run` will connect to application running in development mode. + + + + +### Package flie + +Package file is a json file that provides information about the application. +Json file contains following informations + +``` +{ + name : "[organization].[name]", + description : "Description", + artifact : "groupId:artifactId:version", + className : "your.package.name.YourApplicationClass", + resources : [ + ["resource.name", ":resource.class.name"], + ["alternative.resource.name", ":alternative.class.name"] + ], + icon : "" +} + +``` + +#### name + +Name is a string in '[group].[name]' format. +[group] and [name] allows only [A-Za-z0-9_]. +Group is normally organization name who creates this application. + +#### description + +Short description. about application + +#### artifact + +Location of the jar artifact. +"groupId:artifactId:version" will make load artifact from maven repository. +If jar is in local filesystem, absolute/relative can be used. + +e.g. + +When artifact exists in Maven repository + +`artifact: "org.apache.zeppelin:zeppelin-examples:0.6.0"` + +When artifact exists in local filesystem + +`artifact: "zeppelin-example/target/zeppelin-example-0.6.0.jar"` + + +#### className + +Entry point. Class that extends `org.apache.zeppelin.helium.Application` + + +#### resources + +Two dimensional array that defines required resources by name or by className. Helium Application launcher will compare resources in the ResourcePool with informations in this field and suggest application only when all required resources are available in the ResourcePool. + +Resouce name is a string which will be compared with name of objects in the ResourcePool. className is a string with ":" prepended, which will be compared with className of the objects in the ResourcePool. + +Application may require two or more resources. Required resource can be listed inside of json array. For example, if application requires object "name1", "name2" and "className1" type of object to run, resources field can be + +``` +resources: [ + [ "name1", "name2", ":className1", ...] +] +``` + +If Application can handle alternative combination of required resource, alternative set can be listed as below. + +``` +resources: [ + [ "name", ":className"], + [ "altName", ":altClassName1"], + ... +] +``` + +Easier way of understanding this scheme is + +``` +resources: [ + [ 'resource' AND 'resource' AND ... ] OR + [ 'resource' AND 'resource' AND ... ] OR + ... +] +``` + + +#### icon + +Icon to be used on the application button. String in this field will be rendered as a html. + +e.g. + +``` +icon: "" +``` + From bff2217ee9d6b3432e33a49b1aaa710d5b2aea74 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Wed, 8 Jun 2016 15:59:25 -0700 Subject: [PATCH 46/69] Fix SparkParagraphIT --- .../java/org/apache/zeppelin/integration/SparkParagraphIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/integration/SparkParagraphIT.java b/zeppelin-server/src/test/java/org/apache/zeppelin/integration/SparkParagraphIT.java index 1eadd0ecef9..e4a450f6780 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/integration/SparkParagraphIT.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/integration/SparkParagraphIT.java @@ -165,7 +165,7 @@ public void testSqlSpark() throws Exception { } WebElement paragraph1Result = driver.findElement(By.xpath( - getParagraphXPath(1) + "//div[@class=\"tableDisplay\"]")); + getParagraphXPath(1) + "//div[@class=\"tableDisplay\"]/div/div/div")); collector.checkThat("Paragraph from SparkParagraphIT of testSqlSpark result: ", paragraph1Result.getText().toString(), CoreMatchers.equalTo("age\njob\nmarital\neducation\nbalance\n30" + " unemployed married primary 1,787\nage\njob\nmarital\neducation\nbalance")); From 911089785d137cae20067fd0feeee2e926808040 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 14 Jun 2016 12:46:53 -0700 Subject: [PATCH 47/69] arrange buttons --- .../paragraph/paragraph-chart-selector.html | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph-chart-selector.html b/zeppelin-web/src/app/notebook/paragraph/paragraph-chart-selector.html index fd4e0208f74..14f51651a5d 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph-chart-selector.html +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph-chart-selector.html @@ -68,13 +68,6 @@ ng-click="exportToTSV()">     - - - settings - -
    + + + settings + + From b6811a615d6cd01afb93b54eaf454788615b8238 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Tue, 14 Jun 2016 13:53:43 -0700 Subject: [PATCH 48/69] set SPARK_HOME to find bin/interpreter.sh --- .../zeppelin/helium/HeliumApplicationFactoryTest.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java index c9a41f6d877..641d520ee4d 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java @@ -63,6 +63,14 @@ public void setUp() throws Exception { confDir.mkdirs(); notebookDir = new File(tmpDir + "/notebook"); notebookDir.mkdirs(); + + File home = new File(getClass().getClassLoader().getResource("note").getFile()) // zeppelin/zeppelin-zengine/target/test-classes/note + .getParentFile() // zeppelin/zeppelin-zengine/target/test-classes + .getParentFile() // zeppelin/zeppelin-zengine/target + .getParentFile() // zeppelin/zeppelin-zengine + .getParentFile(); // zeppelin + + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), home.getAbsolutePath()); System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_CONF_DIR.getVarName(), tmpDir.getAbsolutePath() + "/conf"); System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath()); System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETERS.getVarName(), "org.apache.zeppelin.interpreter.mock.MockInterpreter1,org.apache.zeppelin.interpreter.mock.MockInterpreter2"); From 3619afd422b18eac65daa5053cfa36b402127f43 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Wed, 15 Jun 2016 13:54:59 -0700 Subject: [PATCH 49/69] Exclude test data file from rat check --- .../zeppelin-example-horizontalbar/pom.xml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/zeppelin-examples/zeppelin-example-horizontalbar/pom.xml b/zeppelin-examples/zeppelin-example-horizontalbar/pom.xml index cd199c2edc1..f34c55ed7a6 100644 --- a/zeppelin-examples/zeppelin-example-horizontalbar/pom.xml +++ b/zeppelin-examples/zeppelin-example-horizontalbar/pom.xml @@ -106,8 +106,16 @@ - + + + org.apache.rat + apache-rat-plugin + + + **/horizontalbar_mockdata.txt + + + - From 710487b039edf32cb28bbf7e4716e4d1b3664709 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Wed, 15 Jun 2016 15:17:59 -0700 Subject: [PATCH 50/69] fix syntax error --- .../main/java/org/apache/zeppelin/notebook/socket/Message.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java index 3e53553bdf0..08b32359226 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/socket/Message.java @@ -118,7 +118,7 @@ public static enum OP { APP_APPEND_OUTPUT, // [s-c] append output APP_UPDATE_OUTPUT, // [s-c] update (replace) output APP_LOAD, // [s-c] on app load - APP_STATUS_CHANGE // [s-c] on app status change + APP_STATUS_CHANGE, // [s-c] on app status change LIST_NOTEBOOK_JOBS, // [c-s] get notebook job management infomations LIST_UPDATE_NOTEBOOK_JOBS // [c-s] get job management informations for until unixtime From 4571781157a21ad8673446e0a6e13ce22841507a Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Wed, 15 Jun 2016 15:54:59 -0700 Subject: [PATCH 51/69] double quote the keys in json --- .../zeppelin-example-clock.json | 14 +++++++------- .../zeppelin-example-horizontalbar.json | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/zeppelin-examples/zeppelin-example-clock/zeppelin-example-clock.json b/zeppelin-examples/zeppelin-example-clock/zeppelin-example-clock.json index 1deff591b81..dcaed396c98 100644 --- a/zeppelin-examples/zeppelin-example-clock/zeppelin-example-clock.json +++ b/zeppelin-examples/zeppelin-example-clock/zeppelin-example-clock.json @@ -15,11 +15,11 @@ * limitations under the License. */ { - type : "APPLICATION", - name : "zeppelin.clock", - description : "Clock (example)", - artifact : "zeppelin-examples/zeppelin-example-clock/target/zeppelin-example-clock-0.6.0-incubating-SNAPSHOT.jar", - className : "org.apache.zeppelin.example.app.clock.Clock", - resources : [[":java.util.Date"]], - icon : '' + "type" : "APPLICATION", + "name" : "zeppelin.clock", + "description" : "Clock (example)", + "artifact" : "zeppelin-examples/zeppelin-example-clock/target/zeppelin-example-clock-0.6.0-incubating-SNAPSHOT.jar", + "className" : "org.apache.zeppelin.example.app.clock.Clock", + "resources" : [[":java.util.Date"]], + "icon" : '' } diff --git a/zeppelin-examples/zeppelin-example-horizontalbar/zeppelin-example-horizontalbar.json b/zeppelin-examples/zeppelin-example-horizontalbar/zeppelin-example-horizontalbar.json index f61953c7300..7473e47bd30 100644 --- a/zeppelin-examples/zeppelin-example-horizontalbar/zeppelin-example-horizontalbar.json +++ b/zeppelin-examples/zeppelin-example-horizontalbar/zeppelin-example-horizontalbar.json @@ -15,11 +15,11 @@ * limitations under the License. */ { - type : "APPLICATION", - name : "zeppelin.horizontalbar", - description : "Horizontal Bar chart (example)", - artifact : "zeppelin-examples/zeppelin-example-horizontalbar/target/zeppelin-example-horizontalbar-0.6.0-incubating-SNAPSHOT.jar", - className : "org.apache.zeppelin.example.app.horizontalbar.HorizontalBar", - resources : [[":org.apache.zeppelin.interpreter.InterpreterResult"]], - icon : '' + "type" : "APPLICATION", + "name" : "zeppelin.horizontalbar", + "description" : "Horizontal Bar chart (example)", + "artifact" : "zeppelin-examples/zeppelin-example-horizontalbar/target/zeppelin-example-horizontalbar-0.6.0-incubating-SNAPSHOT.jar", + "className" : "org.apache.zeppelin.example.app.horizontalbar.HorizontalBar", + "resources" : [[":org.apache.zeppelin.interpreter.InterpreterResult"]], + "icon" : '' } From 6f1343fed1f02e5381d2bd96dc7e233d6325e9ce Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Wed, 15 Jun 2016 15:55:24 -0700 Subject: [PATCH 52/69] add license header --- .../zeppelin/notebook/NoteEventListener.java | 16 ++++++++++++++++ .../zeppelin/notebook/NotebookEventListener.java | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteEventListener.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteEventListener.java index c89901f51e0..5f98f7017dc 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteEventListener.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteEventListener.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.zeppelin.notebook; import org.apache.zeppelin.scheduler.Job; diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookEventListener.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookEventListener.java index d1c3020e0b2..904eba0c574 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookEventListener.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NotebookEventListener.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.zeppelin.notebook; import org.apache.zeppelin.interpreter.InterpreterSetting; From 88dbc71bf62d445c43e0568522e82f3c9a5d9a77 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Wed, 15 Jun 2016 15:55:40 -0700 Subject: [PATCH 53/69] change style --- .../interpreter/remote/RemoteInterpreter.java | 40 ++++++++++--------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java index 3505abf6304..802bb3b6559 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java @@ -90,15 +90,16 @@ public RemoteInterpreter(Properties property, /** * Connect to existing process */ - public RemoteInterpreter(Properties property, - String noteId, - String className, - String host, - int port, - int connectTimeout, - int maxPoolSize, - RemoteInterpreterProcessListener remoteInterpreterProcessListener, - ApplicationEventListener appListener) { + public RemoteInterpreter( + Properties property, + String noteId, + String className, + String host, + int port, + int connectTimeout, + int maxPoolSize, + RemoteInterpreterProcessListener remoteInterpreterProcessListener, + ApplicationEventListener appListener) { super(property); this.noteId = noteId; this.className = className; @@ -113,16 +114,17 @@ public RemoteInterpreter(Properties property, // VisibleForTesting - public RemoteInterpreter(Properties property, - String noteId, - String className, - String interpreterRunner, - String interpreterPath, - String localRepoPath, - Map env, - int connectTimeout, - RemoteInterpreterProcessListener remoteInterpreterProcessListener, - ApplicationEventListener appListener) { + public RemoteInterpreter( + Properties property, + String noteId, + String className, + String interpreterRunner, + String interpreterPath, + String localRepoPath, + Map env, + int connectTimeout, + RemoteInterpreterProcessListener remoteInterpreterProcessListener, + ApplicationEventListener appListener) { super(property); this.className = className; this.noteId = noteId; From 5bef173af741b391624824f901df787965e30f26 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Wed, 15 Jun 2016 23:40:19 -0700 Subject: [PATCH 54/69] Update NoteTest --- .../test/java/org/apache/zeppelin/notebook/NoteTest.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java index 33a7ef2ed98..9d02ed5d290 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java @@ -54,13 +54,16 @@ public class NoteTest { @Mock Scheduler scheduler; + @Mock + NoteEventListener noteEventListener; + @Test public void runNormalTest() { when(replLoader.get("spark")).thenReturn(interpreter); when(interpreter.getScheduler()).thenReturn(scheduler); String pText = "%spark sc.version"; - Note note = new Note(repo, replLoader, jobListenerFactory, index, credentials); + Note note = new Note(repo, replLoader, jobListenerFactory, index, credentials, noteEventListener); Paragraph p = note.addParagraph(); p.setText(pText); note.run(p.getId()); @@ -79,7 +82,7 @@ public void runJdbcTest() { when(interpreter.getScheduler()).thenReturn(scheduler); String pText = "%mysql show databases"; - Note note = new Note(repo, replLoader, jobListenerFactory, index, credentials); + Note note = new Note(repo, replLoader, jobListenerFactory, index, credentials, noteEventListener); Paragraph p = note.addParagraph(); p.setText(pText); note.run(p.getId()); From 644add4c22f9617e3326d94cbfbd27cf7071ff13 Mon Sep 17 00:00:00 2001 From: Lee moon soo Date: Thu, 16 Jun 2016 12:32:03 -0700 Subject: [PATCH 55/69] Takecare csv/tsv download button position after merge --- .../paragraph/paragraph-chart-selector.html | 42 ++++++++++--------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/zeppelin-web/src/app/notebook/paragraph/paragraph-chart-selector.html b/zeppelin-web/src/app/notebook/paragraph/paragraph-chart-selector.html index 454dd96ee21..507c57f984a 100644 --- a/zeppelin-web/src/app/notebook/paragraph/paragraph-chart-selector.html +++ b/zeppelin-web/src/app/notebook/paragraph/paragraph-chart-selector.html @@ -62,25 +62,6 @@ ng-class="{'active' : app.id == paragraph.config.helium.activeApp}" ng-bind-html="app.pkg.icon"> - - - - - - -