From cbdb22d309948eaa2d7562c66c3b8780a6581647 Mon Sep 17 00:00:00 2001 From: Rusty Phillips Date: Tue, 15 Mar 2016 19:11:20 -0400 Subject: [PATCH 1/6] Added serialization to resourcepoolutils. --- .../org/apache/zeppelin/resource/ResourcePoolUtils.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java index ecd84d4c01f..9768bc11b25 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java @@ -18,6 +18,8 @@ package org.apache.zeppelin.resource; import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService; @@ -54,8 +56,11 @@ public static ResourceSet getAllResourcesExcept(String interpreterGroupExcludsio boolean broken = false; try { client = remoteInterpreterProcess.getClient(); - List resourceList = client.resoucePoolGetAll(); - Gson gson = new Gson(); + List resourceList = client.resourcePoolGetAll(); + GsonBuilder gsonBuilder = new GsonBuilder(); + gsonBuilder.registerTypeAdapter(Resource.class, new ResourceSerializer()); + Gson gson = gsonBuilder.create(); + for (String res : resourceList) { resourceSet.add(gson.fromJson(res, Resource.class)); } From ffd092b6e7df495ecd1f4e753e3e88678bae2ead Mon Sep 17 00:00:00 2001 From: Rusty Phillips Date: Tue, 15 Mar 2016 19:06:35 -0400 Subject: [PATCH 2/6] Added pool persistance. --- .../apache/zeppelin/resource/Resource.java | 2 +- .../zeppelin/resource/ResourceSerializer.java | 46 +++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourceSerializer.java diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/Resource.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/Resource.java index 6988b3ea762..9e38d402302 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/Resource.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/Resource.java @@ -23,7 +23,7 @@ * Information and reference to the resource */ public class Resource { - private final transient Object r; + private final Object r; private final boolean serializable; private final ResourceId resourceId; private final String className; diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourceSerializer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourceSerializer.java new file mode 100644 index 00000000000..a0b35fa63cc --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourceSerializer.java @@ -0,0 +1,46 @@ +package org.apache.zeppelin.resource; + +import java.lang.reflect.Type; + +import com.google.gson.Gson; +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; +/** + * Serializes and Deserializes resources if they are serializable. + */ +public class ResourceSerializer implements JsonDeserializer, JsonSerializer { + + public ResourceSerializer() { + } + + @Override + public JsonElement serialize(Resource src, Type typeOfSrc, JsonSerializationContext context) { + // This is straightforward at the moment. + return context.serialize(src); + } + + @Override + public Resource deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) + throws JsonParseException { + // This requires that we use the class that's stored in the element to deserialize. + JsonObject obj = json.getAsJsonObject(); + String className = obj.getAsJsonPrimitive("className").getAsString(); + + Gson gson = new Gson(); + Object r; + try { + r = gson.fromJson(obj.get("r"), Class.forName(className)); + } catch (ClassNotFoundException e) { + throw new RuntimeException("Unable to deserialize the resource"); + } + ResourceId id = gson.fromJson(obj.get("resourceId"), ResourceId.class); + + return new Resource(id, r); + } + +} \ No newline at end of file From 84cabe2d02a7cf49a82f56d8ae8cbbecb6699aaf Mon Sep 17 00:00:00 2001 From: Rusty Phillips Date: Wed, 16 Mar 2016 10:59:29 -0400 Subject: [PATCH 3/6] Added test. --- .../zeppelin/resource/ResourceSerializer.java | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourceSerializer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourceSerializer.java index a0b35fa63cc..4db1f8742c0 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourceSerializer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourceSerializer.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.zeppelin.resource; import java.lang.reflect.Type; @@ -21,7 +37,8 @@ public ResourceSerializer() { @Override public JsonElement serialize(Resource src, Type typeOfSrc, JsonSerializationContext context) { // This is straightforward at the moment. - return context.serialize(src); + Gson gson = new Gson(); + return gson.toJsonTree(src); } @Override @@ -43,4 +60,4 @@ public Resource deserialize(JsonElement json, Type typeOfT, JsonDeserializationC return new Resource(id, r); } -} \ No newline at end of file +} From f1f0d91dbce47a50d5f9737b052ee1f73cc04590 Mon Sep 17 00:00:00 2001 From: Rusty Phillips Date: Mon, 28 Mar 2016 12:14:36 -0400 Subject: [PATCH 4/6] Changed resource pool utils so that it will work anywhere. --- ...terpreterProcessResourcePoolConnector.java | 65 +++++++++++++++++++ .../apache/zeppelin/resource/Resource.java | 2 +- .../zeppelin/resource/ResourcePoolUtils.java | 12 ++-- 3 files changed, 72 insertions(+), 7 deletions(-) create mode 100644 zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/RemoteInterpreterProcessResourcePoolConnector.java diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/RemoteInterpreterProcessResourcePoolConnector.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/RemoteInterpreterProcessResourcePoolConnector.java new file mode 100644 index 00000000000..7315af7c266 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/RemoteInterpreterProcessResourcePoolConnector.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.resource; + +import java.util.List; + +import org.apache.thrift.TException; +import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; + +import com.google.gson.Gson; +/** + * Makes a remote interpreter service client act as a resource pool connector. + */ +public class RemoteInterpreterProcessResourcePoolConnector implements ResourcePoolConnector { + + private Client client; + + public RemoteInterpreterProcessResourcePoolConnector(Client client) { + this.client = client; + } + + @Override + public ResourceSet getAllResources() { + try { + List resourceList = client.resourcePoolGetAll(); + ResourceSet resources = new ResourceSet(); + Gson gson = new Gson(); + + for (String res : resourceList) { + RemoteResource r = gson.fromJson(res, RemoteResource.class); + r.setResourcePoolConnector(this); + resources.add(r); + } + + return resources; + } catch (TException e) { + throw new RuntimeException(e); + } + } + + @Override + public Object readResource(ResourceId id) { + try { + // TODO(Object): Deserialize object + return client.resourceGet(id.getNoteId(), id.getParagraphId(), id.getName()); + } catch (TException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/Resource.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/Resource.java index 9e38d402302..6988b3ea762 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/Resource.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/Resource.java @@ -23,7 +23,7 @@ * Information and reference to the resource */ public class Resource { - private final Object r; + private final transient Object r; private final boolean serializable; private final ResourceId resourceId; private final String className; diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java index 678b0cc7033..3aadffb8524 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java @@ -56,13 +56,13 @@ public static ResourceSet getAllResourcesExcept(String interpreterGroupExcludsio boolean broken = false; try { client = remoteInterpreterProcess.getClient(); - List resourceList = client.resourcePoolGetAll(); - GsonBuilder gsonBuilder = new GsonBuilder(); - gsonBuilder.registerTypeAdapter(Resource.class, new ResourceSerializer()); - Gson gson = gsonBuilder.create(); + RemoteInterpreterProcessResourcePoolConnector remoteConnector = + new RemoteInterpreterProcessResourcePoolConnector(client); + //List resourceList = client.resourcePoolGetAll(); + //gsonBuilder.registerTypeAdapter(Resource.class, new ResourceSerializer()); - for (String res : resourceList) { - resourceSet.add(gson.fromJson(res, Resource.class)); + for (Resource r: remoteConnector.getAllResources()) { + resourceSet.add(r); } } catch (Exception e) { logger.error(e.getMessage(), e); From a19c48e124cf45b6287656133bd4dca26dcce002 Mon Sep 17 00:00:00 2001 From: Rusty Phillips Date: Mon, 28 Mar 2016 13:42:00 -0400 Subject: [PATCH 5/6] Removed unnecessary serialization. --- .../java/org/apache/zeppelin/resource/ResourcePoolUtils.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java index 3aadffb8524..4eb1d6a8896 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourcePoolUtils.java @@ -58,8 +58,6 @@ public static ResourceSet getAllResourcesExcept(String interpreterGroupExcludsio client = remoteInterpreterProcess.getClient(); RemoteInterpreterProcessResourcePoolConnector remoteConnector = new RemoteInterpreterProcessResourcePoolConnector(client); - //List resourceList = client.resourcePoolGetAll(); - //gsonBuilder.registerTypeAdapter(Resource.class, new ResourceSerializer()); for (Resource r: remoteConnector.getAllResources()) { resourceSet.add(r); From 8fa84a832200351465793f83aafe8d1c0c3c3573 Mon Sep 17 00:00:00 2001 From: Rusty Phillips Date: Mon, 28 Mar 2016 17:02:18 -0400 Subject: [PATCH 6/6] Added explicit save to the serializer. --- .../org/apache/zeppelin/resource/ResourceSerializer.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourceSerializer.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourceSerializer.java index 4db1f8742c0..887f3078525 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourceSerializer.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/resource/ResourceSerializer.java @@ -38,7 +38,12 @@ public ResourceSerializer() { public JsonElement serialize(Resource src, Type typeOfSrc, JsonSerializationContext context) { // This is straightforward at the moment. Gson gson = new Gson(); - return gson.toJsonTree(src); + JsonElement elem = gson.toJsonTree(src); + JsonObject obj = elem.getAsJsonObject(); + if (src.isSerializable()) { + obj.add("r", gson.toJsonTree(src.get())); + } + return obj; } @Override