From 11139788ba9a60eba2a75ba47f30557af1dfbbdb Mon Sep 17 00:00:00 2001 From: ace-n Date: Thu, 12 Mar 2020 23:07:24 -0700 Subject: [PATCH 01/18] Add Slack sample + clean up imports --- functions/snippets/.gitignore | 1 + functions/snippets/config.slack.json | 4 + functions/snippets/pom.xml | 16 +- .../example/functions/SlackSlashCommand.java | 172 ++++++++++++++++++ .../functions/SlackSlashCommandTest.java | 172 ++++++++++++++++++ .../com/example/functions/SnippetsTests.java | 18 -- 6 files changed, 363 insertions(+), 20 deletions(-) create mode 100644 functions/snippets/.gitignore create mode 100644 functions/snippets/config.slack.json create mode 100644 functions/snippets/src/main/java/com/example/functions/SlackSlashCommand.java create mode 100644 functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java diff --git a/functions/snippets/.gitignore b/functions/snippets/.gitignore new file mode 100644 index 00000000000..d344ba6b06c --- /dev/null +++ b/functions/snippets/.gitignore @@ -0,0 +1 @@ +config.json diff --git a/functions/snippets/config.slack.json b/functions/snippets/config.slack.json new file mode 100644 index 00000000000..ca8017b328f --- /dev/null +++ b/functions/snippets/config.slack.json @@ -0,0 +1,4 @@ +{ + "SLACK_SECRET": "[YOUR_SLACK_SIGNING_SECRET]", + "KG_API_KEY": "[YOUR_KG_API_KEY]" +} \ No newline at end of file diff --git a/functions/snippets/pom.xml b/functions/snippets/pom.xml index b94a552825b..3d85e74aac7 100644 --- a/functions/snippets/pom.xml +++ b/functions/snippets/pom.xml @@ -26,20 +26,32 @@ - + com.google.code.gson gson 2.8.6 - + com.google.cloud google-cloud-logging 1.100.0 + + + com.google.apis + google-api-services-kgsearch + v1-rev253-1.25.0 + + + com.github.seratch + jslack + 3.4.1 + + junit diff --git a/functions/snippets/src/main/java/com/example/functions/SlackSlashCommand.java b/functions/snippets/src/main/java/com/example/functions/SlackSlashCommand.java new file mode 100644 index 00000000000..890078c775b --- /dev/null +++ b/functions/snippets/src/main/java/com/example/functions/SlackSlashCommand.java @@ -0,0 +1,172 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.functions; + +import com.github.seratch.jslack.app_backend.SlackSignature; +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.api.client.util.ArrayMap; +import com.google.api.services.kgsearch.v1.Kgsearch; +import com.google.cloud.functions.HttpFunction; +import com.google.cloud.functions.HttpRequest; +import com.google.cloud.functions.HttpResponse; +import com.google.gson.Gson; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import java.io.BufferedWriter; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.GeneralSecurityException; +import java.util.HashMap; +import java.util.List; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +public class SlackSlashCommand implements HttpFunction { + + private Kgsearch kgClient; + private static String API_KEY; + private static String SLACK_SECRET; + private static final Logger LOGGER = Logger.getLogger(HelloHttp.class.getName()); + private SlackSignature.Verifier verifier; + private Gson gson = new Gson(); + + public SlackSlashCommand() throws IOException, GeneralSecurityException { + kgClient = new Kgsearch.Builder( + GoogleNetHttpTransport.newTrustedTransport(), new JacksonFactory(), null).build(); + + // Read + parse config file + Path configPath = Path.of(System.getProperty("user.dir"), "config.json"); + JsonObject configJson = (new Gson()).fromJson(Files.readString(configPath), JsonObject.class); + + SLACK_SECRET = configJson.get("SLACK_SECRET").getAsString(); + API_KEY = configJson.get("KG_API_KEY").getAsString(); + + verifier = new SlackSignature.Verifier(new SlackSignature.Generator(SLACK_SECRET)); + } + + private boolean isValidSlackWebhook(HttpRequest request, String requestBody) throws IOException { + + // Check for headers + HashMap> headers = new HashMap(request.getHeaders()); + if (!headers.containsKey("X-Slack-Request-Timestamp") + || !headers.containsKey("X-Slack-Signature")) { + return false; + } + + return verifier.isValid( + headers.get("X-Slack-Request-Timestamp").get(0), + requestBody, + headers.get("X-Slack-Signature").get(0), + 1L); + } + + private void addPropertyIfPresent( + JsonObject target, String targetName, ArrayMap source, String sourceName) { + if (source.containsKey(sourceName)) { + target.addProperty(targetName, source.get(sourceName).toString()); + } + } + + private String formatSlackMessage(List kgResults, String query) { + JsonObject attachmentJson = new JsonObject(); + JsonArray attachments = new JsonArray(); + + JsonObject responseJson = new JsonObject(); + responseJson.addProperty("response_type", "in_channel"); + responseJson.addProperty("text", String.format("Query: %s", query)); + + // Extract the first entity from the result list, if any + if (kgResults.size() == 0) { + attachmentJson.addProperty("text","No results match your query..."); + + attachments.add(attachmentJson); + responseJson.add("attachments", attachmentJson); + + return gson.toJson(responseJson); + } + + ArrayMap entity = (ArrayMap) ((ArrayMap) kgResults.get(0)).get("result"); + + // Construct Knowledge Graph response attachment + String title = entity.get("name").toString(); + if (entity.containsKey("description")) { + title = String.format("%s: %s", title, entity.get("description").toString()); + } + attachmentJson.addProperty("title", title); + + if (entity.containsKey("detailedDescription")) { + ArrayMap detailedDescJson = (ArrayMap) entity.get("detailedDescription"); + addPropertyIfPresent(attachmentJson, "title_link", detailedDescJson, "url"); + addPropertyIfPresent(attachmentJson, "text", detailedDescJson, "articleBody"); + } + + if (entity.containsKey("image")) { + ArrayMap imageJson = (ArrayMap) entity.get("image"); + addPropertyIfPresent(attachmentJson, "image_url", imageJson, "contentUrl"); + } + + // Construct top level response + attachments.add(attachmentJson); + responseJson.add("attachments", attachmentJson); + + return gson.toJson(responseJson); + } + + private List searchKnowledgeGraph(String query) throws IOException { + Kgsearch.Entities.Search kgRequest = kgClient.entities().search(); + kgRequest.setQuery(query); + kgRequest.setKey(API_KEY); + + return kgRequest.execute().getItemListElement(); + } + + @Override + public void service(HttpRequest request, HttpResponse response) throws IOException { + + // Validate request + if (request.getMethod() != "POST") { + response.setStatusCode(HttpURLConnection.HTTP_BAD_METHOD); + return; + } + + // reader can only be read once per request, so we preserve its contents + String bodyString = request.getReader().lines().collect(Collectors.joining()); + JsonObject body = (new Gson()).fromJson(bodyString, JsonObject.class); + + if (body == null || !body.has("text")) { + response.setStatusCode(HttpURLConnection.HTTP_BAD_REQUEST); + return; + } + + if (!isValidSlackWebhook(request, bodyString)) { + response.setStatusCode(HttpURLConnection.HTTP_UNAUTHORIZED); + return; + } + + String query = body.get("text").getAsString(); + + // Call knowledge graph API + List kgResults = searchKnowledgeGraph(query); + + // Format response to Slack + BufferedWriter writer = response.getWriter(); + writer.write(formatSlackMessage(kgResults, query)); + } +} diff --git a/functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java b/functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java new file mode 100644 index 00000000000..5df91a0d48e --- /dev/null +++ b/functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java @@ -0,0 +1,172 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.functions; + +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.powermock.api.mockito.PowerMockito.mock; +import static org.powermock.api.mockito.PowerMockito.when; + +import com.github.seratch.jslack.app_backend.SlackSignature; +import com.google.api.client.googleapis.json.GoogleJsonResponseException; +import com.google.cloud.functions.HttpRequest; +import com.google.cloud.functions.HttpResponse; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; +import java.net.HttpURLConnection; +import java.security.GeneralSecurityException; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentMatchers; +import org.mockito.Mock; +import org.powermock.reflect.Whitebox; + +public class SlackSlashCommandTest { + + private BufferedWriter writerOut; + private StringWriter responseOut; + + @Mock private HttpRequest request; + @Mock private HttpResponse response; + + @Mock private SlackSignature.Verifier alwaysValidVerifier; + + @Before + public void beforeTest() throws IOException { + request = mock(HttpRequest.class); + when(request.getReader()).thenReturn(new BufferedReader(new StringReader(""))); + + response = mock(HttpResponse.class); + + responseOut = new StringWriter(); + + writerOut = new BufferedWriter(responseOut); + when(response.getWriter()).thenReturn(writerOut); + + alwaysValidVerifier = mock(SlackSignature.Verifier.class); + when(alwaysValidVerifier.isValid( + ArgumentMatchers.any(), + ArgumentMatchers.any(), + ArgumentMatchers.any(), + ArgumentMatchers.anyLong()) + ).thenReturn(true); + + // Construct valid header list + HashMap> validHeaders = new HashMap>(); + String validSlackSignature = System.getenv("SLACK_TEST_SIGNATURE"); + String timestamp = + Long.toString(ZonedDateTime.of(1970, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC) + .toInstant().toEpochMilli()); + + validHeaders.put("X-Slack-Signature", Arrays.asList(validSlackSignature)); + validHeaders.put("X-Slack-Request-Timestamp", Arrays.asList(timestamp)); + + when(request.getHeaders()).thenReturn(validHeaders); + } + + @Test + public void onlyAcceptsPostRequestsTest() throws IOException, GeneralSecurityException { + when(request.getMethod()).thenReturn("GET"); + new SlackSlashCommand().service(request, response); + + writerOut.flush(); + verify(response, times(1)).setStatusCode(HttpURLConnection.HTTP_BAD_METHOD); + } + + @Test + public void requiresSlackAuthHeadersTest() throws IOException, GeneralSecurityException { + StringReader requestReadable = new StringReader("{ \"text\": \"foo\" }\n"); + + when(request.getMethod()).thenReturn("POST"); + when(request.getReader()).thenReturn(new BufferedReader(requestReadable)); + + new SlackSlashCommand().service(request, response); + + // Do NOT look for HTTP_BAD_REQUEST here (that means the request WAS authorized)! + verify(response, times(1)).setStatusCode(HttpURLConnection.HTTP_UNAUTHORIZED); + } + + @Test + public void recognizesValidSlackTokenTest() throws IOException, GeneralSecurityException { + StringReader requestReadable = new StringReader("{}"); + + when(request.getReader()).thenReturn(new BufferedReader(requestReadable)); + when(request.getMethod()).thenReturn("POST"); + + new SlackSlashCommand().service(request, response); + + verify(response, times(1)).setStatusCode(HttpURLConnection.HTTP_BAD_REQUEST); + } + + @Test(expected = GoogleJsonResponseException.class) + public void handlesSearchErrorTest() throws IOException, GeneralSecurityException { + StringReader requestReadable = new StringReader("{ \"text\": \"foo\" }\n"); + + when(request.getReader()).thenReturn(new BufferedReader(requestReadable)); + when(request.getMethod()).thenReturn("POST"); + + SlackSlashCommand functionInstance = new SlackSlashCommand(); + Whitebox.setInternalState(functionInstance, "verifier", alwaysValidVerifier); + Whitebox.setInternalState(SlackSlashCommand.class, "API_KEY", "gibberish"); + + // Should throw a GoogleJsonResponseException (due to invalid API key) + functionInstance.service(request, response); + } + + @Test + public void handlesEmptyKgResultsTest() throws IOException, GeneralSecurityException { + StringReader requestReadable = new StringReader("{ \"text\": \"asdfjkl13579\" }\n"); + + when(request.getReader()).thenReturn(new BufferedReader(requestReadable)); + when(request.getMethod()).thenReturn("POST"); + + SlackSlashCommand functionInstance = new SlackSlashCommand(); + Whitebox.setInternalState(functionInstance, "verifier", alwaysValidVerifier); + + + functionInstance.service(request, response); + + writerOut.flush(); + assertThat(responseOut.toString()).contains("No results match your query..."); + } + + @Test + public void handlesPopulatedKgResultsTest() throws IOException, GeneralSecurityException { + StringReader requestReadable = new StringReader("{ \"text\": \"lion\" }\n"); + + when(request.getReader()).thenReturn(new BufferedReader(requestReadable)); + when(request.getMethod()).thenReturn("POST"); + + SlackSlashCommand functionInstance = new SlackSlashCommand(); + Whitebox.setInternalState(functionInstance, "verifier", alwaysValidVerifier); + + + functionInstance.service(request, response); + + writerOut.flush(); + assertThat(responseOut.toString()).contains("https://en.wikipedia.org/wiki/Lion"); + } +} diff --git a/functions/snippets/src/test/java/com/example/functions/SnippetsTests.java b/functions/snippets/src/test/java/com/example/functions/SnippetsTests.java index ecdf7af7a7f..c13a6aa3b9e 100644 --- a/functions/snippets/src/test/java/com/example/functions/SnippetsTests.java +++ b/functions/snippets/src/test/java/com/example/functions/SnippetsTests.java @@ -20,24 +20,6 @@ import static org.powermock.api.mockito.PowerMockito.mock; import static org.powermock.api.mockito.PowerMockito.when; -import com.example.functions.CorsEnabled; -import com.example.functions.EnvVars; -import com.example.functions.ExecutionCount; -import com.example.functions.FileSystem; -import com.example.functions.FirebaseAuth; -import com.example.functions.HelloBackground; -import com.example.functions.HelloGcs; -import com.example.functions.HelloHttp; -import com.example.functions.HelloPubSub; -import com.example.functions.HelloWorld; -import com.example.functions.LazyFields; -import com.example.functions.LogHelloWorld; -import com.example.functions.ParseContentType; -import com.example.functions.PubSubMessage; -import com.example.functions.RetrieveLogs; -import com.example.functions.Scopes; -import com.example.functions.SendHttpRequest; -import com.example.functions.StackdriverLogging; import com.google.cloud.functions.HttpRequest; import com.google.cloud.functions.HttpResponse; import com.google.common.testing.TestLogHandler; From b84f11b129e955eef71b31ea9f1f7be9c349b091 Mon Sep 17 00:00:00 2001 From: ace-n Date: Fri, 13 Mar 2020 22:11:38 -0700 Subject: [PATCH 02/18] Address comments --- .../example/functions/SlackSlashCommand.java | 57 ++++++++----------- 1 file changed, 24 insertions(+), 33 deletions(-) diff --git a/functions/snippets/src/main/java/com/example/functions/SlackSlashCommand.java b/functions/snippets/src/main/java/com/example/functions/SlackSlashCommand.java index 890078c775b..e4991da499c 100644 --- a/functions/snippets/src/main/java/com/example/functions/SlackSlashCommand.java +++ b/functions/snippets/src/main/java/com/example/functions/SlackSlashCommand.java @@ -19,7 +19,6 @@ import com.github.seratch.jslack.app_backend.SlackSignature; import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; import com.google.api.client.json.jackson2.JacksonFactory; -import com.google.api.client.util.ArrayMap; import com.google.api.services.kgsearch.v1.Kgsearch; import com.google.cloud.functions.HttpFunction; import com.google.cloud.functions.HttpRequest; @@ -30,8 +29,6 @@ import java.io.BufferedWriter; import java.io.IOException; import java.net.HttpURLConnection; -import java.nio.file.Files; -import java.nio.file.Path; import java.security.GeneralSecurityException; import java.util.HashMap; import java.util.List; @@ -41,9 +38,9 @@ public class SlackSlashCommand implements HttpFunction { private Kgsearch kgClient; - private static String API_KEY; - private static String SLACK_SECRET; - private static final Logger LOGGER = Logger.getLogger(HelloHttp.class.getName()); + private static final String API_KEY = System.getenv("KG_API_KEY"); + private static final String SLACK_SECRET = System.getenv("SLACK_SECRET"); + private static final Logger LOGGER = Logger.getLogger(SlackSlashCommand.class.getName()); private SlackSignature.Verifier verifier; private Gson gson = new Gson(); @@ -51,17 +48,10 @@ public SlackSlashCommand() throws IOException, GeneralSecurityException { kgClient = new Kgsearch.Builder( GoogleNetHttpTransport.newTrustedTransport(), new JacksonFactory(), null).build(); - // Read + parse config file - Path configPath = Path.of(System.getProperty("user.dir"), "config.json"); - JsonObject configJson = (new Gson()).fromJson(Files.readString(configPath), JsonObject.class); - - SLACK_SECRET = configJson.get("SLACK_SECRET").getAsString(); - API_KEY = configJson.get("KG_API_KEY").getAsString(); - verifier = new SlackSignature.Verifier(new SlackSignature.Generator(SLACK_SECRET)); } - private boolean isValidSlackWebhook(HttpRequest request, String requestBody) throws IOException { + boolean isValidSlackWebhook(HttpRequest request, String requestBody) throws IOException { // Check for headers HashMap> headers = new HashMap(request.getHeaders()); @@ -69,7 +59,6 @@ private boolean isValidSlackWebhook(HttpRequest request, String requestBody) thr || !headers.containsKey("X-Slack-Signature")) { return false; } - return verifier.isValid( headers.get("X-Slack-Request-Timestamp").get(0), requestBody, @@ -77,14 +66,14 @@ private boolean isValidSlackWebhook(HttpRequest request, String requestBody) thr 1L); } - private void addPropertyIfPresent( - JsonObject target, String targetName, ArrayMap source, String sourceName) { - if (source.containsKey(sourceName)) { - target.addProperty(targetName, source.get(sourceName).toString()); + void addPropertyIfPresent( + JsonObject target, String targetName, JsonObject source, String sourceName) { + if (source.has(sourceName)) { + target.addProperty(targetName, source.get(sourceName).getAsString()); } } - private String formatSlackMessage(List kgResults, String query) { + String formatSlackMessage(JsonObject kgResponse, String query) { JsonObject attachmentJson = new JsonObject(); JsonArray attachments = new JsonArray(); @@ -92,8 +81,10 @@ private String formatSlackMessage(List kgResults, String query) { responseJson.addProperty("response_type", "in_channel"); responseJson.addProperty("text", String.format("Query: %s", query)); + JsonArray entityList = kgResponse.getAsJsonArray("itemListElement"); + // Extract the first entity from the result list, if any - if (kgResults.size() == 0) { + if (entityList.size() == 0) { attachmentJson.addProperty("text","No results match your query..."); attachments.add(attachmentJson); @@ -102,23 +93,23 @@ private String formatSlackMessage(List kgResults, String query) { return gson.toJson(responseJson); } - ArrayMap entity = (ArrayMap) ((ArrayMap) kgResults.get(0)).get("result"); + JsonObject entity = entityList.get(0).getAsJsonObject().getAsJsonObject("result"); // Construct Knowledge Graph response attachment - String title = entity.get("name").toString(); - if (entity.containsKey("description")) { - title = String.format("%s: %s", title, entity.get("description").toString()); + String title = entity.get("name").getAsString(); + if (entity.has("description")) { + title = String.format("%s: %s", title, entity.get("description").getAsString()); } attachmentJson.addProperty("title", title); - if (entity.containsKey("detailedDescription")) { - ArrayMap detailedDescJson = (ArrayMap) entity.get("detailedDescription"); + if (entity.has("detailedDescription")) { + JsonObject detailedDescJson = entity.getAsJsonObject("detailedDescription"); addPropertyIfPresent(attachmentJson, "title_link", detailedDescJson, "url"); addPropertyIfPresent(attachmentJson, "text", detailedDescJson, "articleBody"); } - if (entity.containsKey("image")) { - ArrayMap imageJson = (ArrayMap) entity.get("image"); + if (entity.has("image")) { + JsonObject imageJson = entity.getAsJsonObject("image"); addPropertyIfPresent(attachmentJson, "image_url", imageJson, "contentUrl"); } @@ -129,12 +120,12 @@ private String formatSlackMessage(List kgResults, String query) { return gson.toJson(responseJson); } - private List searchKnowledgeGraph(String query) throws IOException { + JsonObject searchKnowledgeGraph(String query) throws IOException { Kgsearch.Entities.Search kgRequest = kgClient.entities().search(); kgRequest.setQuery(query); kgRequest.setKey(API_KEY); - return kgRequest.execute().getItemListElement(); + return gson.fromJson(kgRequest.execute().toString(), JsonObject.class); } @Override @@ -163,10 +154,10 @@ public void service(HttpRequest request, HttpResponse response) throws IOExcepti String query = body.get("text").getAsString(); // Call knowledge graph API - List kgResults = searchKnowledgeGraph(query); + JsonObject kgResponse = searchKnowledgeGraph(query); // Format response to Slack BufferedWriter writer = response.getWriter(); - writer.write(formatSlackMessage(kgResults, query)); + writer.write(formatSlackMessage(kgResponse, query)); } } From 282749602d18df3c69832d8158ea2068fd552667 Mon Sep 17 00:00:00 2001 From: ace-n Date: Fri, 13 Mar 2020 22:12:18 -0700 Subject: [PATCH 03/18] Remove excess gcloudignore + actually disable tests --- functions/snippets/.gcloudignore | 8 -------- functions/snippets/pom.xml | 13 +++++++++++-- 2 files changed, 11 insertions(+), 10 deletions(-) delete mode 100644 functions/snippets/.gcloudignore diff --git a/functions/snippets/.gcloudignore b/functions/snippets/.gcloudignore deleted file mode 100644 index 0ac51e7dd4e..00000000000 --- a/functions/snippets/.gcloudignore +++ /dev/null @@ -1,8 +0,0 @@ -# Don't run tests when deploying a function using gcloud -# Comment out (with a leading '#') or remove the line below -# if you want to run tests upon deployment with 'gcloud' -src/test/ - -# If you want your code to be automatically tested and -# deployed to Cloud Functions, we recommend using -# a continuous integration pipeline to enable this. \ No newline at end of file diff --git a/functions/snippets/pom.xml b/functions/snippets/pom.xml index 3d85e74aac7..3a79701b67b 100644 --- a/functions/snippets/pom.xml +++ b/functions/snippets/pom.xml @@ -114,10 +114,19 @@ - - + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M3 + + true + sponge_log + false + + + org.apache.maven.plugins maven-compiler-plugin From 6383b60197d0fd0818776938476c4f06ee9d3833 Mon Sep 17 00:00:00 2001 From: ace-n Date: Mon, 16 Mar 2020 01:47:50 -0700 Subject: [PATCH 04/18] Simplify tests + run them on Kokoro. ALSO bugfix unused shellchecks. --- .kokoro/tests/run_tests.sh | 8 +++++--- .../java/com/example/functions/SlackSlashCommandTest.java | 6 +----- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.kokoro/tests/run_tests.sh b/.kokoro/tests/run_tests.sh index c1b9336427d..24f8a6d1335 100755 --- a/.kokoro/tests/run_tests.sh +++ b/.kokoro/tests/run_tests.sh @@ -65,12 +65,14 @@ if [[ "$SCRIPT_DEBUG" != "true" ]]; then source "${KOKORO_GFILE_DIR}/aws-secrets.sh" # shellcheck source=src/storage-hmac-credentials.sh source "${KOKORO_GFILE_DIR}/storage-hmac-credentials.sh" - # shellcheck source=src/dlp_secrets.sh + # shellcheck source=src/dlp_secrets.txt source "${KOKORO_GFILE_DIR}/dlp_secrets.txt" - # shellcheck source=src/bigtable_secrets.sh + # shellcheck source=src/bigtable_secrets.txt source "${KOKORO_GFILE_DIR}/bigtable_secrets.txt" - # shellcheck source=src/automl_secrets.sh + # shellcheck source=src/automl_secrets.txt source "${KOKORO_GFILE_DIR}/automl_secrets.txt" + # shellcheck source=src/functions_secrets.txt + source "${KOKORO_GFILE_DIR}/functions_secrets.txt" # Activate service account gcloud auth activate-service-account \ --key-file="$GOOGLE_APPLICATION_CREDENTIALS" \ diff --git a/functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java b/functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java index 5df91a0d48e..712c85f6cc0 100644 --- a/functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java +++ b/functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java @@ -33,8 +33,6 @@ import java.io.StringWriter; import java.net.HttpURLConnection; import java.security.GeneralSecurityException; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -77,9 +75,7 @@ public void beforeTest() throws IOException { // Construct valid header list HashMap> validHeaders = new HashMap>(); String validSlackSignature = System.getenv("SLACK_TEST_SIGNATURE"); - String timestamp = - Long.toString(ZonedDateTime.of(1970, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC) - .toInstant().toEpochMilli()); + String timestamp = "0"; // start of Unix epoch validHeaders.put("X-Slack-Signature", Arrays.asList(validSlackSignature)); validHeaders.put("X-Slack-Request-Timestamp", Arrays.asList(timestamp)); From 0c2d0ffd33b1192a9c4eddc066768239ddbc4c6b Mon Sep 17 00:00:00 2001 From: ace-n Date: Mon, 16 Mar 2020 18:08:15 -0700 Subject: [PATCH 05/18] Remove extra file --- functions/snippets/config.slack.json | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 functions/snippets/config.slack.json diff --git a/functions/snippets/config.slack.json b/functions/snippets/config.slack.json deleted file mode 100644 index ca8017b328f..00000000000 --- a/functions/snippets/config.slack.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "SLACK_SECRET": "[YOUR_SLACK_SIGNING_SECRET]", - "KG_API_KEY": "[YOUR_KG_API_KEY]" -} \ No newline at end of file From 032e107f06f1b82eb5da399f653be4ab60c72f92 Mon Sep 17 00:00:00 2001 From: ace-n Date: Mon, 16 Mar 2020 18:25:35 -0700 Subject: [PATCH 06/18] HACK: resolve surefire issue via file presence --- functions/snippets/pom.xml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/functions/snippets/pom.xml b/functions/snippets/pom.xml index aff79296e00..08838cc4189 100644 --- a/functions/snippets/pom.xml +++ b/functions/snippets/pom.xml @@ -114,14 +114,31 @@ + + + + + + + .gcloudignore + + + + + true + + + + + - + org.apache.maven.plugins maven-surefire-plugin 3.0.0-M3 - true + ${skipTests} sponge_log false From 38a32f1953bbc343a7c58326a5029f5fda325b1a Mon Sep 17 00:00:00 2001 From: ace-n Date: Wed, 18 Mar 2020 20:07:37 -0700 Subject: [PATCH 07/18] HACK take 2: use a different filepath --- functions/snippets/pom.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/functions/snippets/pom.xml b/functions/snippets/pom.xml index 08838cc4189..5994198689d 100644 --- a/functions/snippets/pom.xml +++ b/functions/snippets/pom.xml @@ -114,20 +114,20 @@ - - + + + + - .gcloudignore + /builder/home - true - From 5005a1bcc7604728d3d4b5bf0d9379120d4c3375 Mon Sep 17 00:00:00 2001 From: ace-n Date: Thu, 19 Mar 2020 17:38:24 -0700 Subject: [PATCH 08/18] HACK take 3: use env var not used by local Cloud Build --- functions/snippets/pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/functions/snippets/pom.xml b/functions/snippets/pom.xml index 5994198689d..42f8895c54c 100644 --- a/functions/snippets/pom.xml +++ b/functions/snippets/pom.xml @@ -121,9 +121,9 @@ - - /builder/home - + + env.NEW_BUILD + true From 1583f6391e14c6c71d6cab715597b90028965738 Mon Sep 17 00:00:00 2001 From: ace-n Date: Thu, 19 Mar 2020 18:24:40 -0700 Subject: [PATCH 09/18] Remove gitignore now that config.json isnt used --- functions/snippets/.gitignore | 1 - 1 file changed, 1 deletion(-) delete mode 100644 functions/snippets/.gitignore diff --git a/functions/snippets/.gitignore b/functions/snippets/.gitignore deleted file mode 100644 index d344ba6b06c..00000000000 --- a/functions/snippets/.gitignore +++ /dev/null @@ -1 +0,0 @@ -config.json From 5ee5f0dd46333a1af5df658ee076d21fdf5fe9d8 Mon Sep 17 00:00:00 2001 From: ace-n Date: Fri, 20 Mar 2020 13:44:05 -0700 Subject: [PATCH 10/18] DBG: print defined env vars --- .../src/test/java/com/example/functions/SnippetsTests.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/functions/snippets/src/test/java/com/example/functions/SnippetsTests.java b/functions/snippets/src/test/java/com/example/functions/SnippetsTests.java index c13a6aa3b9e..39bf25d0071 100644 --- a/functions/snippets/src/test/java/com/example/functions/SnippetsTests.java +++ b/functions/snippets/src/test/java/com/example/functions/SnippetsTests.java @@ -273,6 +273,10 @@ public void helloHttp_bodyParamsPost() throws IOException { writerOut.flush(); assertThat(responseOut.toString()).isEqualTo("Hello Jane!"); + + // DBG + String envStr = String.join("/", System.getenv().keySet()); + GCS_LOGGER.info(envStr); } @Test From 3dc41288916f8c9fb8c4dd577c742f64ab848f12 Mon Sep 17 00:00:00 2001 From: ace-n Date: Fri, 20 Mar 2020 13:53:36 -0700 Subject: [PATCH 11/18] DBG take 2 --- .../java/com/example/functions/SlackSlashCommandTest.java | 5 +++++ .../src/test/java/com/example/functions/SnippetsTests.java | 4 ---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java b/functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java index 712c85f6cc0..2a36acf5a2f 100644 --- a/functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java +++ b/functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java @@ -36,6 +36,7 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; +import java.util.logging.Logger; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentMatchers; @@ -90,6 +91,10 @@ public void onlyAcceptsPostRequestsTest() throws IOException, GeneralSecurityExc writerOut.flush(); verify(response, times(1)).setStatusCode(HttpURLConnection.HTTP_BAD_METHOD); + + // DBG + String envStr = String.join("/", System.getenv().keySet()); + Logger.getAnonymousLogger().info(envStr); } @Test diff --git a/functions/snippets/src/test/java/com/example/functions/SnippetsTests.java b/functions/snippets/src/test/java/com/example/functions/SnippetsTests.java index 39bf25d0071..c13a6aa3b9e 100644 --- a/functions/snippets/src/test/java/com/example/functions/SnippetsTests.java +++ b/functions/snippets/src/test/java/com/example/functions/SnippetsTests.java @@ -273,10 +273,6 @@ public void helloHttp_bodyParamsPost() throws IOException { writerOut.flush(); assertThat(responseOut.toString()).isEqualTo("Hello Jane!"); - - // DBG - String envStr = String.join("/", System.getenv().keySet()); - GCS_LOGGER.info(envStr); } @Test From d3be1ce9e9c0d975fca8ef79f26cb67feb6d5e69 Mon Sep 17 00:00:00 2001 From: ace-n Date: Fri, 20 Mar 2020 14:32:51 -0700 Subject: [PATCH 12/18] DBG take 3 --- .../java/com/example/functions/SlackSlashCommandTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java b/functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java index 2a36acf5a2f..7f74e0774e1 100644 --- a/functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java +++ b/functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java @@ -89,12 +89,12 @@ public void onlyAcceptsPostRequestsTest() throws IOException, GeneralSecurityExc when(request.getMethod()).thenReturn("GET"); new SlackSlashCommand().service(request, response); - writerOut.flush(); - verify(response, times(1)).setStatusCode(HttpURLConnection.HTTP_BAD_METHOD); - // DBG String envStr = String.join("/", System.getenv().keySet()); - Logger.getAnonymousLogger().info(envStr); + assertThat("DBG").isEqualTo(envStr); + + writerOut.flush(); + verify(response, times(1)).setStatusCode(HttpURLConnection.HTTP_BAD_METHOD); } @Test From 4b6ea96655897c0c45ec2b1a9f291090be592818 Mon Sep 17 00:00:00 2001 From: ace-n Date: Fri, 20 Mar 2020 15:30:04 -0700 Subject: [PATCH 13/18] DBG take 4 --- .../java/com/example/functions/SlackSlashCommandTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java b/functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java index 7f74e0774e1..d14adff3da3 100644 --- a/functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java +++ b/functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java @@ -86,13 +86,13 @@ public void beforeTest() throws IOException { @Test public void onlyAcceptsPostRequestsTest() throws IOException, GeneralSecurityException { - when(request.getMethod()).thenReturn("GET"); - new SlackSlashCommand().service(request, response); - // DBG String envStr = String.join("/", System.getenv().keySet()); assertThat("DBG").isEqualTo(envStr); + when(request.getMethod()).thenReturn("GET"); + new SlackSlashCommand().service(request, response); + writerOut.flush(); verify(response, times(1)).setStatusCode(HttpURLConnection.HTTP_BAD_METHOD); } From c6d877a9f0a2d67caabff165b5b1922f5566b7a0 Mon Sep 17 00:00:00 2001 From: ace-n Date: Fri, 20 Mar 2020 16:26:44 -0700 Subject: [PATCH 14/18] DBG take 5 --- .../main/java/com/example/functions/SlackSlashCommand.java | 4 ++++ .../java/com/example/functions/SlackSlashCommandTest.java | 4 ---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/functions/snippets/src/main/java/com/example/functions/SlackSlashCommand.java b/functions/snippets/src/main/java/com/example/functions/SlackSlashCommand.java index e4991da499c..40da42eecd0 100644 --- a/functions/snippets/src/main/java/com/example/functions/SlackSlashCommand.java +++ b/functions/snippets/src/main/java/com/example/functions/SlackSlashCommand.java @@ -122,6 +122,10 @@ String formatSlackMessage(JsonObject kgResponse, String query) { JsonObject searchKnowledgeGraph(String query) throws IOException { Kgsearch.Entities.Search kgRequest = kgClient.entities().search(); + + LOGGER.info("KG KEY"); + LOGGER.info(API_KEY); + kgRequest.setQuery(query); kgRequest.setKey(API_KEY); diff --git a/functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java b/functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java index d14adff3da3..4ce7df37a4b 100644 --- a/functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java +++ b/functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java @@ -86,10 +86,6 @@ public void beforeTest() throws IOException { @Test public void onlyAcceptsPostRequestsTest() throws IOException, GeneralSecurityException { - // DBG - String envStr = String.join("/", System.getenv().keySet()); - assertThat("DBG").isEqualTo(envStr); - when(request.getMethod()).thenReturn("GET"); new SlackSlashCommand().service(request, response); From 3fcc16e1ca1f40875673339965ce8889fb07ca6f Mon Sep 17 00:00:00 2001 From: ace-n Date: Fri, 20 Mar 2020 16:47:19 -0700 Subject: [PATCH 15/18] DBG take 6 --- .../main/java/com/example/functions/SlackSlashCommand.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/functions/snippets/src/main/java/com/example/functions/SlackSlashCommand.java b/functions/snippets/src/main/java/com/example/functions/SlackSlashCommand.java index 40da42eecd0..cf20eb1f80e 100644 --- a/functions/snippets/src/main/java/com/example/functions/SlackSlashCommand.java +++ b/functions/snippets/src/main/java/com/example/functions/SlackSlashCommand.java @@ -123,9 +123,6 @@ String formatSlackMessage(JsonObject kgResponse, String query) { JsonObject searchKnowledgeGraph(String query) throws IOException { Kgsearch.Entities.Search kgRequest = kgClient.entities().search(); - LOGGER.info("KG KEY"); - LOGGER.info(API_KEY); - kgRequest.setQuery(query); kgRequest.setKey(API_KEY); @@ -162,6 +159,6 @@ public void service(HttpRequest request, HttpResponse response) throws IOExcepti // Format response to Slack BufferedWriter writer = response.getWriter(); - writer.write(formatSlackMessage(kgResponse, query)); + writer.write(API_KEY); // formatSlackMessage(kgResponse, query)); } } From 784175883948de45a754ea9241ceace44aed9d29 Mon Sep 17 00:00:00 2001 From: ace-n Date: Fri, 20 Mar 2020 17:12:06 -0700 Subject: [PATCH 16/18] DBG take 7 --- .../src/main/java/com/example/functions/SlackSlashCommand.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/snippets/src/main/java/com/example/functions/SlackSlashCommand.java b/functions/snippets/src/main/java/com/example/functions/SlackSlashCommand.java index cf20eb1f80e..b86a4a54e9f 100644 --- a/functions/snippets/src/main/java/com/example/functions/SlackSlashCommand.java +++ b/functions/snippets/src/main/java/com/example/functions/SlackSlashCommand.java @@ -155,7 +155,7 @@ public void service(HttpRequest request, HttpResponse response) throws IOExcepti String query = body.get("text").getAsString(); // Call knowledge graph API - JsonObject kgResponse = searchKnowledgeGraph(query); + //JsonObject kgResponse = searchKnowledgeGraph(query); // Format response to Slack BufferedWriter writer = response.getWriter(); From 561fb1133689ddf3ddc9b580740f2d4fd64e7229 Mon Sep 17 00:00:00 2001 From: ace-n Date: Fri, 20 Mar 2020 17:15:45 -0700 Subject: [PATCH 17/18] Fix tests...? --- .../test/java/com/example/functions/SlackSlashCommandTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java b/functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java index 4ce7df37a4b..ad10bce3a83 100644 --- a/functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java +++ b/functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java @@ -82,6 +82,9 @@ public void beforeTest() throws IOException { validHeaders.put("X-Slack-Request-Timestamp", Arrays.asList(timestamp)); when(request.getHeaders()).thenReturn(validHeaders); + + // Reset knowledge graph API key + Whitebox.setInternalState(SlackSlashCommand.class, "API_KEY", System.getenv("KG_API_KEY")); } @Test From 18b011eed7458e037663c8fe9a4e190582289653 Mon Sep 17 00:00:00 2001 From: ace-n Date: Fri, 20 Mar 2020 17:29:45 -0700 Subject: [PATCH 18/18] Revert dbg commits + fix tests --- .../main/java/com/example/functions/SlackSlashCommand.java | 5 ++--- .../java/com/example/functions/SlackSlashCommandTest.java | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/functions/snippets/src/main/java/com/example/functions/SlackSlashCommand.java b/functions/snippets/src/main/java/com/example/functions/SlackSlashCommand.java index b86a4a54e9f..e4991da499c 100644 --- a/functions/snippets/src/main/java/com/example/functions/SlackSlashCommand.java +++ b/functions/snippets/src/main/java/com/example/functions/SlackSlashCommand.java @@ -122,7 +122,6 @@ String formatSlackMessage(JsonObject kgResponse, String query) { JsonObject searchKnowledgeGraph(String query) throws IOException { Kgsearch.Entities.Search kgRequest = kgClient.entities().search(); - kgRequest.setQuery(query); kgRequest.setKey(API_KEY); @@ -155,10 +154,10 @@ public void service(HttpRequest request, HttpResponse response) throws IOExcepti String query = body.get("text").getAsString(); // Call knowledge graph API - //JsonObject kgResponse = searchKnowledgeGraph(query); + JsonObject kgResponse = searchKnowledgeGraph(query); // Format response to Slack BufferedWriter writer = response.getWriter(); - writer.write(API_KEY); // formatSlackMessage(kgResponse, query)); + writer.write(formatSlackMessage(kgResponse, query)); } } diff --git a/functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java b/functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java index ad10bce3a83..9b4eaad6fb0 100644 --- a/functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java +++ b/functions/snippets/src/test/java/com/example/functions/SlackSlashCommandTest.java @@ -36,7 +36,6 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; -import java.util.logging.Logger; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentMatchers;