From 3bfa3f1a4802fc52d767ca3aa113215d3e5bfb77 Mon Sep 17 00:00:00 2001 From: Xinyuan Fan Date: Tue, 23 Feb 2016 15:31:17 -0500 Subject: [PATCH] Port LocalServiceHelper into GAX --- .../api/gax/testing/CommandWrapper.java | 103 +++++++ .../testing/DownloadableEmulatorRunner.java | 166 +++++++++++ .../api/gax/testing/EmulatorRunner.java | 29 ++ .../api/gax/testing/GCloudEmulatorRunner.java | 106 +++++++ .../api/gax/testing/LocalServiceHelper.java | 262 ++++++++++++++++++ 5 files changed, 666 insertions(+) create mode 100644 src/main/java/com/google/api/gax/testing/CommandWrapper.java create mode 100644 src/main/java/com/google/api/gax/testing/DownloadableEmulatorRunner.java create mode 100644 src/main/java/com/google/api/gax/testing/EmulatorRunner.java create mode 100644 src/main/java/com/google/api/gax/testing/GCloudEmulatorRunner.java create mode 100644 src/main/java/com/google/api/gax/testing/LocalServiceHelper.java diff --git a/src/main/java/com/google/api/gax/testing/CommandWrapper.java b/src/main/java/com/google/api/gax/testing/CommandWrapper.java new file mode 100644 index 000000000000..8daa930f16f8 --- /dev/null +++ b/src/main/java/com/google/api/gax/testing/CommandWrapper.java @@ -0,0 +1,103 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.api.gax.testing; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +/* Utility class that executes system commands on both Windows and Unix */ +public class CommandWrapper { + private final List prefix; + private List command; + private String nullFilename; + private boolean redirectOutputToNull; + private boolean redirectErrorStream; + private boolean redirectErrorInherit; + private Path directory; + + private CommandWrapper() { + this.prefix = new ArrayList<>(); + if (isWindows()) { + this.prefix.add("cmd"); + this.prefix.add("/C"); + this.nullFilename = "NUL:"; + } else { + this.prefix.add("bash"); + this.nullFilename = "/dev/null"; + } + } + + public CommandWrapper command(List command) { + this.command = new ArrayList<>(command.size() + this.prefix.size()); + this.command.addAll(prefix); + this.command.addAll(command); + return this; + } + + public CommandWrapper redirectOutputToNull() { + this.redirectOutputToNull = true; + return this; + } + + public CommandWrapper redirectErrorStream() { + this.redirectErrorStream = true; + return this; + } + + public CommandWrapper redirectErrorInherit() { + this.redirectErrorInherit = true; + return this; + } + + public CommandWrapper directory(Path directory) { + this.directory = directory; + return this; + } + + public ProcessBuilder builder() { + ProcessBuilder builder = new ProcessBuilder(command); + if (redirectOutputToNull) { + builder.redirectOutput(new File(nullFilename)); + } + if (directory != null) { + builder.directory(directory.toFile()); + } + if (redirectErrorStream) { + builder.redirectErrorStream(true); + } + if (redirectErrorInherit) { + builder.redirectError(ProcessBuilder.Redirect.INHERIT); + } + return builder; + } + + public Process start() throws IOException { + return builder().start(); + } + + public static CommandWrapper create() { + return new CommandWrapper(); + } + + public static boolean isWindows() { + return System.getProperty("os.name").toLowerCase(Locale.ENGLISH).contains("windows"); + } +} diff --git a/src/main/java/com/google/api/gax/testing/DownloadableEmulatorRunner.java b/src/main/java/com/google/api/gax/testing/DownloadableEmulatorRunner.java new file mode 100644 index 000000000000..eac97c7db622 --- /dev/null +++ b/src/main/java/com/google/api/gax/testing/DownloadableEmulatorRunner.java @@ -0,0 +1,166 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.api.gax.testing; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.math.BigInteger; +import java.net.URL; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +/** + * Utility class to start and run an emulator from a download URL. + */ +public class DownloadableEmulatorRunner implements EmulatorRunner { + private final List commandText; + private final String md5CheckSum; + private final URL downloadUrl; + private final String fileName; + private Process process; + private static final Logger log = Logger.getLogger(DownloadableEmulatorRunner.class.getName()); + + public DownloadableEmulatorRunner(List commandText, URL downloadUrl, String md5CheckSum) { + this.commandText = commandText; + this.md5CheckSum = md5CheckSum; + this.downloadUrl = downloadUrl; + String[] splitUrl = downloadUrl.toString().split("/"); + this.fileName = splitUrl[splitUrl.length - 1]; + } + + @Override + public boolean isAvailable() { + try { + downloadZipFile(); + return true; + } catch (IOException e) { + return false; + } + } + + @Override + public void start() throws IOException { + Path emulatorPath = downloadEmulator(); + process = CommandWrapper.create() + .command(commandText) + .directory(emulatorPath) + .start(); + } + + @Override + public void stop() throws InterruptedException { + if (process != null) { + process.destroy(); + process.waitFor(); + } + } + + @Override + public Process getProcess() { + return process; + } + + private Path downloadEmulator() throws IOException { + // Retrieve the file name from the download link + String[] splittedUrl = downloadUrl.toString().split("/"); + String fileName = splittedUrl[splittedUrl.length - 1]; + + // Each run is associated with its own folder that is deleted once test completes. + Path emulatorPath = Files.createTempDirectory(fileName.split("\\.")[0]); + File emulatorFolder = emulatorPath.toFile(); + emulatorFolder.deleteOnExit(); + + File zipFile = downloadZipFile(); + // Unzip the emulator + try (ZipInputStream zipIn = new ZipInputStream(new FileInputStream(zipFile))) { + if (log.isLoggable(Level.FINE)) { + log.fine("Unzipping emulator"); + } + ZipEntry entry = zipIn.getNextEntry(); + while (entry != null) { + File filePath = new File(emulatorPath.toFile(), entry.getName()); + if (!entry.isDirectory()) { + extractFile(zipIn, filePath); + } else { + filePath.mkdir(); + } + zipIn.closeEntry(); + entry = zipIn.getNextEntry(); + } + } + return emulatorPath; + } + + private File downloadZipFile() throws IOException { + // Check if we already have a local copy of the emulator and download it if not. + File zipFile = new File(System.getProperty("java.io.tmpdir"), fileName); + if (!zipFile.exists() || (md5CheckSum != null && !md5CheckSum.equals(md5(zipFile)))) { + if (log.isLoggable(Level.FINE)) { + log.fine("Fetching emulator"); + } + ReadableByteChannel rbc = Channels.newChannel(downloadUrl.openStream()); + try (FileOutputStream fos = new FileOutputStream(zipFile)) { + fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); + } + } else { + if (log.isLoggable(Level.FINE)) { + log.fine("Using cached emulator"); + } + } + return zipFile; + } + + private void extractFile(ZipInputStream zipIn, File filePath) throws IOException { + try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(filePath))) { + byte[] bytesIn = new byte[1024]; + int read; + while ((read = zipIn.read(bytesIn)) != -1) { + bos.write(bytesIn, 0, read); + } + } + } + + private static String md5(File zipFile) throws IOException { + try { + MessageDigest md5 = MessageDigest.getInstance("MD5"); + try (InputStream is = new BufferedInputStream(new FileInputStream(zipFile))) { + byte[] bytes = new byte[4 * 1024 * 1024]; + int len; + while ((len = is.read(bytes)) >= 0) { + md5.update(bytes, 0, len); + } + } + return String.format("%032x", new BigInteger(1, md5.digest())); + } catch (NoSuchAlgorithmException e) { + throw new IOException(e); + } + } +} diff --git a/src/main/java/com/google/api/gax/testing/EmulatorRunner.java b/src/main/java/com/google/api/gax/testing/EmulatorRunner.java new file mode 100644 index 000000000000..8192017149c4 --- /dev/null +++ b/src/main/java/com/google/api/gax/testing/EmulatorRunner.java @@ -0,0 +1,29 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.api.gax.testing; + +import java.io.IOException; + +/** + * Utility interface to start and run an emulator. + */ +public interface EmulatorRunner { + public boolean isAvailable(); + public void start() throws IOException; + public void stop() throws InterruptedException; + public Process getProcess(); +} diff --git a/src/main/java/com/google/api/gax/testing/GCloudEmulatorRunner.java b/src/main/java/com/google/api/gax/testing/GCloudEmulatorRunner.java new file mode 100644 index 000000000000..5a2a76f9a46a --- /dev/null +++ b/src/main/java/com/google/api/gax/testing/GCloudEmulatorRunner.java @@ -0,0 +1,106 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.api.gax.testing; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * Utility class to start and run an emulator from gcloud SDK. + */ +public class GCloudEmulatorRunner implements EmulatorRunner { + private final List commandText; + private final String versionPrefix; + private final String minVersion; + private Process process; + + public GCloudEmulatorRunner(List commandText, String versionPrefix, String minVersion) { + this.commandText = commandText; + this.versionPrefix = versionPrefix; + this.minVersion = minVersion; + } + + @Override + public boolean isAvailable() { + try { + return isGCloudInstalled() && isEmulatorUpdateToDate() && commandText.size() > 0; + } catch (IOException | InterruptedException e) { + e.printStackTrace(System.err); + } + return false; + } + + @Override + public void start() throws IOException { + process = CommandWrapper.create() + .command(commandText) + .start(); + } + + @Override + public void stop() throws InterruptedException { + if (process != null) { + process.destroy(); + process.waitFor(); + } + } + + @Override + public Process getProcess() { + return process; + } + + private boolean isGCloudInstalled() { + Map env = System.getenv(); + for (String envName : env.keySet()) { + if (envName.equals("PATH")) { + return env.get(envName).contains("google-cloud-sdk"); + } + } + return false; + } + + private boolean isEmulatorUpdateToDate() + throws IOException, InterruptedException { + String currentVersion = installedEmulatorVersion(versionPrefix); + return currentVersion.compareTo(minVersion) >= 0; + } + + private String installedEmulatorVersion(String versionPrefix) + throws IOException, InterruptedException { + Process process = + CommandWrapper.create().command(Arrays.asList("gcloud", "version")) + .redirectErrorStream().start(); + process.waitFor(); + try (BufferedReader reader = + new BufferedReader(new InputStreamReader(process.getInputStream()))) { + for (String line = reader.readLine(); line != null; line = reader.readLine()) { + if (line.startsWith(versionPrefix)) { + String[] lineComponents = line.split(" "); + if (lineComponents.length > 1) { + return lineComponents[1]; + } + } + } + return null; + } + } +} diff --git a/src/main/java/com/google/api/gax/testing/LocalServiceHelper.java b/src/main/java/com/google/api/gax/testing/LocalServiceHelper.java new file mode 100644 index 000000000000..1e0f84639cd1 --- /dev/null +++ b/src/main/java/com/google/api/gax/testing/LocalServiceHelper.java @@ -0,0 +1,262 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.api.gax.testing; + +import static com.google.common.base.MoreObjects.firstNonNull; + +import com.google.common.base.Strings; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.ServerSocket; +import java.net.URL; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Utility class to start and stop a local service which is used by unit testing. + */ +public class LocalServiceHelper { + private final int port; + private EmulatorRunner activeRunner; + private List runners; + private ProcessStreamReader processReader; + private ProcessErrorStreamReader processErrorReader; + private static final Logger log = Logger.getLogger(LocalServiceHelper.class.getName()); + + private static final int DEFAULT_PORT = 8080; + private static final int STREAM_READER_SLEEP_INTERVAL_IN_MS = 200; + + public static int findAvailablePort(int defaultPort) { + try (ServerSocket tempSocket = new ServerSocket(0)) { + return tempSocket.getLocalPort(); + } catch (IOException e) { + return defaultPort; + } + } + + private static class ProcessStreamReader extends Thread { + private final BufferedReader reader; + private volatile boolean terminated; + + ProcessStreamReader(InputStream inputStream) { + super("Local InputStream reader"); + setDaemon(true); + reader = new BufferedReader(new InputStreamReader(inputStream)); + } + + void terminate() throws IOException { + terminated = true; + reader.close(); + interrupt(); + } + + @Override + public void run() { + while (!terminated) { + try { + if (reader.ready()) { + String line = reader.readLine(); + if (line == null) { + terminated = true; + } + } else { + sleep(STREAM_READER_SLEEP_INTERVAL_IN_MS); + } + } catch (IOException e) { + e.printStackTrace(System.err); + } catch (InterruptedException e) { + break; + } + } + } + + public static ProcessStreamReader start(InputStream inputStream) { + ProcessStreamReader thread = new ProcessStreamReader(inputStream); + thread.start(); + return thread; + } + } + + private static class ProcessErrorStreamReader extends Thread { + private static final int LOG_LENGTH_LIMIT = 50000; + private static final String LOGGING_CLASS = + "com.google.apphosting.client.serviceapp.BaseApiServlet"; + + private final BufferedReader errorReader; + private StringBuilder currentLog; + private Level currentLogLevel; + private boolean collectionMode; + private volatile boolean terminated; + + ProcessErrorStreamReader(InputStream errorStream, String blockUntil) throws IOException { + super("Local ErrorStream reader"); + setDaemon(true); + errorReader = new BufferedReader(new InputStreamReader(errorStream)); + if (!Strings.isNullOrEmpty(blockUntil)) { + String line; + do { + line = errorReader.readLine(); + } while (line != null && !line.contains(blockUntil)); + } + } + + void terminate() throws IOException { + terminated = true; + errorReader.close(); + interrupt(); + } + + @Override + public void run() { + String previousLine = ""; + String nextLine = ""; + while (!terminated) { + try { + if (errorReader.ready()) { + previousLine = nextLine; + nextLine = errorReader.readLine(); + if (nextLine == null) { + terminated = true; + } else { + processLogLine(previousLine, nextLine); + } + } else { + sleep(STREAM_READER_SLEEP_INTERVAL_IN_MS); + } + } catch (IOException e) { + e.printStackTrace(System.err); + } catch (InterruptedException e) { + break; + } + } + processLogLine(previousLine, firstNonNull(nextLine, "")); + writeLog(currentLogLevel, currentLog); + } + + private void processLogLine(String previousLine, String nextLine) { + // Each log is two lines with the following format: + // [Date] [Time] [LOGGING_CLASS] [method] + // [LEVEL]: error message + // Exceptions and stack traces are included in error stream, separated by a newline + Level nextLogLevel = getLevel(nextLine); + if (nextLogLevel != null) { + writeLog(currentLogLevel, currentLog); + currentLog = new StringBuilder(); + currentLogLevel = nextLogLevel; + collectionMode = previousLine.contains(LOGGING_CLASS); + } else if (collectionMode) { + if (currentLog.length() > LOG_LENGTH_LIMIT) { + collectionMode = false; + } else if (currentLog.length() == 0) { + // strip level out of the line + currentLog.append(previousLine.split(":", 2)[1]); + currentLog.append(System.getProperty("line.separator")); + } else { + currentLog.append(previousLine); + currentLog.append(System.getProperty("line.separator")); + } + } + } + + private static void writeLog(Level level, StringBuilder msg) { + if (level != null && msg != null && msg.length() != 0) { + log.log(level, msg.toString().trim()); + } + } + + private static Level getLevel(String line) { + try { + return Level.parse(line.split(":")[0]); + } catch (IllegalArgumentException e) { + return null; // level wasn't supplied in this log line + } + } + + public static ProcessErrorStreamReader start(InputStream errorStream, String blockUntil) + throws IOException { + ProcessErrorStreamReader thread = new ProcessErrorStreamReader(errorStream, blockUntil); + thread.start(); + return thread; + } + } + + public LocalServiceHelper(List runners, int port) { + this.port = port > 0 ? port : DEFAULT_PORT; + this.runners = runners; + } + + /** + * Starts the local service as a subprocess. + * Block the the execution until |blockUntilOutput| is found from stderr of the emulator. + * @throws IOException + * @throws InterruptedException + */ + public void start(String blockUntilOutput) throws IOException, InterruptedException { + for (EmulatorRunner runner : runners) { + // Iterate through all emulator runners until find first available runner. + if (runner.isAvailable()) { + activeRunner = runner; + runner.start(); + break; + } + } + if (activeRunner != null) { + processReader = ProcessStreamReader.start(activeRunner.getProcess().getInputStream()); + processErrorReader = ProcessErrorStreamReader.start( + activeRunner.getProcess().getErrorStream(), blockUntilOutput); + } else { + // No available runner found. + throw new IOException("No available emulator runner is found."); + } + } + + /** + * Stops the local service + * @throws IOException + * @throws InterruptedException + */ + public void stop() throws IOException, InterruptedException { + if (processReader != null) { + processReader.terminate(); + processReader = null; + } + if (processErrorReader != null) { + processErrorReader.terminate(); + processErrorReader = null; + } + if (activeRunner != null) { + activeRunner.stop(); + activeRunner = null; + } + } + + public void sendPostRequest(String request) throws IOException { + URL url = new URL("http", "localhost", this.port, request); + HttpURLConnection con = (HttpURLConnection) url.openConnection(); + con.setRequestMethod("POST"); + con.setDoOutput(true); + OutputStream out = con.getOutputStream(); + out.write("".getBytes()); + out.flush(); + } +}