From f36a0eac25a495857e2a4c2f396c4d98557d3bf6 Mon Sep 17 00:00:00 2001 From: Christian Himpe Date: Fri, 2 Aug 2024 17:23:28 +0200 Subject: [PATCH 1/5] first step for bulk processing in console --- .../java/com/arcadedb/console/Console.java | 41 +++++++++++-------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/console/src/main/java/com/arcadedb/console/Console.java b/console/src/main/java/com/arcadedb/console/Console.java index 37ce36d71f..ea31cae5f6 100644 --- a/console/src/main/java/com/arcadedb/console/Console.java +++ b/console/src/main/java/com/arcadedb/console/Console.java @@ -128,7 +128,7 @@ public void interactiveMode() throws IOException { } try { - if (!parse(line, false)) + if (!parse(line, false, false)) return; } catch (final Exception e) { // IGNORE (ALREADY PRINTED) @@ -175,16 +175,13 @@ public static void execute(final String[] args) throws IOException { final Console console = new Console(); try { - if (batchMode) { - // BATCH MODE - console.parse(commands.toString(), true); - console.parse("exit", true); - } else { - // INTERACTIVE MODE - if (console.parse(commands.toString(), true)) + if (console.parse(commands.toString(), true, batchMode)) { // COMMANDLINE ARGUMENT COMMANDS + if (batchMode) { // BATCH MODE + console.executeClose(); + } else { // INTERACTIVE MODE console.interactiveMode(); + } } - } finally { // FORCE THE CLOSING console.close(); @@ -223,7 +220,7 @@ public BasicDatabase getDatabase() { return databaseProxy; } - private boolean execute(final String line) throws IOException { + private boolean execute(final String line) throws IOException, RuntimeException { try { if (line == null) @@ -601,8 +598,7 @@ public void onError(Exception exception) { try { resultSet = databaseProxy.command(language, line); } catch (Exception e) { - outputError(e); - return; + throw e; } if (transactionBatchSize > 0) { @@ -696,7 +692,7 @@ private void executeLoad(final String fileName) throws IOException { while (bufferedReader.ready()) { final String line = FileUtils.decodeFromFile(bufferedReader.readLine()); - parse(line, true); + parse(line, true, false); ++executedLines; byteReadFromFile += line.length() + 1; @@ -724,11 +720,15 @@ private void executeLoad(final String fileName) throws IOException { flushOutput(); } - public boolean parse(final String line) throws IOException { - return parse(line, false); + public boolean parse(final String line) throws IOException, RuntimeException { + return parse(line, false, true); } - public boolean parse(final String line, final boolean printCommand) throws IOException { + public boolean parse(final String line, final boolean printCommand) throws IOException, RuntimeException { + return parse(line, printCommand, true); + } + + public boolean parse(final String line, final boolean printCommand, final boolean batchMode) throws IOException, RuntimeException { final ParsedLine parsedLine = parser.parse(line, 0); @@ -739,8 +739,13 @@ public boolean parse(final String line, final boolean printCommand) throws IOExc if (printCommand) output(3, getPrompt() + w); - if (!execute(w)) - return false; + try { + if (!execute(w)) + return false; + } catch (final IOException | RuntimeException e) { + if (batchMode) + throw e; + } } return true; } From 41c4b464c1769e5b078cf615ea42bb816cd11355 Mon Sep 17 00:00:00 2001 From: Roberto Franchini Date: Mon, 16 Dec 2024 18:58:00 +0100 Subject: [PATCH 2/5] move from jansi to jni, add test for batch failing --- console/pom.xml | 2 +- .../java/com/arcadedb/console/Console.java | 27 +++++++++++++------ .../arcadedb/console/ConsoleBatchTest.java | 23 ++++++++++++++-- 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/console/pom.xml b/console/pom.xml index 8af2b96a5c..283436fdf4 100644 --- a/console/pom.xml +++ b/console/pom.xml @@ -73,7 +73,7 @@ org.jline - jline-terminal-jansi + jline-terminal-jni ${jline.version} diff --git a/console/src/main/java/com/arcadedb/console/Console.java b/console/src/main/java/com/arcadedb/console/Console.java index ea31cae5f6..cd5e2b84d6 100644 --- a/console/src/main/java/com/arcadedb/console/Console.java +++ b/console/src/main/java/com/arcadedb/console/Console.java @@ -54,8 +54,18 @@ import org.jline.terminal.Terminal; import org.jline.terminal.TerminalBuilder; -import java.io.*; -import java.util.*; +import java.io.BufferedReader; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; public class Console { private static final String PROMPT = "%n%s> "; @@ -95,7 +105,7 @@ public Console() throws IOException { GlobalConfiguration.PROFILE.setValue("low-cpu"); - terminal = TerminalBuilder.builder().system(system).streams(System.in, System.out).jansi(true).build(); + terminal = TerminalBuilder.builder().system(system).streams(System.in, System.out).jni(true).build(); output(3, "%s Console v.%s - %s (%s)", Constants.PRODUCT, Constants.getRawVersion(), Constants.COPYRIGHT, Constants.URL); } @@ -163,9 +173,9 @@ public static void execute(final String[] args) throws IOException { final String[] parts = value.substring(2).split("="); System.setProperty(parts[0], parts[1]); setGlobalConfiguration(parts[0], parts[1], true); - } else if (value.equalsIgnoreCase("-b")) + } else if (value.equalsIgnoreCase("-b")) { batchMode = true; - else { + } else { commands.append(value); if (!value.endsWith(";")) commands.append(";"); @@ -721,14 +731,15 @@ private void executeLoad(final String fileName) throws IOException { } public boolean parse(final String line) throws IOException, RuntimeException { - return parse(line, false, true); + return parse(line, false, false); } public boolean parse(final String line, final boolean printCommand) throws IOException, RuntimeException { - return parse(line, printCommand, true); + return parse(line, printCommand, false); } - public boolean parse(final String line, final boolean printCommand, final boolean batchMode) throws IOException, RuntimeException { + public boolean parse(final String line, final boolean printCommand, final boolean batchMode) + throws IOException, RuntimeException { final ParsedLine parsedLine = parser.parse(line, 0); diff --git a/console/src/test/java/com/arcadedb/console/ConsoleBatchTest.java b/console/src/test/java/com/arcadedb/console/ConsoleBatchTest.java index d3ff0c7368..e1a53998d5 100644 --- a/console/src/test/java/com/arcadedb/console/ConsoleBatchTest.java +++ b/console/src/test/java/com/arcadedb/console/ConsoleBatchTest.java @@ -21,15 +21,18 @@ import com.arcadedb.GlobalConfiguration; import com.arcadedb.database.Database; import com.arcadedb.database.DatabaseFactory; +import com.arcadedb.exception.CommandSQLParsingException; import com.arcadedb.server.TestServerHelper; import com.arcadedb.utility.FileUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.io.*; +import java.io.File; +import java.io.IOException; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; public class ConsoleBatchTest { @Test @@ -40,6 +43,21 @@ public void batchMode() throws IOException { db.drop(); } + @Test + public void batchModeWithError() throws IOException { + + assertThatThrownBy(() -> Console.execute( + new String[] { "-b", """ + create database console; + create vertex table WRONG_STATEMENT; + create vertex type ConsoleOnlyVertex; + """ })) + .isInstanceOf(CommandSQLParsingException.class); + final Database db = new DatabaseFactory("./target/databases/console").open(); + assertThat(db.getSchema().existsType("ConsoleOnlyVertex")).isFalse(); + db.drop(); + } + @Test public void interactiveMode() throws IOException { Console.execute(new String[] { "create database console; create vertex type ConsoleOnlyVertex;exit" }); @@ -51,7 +69,8 @@ public void interactiveMode() throws IOException { @Test public void swallowSettings() throws IOException { FileUtils.deleteRecursively(new File("./console")); - Console.execute(new String[] { "-Darcadedb.server.databaseDirectory=.", "create database console; create vertex type ConsoleOnlyVertex;exit;" }); + Console.execute(new String[] { "-Darcadedb.server.databaseDirectory=.", + "create database console; create vertex type ConsoleOnlyVertex;exit;" }); final Database db = new DatabaseFactory("./console").open(); assertThat(db.getSchema().existsType("ConsoleOnlyVertex")).isTrue(); db.drop(); From a528c27b9c5c4800aaa135640bbb517a205600fe Mon Sep 17 00:00:00 2001 From: Roberto Franchini Date: Tue, 17 Dec 2024 11:20:51 +0100 Subject: [PATCH 3/5] WIP --- .../java/com/arcadedb/console/Console.java | 42 ++++++++++++------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/console/src/main/java/com/arcadedb/console/Console.java b/console/src/main/java/com/arcadedb/console/Console.java index cd5e2b84d6..e273574201 100644 --- a/console/src/main/java/com/arcadedb/console/Console.java +++ b/console/src/main/java/com/arcadedb/console/Console.java @@ -138,7 +138,7 @@ public void interactiveMode() throws IOException { } try { - if (!parse(line, false, false)) + if (!parse(line, false)) return; } catch (final Exception e) { // IGNORE (ALREADY PRINTED) @@ -165,6 +165,7 @@ public static void main(final String[] args) throws IOException { public static void execute(final String[] args) throws IOException { final StringBuilder commands = new StringBuilder(); boolean batchMode = false; + boolean failAtEnd = false; // PARSE ARGUMENT, EXTRACT SETTING AND BATCH MODE AND COMPILE THE LINES TO EXECUTE for (int i = 0; i < args.length; i++) { final String value = args[i].trim(); @@ -175,6 +176,8 @@ public static void execute(final String[] args) throws IOException { setGlobalConfiguration(parts[0], parts[1], true); } else if (value.equalsIgnoreCase("-b")) { batchMode = true; + } else if (value.equalsIgnoreCase("-fae")) { + failAtEnd = true; } else { commands.append(value); if (!value.endsWith(";")) @@ -185,12 +188,13 @@ public static void execute(final String[] args) throws IOException { final Console console = new Console(); try { - if (console.parse(commands.toString(), true, batchMode)) { // COMMANDLINE ARGUMENT COMMANDS - if (batchMode) { // BATCH MODE - console.executeClose(); - } else { // INTERACTIVE MODE + if (batchMode) { + console.parse(commands.toString(), true, batchMode, failAtEnd); + console.parse("exit", true); + } else { + // INTERACTIVE MODE + if (console.parse(commands.toString(), true)) console.interactiveMode(); - } } } finally { // FORCE THE CLOSING @@ -230,7 +234,7 @@ public BasicDatabase getDatabase() { return databaseProxy; } - private boolean execute(final String line) throws IOException, RuntimeException { + private boolean execute(final String line) throws IOException { try { if (line == null) @@ -608,7 +612,8 @@ public void onError(Exception exception) { try { resultSet = databaseProxy.command(language, line); } catch (Exception e) { - throw e; + outputError(e); + return; } if (transactionBatchSize > 0) { @@ -702,7 +707,7 @@ private void executeLoad(final String fileName) throws IOException { while (bufferedReader.ready()) { final String line = FileUtils.decodeFromFile(bufferedReader.readLine()); - parse(line, true, false); + parse(line, true); ++executedLines; byteReadFromFile += line.length() + 1; @@ -731,14 +736,14 @@ private void executeLoad(final String fileName) throws IOException { } public boolean parse(final String line) throws IOException, RuntimeException { - return parse(line, false, false); + return parse(line, false); } public boolean parse(final String line, final boolean printCommand) throws IOException, RuntimeException { - return parse(line, printCommand, false); + return parse(line, printCommand, false, false); } - public boolean parse(final String line, final boolean printCommand, final boolean batchMode) + public boolean parse(final String line, final boolean printCommand, final boolean batchMode, boolean failAtEnd) throws IOException, RuntimeException { final ParsedLine parsedLine = parser.parse(line, 0); @@ -750,12 +755,17 @@ public boolean parse(final String line, final boolean printCommand, final boolea if (printCommand) output(3, getPrompt() + w); - try { + if (batchMode) { + try { + if (!execute(w)) + return false; + } catch (final Exception e) { + if (!failAtEnd) + throw e; + } + } else { if (!execute(w)) return false; - } catch (final IOException | RuntimeException e) { - if (batchMode) - throw e; } } return true; From 545573ff2f6c52c502b857223cac0e521e5df1e3 Mon Sep 17 00:00:00 2001 From: Roberto Franchini Date: Tue, 17 Dec 2024 13:02:23 +0100 Subject: [PATCH 4/5] WIP --- .../main/java/com/arcadedb/console/Console.java | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/console/src/main/java/com/arcadedb/console/Console.java b/console/src/main/java/com/arcadedb/console/Console.java index e273574201..ce31396908 100644 --- a/console/src/main/java/com/arcadedb/console/Console.java +++ b/console/src/main/java/com/arcadedb/console/Console.java @@ -612,8 +612,9 @@ public void onError(Exception exception) { try { resultSet = databaseProxy.command(language, line); } catch (Exception e) { - outputError(e); - return; +// outputError(e); +// return; + throw e; } if (transactionBatchSize > 0) { @@ -764,8 +765,15 @@ public boolean parse(final String line, final boolean printCommand, final boolea throw e; } } else { - if (!execute(w)) - return false; + try { + if (!execute(w)) + return false; + } catch (final Exception e) { + return true; + +// throw e; + } + } } return true; From a1902b978726a2c19078acb030ffb3b9e608c48d Mon Sep 17 00:00:00 2001 From: Roberto Franchini Date: Thu, 19 Dec 2024 10:39:42 +0100 Subject: [PATCH 5/5] move batchMode and failAtAnd at memeber level --- .../java/com/arcadedb/console/Console.java | 85 ++++++++++--------- .../arcadedb/console/ConsoleBatchTest.java | 20 ++++- 2 files changed, 66 insertions(+), 39 deletions(-) diff --git a/console/src/main/java/com/arcadedb/console/Console.java b/console/src/main/java/com/arcadedb/console/Console.java index ce31396908..a04aeb1f1b 100644 --- a/console/src/main/java/com/arcadedb/console/Console.java +++ b/console/src/main/java/com/arcadedb/console/Console.java @@ -91,12 +91,20 @@ public class Console { private long transactionBatchSize = 0L; protected long currentOperationsInBatch = 0L; private RemoteServer remoteServer; + private boolean batchMode = false; + private boolean failAtEnd = false; public Console(final DatabaseInternal database) throws IOException { this(); this.databaseProxy = database; } + public Console(boolean batchMode, boolean failAtEnd) throws IOException { + this(); + this.batchMode = batchMode; + this.failAtEnd = failAtEnd; + } + public Console() throws IOException { IntegrationUtils.setRootPath(configuration); databaseDirectory = configuration.getValueAsString(GlobalConfiguration.SERVER_DATABASE_DIRECTORY); @@ -185,11 +193,11 @@ public static void execute(final String[] args) throws IOException { } } - final Console console = new Console(); + final Console console = new Console(batchMode, failAtEnd); try { if (batchMode) { - console.parse(commands.toString(), true, batchMode, failAtEnd); + console.parse(commands.toString(), true); console.parse("exit", true); } else { // INTERACTIVE MODE @@ -298,10 +306,12 @@ private void executeSet(final String line) { final String key = parts[0].trim(); final String value = parts[1].trim(); - if ("limit".equalsIgnoreCase(key)) { + switch (key.toLowerCase()) { + case "limit" -> { limit = Integer.parseInt(value); outputLine(3, "Set new limit to %d", limit); - } else if ("asyncMode".equalsIgnoreCase(key)) { + } + case "asyncmode" -> { asyncMode = Boolean.parseBoolean(value); if (asyncMode) { // ENABLE ASYNCHRONOUS PARALLEL MODE @@ -314,28 +324,36 @@ private void executeSet(final String line) { }); } outputLine(3, "Set asyncMode to %s", asyncMode); - } else if ("transactionBatchSize".equalsIgnoreCase(key)) { + } + case "transactionbatchsize" -> { transactionBatchSize = Integer.parseInt(value); outputLine(3, "Set new transactionBatch to %d", transactionBatchSize); - } else if ("language".equalsIgnoreCase(key)) { + } + case "language" -> { language = value; outputLine(3, "Set language to %s", language); - } else if ("expandResultSet".equalsIgnoreCase(key)) { + } + case "expandresultset" -> { expandResultSet = value.equalsIgnoreCase("true"); outputLine(3, "Set expanded result set to %s", expandResultSet); - } else if ("maxMultiValueEntries".equalsIgnoreCase(key)) { + } + case "maxmultivalueentries" -> { maxMultiValueEntries = Integer.parseInt(value); outputLine(3, "Set maximum multi value entries to %d", maxMultiValueEntries); - } else if ("verbose".equalsIgnoreCase(key)) { + } + case "verbose" -> { verboseLevel = Integer.parseInt(value); outputLine(3, "Set verbose level to %d", verboseLevel); - } else if ("maxWidth".equalsIgnoreCase(key)) { + } + case "maxwidth" -> { maxWidth = Integer.parseInt(value); outputLine(3, "Set maximum width to %d", maxWidth); - } else { + } + default -> { if (!setGlobalConfiguration(key, value, false)) outputLine(3, "Setting '%s' is not supported by the console", key); } + } flushOutput(); } @@ -343,10 +361,10 @@ private void executeSet(final String line) { private void executeTransactionStatus() { checkDatabaseIsOpen(); - if (databaseProxy instanceof DatabaseInternal) { - final TransactionContext tx = ((DatabaseInternal) databaseProxy).getTransaction(); + if (databaseProxy instanceof DatabaseInternal db) { + final TransactionContext tx = db.getTransaction(); if (tx.isActive()) { - final ResultInternal row = new ResultInternal((Database) databaseProxy); + final ResultInternal row = new ResultInternal(db); row.setPropertiesFromMap(tx.getStats()); printRecord(row); @@ -561,15 +579,14 @@ else if (rec != null) final List resultSet = new ArrayList<>(); - Object value; for (final String fieldName : currentRecord.getPropertyNames()) { - value = currentRecord.getProperty(fieldName); - if (value instanceof byte[]) - value = "byte[" + ((byte[]) value).length + "]"; - else if (value instanceof Iterator) { + Object value = currentRecord.getProperty(fieldName); + if (value instanceof byte[] bytes) + value = "byte[" + bytes.length + "]"; + else if (value instanceof Iterator iterator) { final List coll = new ArrayList<>(); - while (((Iterator) value).hasNext()) - coll.add(((Iterator) value).next()); + while (iterator.hasNext()) + coll.add(iterator.next()); value = coll; } else if (MultiValue.isMultiValue(value)) { value = TableFormatter.getPrettyFieldMultiValue(MultiValue.getMultiValueIterator(value), maxMultiValueEntries); @@ -608,14 +625,17 @@ public void onError(Exception exception) { outputError(exception); } }); - } else + } else { try { resultSet = databaseProxy.command(language, line); } catch (Exception e) { -// outputError(e); -// return; - throw e; + if (batchMode && !failAtEnd) + throw e; + else + outputError(e); + return; } + } if (transactionBatchSize > 0) { ++currentOperationsInBatch; @@ -736,16 +756,11 @@ private void executeLoad(final String fileName) throws IOException { flushOutput(); } - public boolean parse(final String line) throws IOException, RuntimeException { + public boolean parse(final String line) throws IOException { return parse(line, false); } - public boolean parse(final String line, final boolean printCommand) throws IOException, RuntimeException { - return parse(line, printCommand, false, false); - } - - public boolean parse(final String line, final boolean printCommand, final boolean batchMode, boolean failAtEnd) - throws IOException, RuntimeException { + public boolean parse(final String line, final boolean printCommand) throws IOException { final ParsedLine parsedLine = parser.parse(line, 0); @@ -765,14 +780,8 @@ public boolean parse(final String line, final boolean printCommand, final boolea throw e; } } else { - try { if (!execute(w)) return false; - } catch (final Exception e) { - return true; - -// throw e; - } } } diff --git a/console/src/test/java/com/arcadedb/console/ConsoleBatchTest.java b/console/src/test/java/com/arcadedb/console/ConsoleBatchTest.java index e1a53998d5..0c1aea81a5 100644 --- a/console/src/test/java/com/arcadedb/console/ConsoleBatchTest.java +++ b/console/src/test/java/com/arcadedb/console/ConsoleBatchTest.java @@ -45,7 +45,7 @@ public void batchMode() throws IOException { @Test public void batchModeWithError() throws IOException { - + // This should fail assertThatThrownBy(() -> Console.execute( new String[] { "-b", """ create database console; @@ -53,11 +53,29 @@ public void batchModeWithError() throws IOException { create vertex type ConsoleOnlyVertex; """ })) .isInstanceOf(CommandSQLParsingException.class); + final Database db = new DatabaseFactory("./target/databases/console").open(); + // the ConsoleOnlyVertex should not be created assertThat(db.getSchema().existsType("ConsoleOnlyVertex")).isFalse(); db.drop(); } + @Test + public void batchModeWithFailAtEnd() throws IOException { + // Error is only printed out + Console.execute( + new String[] { "-b", "-fae", """ + create database console; + create vertex table WRONG_STATEMENT; + create vertex type ConsoleOnlyVertex; + """ }); + + final Database db = new DatabaseFactory("./target/databases/console").open(); + // the ConsoleOnlyVertex is created + assertThat(db.getSchema().existsType("ConsoleOnlyVertex")).isTrue(); + db.drop(); + } + @Test public void interactiveMode() throws IOException { Console.execute(new String[] { "create database console; create vertex type ConsoleOnlyVertex;exit" });