From 49baaa9f3c8e62e3bf553c61f1b58d6a5848d2e3 Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Sun, 31 Aug 2025 09:40:05 -0700 Subject: [PATCH 01/35] Sync gRPC related work changes into Forked Repo --- .github/workflows/studio-security-audit.yml | 2 +- e2e-js/package-lock.json | 562 ++--- grpc-client/pom.xml | 136 ++ .../remote/grpc/RemoteGrpcDatabase.java | 1793 ++++++++++++++++ .../RemoteGrpcDatabaseWithCompression.java | 51 + .../RemoteGrpcTransactionExplicitLock.java | 74 + .../test/resources/arcadedb-log.properties | 31 + grpc/gRPC-Readme.txt | 422 ++++ grpc/gRPC-Support-Case.md | 191 ++ grpc/gRPC-vs-HTTP-PerformanceTest.txt | 179 ++ grpc/pom.xml | 140 ++ grpc/src/main/proto/arcadedb-server.proto | 390 ++++ grpcw/pom.xml | 64 + .../server/grpc/ArcadeDbGrpcService.java | 1867 +++++++++++++++++ .../server/grpc/CompressionAwareService.java | 86 + .../server/grpc/GrpcAuthInterceptor.java | 142 ++ .../grpc/GrpcCompressionInterceptor.java | 83 + .../server/grpc/GrpcLoggingInterceptor.java | 81 + .../server/grpc/GrpcMetricsInterceptor.java | 78 + .../server/grpc/GrpcServerPlugin.java | 386 ++++ .../test/resources/arcadedb-log.properties | 31 + network/pom.xml | 6 +- .../com/arcadedb/remote/RemoteDatabase.java | 6 +- .../arcadedb/remote/RemoteDocumentType.java | 6 +- .../remote/RemoteImmutableDocument.java | 2 +- .../remote/RemoteImmutableVertex.java | 2 +- network/src/main/proto/arcadedb-server.proto | 281 +++ package/pom.xml | 12 + pom.xml | 57 + server/pom.xml | 22 +- .../server/monitor/ServerMonitor.java | 357 +++- server/src/main/proto/arcadedb-server.proto | 281 +++ 32 files changed, 7405 insertions(+), 416 deletions(-) create mode 100644 grpc-client/pom.xml create mode 100644 grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java create mode 100644 grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseWithCompression.java create mode 100644 grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcTransactionExplicitLock.java create mode 100644 grpc-client/src/test/resources/arcadedb-log.properties create mode 100644 grpc/gRPC-Readme.txt create mode 100644 grpc/gRPC-Support-Case.md create mode 100644 grpc/gRPC-vs-HTTP-PerformanceTest.txt create mode 100644 grpc/pom.xml create mode 100644 grpc/src/main/proto/arcadedb-server.proto create mode 100644 grpcw/pom.xml create mode 100644 grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java create mode 100644 grpcw/src/main/java/com/arcadedb/server/grpc/CompressionAwareService.java create mode 100644 grpcw/src/main/java/com/arcadedb/server/grpc/GrpcAuthInterceptor.java create mode 100644 grpcw/src/main/java/com/arcadedb/server/grpc/GrpcCompressionInterceptor.java create mode 100644 grpcw/src/main/java/com/arcadedb/server/grpc/GrpcLoggingInterceptor.java create mode 100644 grpcw/src/main/java/com/arcadedb/server/grpc/GrpcMetricsInterceptor.java create mode 100644 grpcw/src/main/java/com/arcadedb/server/grpc/GrpcServerPlugin.java create mode 100644 grpcw/src/test/resources/arcadedb-log.properties create mode 100644 network/src/main/proto/arcadedb-server.proto create mode 100644 server/src/main/proto/arcadedb-server.proto diff --git a/.github/workflows/studio-security-audit.yml b/.github/workflows/studio-security-audit.yml index c0428a025a..8c34311c1d 100644 --- a/.github/workflows/studio-security-audit.yml +++ b/.github/workflows/studio-security-audit.yml @@ -146,7 +146,7 @@ jobs: uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: "Dependency Review" - uses: actions/dependency-review-action@595b5aeba73380359d98a5e087f648dbb0edce1b # v4.7.3 + uses: actions/dependency-review-action@bc41886e18ea39df68b1b1245f4184881938e050 # v4.7.2 with: fail-on-severity: moderate allow-dependencies-licenses: "MIT, Apache-2.0, BSD-2-Clause, BSD-3-Clause, ISC, 0BSD" diff --git a/e2e-js/package-lock.json b/e2e-js/package-lock.json index ca0a6a7a77..026a23e709 100644 --- a/e2e-js/package-lock.json +++ b/e2e-js/package-lock.json @@ -57,22 +57,22 @@ } }, "node_modules/@babel/core": { - "version": "7.28.3", - "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.28.3.tgz", - "integrity": "sha512-yDBHV9kQNcr2/sUr9jghVyz9C3Y5G2zUM2H2lo+9mKv4sFgbA8s8Z9t8D1jiTkGoO/NoIfKMyKWr4s6CN23ZwQ==", + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.28.0.tgz", + "integrity": "sha512-UlLAnTPrFdNGoFtbSXwcGFQBtQZJCNjaN6hQNP3UPvuNXT1i82N26KL3dZeIpNalWywr9IuQuncaAfUaS1g6sQ==", "dev": true, "license": "MIT", "dependencies": { "@ampproject/remapping": "^2.2.0", "@babel/code-frame": "^7.27.1", - "@babel/generator": "^7.28.3", + "@babel/generator": "^7.28.0", "@babel/helper-compilation-targets": "^7.27.2", - "@babel/helper-module-transforms": "^7.28.3", - "@babel/helpers": "^7.28.3", - "@babel/parser": "^7.28.3", + "@babel/helper-module-transforms": "^7.27.3", + "@babel/helpers": "^7.27.6", + "@babel/parser": "^7.28.0", "@babel/template": "^7.27.2", - "@babel/traverse": "^7.28.3", - "@babel/types": "^7.28.2", + "@babel/traverse": "^7.28.0", + "@babel/types": "^7.28.0", "convert-source-map": "^2.0.0", "debug": "^4.1.0", "gensync": "^1.0.0-beta.2", @@ -88,14 +88,14 @@ } }, "node_modules/@babel/generator": { - "version": "7.28.3", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.28.3.tgz", - "integrity": "sha512-3lSpxGgvnmZznmBkCRnVREPUFJv2wrv9iAoFDvADJc0ypmdOxdUtcLeBgBJ6zE0PMeTKnxeQzyk0xTBq4Ep7zw==", + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.28.0.tgz", + "integrity": "sha512-lJjzvrbEeWrhB4P3QBsH7tey117PjLZnDbLiQEKjQ/fNJTjuq4HSqgFA+UNSwZT8D7dxxbnuSBMsa1lrWzKlQg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/parser": "^7.28.3", - "@babel/types": "^7.28.2", + "@babel/parser": "^7.28.0", + "@babel/types": "^7.28.0", "@jridgewell/gen-mapping": "^0.3.12", "@jridgewell/trace-mapping": "^0.3.28", "jsesc": "^3.0.2" @@ -146,15 +146,15 @@ } }, "node_modules/@babel/helper-module-transforms": { - "version": "7.28.3", - "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.28.3.tgz", - "integrity": "sha512-gytXUbs8k2sXS9PnQptz5o0QnpLL51SwASIORY6XaBKF88nsOT0Zw9szLqlSGQDP/4TljBAD5y98p2U1fqkdsw==", + "version": "7.27.3", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.27.3.tgz", + "integrity": "sha512-dSOvYwvyLsWBeIRyOeHXp5vPj5l1I011r52FM1+r1jCERv+aFXYk4whgQccYEGYxK2H3ZAIA8nuPkQ0HaUo3qg==", "dev": true, "license": "MIT", "dependencies": { "@babel/helper-module-imports": "^7.27.1", "@babel/helper-validator-identifier": "^7.27.1", - "@babel/traverse": "^7.28.3" + "@babel/traverse": "^7.27.3" }, "engines": { "node": ">=6.9.0" @@ -204,9 +204,9 @@ } }, "node_modules/@babel/helpers": { - "version": "7.28.3", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.28.3.tgz", - "integrity": "sha512-PTNtvUQihsAsDHMOP5pfobP8C6CM4JWXmP8DrEIt46c3r2bf87Ua1zoqevsMo9g+tWDwgWrFP5EIxuBx5RudAw==", + "version": "7.28.2", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.28.2.tgz", + "integrity": "sha512-/V9771t+EgXz62aCcyofnQhGM8DQACbRhvzKFsXKC9QM+5MadF8ZmIm0crDMaz3+o0h0zXfJnd4EhbYbxsrcFw==", "dev": true, "license": "MIT", "dependencies": { @@ -218,13 +218,13 @@ } }, "node_modules/@babel/parser": { - "version": "7.28.3", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.28.3.tgz", - "integrity": "sha512-7+Ey1mAgYqFAx2h0RuoxcQT5+MlG3GTV0TQrgr7/ZliKsm/MNDxVVutlWaziMq7wJNAz8MTqz55XLpWvva6StA==", + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.28.0.tgz", + "integrity": "sha512-jVZGvOxOuNSsuQuLRTh13nU0AogFlw32w/MT+LV6D3sP5WdbW61E77RnkbaO2dUvmPAYrBDJXGn5gGS6tH4j8g==", "dev": true, "license": "MIT", "dependencies": { - "@babel/types": "^7.28.2" + "@babel/types": "^7.28.0" }, "bin": { "parser": "bin/babel-parser.js" @@ -488,18 +488,18 @@ } }, "node_modules/@babel/traverse": { - "version": "7.28.3", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.28.3.tgz", - "integrity": "sha512-7w4kZYHneL3A6NP2nxzHvT3HCZ7puDZZjFMqDpBPECub79sTtSO5CGXDkKrTQq8ksAwfD/XI2MRFX23njdDaIQ==", + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.28.0.tgz", + "integrity": "sha512-mGe7UK5wWyh0bKRfupsUchrQGqvDbZDbKJw+kcRGSmdHVYrv+ltd0pnpDTVpiTqnaBru9iEvA8pz8W46v0Amwg==", "dev": true, "license": "MIT", "dependencies": { "@babel/code-frame": "^7.27.1", - "@babel/generator": "^7.28.3", + "@babel/generator": "^7.28.0", "@babel/helper-globals": "^7.28.0", - "@babel/parser": "^7.28.3", + "@babel/parser": "^7.28.0", "@babel/template": "^7.27.2", - "@babel/types": "^7.28.2", + "@babel/types": "^7.28.0", "debug": "^4.3.1" }, "engines": { @@ -534,21 +534,21 @@ "license": "MIT" }, "node_modules/@emnapi/core": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.5.0.tgz", - "integrity": "sha512-sbP8GzB1WDzacS8fgNPpHlp6C9VZe+SJP3F90W9rLemaQj2PzIuTEl1qDOYQf58YIpyjViI24y9aPWCjEzY2cg==", + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.4.5.tgz", + "integrity": "sha512-XsLw1dEOpkSX/WucdqUhPWP7hDxSvZiY+fsUC14h+FtQ2Ifni4znbBt8punRX+Uj2JG/uDb8nEHVKvrVlvdZ5Q==", "dev": true, "license": "MIT", "optional": true, "dependencies": { - "@emnapi/wasi-threads": "1.1.0", + "@emnapi/wasi-threads": "1.0.4", "tslib": "^2.4.0" } }, "node_modules/@emnapi/runtime": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.5.0.tgz", - "integrity": "sha512-97/BJ3iXHww3djw6hYIfErCZFee7qCtrneuLa20UXFCOTCfBM2cvQHjWJ2EG0s0MtdNwInarqCTz35i4wWXHsQ==", + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.4.5.tgz", + "integrity": "sha512-++LApOtY0pEEz1zrd9vy1/zXVaVJJ/EbAF3u0fXIzPJEDtnITsBGbbK0EkM72amhl/R5b+5xx0Y/QhcVOpuulg==", "dev": true, "license": "MIT", "optional": true, @@ -557,9 +557,9 @@ } }, "node_modules/@emnapi/wasi-threads": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.1.0.tgz", - "integrity": "sha512-WI0DdZ8xFSbgMjR1sFsKABJ/C5OnRrjT06JXbZKexJGrDuPTzZdDYfFlsgcCXCyf+suG5QU2e/y1Wo2V/OapLQ==", + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.0.4.tgz", + "integrity": "sha512-PJR+bOmMOPH8AtcTGAyYNiuJ3/Fcoj2XN/gBEWzDIKh254XO+mM9XoXHk5GNEhodxeMznbg7BlRojVbKN+gC6g==", "dev": true, "license": "MIT", "optional": true, @@ -729,16 +729,16 @@ } }, "node_modules/@jest/console": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/@jest/console/-/console-30.1.1.tgz", - "integrity": "sha512-f7TGqR1k4GtN5pyFrKmq+ZVndesiwLU33yDpJIGMS9aW+j6hKjue7ljeAdznBsH9kAnxUWe2Y+Y3fLV/FJt3gA==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/@jest/console/-/console-30.0.5.tgz", + "integrity": "sha512-xY6b0XiL0Nav3ReresUarwl2oIz1gTnxGbGpho9/rbUWsLH0f1OD/VT84xs8c7VmH7MChnLb0pag6PhZhAdDiA==", "dev": true, "license": "MIT", "dependencies": { "@jest/types": "30.0.5", "@types/node": "*", "chalk": "^4.1.2", - "jest-message-util": "30.1.0", + "jest-message-util": "30.0.5", "jest-util": "30.0.5", "slash": "^3.0.0" }, @@ -747,17 +747,17 @@ } }, "node_modules/@jest/core": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/@jest/core/-/core-30.1.1.tgz", - "integrity": "sha512-3ncU9peZ3D2VdgRkdZtUceTrDgX5yiDRwAFjtxNfU22IiZrpVWlv/FogzDLYSJQptQGfFo3PcHK86a2oG6WUGg==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/@jest/core/-/core-30.0.5.tgz", + "integrity": "sha512-fKD0OulvRsXF1hmaFgHhVJzczWzA1RXMMo9LTPuFXo9q/alDbME3JIyWYqovWsUBWSoBcsHaGPSLF9rz4l9Qeg==", "dev": true, "license": "MIT", "dependencies": { - "@jest/console": "30.1.1", + "@jest/console": "30.0.5", "@jest/pattern": "30.0.1", - "@jest/reporters": "30.1.1", - "@jest/test-result": "30.1.1", - "@jest/transform": "30.1.1", + "@jest/reporters": "30.0.5", + "@jest/test-result": "30.0.5", + "@jest/transform": "30.0.5", "@jest/types": "30.0.5", "@types/node": "*", "ansi-escapes": "^4.3.2", @@ -766,18 +766,18 @@ "exit-x": "^0.2.2", "graceful-fs": "^4.2.11", "jest-changed-files": "30.0.5", - "jest-config": "30.1.1", - "jest-haste-map": "30.1.0", - "jest-message-util": "30.1.0", + "jest-config": "30.0.5", + "jest-haste-map": "30.0.5", + "jest-message-util": "30.0.5", "jest-regex-util": "30.0.1", - "jest-resolve": "30.1.0", - "jest-resolve-dependencies": "30.1.1", - "jest-runner": "30.1.1", - "jest-runtime": "30.1.1", - "jest-snapshot": "30.1.1", + "jest-resolve": "30.0.5", + "jest-resolve-dependencies": "30.0.5", + "jest-runner": "30.0.5", + "jest-runtime": "30.0.5", + "jest-snapshot": "30.0.5", "jest-util": "30.0.5", - "jest-validate": "30.1.0", - "jest-watcher": "30.1.1", + "jest-validate": "30.0.5", + "jest-watcher": "30.0.5", "micromatch": "^4.0.8", "pretty-format": "30.0.5", "slash": "^3.0.0" @@ -805,13 +805,13 @@ } }, "node_modules/@jest/environment": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-30.1.1.tgz", - "integrity": "sha512-yWHbU+3j7ehQE+NRpnxRvHvpUhoohIjMePBbIr8lfe0cWVb0WeTf80DNux1GPJa18CDHiIU5DtksGUfxcDE+Rw==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-30.0.5.tgz", + "integrity": "sha512-aRX7WoaWx1oaOkDQvCWImVQ8XNtdv5sEWgk4gxR6NXb7WBUnL5sRak4WRzIQRZ1VTWPvV4VI4mgGjNL9TeKMYA==", "dev": true, "license": "MIT", "dependencies": { - "@jest/fake-timers": "30.1.1", + "@jest/fake-timers": "30.0.5", "@jest/types": "30.0.5", "@types/node": "*", "jest-mock": "30.0.5" @@ -821,43 +821,43 @@ } }, "node_modules/@jest/expect": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-30.1.1.tgz", - "integrity": "sha512-3vHIHsF+qd3D8FU2c7U5l3rg1fhDwAYcGyHyZAi94YIlTwcJ+boNhRyJf373cl4wxbOX+0Q7dF40RTrTFTSuig==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-30.0.5.tgz", + "integrity": "sha512-6udac8KKrtTtC+AXZ2iUN/R7dp7Ydry+Fo6FPFnDG54wjVMnb6vW/XNlf7Xj8UDjAE3aAVAsR4KFyKk3TCXmTA==", "dev": true, "license": "MIT", "dependencies": { - "expect": "30.1.1", - "jest-snapshot": "30.1.1" + "expect": "30.0.5", + "jest-snapshot": "30.0.5" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/expect-utils": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-30.1.1.tgz", - "integrity": "sha512-5YUHr27fpJ64dnvtu+tt11ewATynrHkGYD+uSFgRr8V2eFJis/vEXgToyLwccIwqBihVfz9jwio+Zr1ab1Zihw==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-30.0.5.tgz", + "integrity": "sha512-F3lmTT7CXWYywoVUGTCmom0vXq3HTTkaZyTAzIy+bXSBizB7o5qzlC9VCtq0arOa8GqmNsbg/cE9C6HLn7Szew==", "dev": true, "license": "MIT", "dependencies": { - "@jest/get-type": "30.1.0" + "@jest/get-type": "30.0.1" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/fake-timers": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-30.1.1.tgz", - "integrity": "sha512-fK/25dNgBNYPw3eLi2CRs57g1H04qBAFNMsUY3IRzkfx/m4THe0E1zF+yGQBOMKKc2XQVdc9EYbJ4hEm7/2UtA==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-30.0.5.tgz", + "integrity": "sha512-ZO5DHfNV+kgEAeP3gK3XlpJLL4U3Sz6ebl/n68Uwt64qFFs5bv4bfEEjyRGK5uM0C90ewooNgFuKMdkbEoMEXw==", "dev": true, "license": "MIT", "dependencies": { "@jest/types": "30.0.5", "@sinonjs/fake-timers": "^13.0.0", "@types/node": "*", - "jest-message-util": "30.1.0", + "jest-message-util": "30.0.5", "jest-mock": "30.0.5", "jest-util": "30.0.5" }, @@ -866,9 +866,9 @@ } }, "node_modules/@jest/get-type": { - "version": "30.1.0", - "resolved": "https://registry.npmjs.org/@jest/get-type/-/get-type-30.1.0.tgz", - "integrity": "sha512-eMbZE2hUnx1WV0pmURZY9XoXPkUYjpc55mb0CrhtdWLtzMQPFvu/rZkTLZFTsdaVQa+Tr4eWAteqcUzoawq/uA==", + "version": "30.0.1", + "resolved": "https://registry.npmjs.org/@jest/get-type/-/get-type-30.0.1.tgz", + "integrity": "sha512-AyYdemXCptSRFirI5EPazNxyPwAL0jXt3zceFjaj8NFiKP9pOi0bfXonf6qkf82z2t3QWPeLCWWw4stPBzctLw==", "dev": true, "license": "MIT", "engines": { @@ -876,14 +876,14 @@ } }, "node_modules/@jest/globals": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-30.1.1.tgz", - "integrity": "sha512-NNUUkHT2TU/xztZl6r1UXvJL+zvCwmZsQDmK69fVHHcB9fBtlu3FInnzOve/ZoyKnWY8JXWJNT+Lkmu1+ubXUA==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-30.0.5.tgz", + "integrity": "sha512-7oEJT19WW4oe6HR7oLRvHxwlJk2gev0U9px3ufs8sX9PoD1Eza68KF0/tlN7X0dq/WVsBScXQGgCldA1V9Y/jA==", "dev": true, "license": "MIT", "dependencies": { - "@jest/environment": "30.1.1", - "@jest/expect": "30.1.1", + "@jest/environment": "30.0.5", + "@jest/expect": "30.0.5", "@jest/types": "30.0.5", "jest-mock": "30.0.5" }, @@ -906,16 +906,16 @@ } }, "node_modules/@jest/reporters": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-30.1.1.tgz", - "integrity": "sha512-Hb2Bq80kahOC6Sv2waEaH1rEU6VdFcM6WHaRBWQF9tf30+nJHxhl/Upbgo9+25f0mOgbphxvbwSMjSgy9gW/FA==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-30.0.5.tgz", + "integrity": "sha512-mafft7VBX4jzED1FwGC1o/9QUM2xebzavImZMeqnsklgcyxBto8mV4HzNSzUrryJ+8R9MFOM3HgYuDradWR+4g==", "dev": true, "license": "MIT", "dependencies": { "@bcoe/v8-coverage": "^0.2.3", - "@jest/console": "30.1.1", - "@jest/test-result": "30.1.1", - "@jest/transform": "30.1.1", + "@jest/console": "30.0.5", + "@jest/test-result": "30.0.5", + "@jest/transform": "30.0.5", "@jest/types": "30.0.5", "@jridgewell/trace-mapping": "^0.3.25", "@types/node": "*", @@ -929,9 +929,9 @@ "istanbul-lib-report": "^3.0.0", "istanbul-lib-source-maps": "^5.0.0", "istanbul-reports": "^3.1.3", - "jest-message-util": "30.1.0", + "jest-message-util": "30.0.5", "jest-util": "30.0.5", - "jest-worker": "30.1.0", + "jest-worker": "30.0.5", "slash": "^3.0.0", "string-length": "^4.0.2", "v8-to-istanbul": "^9.0.1" @@ -962,9 +962,9 @@ } }, "node_modules/@jest/snapshot-utils": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/@jest/snapshot-utils/-/snapshot-utils-30.1.1.tgz", - "integrity": "sha512-TkVBc9wuN22TT8hESRFmjjg/xIMu7z0J3UDYtIRydzCqlLPTB7jK1DDBKdnTUZ4zL3z3rnPpzV6rL1Uzh87sXg==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/@jest/snapshot-utils/-/snapshot-utils-30.0.5.tgz", + "integrity": "sha512-XcCQ5qWHLvi29UUrowgDFvV4t7ETxX91CbDczMnoqXPOIcZOxyNdSjm6kV5XMc8+HkxfRegU/MUmnTbJRzGrUQ==", "dev": true, "license": "MIT", "dependencies": { @@ -993,13 +993,13 @@ } }, "node_modules/@jest/test-result": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-30.1.1.tgz", - "integrity": "sha512-bMdj7fNu8iZuBPSnbVir5ezvWmVo4jrw7xDE+A33Yb3ENCoiJK9XgOLgal+rJ9XSKjsL7aPUMIo87zhN7I5o2w==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-30.0.5.tgz", + "integrity": "sha512-wPyztnK0gbDMQAJZ43tdMro+qblDHH1Ru/ylzUo21TBKqt88ZqnKKK2m30LKmLLoKtR2lxdpCC/P3g1vfKcawQ==", "dev": true, "license": "MIT", "dependencies": { - "@jest/console": "30.1.1", + "@jest/console": "30.0.5", "@jest/types": "30.0.5", "@types/istanbul-lib-coverage": "^2.0.6", "collect-v8-coverage": "^1.0.2" @@ -1009,15 +1009,15 @@ } }, "node_modules/@jest/test-sequencer": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-30.1.1.tgz", - "integrity": "sha512-yruRdLXSA3HYD/MTNykgJ6VYEacNcXDFRMqKVAwlYegmxICUiT/B++CNuhJnYJzKYks61iYnjVsMwbUqmmAYJg==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-30.0.5.tgz", + "integrity": "sha512-Aea/G1egWoIIozmDD7PBXUOxkekXl7ueGzrsGGi1SbeKgQqCYCIf+wfbflEbf2LiPxL8j2JZGLyrzZagjvW4YQ==", "dev": true, "license": "MIT", "dependencies": { - "@jest/test-result": "30.1.1", + "@jest/test-result": "30.0.5", "graceful-fs": "^4.2.11", - "jest-haste-map": "30.1.0", + "jest-haste-map": "30.0.5", "slash": "^3.0.0" }, "engines": { @@ -1025,9 +1025,9 @@ } }, "node_modules/@jest/transform": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-30.1.1.tgz", - "integrity": "sha512-PHIA2AbAASBfk6evkNifvmx9lkOSkmvaQoO6VSpuL8+kQqDMHeDoJ7RU3YP1wWAMD7AyQn9UL5iheuFYCC4lqQ==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-30.0.5.tgz", + "integrity": "sha512-Vk8amLQCmuZyy6GbBht1Jfo9RSdBtg7Lks+B0PecnjI8J+PCLQPGh7uI8Q/2wwpW2gLdiAfiHNsmekKlywULqg==", "dev": true, "license": "MIT", "dependencies": { @@ -1039,7 +1039,7 @@ "convert-source-map": "^2.0.0", "fast-json-stable-stringify": "^2.1.0", "graceful-fs": "^4.2.11", - "jest-haste-map": "30.1.0", + "jest-haste-map": "30.0.5", "jest-regex-util": "30.0.1", "jest-util": "30.0.5", "micromatch": "^4.0.8", @@ -1071,9 +1071,9 @@ } }, "node_modules/@jridgewell/gen-mapping": { - "version": "0.3.13", - "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", - "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "version": "0.3.12", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.12.tgz", + "integrity": "sha512-OuLGC46TjB5BbN1dH8JULVVZY4WTdkF7tV9Ys6wLL1rubZnCMstOhNHueU5bLCrnRuDhKPDM4g6sw4Bel5Gzqg==", "dev": true, "license": "MIT", "dependencies": { @@ -1092,16 +1092,16 @@ } }, "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.5", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", - "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.4.tgz", + "integrity": "sha512-VT2+G1VQs/9oz078bLrYbecdZKs912zQlkelYpuf+SXF+QvZDYJlbx/LSx+meSAwdDFnF8FVXW92AVjjkVmgFw==", "dev": true, "license": "MIT" }, "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.30", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.30.tgz", - "integrity": "sha512-GQ7Nw5G2lTu/BtHTKfXhKHok2WGetd4XYcVKGx00SjAk8GMwgJM3zr6zORiPGuOE+/vkc90KtTosSSvaCjKb2Q==", + "version": "0.3.29", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.29.tgz", + "integrity": "sha512-uw6guiW/gcAGPDhLmd77/6lW8QLeiV5RUTsAX46Db6oLhGaVj4lhnPwb184s1bkc8kdVg/+h988dro8GRDpmYQ==", "dev": true, "license": "MIT", "dependencies": { @@ -1221,9 +1221,9 @@ "dev": true }, "node_modules/@sinclair/typebox": { - "version": "0.34.41", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.34.41.tgz", - "integrity": "sha512-6gS8pZzSXdyRHTIqoqSVknxolr1kzfy4/CeDnrzsVz8TTIWUbOBr6gnzOmTYJ3eXQNh4IYHIGi5aIL7sOZ2G/g==", + "version": "0.34.38", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.34.38.tgz", + "integrity": "sha512-HpkxMmc2XmZKhvaKIZZThlHmx1L0I/V1hWK1NubtlFnr6ZqdiOpV72TKudZUNQjZNsyDBay72qFEhEvb+bcwcA==", "dev": true, "license": "MIT" }, @@ -1294,13 +1294,13 @@ } }, "node_modules/@types/babel__traverse": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.28.0.tgz", - "integrity": "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==", + "version": "7.20.7", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.20.7.tgz", + "integrity": "sha512-dkO5fhS7+/oos4ciWxyEyjWe48zmG6wbCheo/G2ZnHx4fs3EU6YC6UM8rk56gAjNJ9P3MTH2jo5jb92/K6wbng==", "dev": true, "license": "MIT", "dependencies": { - "@babel/types": "^7.28.2" + "@babel/types": "^7.20.7" } }, "node_modules/@types/docker-modem": { @@ -1844,13 +1844,13 @@ "license": "Apache-2.0" }, "node_modules/babel-jest": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-30.1.1.tgz", - "integrity": "sha512-1bZfC/V03qBCzASvZpNFhx3Ouj6LgOd4KFJm4br/fYOS+tSSvVCE61QmcAVbMTwq/GoB7KN4pzGMoyr9cMxSvQ==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-30.0.5.tgz", + "integrity": "sha512-mRijnKimhGDMsizTvBTWotwNpzrkHr+VvZUQBof2AufXKB8NXrL1W69TG20EvOz7aevx6FTJIaBuBkYxS8zolg==", "dev": true, "license": "MIT", "dependencies": { - "@jest/transform": "30.1.1", + "@jest/transform": "30.0.5", "@types/babel__core": "^7.20.5", "babel-plugin-istanbul": "^7.0.0", "babel-preset-jest": "30.0.1", @@ -1898,9 +1898,9 @@ } }, "node_modules/babel-preset-current-node-syntax": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/babel-preset-current-node-syntax/-/babel-preset-current-node-syntax-1.2.0.tgz", - "integrity": "sha512-E/VlAEzRrsLEb2+dv8yp3bo4scof3l9nR4lrld+Iy5NyVqgVYUJnDAmunkhPMisRI32Qc4iRiz425d8vM++2fg==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/babel-preset-current-node-syntax/-/babel-preset-current-node-syntax-1.1.0.tgz", + "integrity": "sha512-ldYss8SbBlWva1bs28q78Ju5Zq1F+8BrqBZZ0VFhLBvhh6lCpC2o3gDJi/5DRLs9FgYZCnmPYIVFU4lRXCkyUw==", "dev": true, "license": "MIT", "dependencies": { @@ -1921,7 +1921,7 @@ "@babel/plugin-syntax-top-level-await": "^7.14.5" }, "peerDependencies": { - "@babel/core": "^7.0.0 || ^8.0.0-0" + "@babel/core": "^7.0.0" } }, "node_modules/babel-preset-jest": { @@ -2125,9 +2125,9 @@ } }, "node_modules/browserslist": { - "version": "4.25.4", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.25.4.tgz", - "integrity": "sha512-4jYpcjabC606xJ3kw2QwGEZKX0Aw7sgQdZCvIK9dhVSPh76BKo+C+btT1RRofH7B+8iNpEbgGNVWiLki5q93yg==", + "version": "4.25.1", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.25.1.tgz", + "integrity": "sha512-KGj0KoOMXLpSNkkEI6Z6mShmQy0bc1I+T7K9N81k4WWMrfz+6fQ6es80B/YLAeRoKvjYE1YSHHOW1qe9xIVzHw==", "dev": true, "funding": [ { @@ -2145,8 +2145,8 @@ ], "license": "MIT", "dependencies": { - "caniuse-lite": "^1.0.30001737", - "electron-to-chromium": "^1.5.211", + "caniuse-lite": "^1.0.30001726", + "electron-to-chromium": "^1.5.173", "node-releases": "^2.0.19", "update-browserslist-db": "^1.1.3" }, @@ -2250,9 +2250,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001739", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001739.tgz", - "integrity": "sha512-y+j60d6ulelrNSwpPyrHdl+9mJnQzHBr08xm48Qno0nSk4h3Qojh+ziv2qE6rXf4k3tadF4o1J/1tAbVm1NtnA==", + "version": "1.0.30001727", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001727.tgz", + "integrity": "sha512-pB68nIHmbN6L/4C6MH1DokyR3bYqFwjaSs/sWDHGj4CTcFtQUQMuJftVwWkXq7mNWOybD3KhUv3oWHoGxgP14Q==", "dev": true, "funding": [ { @@ -2649,9 +2649,9 @@ "license": "MIT" }, "node_modules/electron-to-chromium": { - "version": "1.5.211", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.211.tgz", - "integrity": "sha512-IGBvimJkotaLzFnwIVgW9/UD/AOJ2tByUmeOrtqBfACSbAw5b1G0XpvdaieKyc7ULmbwXVx+4e4Be8pOPBrYkw==", + "version": "1.5.191", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.191.tgz", + "integrity": "sha512-xcwe9ELcuxYLUFqZZxL19Z6HVKcvNkIwhbHUz7L3us6u12yR+7uY89dSl570f/IqNthx8dAw3tojG7i4Ni4tDA==", "dev": true, "license": "ISC" }, @@ -2783,16 +2783,16 @@ } }, "node_modules/expect": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/expect/-/expect-30.1.1.tgz", - "integrity": "sha512-OKe7cdic4qbfWd/CcgwJvvCrNX2KWfuMZee9AfJHL1gTYmvqjBjZG1a2NwfhspBzxzlXwsN75WWpKTYfsJpBxg==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/expect/-/expect-30.0.5.tgz", + "integrity": "sha512-P0te2pt+hHI5qLJkIR+iMvS+lYUZml8rKKsohVHAGY+uClp9XVbdyYNJOIjSRpHVp8s8YqxJCiHUkSYZGr8rtQ==", "dev": true, "license": "MIT", "dependencies": { - "@jest/expect-utils": "30.1.1", - "@jest/get-type": "30.1.0", - "jest-matcher-utils": "30.1.1", - "jest-message-util": "30.1.0", + "@jest/expect-utils": "30.0.5", + "@jest/get-type": "30.0.1", + "jest-matcher-utils": "30.0.5", + "jest-message-util": "30.0.5", "jest-mock": "30.0.5", "jest-util": "30.0.5" }, @@ -3225,9 +3225,9 @@ } }, "node_modules/istanbul-reports": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", - "integrity": "sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==", + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.1.7.tgz", + "integrity": "sha512-BewmUXImeuRk2YY0PVbxgKAysvhRPUQE0h5QRM++nVWyubKGV0l8qQ5op8+B2DOmwSe63Jivj0BjkPQVf8fP5g==", "dev": true, "license": "BSD-3-Clause", "dependencies": { @@ -3255,16 +3255,16 @@ } }, "node_modules/jest": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/jest/-/jest-30.1.1.tgz", - "integrity": "sha512-yC3JvpP/ZcAZX5rYCtXO/g9k6VTCQz0VFE2v1FpxytWzUqfDtu0XL/pwnNvptzYItvGwomh1ehomRNMOyhCJKw==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest/-/jest-30.0.5.tgz", + "integrity": "sha512-y2mfcJywuTUkvLm2Lp1/pFX8kTgMO5yyQGq/Sk/n2mN7XWYp4JsCZ/QXW34M8YScgk8bPZlREH04f6blPnoHnQ==", "dev": true, "license": "MIT", "dependencies": { - "@jest/core": "30.1.1", + "@jest/core": "30.0.5", "@jest/types": "30.0.5", "import-local": "^3.2.0", - "jest-cli": "30.1.1" + "jest-cli": "30.0.5" }, "bin": { "jest": "bin/jest.js" @@ -3297,26 +3297,26 @@ } }, "node_modules/jest-circus": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/jest-circus/-/jest-circus-30.1.1.tgz", - "integrity": "sha512-M3Vd4x5wD7eSJspuTvRF55AkOOBndRxgW3gqQBDlFvbH3X+ASdi8jc+EqXEeAFd/UHulVYIlC4XKJABOhLw6UA==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-circus/-/jest-circus-30.0.5.tgz", + "integrity": "sha512-h/sjXEs4GS+NFFfqBDYT7y5Msfxh04EwWLhQi0F8kuWpe+J/7tICSlswU8qvBqumR3kFgHbfu7vU6qruWWBPug==", "dev": true, "license": "MIT", "dependencies": { - "@jest/environment": "30.1.1", - "@jest/expect": "30.1.1", - "@jest/test-result": "30.1.1", + "@jest/environment": "30.0.5", + "@jest/expect": "30.0.5", + "@jest/test-result": "30.0.5", "@jest/types": "30.0.5", "@types/node": "*", "chalk": "^4.1.2", "co": "^4.6.0", "dedent": "^1.6.0", "is-generator-fn": "^2.1.0", - "jest-each": "30.1.0", - "jest-matcher-utils": "30.1.1", - "jest-message-util": "30.1.0", - "jest-runtime": "30.1.1", - "jest-snapshot": "30.1.1", + "jest-each": "30.0.5", + "jest-matcher-utils": "30.0.5", + "jest-message-util": "30.0.5", + "jest-runtime": "30.0.5", + "jest-snapshot": "30.0.5", "jest-util": "30.0.5", "p-limit": "^3.1.0", "pretty-format": "30.0.5", @@ -3329,21 +3329,21 @@ } }, "node_modules/jest-cli": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-30.1.1.tgz", - "integrity": "sha512-xm9llxuh5OoI5KZaYzlMhklryHBwg9LZy/gEaaMlXlxb+cZekGNzukU0iblbDo3XOBuN6N0CgK4ykgNRYSEb6g==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-30.0.5.tgz", + "integrity": "sha512-Sa45PGMkBZzF94HMrlX4kUyPOwUpdZasaliKN3mifvDmkhLYqLLg8HQTzn6gq7vJGahFYMQjXgyJWfYImKZzOw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/core": "30.1.1", - "@jest/test-result": "30.1.1", + "@jest/core": "30.0.5", + "@jest/test-result": "30.0.5", "@jest/types": "30.0.5", "chalk": "^4.1.2", "exit-x": "^0.2.2", "import-local": "^3.2.0", - "jest-config": "30.1.1", + "jest-config": "30.0.5", "jest-util": "30.0.5", - "jest-validate": "30.1.0", + "jest-validate": "30.0.5", "yargs": "^17.7.2" }, "bin": { @@ -3362,31 +3362,31 @@ } }, "node_modules/jest-config": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/jest-config/-/jest-config-30.1.1.tgz", - "integrity": "sha512-xuPGUGDw+9fPPnGmddnLnHS/mhKUiJOW7K65vErYmglEPKq65NKwSRchkQ7iv6gqjs2l+YNEsAtbsplxozdOWg==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-config/-/jest-config-30.0.5.tgz", + "integrity": "sha512-aIVh+JNOOpzUgzUnPn5FLtyVnqc3TQHVMupYtyeURSb//iLColiMIR8TxCIDKyx9ZgjKnXGucuW68hCxgbrwmA==", "dev": true, "license": "MIT", "dependencies": { "@babel/core": "^7.27.4", - "@jest/get-type": "30.1.0", + "@jest/get-type": "30.0.1", "@jest/pattern": "30.0.1", - "@jest/test-sequencer": "30.1.1", + "@jest/test-sequencer": "30.0.5", "@jest/types": "30.0.5", - "babel-jest": "30.1.1", + "babel-jest": "30.0.5", "chalk": "^4.1.2", "ci-info": "^4.2.0", "deepmerge": "^4.3.1", "glob": "^10.3.10", "graceful-fs": "^4.2.11", - "jest-circus": "30.1.1", + "jest-circus": "30.0.5", "jest-docblock": "30.0.1", - "jest-environment-node": "30.1.1", + "jest-environment-node": "30.0.5", "jest-regex-util": "30.0.1", - "jest-resolve": "30.1.0", - "jest-runner": "30.1.1", + "jest-resolve": "30.0.5", + "jest-runner": "30.0.5", "jest-util": "30.0.5", - "jest-validate": "30.1.0", + "jest-validate": "30.0.5", "micromatch": "^4.0.8", "parse-json": "^5.2.0", "pretty-format": "30.0.5", @@ -3414,14 +3414,14 @@ } }, "node_modules/jest-diff": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.1.1.tgz", - "integrity": "sha512-LUU2Gx8EhYxpdzTR6BmjL1ifgOAQJQELTHOiPv9KITaKjZvJ9Jmgigx01tuZ49id37LorpGc9dPBPlXTboXScw==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.0.5.tgz", + "integrity": "sha512-1UIqE9PoEKaHcIKvq2vbibrCog4Y8G0zmOxgQUVEiTqwR5hJVMCoDsN1vFvI5JvwD37hjueZ1C4l2FyGnfpE0A==", "dev": true, "license": "MIT", "dependencies": { "@jest/diff-sequences": "30.0.1", - "@jest/get-type": "30.1.0", + "@jest/get-type": "30.0.1", "chalk": "^4.1.2", "pretty-format": "30.0.5" }, @@ -3443,13 +3443,13 @@ } }, "node_modules/jest-each": { - "version": "30.1.0", - "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-30.1.0.tgz", - "integrity": "sha512-A+9FKzxPluqogNahpCv04UJvcZ9B3HamqpDNWNKDjtxVRYB8xbZLFuCr8JAJFpNp83CA0anGQFlpQna9Me+/tQ==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-30.0.5.tgz", + "integrity": "sha512-dKjRsx1uZ96TVyejD3/aAWcNKy6ajMaN531CwWIsrazIqIoXI9TnnpPlkrEYku/8rkS3dh2rbH+kMOyiEIv0xQ==", "dev": true, "license": "MIT", "dependencies": { - "@jest/get-type": "30.1.0", + "@jest/get-type": "30.0.1", "@jest/types": "30.0.5", "chalk": "^4.1.2", "jest-util": "30.0.5", @@ -3460,28 +3460,28 @@ } }, "node_modules/jest-environment-node": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-30.1.1.tgz", - "integrity": "sha512-IaMoaA6saxnJimqCppUDqKck+LKM0Jg+OxyMUIvs1yGd2neiC22o8zXo90k04+tO+49OmgMR4jTgM5e4B0S62Q==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-30.0.5.tgz", + "integrity": "sha512-ppYizXdLMSvciGsRsMEnv/5EFpvOdXBaXRBzFUDPWrsfmog4kYrOGWXarLllz6AXan6ZAA/kYokgDWuos1IKDA==", "dev": true, "license": "MIT", "dependencies": { - "@jest/environment": "30.1.1", - "@jest/fake-timers": "30.1.1", + "@jest/environment": "30.0.5", + "@jest/fake-timers": "30.0.5", "@jest/types": "30.0.5", "@types/node": "*", "jest-mock": "30.0.5", "jest-util": "30.0.5", - "jest-validate": "30.1.0" + "jest-validate": "30.0.5" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-haste-map": { - "version": "30.1.0", - "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-30.1.0.tgz", - "integrity": "sha512-JLeM84kNjpRkggcGpQLsV7B8W4LNUWz7oDNVnY1Vjj22b5/fAb3kk3htiD+4Na8bmJmjJR7rBtS2Rmq/NEcADg==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-30.0.5.tgz", + "integrity": "sha512-dkmlWNlsTSR0nH3nRfW5BKbqHefLZv0/6LCccG0xFCTWcJu8TuEwG+5Cm75iBfjVoockmO6J35o5gxtFSn5xeg==", "dev": true, "license": "MIT", "dependencies": { @@ -3492,7 +3492,7 @@ "graceful-fs": "^4.2.11", "jest-regex-util": "30.0.1", "jest-util": "30.0.5", - "jest-worker": "30.1.0", + "jest-worker": "30.0.5", "micromatch": "^4.0.8", "walker": "^1.0.8" }, @@ -3520,13 +3520,13 @@ } }, "node_modules/jest-leak-detector": { - "version": "30.1.0", - "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-30.1.0.tgz", - "integrity": "sha512-AoFvJzwxK+4KohH60vRuHaqXfWmeBATFZpzpmzNmYTtmRMiyGPVhkXpBqxUQunw+dQB48bDf4NpUs6ivVbRv1g==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-30.0.5.tgz", + "integrity": "sha512-3Uxr5uP8jmHMcsOtYMRB/zf1gXN3yUIc+iPorhNETG54gErFIiUhLvyY/OggYpSMOEYqsmRxmuU4ZOoX5jpRFg==", "dev": true, "license": "MIT", "dependencies": { - "@jest/get-type": "30.1.0", + "@jest/get-type": "30.0.1", "pretty-format": "30.0.5" }, "engines": { @@ -3534,15 +3534,15 @@ } }, "node_modules/jest-matcher-utils": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-30.1.1.tgz", - "integrity": "sha512-SuH2QVemK48BNTqReti6FtjsMPFsSOD/ZzRxU1TttR7RiRsRSe78d03bb4Cx6D4bQC/80Q8U4VnaaAH9FlbZ9w==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-30.0.5.tgz", + "integrity": "sha512-uQgGWt7GOrRLP1P7IwNWwK1WAQbq+m//ZY0yXygyfWp0rJlksMSLQAA4wYQC3b6wl3zfnchyTx+k3HZ5aPtCbQ==", "dev": true, "license": "MIT", "dependencies": { - "@jest/get-type": "30.1.0", + "@jest/get-type": "30.0.1", "chalk": "^4.1.2", - "jest-diff": "30.1.1", + "jest-diff": "30.0.5", "pretty-format": "30.0.5" }, "engines": { @@ -3550,9 +3550,9 @@ } }, "node_modules/jest-message-util": { - "version": "30.1.0", - "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.1.0.tgz", - "integrity": "sha512-HizKDGG98cYkWmaLUHChq4iN+oCENohQLb7Z5guBPumYs+/etonmNFlg1Ps6yN9LTPyZn+M+b/9BbnHx3WTMDg==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.0.5.tgz", + "integrity": "sha512-NAiDOhsK3V7RU0Aa/HnrQo+E4JlbarbmI3q6Pi4KcxicdtjV82gcIUrejOtczChtVQR4kddu1E1EJlW6EN9IyA==", "dev": true, "license": "MIT", "dependencies": { @@ -3614,18 +3614,18 @@ } }, "node_modules/jest-resolve": { - "version": "30.1.0", - "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-30.1.0.tgz", - "integrity": "sha512-hASe7D/wRtZw8Cm607NrlF7fi3HWC5wmA5jCVc2QjQAB2pTwP9eVZILGEi6OeSLNUtE1zb04sXRowsdh5CUjwA==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-30.0.5.tgz", + "integrity": "sha512-d+DjBQ1tIhdz91B79mywH5yYu76bZuE96sSbxj8MkjWVx5WNdt1deEFRONVL4UkKLSrAbMkdhb24XN691yDRHg==", "dev": true, "license": "MIT", "dependencies": { "chalk": "^4.1.2", "graceful-fs": "^4.2.11", - "jest-haste-map": "30.1.0", + "jest-haste-map": "30.0.5", "jest-pnp-resolver": "^1.2.3", "jest-util": "30.0.5", - "jest-validate": "30.1.0", + "jest-validate": "30.0.5", "slash": "^3.0.0", "unrs-resolver": "^1.7.11" }, @@ -3634,30 +3634,30 @@ } }, "node_modules/jest-resolve-dependencies": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-30.1.1.tgz", - "integrity": "sha512-tRtaaoH8Ws1Gn1o/9pedt19dvVgr81WwdmvJSP9Ow3amOUOP2nN9j94u5jC9XlIfa2Q1FQKIWWQwL4ajqsjCGQ==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-30.0.5.tgz", + "integrity": "sha512-/xMvBR4MpwkrHW4ikZIWRttBBRZgWK4d6xt3xW1iRDSKt4tXzYkMkyPfBnSCgv96cpkrctfXs6gexeqMYqdEpw==", "dev": true, "license": "MIT", "dependencies": { "jest-regex-util": "30.0.1", - "jest-snapshot": "30.1.1" + "jest-snapshot": "30.0.5" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-runner": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/jest-runner/-/jest-runner-30.1.1.tgz", - "integrity": "sha512-ATe6372SOfJvCRExtCAr06I4rGujwFdKg44b6i7/aOgFnULwjxzugJ0Y4AnG+jeSeQi8dU7R6oqLGmsxRUbErQ==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-runner/-/jest-runner-30.0.5.tgz", + "integrity": "sha512-JcCOucZmgp+YuGgLAXHNy7ualBx4wYSgJVWrYMRBnb79j9PD0Jxh0EHvR5Cx/r0Ce+ZBC4hCdz2AzFFLl9hCiw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/console": "30.1.1", - "@jest/environment": "30.1.1", - "@jest/test-result": "30.1.1", - "@jest/transform": "30.1.1", + "@jest/console": "30.0.5", + "@jest/environment": "30.0.5", + "@jest/test-result": "30.0.5", + "@jest/transform": "30.0.5", "@jest/types": "30.0.5", "@types/node": "*", "chalk": "^4.1.2", @@ -3665,15 +3665,15 @@ "exit-x": "^0.2.2", "graceful-fs": "^4.2.11", "jest-docblock": "30.0.1", - "jest-environment-node": "30.1.1", - "jest-haste-map": "30.1.0", - "jest-leak-detector": "30.1.0", - "jest-message-util": "30.1.0", - "jest-resolve": "30.1.0", - "jest-runtime": "30.1.1", + "jest-environment-node": "30.0.5", + "jest-haste-map": "30.0.5", + "jest-leak-detector": "30.0.5", + "jest-message-util": "30.0.5", + "jest-resolve": "30.0.5", + "jest-runtime": "30.0.5", "jest-util": "30.0.5", - "jest-watcher": "30.1.1", - "jest-worker": "30.1.0", + "jest-watcher": "30.0.5", + "jest-worker": "30.0.5", "p-limit": "^3.1.0", "source-map-support": "0.5.13" }, @@ -3682,18 +3682,18 @@ } }, "node_modules/jest-runtime": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-30.1.1.tgz", - "integrity": "sha512-7sOyR0Oekw4OesQqqBHuYJRB52QtXiq0NNgLRzVogiMSxKCMiliUd6RrXHCnG5f12Age/ggidCBiQftzcA9XKw==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-30.0.5.tgz", + "integrity": "sha512-7oySNDkqpe4xpX5PPiJTe5vEa+Ak/NnNz2bGYZrA1ftG3RL3EFlHaUkA1Cjx+R8IhK0Vg43RML5mJedGTPNz3A==", "dev": true, "license": "MIT", "dependencies": { - "@jest/environment": "30.1.1", - "@jest/fake-timers": "30.1.1", - "@jest/globals": "30.1.1", + "@jest/environment": "30.0.5", + "@jest/fake-timers": "30.0.5", + "@jest/globals": "30.0.5", "@jest/source-map": "30.0.1", - "@jest/test-result": "30.1.1", - "@jest/transform": "30.1.1", + "@jest/test-result": "30.0.5", + "@jest/transform": "30.0.5", "@jest/types": "30.0.5", "@types/node": "*", "chalk": "^4.1.2", @@ -3701,12 +3701,12 @@ "collect-v8-coverage": "^1.0.2", "glob": "^10.3.10", "graceful-fs": "^4.2.11", - "jest-haste-map": "30.1.0", - "jest-message-util": "30.1.0", + "jest-haste-map": "30.0.5", + "jest-message-util": "30.0.5", "jest-mock": "30.0.5", "jest-regex-util": "30.0.1", - "jest-resolve": "30.1.0", - "jest-snapshot": "30.1.1", + "jest-resolve": "30.0.5", + "jest-snapshot": "30.0.5", "jest-util": "30.0.5", "slash": "^3.0.0", "strip-bom": "^4.0.0" @@ -3716,9 +3716,9 @@ } }, "node_modules/jest-snapshot": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-30.1.1.tgz", - "integrity": "sha512-7/iBEzoJqEt2TjkQY+mPLHP8cbPhLReZVkkxjTMzIzoTC4cZufg7HzKo/n9cIkXKj2LG0x3mmBHsZto+7TOmFg==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-30.0.5.tgz", + "integrity": "sha512-T00dWU/Ek3LqTp4+DcW6PraVxjk28WY5Ua/s+3zUKSERZSNyxTqhDXCWKG5p2HAJ+crVQ3WJ2P9YVHpj1tkW+g==", "dev": true, "license": "MIT", "dependencies": { @@ -3727,18 +3727,18 @@ "@babel/plugin-syntax-jsx": "^7.27.1", "@babel/plugin-syntax-typescript": "^7.27.1", "@babel/types": "^7.27.3", - "@jest/expect-utils": "30.1.1", - "@jest/get-type": "30.1.0", - "@jest/snapshot-utils": "30.1.1", - "@jest/transform": "30.1.1", + "@jest/expect-utils": "30.0.5", + "@jest/get-type": "30.0.1", + "@jest/snapshot-utils": "30.0.5", + "@jest/transform": "30.0.5", "@jest/types": "30.0.5", "babel-preset-current-node-syntax": "^1.1.0", "chalk": "^4.1.2", - "expect": "30.1.1", + "expect": "30.0.5", "graceful-fs": "^4.2.11", - "jest-diff": "30.1.1", - "jest-matcher-utils": "30.1.1", - "jest-message-util": "30.1.0", + "jest-diff": "30.0.5", + "jest-matcher-utils": "30.0.5", + "jest-message-util": "30.0.5", "jest-util": "30.0.5", "pretty-format": "30.0.5", "semver": "^7.7.2", @@ -3793,13 +3793,13 @@ } }, "node_modules/jest-validate": { - "version": "30.1.0", - "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-30.1.0.tgz", - "integrity": "sha512-7P3ZlCFW/vhfQ8pE7zW6Oi4EzvuB4sgR72Q1INfW9m0FGo0GADYlPwIkf4CyPq7wq85g+kPMtPOHNAdWHeBOaA==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-30.0.5.tgz", + "integrity": "sha512-ouTm6VFHaS2boyl+k4u+Qip4TSH7Uld5tyD8psQ8abGgt2uYYB8VwVfAHWHjHc0NWmGGbwO5h0sCPOGHHevefw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/get-type": "30.1.0", + "@jest/get-type": "30.0.1", "@jest/types": "30.0.5", "camelcase": "^6.3.0", "chalk": "^4.1.2", @@ -3824,13 +3824,13 @@ } }, "node_modules/jest-watcher": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-30.1.1.tgz", - "integrity": "sha512-CrAQ73LlaS6KGQQw6NBi71g7qvP7scy+4+2c0jKX6+CWaYg85lZiig5nQQVTsS5a5sffNPL3uxXnaE9d7v9eQg==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-30.0.5.tgz", + "integrity": "sha512-z9slj/0vOwBDBjN3L4z4ZYaA+pG56d6p3kTUhFRYGvXbXMWhXmb/FIxREZCD06DYUwDKKnj2T80+Pb71CQ0KEg==", "dev": true, "license": "MIT", "dependencies": { - "@jest/test-result": "30.1.1", + "@jest/test-result": "30.0.5", "@jest/types": "30.0.5", "@types/node": "*", "ansi-escapes": "^4.3.2", @@ -3844,9 +3844,9 @@ } }, "node_modules/jest-worker": { - "version": "30.1.0", - "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-30.1.0.tgz", - "integrity": "sha512-uvWcSjlwAAgIu133Tt77A05H7RIk3Ho8tZL50bQM2AkvLdluw9NG48lRCl3Dt+MOH719n/0nnb5YxUwcuJiKRA==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-30.0.5.tgz", + "integrity": "sha512-ojRXsWzEP16NdUuBw/4H/zkZdHOa7MMYCk4E430l+8fELeLg/mqmMlRhjL7UNZvQrDmnovWZV4DxX03fZF48fQ==", "dev": true, "license": "MIT", "dependencies": { @@ -4166,9 +4166,9 @@ "optional": true }, "node_modules/napi-postinstall": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/napi-postinstall/-/napi-postinstall-0.3.3.tgz", - "integrity": "sha512-uTp172LLXSxuSYHv/kou+f6KW3SMppU9ivthaVTXian9sOt3XM/zHYHpRZiLgQoxeWfYUnslNWQHF1+G71xcow==", + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/napi-postinstall/-/napi-postinstall-0.3.2.tgz", + "integrity": "sha512-tWVJxJHmBWLy69PvO96TZMZDrzmw5KeiZBz3RHmiM2XZ9grBJ2WgMAFVVg25nqp3ZjTFUs2Ftw1JhscL3Teliw==", "dev": true, "license": "MIT", "bin": { diff --git a/grpc-client/pom.xml b/grpc-client/pom.xml new file mode 100644 index 0000000000..eca96e0662 --- /dev/null +++ b/grpc-client/pom.xml @@ -0,0 +1,136 @@ + + + + 4.0.0 + + + com.arcadedb + arcadedb-parent + 25.8.1-SNAPSHOT + ../pom.xml + + + arcadedb-grpc-client + jar + ArcadeDB gRPC Client + + + 5.15.2 + + + + + + com.arcadedb + arcadedb-engine + ${project.parent.version} + compile + + + + com.arcadedb + arcadedb-network + ${project.parent.version} + compile + + + + com.arcadedb + arcadedb-grpc + ${project.parent.version} + grpc-interface + + + * + * + + + + + + + io.grpc + grpc-netty-shaded + ${grpc.version} + + + io.grpc + grpc-protobuf + ${grpc.version} + + + io.grpc + grpc-stub + ${grpc.version} + + + io.grpc + grpc-services + ${grpc.version} + + + + + com.google.protobuf + protobuf-java + ${protobuf-java.version} + + + com.google.protobuf + protobuf-java-util + ${protobuf-java.version} + + + + + com.google.api.grpc + proto-google-common-protos + ${com.google.api.grpc.version} + + + + + javax.annotation + javax.annotation-api + ${javax.annotation-api.version} + + + + + io.grpc + grpc-testing + ${grpc.version} + test + + + + + + + + + + + + + + diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java new file mode 100644 index 0000000000..b2a2a09edf --- /dev/null +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java @@ -0,0 +1,1793 @@ +package com.arcadedb.remote.grpc; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.UUID; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; + +import com.arcadedb.ContextConfiguration; +import com.arcadedb.database.Database; +import com.arcadedb.database.MutableDocument; +import com.arcadedb.database.RID; +import com.arcadedb.database.Record; +import com.arcadedb.exception.ConcurrentModificationException; +import com.arcadedb.exception.DatabaseOperationException; +import com.arcadedb.exception.DuplicatedKeyException; +import com.arcadedb.exception.NeedRetryException; +import com.arcadedb.exception.RecordNotFoundException; +import com.arcadedb.exception.TimeoutException; +import com.arcadedb.exception.TransactionException; +import com.arcadedb.query.sql.executor.InternalResultSet; +import com.arcadedb.query.sql.executor.Result; +import com.arcadedb.query.sql.executor.ResultInternal; +import com.arcadedb.query.sql.executor.ResultSet; +import com.arcadedb.remote.RemoteDatabase; +import com.arcadedb.remote.RemoteException; +import com.arcadedb.remote.RemoteImmutableDocument; +import com.arcadedb.remote.RemoteImmutableEdge; +import com.arcadedb.remote.RemoteImmutableVertex; +import com.arcadedb.remote.RemoteSchema; +import com.arcadedb.remote.RemoteTransactionExplicitLock; +import com.arcadedb.server.grpc.ArcadeDbServiceGrpc; +import com.arcadedb.server.grpc.BatchAck; +import com.arcadedb.server.grpc.BeginTransactionRequest; +import com.arcadedb.server.grpc.BeginTransactionResponse; +import com.arcadedb.server.grpc.CommitInsert; +import com.arcadedb.server.grpc.CommitTransactionRequest; +import com.arcadedb.server.grpc.CommitTransactionResponse; +import com.arcadedb.server.grpc.CreateRecordRequest; +import com.arcadedb.server.grpc.CreateRecordResponse; +import com.arcadedb.server.grpc.DatabaseCredentials; +import com.arcadedb.server.grpc.DeleteRecordRequest; +import com.arcadedb.server.grpc.DeleteRecordResponse; +import com.arcadedb.server.grpc.DropDatabaseRequest; +import com.arcadedb.server.grpc.DropDatabaseResponse; +import com.arcadedb.server.grpc.ExecuteCommandRequest; +import com.arcadedb.server.grpc.ExecuteCommandResponse; +import com.arcadedb.server.grpc.ExecuteQueryRequest; +import com.arcadedb.server.grpc.ExecuteQueryResponse; +import com.arcadedb.server.grpc.GetRecordRequest; +import com.arcadedb.server.grpc.GetRecordResponse; +import com.arcadedb.server.grpc.InsertChunk; +import com.arcadedb.server.grpc.InsertOptions; +import com.arcadedb.server.grpc.InsertRequest; +import com.arcadedb.server.grpc.InsertResponse; +import com.arcadedb.server.grpc.InsertSummary; +import com.arcadedb.server.grpc.QueryResult; +import com.arcadedb.server.grpc.RollbackTransactionRequest; +import com.arcadedb.server.grpc.RollbackTransactionResponse; +import com.arcadedb.server.grpc.StartInsert; +import com.arcadedb.server.grpc.StreamQueryRequest; +import com.arcadedb.server.grpc.TransactionContext; +import com.arcadedb.server.grpc.TransactionIsolation; +import com.arcadedb.server.grpc.UpdateRecordRequest; +import com.arcadedb.server.grpc.UpdateRecordResponse; +import com.google.protobuf.Struct; +import com.google.protobuf.Value; + +import io.grpc.CallCredentials; +import io.grpc.CompressorRegistry; +import io.grpc.DecompressorRegistry; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.Metadata; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; + +/** + * Remote Database implementation using gRPC protocol instead of HTTP. Extends + * RemoteDatabase to maintain compatibility while overriding network operations. + * It's not thread safe. For multi-thread usage create one instance of + * RemoteGrpcDatabase per thread. + * + * @author Oleg Cohen (oleg.cohen@gmail.com) + */ +public class RemoteGrpcDatabase extends RemoteDatabase { + + private final ManagedChannel channel; + + private final ArcadeDbServiceGrpc.ArcadeDbServiceBlockingStub blockingStub; + private final ArcadeDbServiceGrpc.ArcadeDbServiceStub asyncStub; + + private String transactionId; + + private final RemoteSchema schema; + + private final String userName; + private final String userPassword; + private String databaseName; + private RemoteTransactionExplicitLock explicitLock; + + public RemoteGrpcDatabase(final String server, final int port, final String databaseName, final String userName, final String userPassword) { + this(server, port, databaseName, userName, userPassword, new ContextConfiguration()); + } + + public RemoteGrpcDatabase(final String server, final int port, final String databaseName, final String userName, final String userPassword, + final ContextConfiguration configuration) { + + super(server, 2480, databaseName, userName, userPassword, configuration); + + this.userName = userName; + this.userPassword = userPassword; + + this.databaseName = databaseName; + + // Create gRPC channel + this.channel = createChannel(server, port); + + // Create stubs with authentication + CallCredentials credentials = createCallCredentials(userName, userPassword); + + this.blockingStub = createBlockingStub(channel); + + this.asyncStub = createAsyncStub(channel); + + // Create gRPC-specific schema + this.schema = new RemoteSchema(this); + } + + /** + * Creates the gRPC channel. Override this for custom channel configuration. + */ + protected ManagedChannel createChannel(String server, int port) { + return ManagedChannelBuilder.forAddress(server, port).usePlaintext() // No TLS by default + .build(); + } + + /** + * Creates call credentials for authentication + */ + protected CallCredentials createCallCredentials(String userName, String userPassword) { + return new CallCredentials() { + @Override + public void applyRequestMetadata(RequestInfo requestInfo, Executor appExecutor, MetadataApplier applier) { + Metadata headers = new Metadata(); + headers.put(Metadata.Key.of("username", Metadata.ASCII_STRING_MARSHALLER), userName); + headers.put(Metadata.Key.of("password", Metadata.ASCII_STRING_MARSHALLER), userPassword); + headers.put(Metadata.Key.of("x-arcade-user", Metadata.ASCII_STRING_MARSHALLER), userName); + headers.put(Metadata.Key.of("x-arcade-password", Metadata.ASCII_STRING_MARSHALLER), userPassword); + headers.put(Metadata.Key.of("x-arcade-database", Metadata.ASCII_STRING_MARSHALLER), databaseName); + applier.apply(headers); + } + + @Override + public void thisUsesUnstableApi() { + // Required by the interface + } + }; + } + + protected CallCredentials createCredentials() { + return new CallCredentials() { + @Override + public void applyRequestMetadata(RequestInfo requestInfo, Executor appExecutor, MetadataApplier applier) { + Metadata headers = new Metadata(); + headers.put(Metadata.Key.of("username", Metadata.ASCII_STRING_MARSHALLER), userName); + headers.put(Metadata.Key.of("password", Metadata.ASCII_STRING_MARSHALLER), userPassword); + headers.put(Metadata.Key.of("x-arcade-user", Metadata.ASCII_STRING_MARSHALLER), userName); + headers.put(Metadata.Key.of("x-arcade-password", Metadata.ASCII_STRING_MARSHALLER), userPassword); + headers.put(Metadata.Key.of("x-arcade-database", Metadata.ASCII_STRING_MARSHALLER), databaseName); + applier.apply(headers); + } + + // x-arcade-user: root" -H "x-arcade-password: oY9uU2uJ8nD8iY7t" -H "x-arcade-database: local_shakeiq_curonix_poc-app" + @Override + public void thisUsesUnstableApi() { + // Required by the interface + } + }; + } + + /** + * Override this method to customize blocking stub creation + */ + protected ArcadeDbServiceGrpc.ArcadeDbServiceBlockingStub createBlockingStub(ManagedChannel channel) { + return ArcadeDbServiceGrpc.newBlockingStub(channel).withCallCredentials(createCredentials()); + } + + protected ArcadeDbServiceGrpc.ArcadeDbServiceStub createAsyncStub(ManagedChannel channel) { + return ArcadeDbServiceGrpc.newStub(channel).withCallCredentials(createCredentials()).withCompression("gzip"); + } + + @Override + public String getDatabasePath() { + return "grpc://" + channel.authority() + "/" + getName(); + } + + @Override + public RemoteSchema getSchema() { + return schema; + } + + @Override + public void close() { + if (transactionId != null) { + rollback(); + } + super.close(); + + // Shutdown gRPC channel + channel.shutdown(); + try { + if (!channel.awaitTermination(5, TimeUnit.SECONDS)) { + channel.shutdownNow(); + } + } + catch (InterruptedException e) { + channel.shutdownNow(); + } + } + + @Override + public void drop() { + checkDatabaseIsOpen(); + + DropDatabaseRequest request = DropDatabaseRequest.newBuilder().setDatabaseName(getName()).setCredentials(buildCredentials()).build(); + + try { + DropDatabaseResponse response = blockingStub + .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .dropDatabase(request); + if (!response.getSuccess()) { + throw new DatabaseOperationException("Failed to drop database: " + response.getMessage()); + } + } + catch (StatusRuntimeException e) { + throw new DatabaseOperationException("Error on deleting database", e); + } + + close(); + } + + @Override + public void begin(final Database.TRANSACTION_ISOLATION_LEVEL isolationLevel) { + checkDatabaseIsOpen(); + if (transactionId != null) + throw new TransactionException("Transaction already begun"); + + BeginTransactionRequest request = BeginTransactionRequest.newBuilder().setDatabase(getName()).setCredentials(buildCredentials()) + .setIsolation(mapIsolationLevel(isolationLevel)).build(); + + try { + BeginTransactionResponse response = blockingStub + .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .beginTransaction(request); + transactionId = response.getTransactionId(); + // Store transaction ID in parent class session management + setSessionId(transactionId); + } + catch (StatusRuntimeException e) { + throw new TransactionException("Error on transaction begin", e); + } + } + + @Override + public void commit() { + checkDatabaseIsOpen(); + stats.txCommits.incrementAndGet(); + + if (transactionId == null) + throw new TransactionException("Transaction not begun"); + + CommitTransactionRequest request = CommitTransactionRequest.newBuilder() + .setTransaction(TransactionContext.newBuilder().setTransactionId(transactionId).setDatabase(getName()).build()) + .setCredentials(buildCredentials()).build(); + + try { + CommitTransactionResponse response = blockingStub + .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .commitTransaction(request); + if (!response.getSuccess()) { + throw new TransactionException("Failed to commit transaction: " + response.getMessage()); + } + } + catch (StatusRuntimeException e) { + handleGrpcException(e); + } + finally { + transactionId = null; + setSessionId(null); + } + } + + @Override + public void rollback() { + checkDatabaseIsOpen(); + stats.txRollbacks.incrementAndGet(); + + if (transactionId == null) + throw new TransactionException("Transaction not begun"); + + RollbackTransactionRequest request = RollbackTransactionRequest.newBuilder() + .setTransaction(TransactionContext.newBuilder().setTransactionId(transactionId).setDatabase(getName()).build()) + .setCredentials(buildCredentials()).build(); + + try { + RollbackTransactionResponse response = blockingStub + .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .rollbackTransaction(request); + if (!response.getSuccess()) { + throw new TransactionException("Failed to rollback transaction: " + response.getMessage()); + } + } + catch (StatusRuntimeException e) { + throw new TransactionException("Error on transaction rollback", e); + } + finally { + transactionId = null; + setSessionId(null); + } + } + + @Override + public Record lookupByRID(final RID rid, final boolean loadContent) { + checkDatabaseIsOpen(); + stats.readRecord.incrementAndGet(); + + if (rid == null) + throw new IllegalArgumentException("Record is null"); + + GetRecordRequest request = GetRecordRequest.newBuilder().setDatabase(getName()).setRid(rid.toString()).setCredentials(buildCredentials()) + .build(); + + try { + GetRecordResponse response = blockingStub + .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .getRecord(request); + if (!response.getFound()) { + throw new RecordNotFoundException("Record " + rid + " not found", rid); + } + return grpcRecordToRecord(response.getRecord()); + } + catch (StatusRuntimeException e) { + handleGrpcException(e); + return null; + } + } + + @Override + public void deleteRecord(final Record record) { + checkDatabaseIsOpen(); + stats.deleteRecord.incrementAndGet(); + + if (record.getIdentity() == null) + throw new IllegalArgumentException("Cannot delete a non persistent record"); + + DeleteRecordRequest request = DeleteRecordRequest.newBuilder().setDatabase(getName()).setRid(record.getIdentity().toString()) + .setCredentials(buildCredentials()).build(); + + try { + DeleteRecordResponse response = blockingStub + .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .deleteRecord(request); + if (!response.getSuccess()) { + throw new DatabaseOperationException("Failed to delete record: " + response.getMessage()); + } + } + catch (StatusRuntimeException e) { + handleGrpcException(e); + } + } + + @Override + public Iterator iterateType(final String typeName, final boolean polymorphic) { + String query = "select from `" + typeName + "`"; + if (!polymorphic) + query += " where @type = '" + typeName + "'"; + return streamQuery(query); + } + + @Override + public Iterator iterateBucket(final String bucketName) { + return streamQuery("select from bucket:`" + bucketName + "`"); + } + + @Override + public ResultSet query(final String language, final String query, final Map params) { + checkDatabaseIsOpen(); + stats.queries.incrementAndGet(); + + ExecuteQueryRequest.Builder requestBuilder = ExecuteQueryRequest.newBuilder().setDatabase(getName()).setQuery(query) + .setCredentials(buildCredentials()); + + if (transactionId != null) { + requestBuilder.setTransaction(TransactionContext.newBuilder().setTransactionId(transactionId).setDatabase(getName()).build()); + } + + if (params != null && !params.isEmpty()) { + requestBuilder.putAllParameters(convertParamsToProto(params)); + } + + try { + ExecuteQueryResponse response = blockingStub + .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .executeQuery(requestBuilder.build()); + return createGrpcResultSet(response); + } + catch (StatusRuntimeException e) { + handleGrpcException(e); + return new InternalResultSet(); + } + } + + @Override + public ResultSet query(final String language, final String query, final Object... args) { + checkDatabaseIsOpen(); + stats.queries.incrementAndGet(); + + final Map params = mapArgs(args); + + return query(language, query, params); + } + + @Override + public ResultSet command(final String language, final String command, final Map params) { + checkDatabaseIsOpen(); + stats.commands.incrementAndGet(); + + ExecuteCommandRequest.Builder requestBuilder = ExecuteCommandRequest.newBuilder().setDatabase(getName()).setCommand(command) + .setCredentials(buildCredentials()); + + if (transactionId != null) { + requestBuilder.setTransaction(TransactionContext.newBuilder().setTransactionId(transactionId).setDatabase(getName()).build()); + } + + if (params != null && !params.isEmpty()) { + requestBuilder.putAllParameters(convertParamsToProto(params)); + } + + try { + ExecuteCommandResponse response = blockingStub + .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .executeCommand(requestBuilder.build()); + + // Create result set with command execution info + InternalResultSet resultSet = new InternalResultSet(); + if (response.getAffectedRecords() > 0) { + Map result = new HashMap<>(); + result.put("affected", response.getAffectedRecords()); + result.put("executionTime", response.getExecutionTimeMs()); + resultSet.add(new ResultInternal(result)); + } + return resultSet; + } + catch (StatusRuntimeException e) { + handleGrpcException(e); + return new InternalResultSet(); + } + } + + public com.arcadedb.server.grpc.ExecuteCommandResponse execSql(String db, String sql, Map params, long timeoutMs) { + return executeCommand(db, sql, params, "sql", /*returnRows*/ false, /*maxRows*/ 0, txBeginCommit(), timeoutMs); + } + + public com.arcadedb.server.grpc.ExecuteCommandResponse executeCommand( + String database, + String command, + Map params, + String language, + boolean returnRows, + int maxRows, + com.arcadedb.server.grpc.TransactionContext tx, + long timeoutMs) { + + var reqB = com.arcadedb.server.grpc.ExecuteCommandRequest.newBuilder() + .setDatabase(database) + .setCommand(command) + .putAllParameters(convertParamsToProto(params)) + .setLanguage(langOrDefault(language)) + .setReturnRows(returnRows) + .setMaxRows(maxRows > 0 ? maxRows : 0); + + if (tx != null) reqB.setTransaction(tx); + // credentials: if your stub builds creds implicitly, set here if required + reqB.setCredentials(buildCredentials()); + + return blockingStub + .withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS) + .executeCommand(reqB.build()); + } + + @Override + protected RID saveRecord(final MutableDocument record) { + stats.createRecord.incrementAndGet(); + + RID rid = record.getIdentity(); + if (rid != null) { + // Update existing record + UpdateRecordRequest request = UpdateRecordRequest.newBuilder().setDatabase(getName()).setRid(rid.toString()) + .putAllProperties(convertParamsToProto(record.toMap(false))).setCredentials(buildCredentials()).build(); + + try { + UpdateRecordResponse response = blockingStub + .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .updateRecord(request); + + if (!response.getSuccess()) { + throw new DatabaseOperationException("Failed to update record"); + } + return rid; + } + catch (StatusRuntimeException e) { + handleGrpcException(e); + return null; + } + } + else { + // Create new record + CreateRecordRequest request = CreateRecordRequest.newBuilder().setDatabase(getName()).setType(record.getTypeName()) + .putAllProperties(convertParamsToProto(record.toMap(false))).setCredentials(buildCredentials()).build(); + + try { + CreateRecordResponse response = blockingStub + .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .createRecord(request); + + if (!response.getSuccess()) { + throw new DatabaseOperationException("Failed to create record"); + } + return new RID(this, response.getRecord().getRid()); + } + catch (StatusRuntimeException e) { + handleGrpcException(e); + return null; + } + } + } + +// /** +// * Executes a streaming query that returns results in batches. +// * Useful for processing large result sets without loading everything into memory. +// * +// * @param language the query language (typically "sql") +// * @param query the query string +// * @param batchSize the number of records per batch (default 100 if <= 0) +// * @return Iterator of records that fetches results in batches +// */ +// public Iterator queryStream(final String language, final String query, final int batchSize) { +// checkDatabaseIsOpen(); +// stats.queries.incrementAndGet(); +// +// StreamQueryRequest request = StreamQueryRequest.newBuilder() +// .setDatabase(getName()) +// .setQuery(query) +// .setCredentials(buildCredentials()) +// .setBatchSize(batchSize > 0 ? batchSize : 100) +// .build(); +// +// Iterator responseIterator = blockingStub +// .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) +// .streamQuery(request); +// +// return new Iterator() { +// private Iterator currentBatch = Collections.emptyIterator(); +// +// @Override +// public boolean hasNext() { +// if (currentBatch.hasNext()) { +// return true; +// } +// if (responseIterator.hasNext()) { +// QueryResult result = responseIterator.next(); +// currentBatch = result.getRecordsList().iterator(); +// return currentBatch.hasNext(); +// } +// return false; +// } +// +// @Override +// public Record next() { +// if (!hasNext()) { +// throw new NoSuchElementException(); +// } +// return grpcRecordToRecord(currentBatch.next()); +// } +// }; +// } +// +// /** +// * Executes a streaming query with parameters. +// */ +// public Iterator queryStream(final String language, final String query, +// final Map params, final int batchSize) { +// checkDatabaseIsOpen(); +// stats.queries.incrementAndGet(); +// +// StreamQueryRequest.Builder requestBuilder = StreamQueryRequest.newBuilder() +// .setDatabase(getName()) +// .setQuery(query) +// .setCredentials(buildCredentials()) +// .setBatchSize(batchSize > 0 ? batchSize : 100); +// +// if (params != null && !params.isEmpty()) { +// requestBuilder.putAllParameters(convertParamsToProto(params)); +// } +// +// Iterator responseIterator = blockingStub +// .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) +// .streamQuery(requestBuilder.build()); +// +// // Same iterator implementation as above +// return new Iterator() { +// private Iterator currentBatch = Collections.emptyIterator(); +// +// @Override +// public boolean hasNext() { +// if (currentBatch.hasNext()) { +// return true; +// } +// if (responseIterator.hasNext()) { +// QueryResult result = responseIterator.next(); +// currentBatch = result.getRecordsList().iterator(); +// return currentBatch.hasNext(); +// } +// return false; +// } +// +// @Override +// public Record next() { +// if (!hasNext()) { +// throw new NoSuchElementException(); +// } +// return grpcRecordToRecord(currentBatch.next()); +// } +// }; +// } +// +// /** +// * Convenience method with default batch size +// */ +// public Iterator queryStream(final String language, final String query) { +// return queryStream(language, query, 100); +// } + + /* +StreamQueryRequest req = StreamQueryRequest.newBuilder() + .setDatabase(db) + .setQuery(sql) + .setBatchSize(500) + .setRetrievalMode(StreamQueryRequest.RetrievalMode.PAGED) + .setTransaction(TransactionContext.newBuilder() + .setBegin(true) + .setCommit(true) // or setRollback(true) + .build()) + .build(); + */ + + // Convenience: default batch size stays 100, default mode = CURSOR + public Iterator queryStream(final String language, + final String query) { + return queryStream(language, query, /*batchSize*/100, + StreamQueryRequest.RetrievalMode.CURSOR); + } + + public Iterator queryStream(final String language, + final String query, + final int batchSize) { + return queryStream(language, query, batchSize, + StreamQueryRequest.RetrievalMode.CURSOR); + } + + // NEW: choose retrieval mode + public Iterator queryStream(final String language, + final String query, + final int batchSize, + final StreamQueryRequest.RetrievalMode mode) { + checkDatabaseIsOpen(); + stats.queries.incrementAndGet(); + + StreamQueryRequest request = StreamQueryRequest.newBuilder() + .setDatabase(getName()) + .setQuery(query) + .setCredentials(buildCredentials()) + .setBatchSize(batchSize > 0 ? batchSize : 100) + .setRetrievalMode(mode) // <--- NEW + .build(); + + Iterator responseIterator = blockingStub + .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .streamQuery(request); + + return new Iterator() { + private Iterator currentBatch = Collections.emptyIterator(); + + @Override + public boolean hasNext() { + if (currentBatch.hasNext()) return true; + + while (responseIterator.hasNext()) { + QueryResult result = responseIterator.next(); + + // Defensive: skip any empty batches the server might send + if (result.getRecordsCount() == 0) { + if (result.getIsLastBatch()) return false; + continue; + } + + currentBatch = result.getRecordsList().iterator(); + return true; + } + return false; + } + + @Override + public Record next() { + if (!hasNext()) throw new NoSuchElementException(); + return grpcRecordToRecord(currentBatch.next()); + } + }; + } + + // PARAMETERIZED variant with retrieval mode + public Iterator queryStream(final String language, + final String query, + final Map params, + final int batchSize, + final StreamQueryRequest.RetrievalMode mode) { + checkDatabaseIsOpen(); + stats.queries.incrementAndGet(); + + StreamQueryRequest.Builder b = StreamQueryRequest.newBuilder() + .setDatabase(getName()) + .setQuery(query) + .setCredentials(buildCredentials()) + .setBatchSize(batchSize > 0 ? batchSize : 100) + .setRetrievalMode(mode); // <--- NEW + + if (params != null && !params.isEmpty()) { + b.putAllParameters(convertParamsToProto(params)); + } + + Iterator responseIterator = blockingStub + .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .streamQuery(b.build()); + + return new Iterator() { + private Iterator currentBatch = Collections.emptyIterator(); + + @Override + public boolean hasNext() { + if (currentBatch.hasNext()) return true; + + while (responseIterator.hasNext()) { + QueryResult result = responseIterator.next(); + + if (result.getRecordsCount() == 0) { + if (result.getIsLastBatch()) return false; + continue; + } + + currentBatch = result.getRecordsList().iterator(); + return true; + } + return false; + } + + @Override + public Record next() { + if (!hasNext()) throw new NoSuchElementException(); + return grpcRecordToRecord(currentBatch.next()); + } + }; + } + + // Keep the old signature working (defaults to CURSOR) + public Iterator queryStream(final String language, + final String query, + final Map params, + final int batchSize) { + return queryStream(language, query, params, batchSize, + StreamQueryRequest.RetrievalMode.CURSOR); + } + + public static final class QueryBatch { + private final List records; + private final int totalInBatch; + private final long runningTotal; + private final boolean lastBatch; + + public QueryBatch(List list, int totalInBatch, long runningTotal, boolean lastBatch) { + this.records = list; + this.totalInBatch = totalInBatch; + this.runningTotal = runningTotal; + this.lastBatch = lastBatch; + } + public List records() { return records; } + public int totalInBatch() { return totalInBatch; } + public long runningTotal() { return runningTotal; } + public boolean isLastBatch() { return lastBatch; } + } + + public Iterator queryStreamBatches(final String language, + final String query, + final Map params, + final int batchSize, + final StreamQueryRequest.RetrievalMode mode) { + checkDatabaseIsOpen(); + stats.queries.incrementAndGet(); + + StreamQueryRequest.Builder b = StreamQueryRequest.newBuilder() + .setDatabase(getName()) + .setQuery(query) + .setCredentials(buildCredentials()) + .setBatchSize(batchSize > 0 ? batchSize : 100) + .setRetrievalMode(mode); + + if (params != null && !params.isEmpty()) { + b.putAllParameters(convertParamsToProto(params)); + } + + Iterator responseIterator = blockingStub + .withWaitForReady() // optional, improves robustness + .withDeadlineAfter(/* e.g. */ 10, TimeUnit.MINUTES) + .streamQuery(b.build()); + + return new Iterator() { + private QueryBatch nextBatch = null; + private boolean drained = false; + + @Override + public boolean hasNext() { + if (nextBatch != null) return true; + if (drained) return false; + + while (responseIterator.hasNext()) { + QueryResult qr = responseIterator.next(); + int n = qr.getTotalRecordsInBatch(); // server-populated + // Guard: some servers could omit this; fallback to list size. + if (n == 0) n = qr.getRecordsCount(); + + if (qr.getRecordsCount() == 0 && !qr.getIsLastBatch()) { + // skip empty non-terminal batch + continue; + } + + List converted = new ArrayList<>(qr.getRecordsCount()); + for (com.arcadedb.server.grpc.Record gr : qr.getRecordsList()) { + converted.add(grpcRecordToRecord(gr)); + } + + nextBatch = new QueryBatch( + converted, + n, + qr.getRunningTotalEmitted(), + qr.getIsLastBatch() + ); + + if (qr.getIsLastBatch()) { + drained = true; // no more after this (even if server sent empty terminal) + } + return true; + } + + drained = true; + return false; + } + + @Override + public QueryBatch next() { + if (!hasNext()) throw new NoSuchElementException(); + QueryBatch out = nextBatch; + nextBatch = null; + return out; + } + }; + } + + public Iterator queryStream( + String database, + String sql, + Map params, + int batchSize, + com.arcadedb.server.grpc.StreamQueryRequest.RetrievalMode mode, + com.arcadedb.server.grpc.TransactionContext tx, + long timeoutMs) { + + var reqB = com.arcadedb.server.grpc.StreamQueryRequest.newBuilder() + .setDatabase(database) + .setQuery(sql) + .putAllParameters(convertParamsToProto(params)) + .setCredentials(buildCredentials()) + .setBatchSize(batchSize > 0 ? batchSize : 100) + .setRetrievalMode(mode); + + if (tx != null) reqB.setTransaction(tx); + + Iterator it = blockingStub + .withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS) + .streamQuery(reqB.build()); + + return new Iterator<>() { + private Iterator curr = java.util.Collections.emptyIterator(); + public boolean hasNext() { + if (curr.hasNext()) return true; + if (it.hasNext()) { curr = it.next().getRecordsList().iterator(); return hasNext(); } + return false; + } + public com.arcadedb.server.grpc.Record next() { + if (!hasNext()) throw new java.util.NoSuchElementException(); + return curr.next(); + } + }; + } + + public Iterator queryStreamBatches( + String passLabel, + String sql, + Map params, + int batchSize, + com.arcadedb.server.grpc.StreamQueryRequest.RetrievalMode mode, + com.arcadedb.server.grpc.TransactionContext tx, + long timeoutMs) { + + var reqB = com.arcadedb.server.grpc.StreamQueryRequest.newBuilder() + .setDatabase(getName()) + .setQuery(sql) + .putAllParameters(convertParamsToProto(params)) + .setCredentials(buildCredentials()) + .setBatchSize(batchSize > 0 ? batchSize : 100) + .setRetrievalMode(mode); + + if (tx != null) reqB.setTransaction(tx); + + Iterator respIter = blockingStub + .withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS) + .streamQuery(reqB.build()); + + return new Iterator<>() { + public boolean hasNext() { return respIter.hasNext(); } + public QueryBatch next() { + + var qr = respIter.next(); + + + List converted = new ArrayList<>(qr.getRecordsCount()); + for (com.arcadedb.server.grpc.Record gr : qr.getRecordsList()) { + converted.add(grpcRecordToRecord(gr)); + } + + return new QueryBatch( + converted, + qr.getTotalRecordsInBatch(), // int totalInBatch + qr.getRunningTotalEmitted(), // long runningTotal + qr.getIsLastBatch() // boolean lastBatch + ); + } + }; + } + + // public QueryBatch(List records, int totalInBatch, long runningTotal, boolean lastBatch) { + + public String createVertex(String cls, Map props, long timeoutMs) { + var req = com.arcadedb.server.grpc.CreateRecordRequest.newBuilder() + .setDatabase(getName()) + .setType(cls) + .putAllProperties(convertParamsToProto(props)) + .setCredentials(buildCredentials()) + .build(); + var res = blockingStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS).createRecord(req); + // adjust to your response field name + return res.getIdentity(); // e.g., "#12:0" + } + + public boolean updateRecord(String rid, Map props, long timeoutMs) { + var req = com.arcadedb.server.grpc.UpdateRecordRequest.newBuilder() + .setDatabase(getName()) + .setRid(rid) + .putAllProperties(convertParamsToProto(props)) + .setCredentials(buildCredentials()) + // .setTransaction(txBeginCommit()) // if you add tx to this RPC + .build(); + var res = blockingStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS).updateRecord(req); + return res.getSuccess(); + } + + public boolean deleteRecord(String rid, long timeoutMs) { + var req = com.arcadedb.server.grpc.DeleteRecordRequest.newBuilder() + .setDatabase(getName()) + .setRid(rid) + .setCredentials(buildCredentials()) + .build(); + var res = blockingStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS).deleteRecord(req); + return res.getDeleted(); + } + + @Override + public RemoteTransactionExplicitLock acquireLock() { + // Need gRPC-specific implementation + if (explicitLock == null) + explicitLock = new RemoteGrpcTransactionExplicitLock(this); + + return explicitLock; + } + + @Override + public long countBucket(final String bucketName) { + checkDatabaseIsOpen(); + stats.countBucket.incrementAndGet(); + ResultSet result = query("sql", "select count(*) as count from bucket:" + bucketName); + if (result.hasNext()) { + Number count = result.next().getProperty("count"); + return count != null ? count.longValue() : 0; + } + return 0; + } + + @Override + public long countType(final String typeName, final boolean polymorphic) { + checkDatabaseIsOpen(); + stats.countType.incrementAndGet(); + final String appendix = polymorphic ? "" : " where @type = '" + typeName + "'"; + ResultSet result = query("sql", "select count(*) as count from " + typeName + appendix); + if (result.hasNext()) { + Number count = result.next().getProperty("count"); + return count != null ? count.longValue() : 0; + } + return 0; + } + + @Override + public Record lookupByRID(final RID rid) { + return lookupByRID(rid, true); + } + + @Override + public boolean existsRecord(RID rid) { + stats.existsRecord.incrementAndGet(); + if (rid == null) + throw new IllegalArgumentException("Record is null"); + try { + return lookupByRID(rid, false) != null; + } catch (RecordNotFoundException e) { + return false; + } + } + + + public com.arcadedb.server.grpc.InsertSummary insertBulk( + final com.arcadedb.server.grpc.InsertOptions options, + final java.util.List protoRows, + final long timeoutMs) { + + // Ensure options carry DB + credentials as the server expects + com.arcadedb.server.grpc.InsertOptions.Builder ob = options.toBuilder(); + + if (options.getDatabase() == null || options.getDatabase().isEmpty()) { + ob.setDatabase(getName()); // your wrapper's DB name + } + + boolean missingCreds = + !options.hasCredentials() + || options.getCredentials().getUsername().isEmpty(); + if (missingCreds) { + ob.setCredentials(buildCredentials()); // your existing helper used by queries + } + + InsertOptions newOptions = ob.build(); + + com.arcadedb.server.grpc.BulkInsertRequest req = + com.arcadedb.server.grpc.BulkInsertRequest.newBuilder() + .setOptions(newOptions) + .addAllRows(protoRows) + .build(); + + return blockingStub + .withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS) + .bulkInsert(req); + } + + // Convenience overload that accepts domain rows (convert first) + public com.arcadedb.server.grpc.InsertSummary insertBulkAsListOfMaps( + final com.arcadedb.server.grpc.InsertOptions options, + final java.util.List> rows, + final long timeoutMs) { + + java.util.List protoRows = + rows.stream() + .map(this::toProtoRecordFromMap) // your converter + .collect(java.util.stream.Collectors.toList()); + + return insertBulk(options, protoRows, timeoutMs); + } + + public com.arcadedb.server.grpc.InsertSummary ingestStream( + final com.arcadedb.server.grpc.InsertOptions options, // note: InsertStream RPC does not carry options + final java.util.List protoRows, + final int chunkSize, + final long timeoutMs) throws InterruptedException { + + final java.util.concurrent.CountDownLatch done = new java.util.concurrent.CountDownLatch(1); + final java.util.concurrent.atomic.AtomicReference summaryRef = + new java.util.concurrent.atomic.AtomicReference<>(); + + // Ensure options carry DB + credentials as the server expects + com.arcadedb.server.grpc.InsertOptions.Builder ob = options.toBuilder(); + + if (options.getDatabase() == null || options.getDatabase().isEmpty()) { + ob.setDatabase(getName()); // your wrapper's DB name + } + + boolean missingCreds = + !options.hasCredentials() + || options.getCredentials().getUsername().isEmpty(); + if (missingCreds) { + ob.setCredentials(buildCredentials()); // your existing helper used by queries + } + + io.grpc.stub.StreamObserver resp = + new io.grpc.stub.StreamObserver<>() { + @Override public void onNext(com.arcadedb.server.grpc.InsertSummary value) { summaryRef.set(value); } + @Override public void onError(Throwable t) { done.countDown(); } + @Override public void onCompleted() { done.countDown(); } + }; + + // Use the write service async stub, per-call deadline + var stub = + this.asyncStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS); + + final io.grpc.stub.StreamObserver req = stub.insertStream(resp); + + // Stream chunks + final String sessionId = "sess-" + System.nanoTime(); + long seq = 1; + for (int i = 0; i < protoRows.size(); i += chunkSize) { + final int end = Math.min(i + chunkSize, protoRows.size()); + final com.arcadedb.server.grpc.InsertChunk chunk = + com.arcadedb.server.grpc.InsertChunk.newBuilder() + .setSessionId(sessionId) + .setChunkSeq(seq++) + .addAllRows(protoRows.subList(i, end)) + .build(); + req.onNext(chunk); + } + req.onCompleted(); + + done.await(timeoutMs + 5_000, java.util.concurrent.TimeUnit.MILLISECONDS); + + final com.arcadedb.server.grpc.InsertSummary s = summaryRef.get(); + return (s != null) + ? s + : com.arcadedb.server.grpc.InsertSummary.newBuilder().setReceived(protoRows.size()).build(); + } + + // Convenience overload + public com.arcadedb.server.grpc.InsertSummary ingestStreamAsListOfMaps( + final com.arcadedb.server.grpc.InsertOptions options, + final java.util.List> rows, + final int chunkSize, + final long timeoutMs) throws InterruptedException { + + java.util.List protoRows = + rows.stream().map(this::toProtoRecordFromMap).collect(java.util.stream.Collectors.toList()); + return ingestStream(options, protoRows, chunkSize, timeoutMs); + } + + /** Push records using InsertBidirectional with per-batch ACKs. */ + public InsertSummary ingestBidi(List rows, InsertOptions opts, int chunkSize, int maxInflight) throws InterruptedException { + + final String sessionId = UUID.randomUUID().toString(); + + // Ensure options carry DB + credentials as the server expects + com.arcadedb.server.grpc.InsertOptions.Builder ob = opts.toBuilder(); + + if (opts.getDatabase() == null || opts.getDatabase().isEmpty()) { + ob.setDatabase(getName()); // your wrapper's DB name + } + + boolean missingCreds = + !opts.hasCredentials() + || opts.getCredentials().getUsername().isEmpty(); + if (missingCreds) { + ob.setCredentials(buildCredentials()); // your existing helper used by queries + } + + final java.util.concurrent.CountDownLatch done = new java.util.concurrent.CountDownLatch(1); + final java.util.concurrent.atomic.AtomicLong seq = new java.util.concurrent.atomic.AtomicLong(1); + final java.util.concurrent.atomic.AtomicInteger sent = new java.util.concurrent.atomic.AtomicInteger(0); + final java.util.concurrent.atomic.AtomicInteger acked = new java.util.concurrent.atomic.AtomicInteger(0); + final java.util.List acks = java.util.Collections.synchronizedList(new java.util.ArrayList<>()); + final java.util.concurrent.atomic.AtomicReference committed = new java.util.concurrent.atomic.AtomicReference<>(); + + io.grpc.stub.ClientResponseObserver obs = new io.grpc.stub.ClientResponseObserver<>() { + + io.grpc.stub.ClientCallStreamObserver req; + + @Override + public void beforeStart(io.grpc.stub.ClientCallStreamObserver r) { + this.req = r; + r.disableAutoInboundFlowControl(); + r.setOnReadyHandler(this::drain); + } + + private int cursor = 0; + + private void drain() { + if (!req.isReady()) + return; + if (seq.get() == 1) { // send START once + req.onNext(InsertRequest.newBuilder().setStart(StartInsert.newBuilder().setOptions(ob.build())).build()); + } + while (req.isReady() && (sent.get() - acked.get()) < maxInflight) { + + if (cursor >= rows.size()) + break; + + int end = Math.min(cursor + chunkSize, rows.size()); + + List protoRows = rows.stream() + .map(RemoteGrpcDatabase.this::toProtoRecordFromDbRecord) + .toList(); + + InsertChunk chunk = InsertChunk.newBuilder() + .setSessionId(sessionId) + .setChunkSeq(seq.getAndIncrement()) + .addAllRows(protoRows) + .build(); + + req.onNext(InsertRequest.newBuilder().setChunk(chunk).build()); + cursor = end; + sent.incrementAndGet(); + } + req.request(1); + if (cursor >= rows.size() && acked.get() >= sent.get()) { + req.onNext(InsertRequest.newBuilder().setCommit(CommitInsert.newBuilder().setSessionId(sessionId)).build()); + req.onCompleted(); + } + } + + @Override + public void onNext(InsertResponse v) { + switch (v.getMsgCase()) { + case STARTED -> { + /* ok */ } + case BATCH_ACK -> { + acks.add(v.getBatchAck()); + acked.incrementAndGet(); + drain(); + } + case COMMITTED -> committed.set(v.getCommitted().getSummary()); + case MSG_NOT_SET -> { + } + } + req.request(1); + } + + @Override + public void onError(Throwable t) { + done.countDown(); + } + + @Override + public void onCompleted() { + done.countDown(); + } + }; + + asyncStub.insertBidirectional(obs); + + done.await(5, java.util.concurrent.TimeUnit.MINUTES); + + InsertSummary s = committed.get(); + if (s != null) + return s; + + long ins = acks.stream().mapToLong(BatchAck::getInserted).sum(); + long upd = acks.stream().mapToLong(BatchAck::getUpdated).sum(); + long ign = acks.stream().mapToLong(BatchAck::getIgnored).sum(); + long fail = acks.stream().mapToLong(BatchAck::getFailed).sum(); + + return InsertSummary.newBuilder().setReceived(rows.size()).setInserted(ins).setUpdated(upd).setIgnored(ign).setFailed(fail).build(); + } + + /** + * Bidirectional streaming ingest with per-batch ACKs. + * + * @param options InsertOptions (database, target class, conflict mode, tx mode, etc.) + * @param rowsProto list of PROTO Records (com.arcadedb.server.grpc.Record) + * @param chunkSize rows per chunk + * @param maxInflight max unacked chunks + * @param timeoutMs overall timeout for the ingest (client-side) + */ + public com.arcadedb.server.grpc.InsertSummary ingestBidi( + final com.arcadedb.server.grpc.InsertOptions options, + final java.util.List> rows, + final int chunkSize, + final int maxInflight, + final long timeoutMs) throws InterruptedException { + + // Ensure options carry DB + credentials as the server expects + com.arcadedb.server.grpc.InsertOptions.Builder ob = options.toBuilder(); + + if (options.getDatabase() == null || options.getDatabase().isEmpty()) { + ob.setDatabase(getName()); // your wrapper's DB name + } + + boolean missingCreds = + !options.hasCredentials() + || options.getCredentials().getUsername().isEmpty(); + if (missingCreds) { + ob.setCredentials(buildCredentials()); // your existing helper used by queries + } + + // Pre-convert to PROTO records (outside the observer!) + final java.util.List protoRows = + rows.stream() + .map(RemoteGrpcDatabase.this::toProtoRecordFromMap) + .collect(java.util.stream.Collectors.toList()); + + final String sessionId = "sess-" + System.nanoTime(); + final java.util.concurrent.CountDownLatch done = new java.util.concurrent.CountDownLatch(1); + final java.util.concurrent.atomic.AtomicLong seq = new java.util.concurrent.atomic.AtomicLong(1); + final java.util.concurrent.atomic.AtomicInteger cursor = new java.util.concurrent.atomic.AtomicInteger(0); + final java.util.concurrent.atomic.AtomicInteger sent = new java.util.concurrent.atomic.AtomicInteger(0); + final java.util.concurrent.atomic.AtomicInteger acked = new java.util.concurrent.atomic.AtomicInteger(0); + final java.util.List acks = + java.util.Collections.synchronizedList(new java.util.ArrayList<>()); + final java.util.concurrent.atomic.AtomicReference committed = + new java.util.concurrent.atomic.AtomicReference<>(); + + io.grpc.stub.ClientResponseObserver observer = + new io.grpc.stub.ClientResponseObserver<>() { + io.grpc.stub.ClientCallStreamObserver req; + + @Override + public void beforeStart(io.grpc.stub.ClientCallStreamObserver r) { + this.req = r; + r.disableAutoInboundFlowControl(); + r.setOnReadyHandler(this::drain); + } + + private void drain() { + if (!req.isReady()) return; + + if (seq.get() == 1) { + req.onNext(com.arcadedb.server.grpc.InsertRequest.newBuilder() + .setStart(com.arcadedb.server.grpc.StartInsert.newBuilder().setOptions(ob.build())) + .build()); + } + + while (req.isReady() && (sent.get() - acked.get()) < maxInflight) { + int start = cursor.get(); + if (start >= protoRows.size()) break; + int end = Math.min(start + chunkSize, protoRows.size()); + + com.arcadedb.server.grpc.InsertChunk chunk = + com.arcadedb.server.grpc.InsertChunk.newBuilder() + .setSessionId(sessionId) + .setChunkSeq(seq.getAndIncrement()) + .addAllRows(protoRows.subList(start, end)) + .build(); + + req.onNext(com.arcadedb.server.grpc.InsertRequest.newBuilder().setChunk(chunk).build()); + cursor.set(end); + sent.incrementAndGet(); + } + + req.request(1); + + if (cursor.get() >= protoRows.size() && acked.get() >= sent.get()) { + req.onNext(com.arcadedb.server.grpc.InsertRequest.newBuilder() + .setCommit(com.arcadedb.server.grpc.CommitInsert.newBuilder().setSessionId(sessionId)) + .build()); + req.onCompleted(); + } + } + + @Override + public void onNext(com.arcadedb.server.grpc.InsertResponse v) { + switch (v.getMsgCase()) { + case STARTED -> { /* ok */ } + case BATCH_ACK -> { + acks.add(v.getBatchAck()); + acked.incrementAndGet(); + drain(); + } + case COMMITTED -> committed.set(v.getCommitted().getSummary()); + case MSG_NOT_SET -> {} + } + req.request(1); + } + + @Override public void onError(Throwable t) { done.countDown(); } + @Override public void onCompleted() { done.countDown(); } + }; + + var stub = this.asyncStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS); + stub.insertBidirectional(observer); + + done.await(timeoutMs + 5_000, java.util.concurrent.TimeUnit.MILLISECONDS); + + var summary = committed.get(); + if (summary != null) return summary; + + long ins = acks.stream().mapToLong(com.arcadedb.server.grpc.BatchAck::getInserted).sum(); + long upd = acks.stream().mapToLong(com.arcadedb.server.grpc.BatchAck::getUpdated).sum(); + long ign = acks.stream().mapToLong(com.arcadedb.server.grpc.BatchAck::getIgnored).sum(); + long fail = acks.stream().mapToLong(com.arcadedb.server.grpc.BatchAck::getFailed).sum(); + + return com.arcadedb.server.grpc.InsertSummary.newBuilder() + .setReceived(protoRows.size()) + .setInserted(ins) + .setUpdated(upd) + .setIgnored(ign) + .setFailed(fail) + .build(); + } + + // Map -> PROTO Value + private com.arcadedb.server.grpc.Record toProtoRecord(Map row) { + com.arcadedb.server.grpc.Record.Builder b = com.arcadedb.server.grpc.Record.newBuilder(); + // Example if your proto uses a map fields/properties + row.forEach((k, v) -> b.putProperties(k, toProtoValue(v))); + return b.build(); + } + + // Map -> PROTO Record + private com.arcadedb.server.grpc.Record toProtoRecordFromMap(java.util.Map row) { + com.arcadedb.server.grpc.Record.Builder b = com.arcadedb.server.grpc.Record.newBuilder(); + row.forEach((k, v) -> b.putProperties(k, toProtoValue(v))); + return b.build(); + } + + // Domain Record (storage) -> PROTO Record + private com.arcadedb.server.grpc.Record toProtoRecordFromDbRecord( + com.arcadedb.database.Record rec) { + + // Promote to a read-only Document; null if this record isn't a document + com.arcadedb.database.Document doc = rec.asDocument(); + if (doc == null) { + return com.arcadedb.server.grpc.Record.getDefaultInstance(); + } + + com.arcadedb.server.grpc.Record.Builder b = com.arcadedb.server.grpc.Record.newBuilder(); + + // Iterate document properties + for (String name : doc.getPropertyNames()) { + // Depending on your ArcadeDB version, use ONE of these: + Object v = doc.get(name); // <-- common in ArcadeDB + // Object v = doc.getProperty(name); // <-- use this if doc.get(...) doesn't exist in your API + b.putProperties(name, toProtoValue(v)); // or putFields(...) if your proto uses "fields" + } + return b.build(); + } + + + // google.protobuf.Value -> Java (mirror of your toProtoValue) + private static Object fromProtoValue(com.google.protobuf.Value v) { + if (v == null) return null; + switch (v.getKindCase()) { + case NULL_VALUE: return null; + case BOOL_VALUE: return v.getBoolValue(); + case NUMBER_VALUE: return v.getNumberValue(); + case STRING_VALUE: return v.getStringValue(); + case LIST_VALUE: { + java.util.List out = new java.util.ArrayList<>(); + for (com.google.protobuf.Value item : v.getListValue().getValuesList()) { + out.add(fromProtoValue(item)); + } + return out; + } + case STRUCT_VALUE: { + java.util.Map m = new java.util.LinkedHashMap<>(); + v.getStructValue().getFieldsMap().forEach((k,vv) -> m.put(k, fromProtoValue(vv))); + return m; // server-side schema can later coerce EMBEDDED if needed + } + case KIND_NOT_SET: + default: + return null; + } + } + + // Query Result -> PROTO Record + private com.arcadedb.server.grpc.Record toProtoRecordFromResult(Result res) { + com.arcadedb.server.grpc.Record.Builder b = com.arcadedb.server.grpc.Record.newBuilder(); + for (String name : res.getPropertyNames()) { + Object v = res.getProperty(name); + b.putProperties(name, toProtoValue(v)); // or putFields(...) + } + return b.build(); + } + + // Convert Java object -> protobuf Value (extend as needed) + private com.google.protobuf.Value toProtoValue(Object v) { + com.google.protobuf.Value.Builder vb = com.google.protobuf.Value.newBuilder(); + if (v == null) return vb.setNullValue(com.google.protobuf.NullValue.NULL_VALUE).build(); + if (v instanceof String s) return vb.setStringValue(s).build(); + if (v instanceof Boolean b) return vb.setBoolValue(b).build(); + if (v instanceof Number n) return vb.setNumberValue(n.doubleValue()).build(); + if (v instanceof java.time.Instant t) + return vb.setStringValue(t.toString()).build(); // or a Struct for Timestamp if you prefer + if (v instanceof java.util.Map m) { + var sb = com.google.protobuf.Struct.newBuilder(); + for (var e : m.entrySet()) + sb.putFields(String.valueOf(e.getKey()), toProtoValue(e.getValue())); + return vb.setStructValue(sb.build()).build(); + } + if (v instanceof java.util.List list) { + var lb = com.google.protobuf.ListValue.newBuilder(); + for (Object e : list) lb.addValues(toProtoValue(e)); + return vb.setListValue(lb.build()).build(); + } + return vb.setStringValue(String.valueOf(v)).build(); + } + + // Override HTTP-specific methods to prevent usage +// @Override +// HttpRequest.Builder createRequestBuilder(String httpMethod, String url) { +// throw new UnsupportedOperationException("HTTP operations not supported in gRPC implementation"); +// } +// +// @Override +// protected ResultSet createResultSet(JSONObject response) { +// throw new UnsupportedOperationException("JSON operations not supported in gRPC implementation"); +// } +// +// @Override +// protected Result json2Result(JSONObject result) { +// throw new UnsupportedOperationException("JSON operations not supported in gRPC implementation"); +// } +// +// @Override +// protected Record json2Record(JSONObject result) { +// throw new UnsupportedOperationException("JSON operations not supported in gRPC implementation"); +// } + + // gRPC-specific helper methods + + private DatabaseCredentials buildCredentials() { + return DatabaseCredentials.newBuilder().setUsername(getUserName()).setPassword(getUserPassword()).build(); + } + + private TransactionIsolation mapIsolationLevel(Database.TRANSACTION_ISOLATION_LEVEL level) { + switch (level) { +// case READ_UNCOMMITTED: +// return TransactionIsolation.READ_UNCOMMITTED; + case READ_COMMITTED: + return TransactionIsolation.READ_COMMITTED; + case REPEATABLE_READ: + return TransactionIsolation.REPEATABLE_READ; +// case SERIALIZABLE: +// return TransactionIsolation.SERIALIZABLE; + default: + return TransactionIsolation.READ_COMMITTED; + } + } + + // ---- TX helpers ------------------------------------------------------------- + private static com.arcadedb.server.grpc.TransactionContext txBeginCommit() { + return com.arcadedb.server.grpc.TransactionContext.newBuilder() + .setBegin(true).setCommit(true).build(); + } + private static com.arcadedb.server.grpc.TransactionContext txNone() { + return com.arcadedb.server.grpc.TransactionContext.getDefaultInstance(); + } + + // Optional: language defaulting (server defaults to "sql" too) + private static String langOrDefault(String language) { + return (language == null || language.isEmpty()) ? "sql" : language; + } + + private Iterator streamQuery(final String query) { + StreamQueryRequest request = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(query).setCredentials(buildCredentials()) + .setBatchSize(100).build(); + + Iterator responseIterator = blockingStub + .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .streamQuery(request); + + return new Iterator() { + private Iterator currentBatch = Collections.emptyIterator(); + + @Override + public boolean hasNext() { + if (currentBatch.hasNext()) { + return true; + } + if (responseIterator.hasNext()) { + QueryResult result = responseIterator.next(); + currentBatch = result.getRecordsList().iterator(); + return currentBatch.hasNext(); + } + return false; + } + + @Override + public Record next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return grpcRecordToRecord(currentBatch.next()); + } + }; + } + + private ResultSet createGrpcResultSet(ExecuteQueryResponse response) { + InternalResultSet resultSet = new InternalResultSet(); + for (QueryResult queryResult : response.getResultsList()) { + for (com.arcadedb.server.grpc.Record record : queryResult.getRecordsList()) { + resultSet.add(grpcRecordToResult(record)); + } + } + return resultSet; + } + + private Result grpcRecordToResult(com.arcadedb.server.grpc.Record grpcRecord) { + Record record = grpcRecordToRecord(grpcRecord); + if (record == null) { + Map properties = new HashMap<>(); + grpcRecord.getPropertiesMap().forEach((k, v) -> properties.put(k, valueToObject(v))); + return new ResultInternal(properties); + } + return new ResultInternal(record); + } + + private Record grpcRecordToRecord(com.arcadedb.server.grpc.Record grpcRecord) { + Map map = new HashMap<>(); + + // Convert properties + grpcRecord.getPropertiesMap().forEach((k, v) -> map.put(k, valueToObject(v))); + + // Add metadata + map.put("@rid", grpcRecord.getRid()); + map.put("@type", grpcRecord.getType()); + map.put("@cat", mapRecordType(grpcRecord)); + + String cat = (String) map.get("@cat"); + switch (cat) { + case "d": + return new RemoteImmutableDocument(this, map); + case "v": + return new RemoteImmutableVertex(this, map); + case "e": + return new RemoteImmutableEdge(this, map); + default: + return null; + } + } + + private String mapRecordType(com.arcadedb.server.grpc.Record grpcRecord) { + // Determine record category from type name + String typeName = grpcRecord.getType(); + + // Check schema to determine actual type + try { + if (getSchema().existsType(typeName)) { + Object type = getSchema().getType(typeName); + if (type instanceof com.arcadedb.schema.VertexType) { + return "v"; + } + else if (type instanceof com.arcadedb.schema.EdgeType) { + return "e"; + } + } + } + catch (Exception e) { + // Fall back to name-based detection + } + + // Fall back to name-based detection + if (typeName.contains("Vertex") || typeName.startsWith("V_")) { + return "v"; + } + else if (typeName.contains("Edge") || typeName.startsWith("E_")) { + return "e"; + } + else { + return "d"; + } + } + + private Map convertParamsToProto(Map params) { + Map protoParams = new HashMap<>(); + for (Map.Entry entry : params.entrySet()) { + protoParams.put(entry.getKey(), objectToValue(entry.getValue())); + } + return protoParams; + } + + private Value objectToValue(Object obj) { + Value.Builder builder = Value.newBuilder(); + if (obj == null) { + builder.setNullValue(com.google.protobuf.NullValue.NULL_VALUE); + } + else if (obj instanceof Boolean) { + builder.setBoolValue((Boolean) obj); + } + else if (obj instanceof Number) { + builder.setNumberValue(((Number) obj).doubleValue()); + } + else if (obj instanceof String) { + builder.setStringValue((String) obj); + } + else if (obj instanceof RID) { + builder.setStringValue(obj.toString()); + } + else if (obj instanceof Map) { + Struct.Builder structBuilder = Struct.newBuilder(); + ((Map) obj).forEach((k, v) -> structBuilder.putFields(k.toString(), objectToValue(v))); + builder.setStructValue(structBuilder.build()); + } + else if (obj instanceof Collection) { + com.google.protobuf.ListValue.Builder listBuilder = com.google.protobuf.ListValue.newBuilder(); + ((Collection) obj).forEach(item -> listBuilder.addValues(objectToValue(item))); + builder.setListValue(listBuilder.build()); + } + else { + builder.setStringValue(obj.toString()); + } + return builder.build(); + } + + private Object valueToObject(Value value) { + switch (value.getKindCase()) { + case NULL_VALUE: + return null; + case NUMBER_VALUE: + return value.getNumberValue(); + case STRING_VALUE: + return value.getStringValue(); + case BOOL_VALUE: + return value.getBoolValue(); + case STRUCT_VALUE: + Map map = new HashMap<>(); + value.getStructValue().getFieldsMap().forEach((k, v) -> map.put(k, valueToObject(v))); + return map; + case LIST_VALUE: + List list = new ArrayList<>(); + value.getListValue().getValuesList().forEach(v -> list.add(valueToObject(v))); + return list; + default: + return null; + } + } + + private void handleGrpcException(StatusRuntimeException e) { + Status status = e.getStatus(); + switch (status.getCode()) { + case NOT_FOUND: + throw new RecordNotFoundException(status.getDescription(), null); + case ALREADY_EXISTS: + throw new DuplicatedKeyException("", "", null); + case ABORTED: + throw new ConcurrentModificationException(status.getDescription()); + case DEADLINE_EXCEEDED: + throw new TimeoutException(status.getDescription()); + case PERMISSION_DENIED: + throw new SecurityException(status.getDescription()); + case UNAVAILABLE: + throw new NeedRetryException(status.getDescription()); + default: + throw new RemoteException("gRPC error: " + status.getDescription(), e); + } + } + + /** + * Extended version with compression support + */ + public static class RemoteGrpcDatabaseWithCompression extends RemoteGrpcDatabase { + + public RemoteGrpcDatabaseWithCompression(String server, int port, String databaseName, String userName, String userPassword, + ContextConfiguration configuration) { + super(server, port, databaseName, userName, userPassword, configuration); + } + + @Override + protected ManagedChannel createChannel(String server, int port) { + + return ManagedChannelBuilder.forAddress(server, port).usePlaintext() // No TLS/SSL + .compressorRegistry(CompressorRegistry.getDefaultInstance()) + .decompressorRegistry(DecompressorRegistry.getDefaultInstance()) + .maxInboundMessageSize(100 * 1024 * 1024) // 100MB max message size + .keepAliveTime(30, TimeUnit.SECONDS) // Keep-alive configuration + .keepAliveTimeout(10, TimeUnit.SECONDS).keepAliveWithoutCalls(true).build(); + } + } +} \ No newline at end of file diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseWithCompression.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseWithCompression.java new file mode 100644 index 0000000000..3965b6f205 --- /dev/null +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseWithCompression.java @@ -0,0 +1,51 @@ +package com.arcadedb.remote.grpc; + +import java.util.concurrent.TimeUnit; + +import com.arcadedb.ContextConfiguration; +import com.arcadedb.server.grpc.ArcadeDbServiceGrpc; + +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; + +/** + * Extended RemoteGrpcDatabase with compression support + */ +public class RemoteGrpcDatabaseWithCompression extends RemoteGrpcDatabase { + + public RemoteGrpcDatabaseWithCompression(String server, int port, + String databaseName, + String userName, + String userPassword, + ContextConfiguration configuration) { + super(server, port, databaseName, userName, userPassword, configuration); + } + + @Override + protected ManagedChannel createChannel(String server, int port) { + + return ManagedChannelBuilder.forAddress(server, port) + .usePlaintext() // No TLS/SSL + .maxInboundMessageSize(100 * 1024 * 1024) // 100MB max message size + .keepAliveTime(30, TimeUnit.SECONDS) // Keep-alive configuration + .keepAliveTimeout(10, TimeUnit.SECONDS) + .keepAliveWithoutCalls(true) + .build(); + } + + @Override + protected ArcadeDbServiceGrpc.ArcadeDbServiceBlockingStub createBlockingStub(ManagedChannel channel) { + return ArcadeDbServiceGrpc.newBlockingStub(channel) + .withCompression("gzip") // Enable GZIP compression + .withCallCredentials(createCredentials()) + .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS); + } + + @Override + protected ArcadeDbServiceGrpc.ArcadeDbServiceStub createAsyncStub(ManagedChannel channel) { + return ArcadeDbServiceGrpc.newStub(channel) + .withCompression("gzip") // Enable GZIP compression + .withCallCredentials(createCredentials()) + .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS); + } +} diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcTransactionExplicitLock.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcTransactionExplicitLock.java new file mode 100644 index 0000000000..ecea2ac893 --- /dev/null +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcTransactionExplicitLock.java @@ -0,0 +1,74 @@ +/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com) + * SPDX-License-Identifier: Apache-2.0 + */ +/* Generated By:JJTree: Do not edit this line. OAlterClassStatement.java Version 4.3 */ +/* JavaCCOptions:MULTI=true,NODE_USES_PARSER=false,VISITOR=true,TRACK_TOKENS=true,NODE_PREFIX=O,NODE_EXTENDS=,NODE_FACTORY=,SUPPORT_USERTYPE_VISIBILITY_PUBLIC=true */ +package com.arcadedb.remote.grpc; + +import com.arcadedb.database.TransactionExplicitLock; +import com.arcadedb.remote.RemoteTransactionExplicitLock; + +import java.util.*; + +/** + * Explicit lock on a transaction to lock buckets and types in pessimistic way. + * This avoids the retry mechanism of default implicit locking. + * + * @author Luca Garulli (l.garulli@arcadedata.com) + */ +public class RemoteGrpcTransactionExplicitLock extends RemoteTransactionExplicitLock implements TransactionExplicitLock { + private final RemoteGrpcDatabase database; + private final Set lockedTypeNames = new HashSet<>(); + private final Set lockedBucketNames = new HashSet<>(); + + public RemoteGrpcTransactionExplicitLock(final RemoteGrpcDatabase database) { + super(database); + this.database = database; + } + + @Override + public RemoteGrpcTransactionExplicitLock bucket(final String bucketName) { + lockedBucketNames.add(bucketName); + return this; + } + + @Override + public RemoteGrpcTransactionExplicitLock type(final String typeName) { + lockedTypeNames.add(typeName); + return this; + } + + @Override + public void lock() { + final StringBuilder command = new StringBuilder(); + command.append("LOCK"); + if (!lockedTypeNames.isEmpty()) { + command.append(" TYPE "); + command.append(String.join(", ", lockedTypeNames)); + } + if (!lockedBucketNames.isEmpty()) { + command.append(" BUCKET "); + command.append(String.join(", ", lockedBucketNames)); + } + + lockedBucketNames.clear(); + lockedTypeNames.clear(); + + database.command("sql", command.toString()); + } +} diff --git a/grpc-client/src/test/resources/arcadedb-log.properties b/grpc-client/src/test/resources/arcadedb-log.properties new file mode 100644 index 0000000000..536f217457 --- /dev/null +++ b/grpc-client/src/test/resources/arcadedb-log.properties @@ -0,0 +1,31 @@ +# +# Copyright 2021-present Arcade Data Ltd +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Specify the handlers to create in the root logger +# (all loggers are children of the root logger) +# The following creates two handlers +handlers=java.util.logging.ConsoleHandler +# Set the default logging level for the root logger +.level=WARNING +com.arcadedb.level=WARNING +# Set the default logging level for new ConsoleHandler instances +java.util.logging.ConsoleHandler.level=WARNING +# Set the default formatter for new ConsoleHandler instances +java.util.logging.ConsoleHandler.formatter=com.arcadedb.utility.AnsiLogFormatter diff --git a/grpc/gRPC-Readme.txt b/grpc/gRPC-Readme.txt new file mode 100644 index 0000000000..bc24c8bc07 --- /dev/null +++ b/grpc/gRPC-Readme.txt @@ -0,0 +1,422 @@ +# ArcadeDB gRPC Server Configuration Examples + +# ========================================== +# Basic Configuration (arcadedb-server.properties) +# ========================================== + +# Enable gRPC server (default: true) +grpc.enabled=true + +# Server mode: standard, xds, or both (default: standard) +grpc.mode=standard + +# Standard gRPC server port (default: 50051) +grpc.port=50051 + +# Bind host (default: 0.0.0.0) +grpc.host=0.0.0.0 + +# Max message size in MB (default: 100) +grpc.maxMessageSize=100 + +# Enable gRPC reflection for debugging with grpcurl (default: true) +grpc.reflection.enabled=true + +# Enable health checking endpoint (default: true) +grpc.health.enabled=true + +# ========================================== +# XDS Configuration (for Service Mesh) +# ========================================== + +# Enable XDS mode for Envoy/Istio integration +grpc.mode=xds +# or run both standard and XDS +# grpc.mode=both + +# XDS server port (default: 50052) +grpc.xds.port=50052 + +# ========================================== +# TLS Configuration +# ========================================== + +# Enable TLS (default: false) +grpc.tls.enabled=true + +# Path to TLS certificate +grpc.tls.cert=/path/to/server.crt + +# Path to TLS private key +grpc.tls.key=/path/to/server.key + + +arcadedb.grpc.compression.enabled=true # Enable compression support +arcadedb.grpc.compression.force=false # Force compression on all responses +arcadedb.grpc.compression.type=gzip # Compression algorithm + + + +# ========================================== +# Docker Compose Example +# ========================================== +# docker-compose.yml: +# +# version: '3.8' +# services: +# arcadedb: +# image: arcadedb/arcadedb:latest +# ports: +# - "2480:2480" # HTTP +# - "2424:2424" # Binary +# - "50051:50051" # gRPC +# environment: +# - ARCADEDB_OPTS_MEMORY=-Xms2G -Xmx2G +# - arcadedb.grpc.enabled=true +# - arcadedb.grpc.port=50051 +# volumes: +# - ./databases:/var/arcadedb/databases + +# ========================================== +# Kubernetes ConfigMap Example +# ========================================== +# apiVersion: v1 +# kind: ConfigMap +# metadata: +# name: arcadedb-config +# data: +# arcadedb.properties: | +# grpc.enabled=true +# grpc.mode=xds +# grpc.xds.port=50052 +# grpc.health.enabled=true +# grpc.reflection.enabled=false + +# ========================================== +# Istio Service Mesh Example +# ========================================== +# apiVersion: v1 +# kind: Service +# metadata: +# name: arcadedb-grpc +# spec: +# ports: +# - name: grpc-xds +# port: 50052 +# protocol: TCP +# - name: grpc +# port: 50051 +# protocol: TCP +# selector: +# app: arcadedb + + + + + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.6.0 + + + jar-with-dependencies + + + + com.arcadedb.server.grpc.GrpcStandaloneLauncher + + + + + + make-grpc-standalone + package + + single + + + arcadedb-grpc-server-${project.version} + false + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.5.1 + + + package + + shade + + + false + + + com.arcadedb.server.grpc.GrpcStandaloneLauncher + + + + + + META-INF/services/io.grpc.ServerProvider + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + true + grpc-standalone + + + + + + + +properties# Basic +grpc.enabled=true +grpc.port=50051 +grpc.host=0.0.0.0 + +# Advanced +grpc.mode=standard|xds|both +grpc.maxMessageSize=100 +grpc.reflection.enabled=true +grpc.health.enabled=true + +# Security +grpc.tls.enabled=true +grpc.tls.cert=/path/to/cert +grpc.tls.key=/path/to/key + + + + +version: '3.8' +services: + arcadedb: + image: arcadedb/arcadedb:latest + ports: + - "50051:50051" # gRPC + environment: + - arcadedb.grpc.enabled=true + +apiVersion: v1 +kind: Service +metadata: + name: arcadedb-grpc +spec: + ports: + - name: grpc-xds + port: 50052 + protocol: TCP + + + + +# Development +mvn exec:java -Dexec.mainClass="com.arcadedb.server.grpc.GrpcStandaloneLauncher" + +# Production +java -jar arcadedb-grpc-server.jar --config /etc/arcadedb/grpc.properties + + + + +# Using grpcurl +grpcurl -plaintext localhost:50051 list +grpcurl -plaintext localhost:50051 describe com.arcadedb.grpc.ArcadeDbService + +# Health check +grpcurl -plaintext localhost:50051 grpc.health.v1.Health/Check + + + +# Basic Authentication +grpcurl -H "x-arcade-user: admin" \ + -H "x-arcade-password: admin" \ + -H "x-arcade-database: mydb" \ + -plaintext localhost:50051 \ + com.arcadedb.grpc.ArcadeDbService/Ping + +# Bearer Token (for future implementation) +grpcurl -H "authorization: Bearer " \ + -plaintext localhost:50051 \ + com.arcadedb.grpc.ArcadeDbService/Ping + + +server.registerPlugin(new GrpcServerPlugin()); + +grpc.enabled=true +grpc.port=50051 +grpc.mode=standard +grpc.reflection.enabled=true +grpc.health.enabled=true + + + +# Check health +grpcurl -plaintext localhost:50051 grpc.health.v1.Health/Check + +# List services +grpcurl -plaintext localhost:50051 list + +# Test with authentication +grpcurl -H "x-arcade-user: admin" \ + -H "x-arcade-password: admin" \ + -H "x-arcade-database: mydb" \ + -plaintext localhost:50051 \ + com.arcadedb.grpc.ArcadeDbService/Ping + + + + +arcadedb.grpc.enabled=true +arcadedb.grpc.port=50051 +arcadedb.grpc.host=0.0.0.0 +arcadedb.grpc.mode=standard +arcadedb.grpc.reflection.enabled=true +arcadedb.grpc.health.enabled=true +arcadedb.grpc.maxMessageSize=100 + + + +java -Darcadedb.grpc.enabled=true \ + -Darcadedb.grpc.port=50051 \ + -Darcadedb.grpc.mode=standard \ + -jar arcadedb-server.jar + +# Check if gRPC is running +grpcurl -plaintext localhost:50051 list + +# Check health +grpcurl -plaintext localhost:50051 grpc.health.v1.Health/Check + + + + + +# Complete build with Docker images +mvn clean install -DskipTests && \ +cd package && \ +mvn package -Pdocker + +# Multi-platform build (slower but supports ARM and x64) +mvn package -Pdocker,multiplatform + +# Build and push to registry +mvn deploy -Pdocker -Ddocker.username=youruser -Ddocker.password=yourpass + +# Skip tests and build quickly +mvn package -Pdocker -DskipTests -Dmaven.test.skip=true + + + + +# Run the Docker image with gRPC enabled +docker run -d \ + --name arcadedb-test \ + -p 2480:2480 \ + -p 50051:50051 \ + -e arcadedb.grpc.enabled=true \ + arcadedata/arcadedb:latest + + +# Run the Docker image with gRPC enabled +docker run -d \ + --name arcadedb-test \ + -p 2480:2480 \ + -p 50051:50051 \ + -e JAVA_OPTS="-Darcadedb.server.rootPassword=oY9uU2uJ8nD8iY7t -Darcadedb.server.name=ArcadeDB1 -Darcadedb.dumpConfigAtStartup=true -Darcadedb.server.mode=development -Darcadedb.server.plugins=GRPC:com.arcadedb.server.grpc.GrpcServerPlugin -Darcadedb.grpc.enabled=true -Darcadedb.grpc.mode=standard -Darcadedb.grpc.port=50051" \ + -e ARCADEDB_OPTS_MEMORY="-Xms4g -Xmx8g -XX:InitialRAMPercentage=50.0 -XX:MaxRAMPercentage=75.0" \ + arcadedata/arcadedb:25.8.1-SNAPSHOT + + + +# Test gRPC connection +grpcurl -plaintext localhost:50051 list + +# Check health +grpcurl -plaintext localhost:50051 grpc.health.v1.Health/Check + +# Stop and remove container +docker stop arcadedb-test +docker rm arcadedb-test + + + + + + + +version: '3.8' + +services: + arcadedb: + image: arcadedata/arcadedb:25.8.1-SNAPSHOT + container_name: arcadedb-grpc + ports: + - "2480:2480" # HTTP API + - "2424:2424" # Binary protocol + - "50051:50051" # gRPC + - "50052:50052" # gRPC XDS (if enabled) + environment: + - ARCADEDB_OPTS_MEMORY=-Xms2G -Xmx2G + - arcadedb.grpc.enabled=true + - arcadedb.grpc.port=50051 + - arcadedb.grpc.mode=standard + - arcadedb.grpc.reflection.enabled=true + - arcadedb.grpc.health.enabled=true + volumes: + - ./databases:/var/arcadedb/databases + - ./config:/var/arcadedb/config + networks: + - arcadedb-network + + # Optional: gRPC UI for testing + grpcui: + image: fullstorydev/grpcui:latest + container_name: grpcui + ports: + - "8080:8080" + command: + - -plaintext + - -port + - "8080" + - arcadedb:50051 + depends_on: + - arcadedb + networks: + - arcadedb-network + +networks: + arcadedb-network: + driver: bridge + + + + +Docker: + + + +grpcui -plaintext -bind 0.0.0.0 -port 8080 localhost:50051 + +mvn clean package -Pdocker \ No newline at end of file diff --git a/grpc/gRPC-Support-Case.md b/grpc/gRPC-Support-Case.md new file mode 100644 index 0000000000..1d0971d938 --- /dev/null +++ b/grpc/gRPC-Support-Case.md @@ -0,0 +1,191 @@ +# Make gRPC a First-Class API in ArcadeDB + +## TL;DR + +- **Unary ops (local, HTTP/2):** gRPC ≈ JSON/HTTP/2 on localhost—expected because ArcadeDB already moved to HTTP/2 (Undertow), removing most HTTP/1.1 overhead. +- **Streaming ops:** gRPC is a step-change: + - **Time-to-first-row:** 0–3 ms (vs ~175–227 ms with batched JSON). + - **Total time (7,746 rows):** 50–74 ms (vs 175–227 ms). + - That’s roughly a **3.1×–3.6× throughput gain** plus near-instant first byte. + +**Bottom line:** gRPC streaming enables real-time, low-latency graph I/O and unlocks new product capabilities, while unary parity on localhost is normal given HTTP/2 parity. + +## What I Measured (Local, Same Machine & Query) + +| Scenario | Transport | Passes | Time to First Response | Time to All | Rows | +|----------|-----------|--------|-----------------------|-------------|------| +| Large single query (batch JSON vs streamed gRPC) | JSON/HTTP (HTTP/2) | 4 | ~175–227 ms (first byte bundled with full payload) | 175–227 ms | 7,746 | +| | gRPC streaming (batchSize 100/500/800) | 3 | 0–3 ms | 50–74 ms | 7,746 | +| Many smaller queries (unary) | JSON/HTTP | 1 | — | 544 ms | — | +| | gRPC (unary) | 1 | — | 694 ms | — | + +**Why unary parity/slight regressions locally?** With HTTP/2 (Undertow), the big wins over HTTP/1.1 (handshakes, head-of-line blocking, connection churn) are already gone. On localhost, JSON parse costs are negligible; gRPC’s binary/framing advantages tend to pay off with network latency, concurrency, or streaming. + +## Why Leadership Should Care (Business Outcomes) + +- **Differentiate with “real-time graph.”** “Milliseconds to first result” demos incredibly well. Streaming queries feel instant—a clear edge vs REST-only graph stores. +- **Lower TCO under load.** Streaming reduces server memory pressure (no giant JSON payloads), tail latencies, and egress bytes—fewer cores, less RAM, less bandwidth for the same work. +- **Mobile & edge advantage.** On lossy/WAN links, HTTP/2 + gRPC streams tolerate jitter and deliver partial results promptly—snappier UX for apps and agents. +- **Enterprise readiness.** gRPC offers typed SDKs (Java, Go, Python, Node, Rust, Swift), mTLS, and OpenTelemetry—accelerating evals, POCs, and platform integrations. +- **New revenue surfaces.** Package Real-time Subscriptions, Live Traversal Streams, and Bulk Ingest via Client-Streaming as enterprise add-ons. + +## Why Engineering Should Care (Technical Benefits) + +1. **Server/Client Streaming** + - Time-to-first-row in ms, not 100s of ms. + - Backpressure built-in (HTTP/2 flow control); avoids buffering entire result sets. + - Enables live queries / CDC / tailing (e.g., “stream new edges matching X”). + +2. **Resource Efficiency** + - Smaller on-wire frames (binary protobuf) vs verbose JSON. + - Lower GC churn (no giant JSON strings/DOMs); steadier memory profile. + +3. **Concurrency at Scale** + - Single multiplexed channel per client; avoids socket storms. + - No app-layer head-of-line blocking; better tail latencies under load. + +4. **Typed Contracts & Evolvability** + - Proto schemas → generated clients, fewer integration bugs. + - Back/forward compatibility via field numbering and optionality. + +5. **Observability & Control** + - Standard interceptors for auth, quotas, retries, deadlines, metrics, tracing. + - First-class OpenTelemetry spans per RPC for SLOs and latency budgets. + +6. **Security & Networking** + - mTLS/ALPN by default; tight per-RPC auth (JWT, service tokens). + - Works seamlessly with Envoy/Istio and gRPC-Web for browsers. + +7. **Mesh-Native Traffic Policy via gRPC xDS (Kubernetes/Istio)** + - With Istio (or any xDS control plane), gRPC can consume xDS (CDS/EDS/LDS/RDS) to get dynamic, centrally managed traffic policy—no app redeploys. + - Smart LB & resilience: locality-aware load-balancing, fast failover, retries/hedging/timeouts/circuit-breaking/outlier detection by config. + - Progressive delivery: weighted canaries, blue-green, traffic mirroring, and fault injection for chaos tests—uniformly applied to gRPC traffic. + - Security & observability: mesh-wide mTLS (SPIFFE), consistent authz, and unified metrics/traces (Envoy/Otel). + +## Product Capabilities gRPC Unlocks + +- **Live Results / Progressive UI**: Stream traversals and long paths as they’re discovered (great for GraphRAG, visual explorers, dashboards). +- **Subscription Queries / CDC**: Subscribe to patterns (“new Person→Facility edges in Region=…”) and receive server-push updates in real time. +- **Bulk Write Pipelines**: Client-streaming for batch insert/upsert; server replies with a summary (counts, errors) at stream end. +- **Chunked Binary Blocks**: Stream rows in row-block or columnar frames for faster client deserialization and vectorization. +- **Back-office & Intra-cluster RPC**: Use the same gRPC surface for internal services (analytics, ingestion, connectors) to remove REST glue code. + +## Kubernetes + Istio + gRPC xDS (Advanced Benefit) + +When ArcadeDB runs in Kubernetes with Istio (or another xDS control plane), you can apply centralized, dynamic traffic policy to gRPC without code changes. + +### Two deployment patterns +1. **Sidecar-terminated gRPC (most common)**: App uses standard gRPC; Envoy sidecars enforce policy, mTLS, and routing—no client changes. +2. **Client-side xDS (sidecar-optional)**: Enable the gRPC xDS client to pull policy directly from the mesh’s xDS server (via a bootstrap file/env). Useful for sidecarless or off-mesh clients that still need mesh policy. + +### Example Istio policy (weighted canary + retry tuning for gRPC) + +``` +apiVersion: networking.istio.io/v1beta1 +kind: DestinationRule +metadata: + name: arcadedb +spec: + host: arcadedb.svc.cluster.local + trafficPolicy: + outlierDetection: + consecutive5xx: 5 + interval: 5s + baseEjectionTime: 30s + subsets: + - name: v1 + labels: { version: v1 } + - name: v2 + labels: { version: v2 } +--- +apiVersion: networking.istio.io/v1beta1 +kind: VirtualService +metadata: + name: arcadedb +spec: + hosts: [arcadedb.svc.cluster.local] + http: + - retries: + attempts: 3 + perTryTimeout: 200ms + retryOn: > + cancelled,connect-failure,refused-stream,resource-exhausted,unavailable + route: + - destination: { host: arcadedb.svc.cluster.local, subset: v1 } + weight: 90 + - destination: { host: arcadedb.svc.cluster.local, subset: v2 } + weight: 10 +``` + +**Notes:** +- Apply retries only to idempotent operations. +- Tune timeouts for long-running traversals; consider hedging for tail-latency control. +- Enable locality-weighted LB for multi-zone clusters. +- For browser apps, expose gRPC via gRPC-Web through the Istio ingress gateway. + +## Suggested MVP (Fast Path) + +### Services +- `QueryService.ExecuteStream(Query, Params) -> stream` + - Row blocks carry N rows + schema, flow-controlled. +- `IngestService.StreamWrites(stream) -> Summary` +- `SubscriptionService.Subscribe(Query) -> stream` +- `Health` (gRPC health checking), `Auth` (JWT/mTLS). + +### Wire Format +- Protobuf frames by default; allow JSON payloads only for interop, not default. + +### Packaging +- Ship `.proto`; publish official clients for Java & Node first. +- Provide gRPC-Web via Envoy for browser apps. + +### Config +- Toggle gRPC alongside REST; shared auth; TLS on by default. + +## Benchmark Plan (Beyond Localhost) + +- **Axes**: LAN vs WAN, concurrency (1, 16, 128 clients), payloads (1K, 10K, 100K rows), TTFB vs total time, p95/p99. +- **Controls**: warm-ups, stabilized JIT, pinned CPU governor, fixed GC (G1/ZGC), channel reuse, compression on/off. +- **Server**: Undertow HTTP/2 vs Netty gRPC; identical query engine/result materialization; measure server CPU/RAM and egress bytes. + +**Expectation**: Streaming gRPC yields lower p95/p99, smaller egress, and better CPU-per-result at medium/high concurrency; unary parity remains close to HTTP/2 JSON. + +## Risks & Mitigations + +- **Browser support**: Use gRPC-Web via Envoy; keep REST for simple use cases. +- **Schema evolution**: Lock proto CI checks; reserve tags; add compatibility tests. +- **Operational complexity**: Ship Helm charts + sample Envoy config; provide OpenTelemetry defaults. +- **Client fragmentation**: Provide first-party clients and examples; keep REST for long tail. + +## Call to Action + +1. Green-light a gRPC Streaming MVP (services above). +2. Publish proto + Java/Node SDKs with examples (streaming query, subscription, bulk ingest). +3. Add docs & a “Real-Time Streaming API” page featuring the metrics below (show 0–3 ms TTFB demo and ~3× throughput). +4. Run external benchmarks and publish guidance on when to choose streaming gRPC vs REST. + +## Appendix: Raw Local Test Results + +``` +testDBPerformance(): Case 1 [Pass 1] - JSON/HTTP: Query Execution Time = 227 ms; # of results = 7746 rows +testDBPerformance(): Case 1 [Pass 2] - JSON/HTTP: Query Execution Time = 179 ms; # of results = 7746 rows +testDBPerformance(): Case 1 [Pass 3] - JSON/HTTP: Query Execution Time = 177 ms; # of results = 7746 rows +testDBPerformance(): Case 1 [Pass 4] - JSON/HTTP: Query Execution Time = 175 ms; # of results = 7746 rows +testDBPerformance(): Case 1 [Pass 1] - gRPC. Query Execution Time = 327 ms; # of results = 7746 rows +testDBPerformance(): Case 1 [Pass 2] - gRPC. Query Execution Time = 202 ms; # of results = 7746 rows +testDBPerformance(): Case 1 [Pass 3] - gRPC. Query Execution Time = 184 ms; # of results = 7746 rows +testDBPerformance(): Case 1 [Pass 4] - gRPC. Query Execution Time = 185 ms; # of results = 7746 rows + +testGRPCLargeSingleQueryWithStreaming(): [Pass 1; batchSize = 100] - gRPC. To 1st response => Query Execution Time = 3 ms +testGRPCLargeSingleQueryWithStreaming(): [Pass 1] - gRPC. To all responses => Query Execution Time = 74 ms; Total Records = 7746 +testGRPCLargeSingleQueryWithStreaming(): [Pass 2; batchSize = 500] - gRPC. To 1st response => Query Execution Time = 0 ms +testGRPCLargeSingleQueryWithStreaming(): [Pass 2] - gRPC. To all responses => Query Execution Time = 52 ms; Total Records = 7746 +testGRPCLargeSingleQueryWithStreaming(): [Pass 3; batchSize = 800] - gRPC. To 1st response => Query Execution Time = 0 ms +testGRPCLargeSingleQueryWithStreaming(): [Pass 3] - gRPC. To all responses => Query Execution Time = 50 ms; Total Records = 7746 + +testHTTPManySmallerQueries(): Entry ... +testDBPerformance(): Case 2 - JSON/HTTP: Query Execution Time = 544 ms + +testGRPCManySmallerQueries(): Entry ... +testDBPerformance(): Case 2 - gRPC: Query Execution Time = 694 ms +``` \ No newline at end of file diff --git a/grpc/gRPC-vs-HTTP-PerformanceTest.txt b/grpc/gRPC-vs-HTTP-PerformanceTest.txt new file mode 100644 index 0000000000..65ad7e0f12 --- /dev/null +++ b/grpc/gRPC-vs-HTTP-PerformanceTest.txt @@ -0,0 +1,179 @@ +QUERY: +====== + + +TRAVERSE out(), in() FROM (SELECT FROM SalesShakeInstance WHERE @rid=:rid) +WHILE $depth <= 4 +STRATEGY BREADTH_FIRST + + +MATCH {class: SalesShakeInstance, as: s, where: (@rid=:rid)} + .out(){as:v1} + .out(){as:v2} +RETURN s, collectDistinct(v1) as lvl1, collectDistinct(v2) as lvl2 + + + +SELECT + $vertices AS vertices, + $edges AS edges +LET + $root = (SELECT FROM SalesShakeInstance WHERE @rid = :rid), + $walk = (TRAVERSE out(), in(), outE(), inE() + FROM $root + WHILE $depth <= 4), -- adjust depth as needed + $vertices = (SELECT FROM $walk WHERE @type = 'd' AND class() IN ['SalesInsight','SalesObjectiveInstance','SalesShakeInstance','SalesRep','SalesShakeAssist','Hospital','Physician']), + $edges = (SELECT FROM $walk WHERE @type = 'e'); + + + +RID = #1137:5 + + +SELECT + $vertices AS vertices, + $edges AS edges +LET + $root = (SELECT FROM SalesShakeInstance WHERE @rid = #1137:5), + $walk = (TRAVERSE out(), in(), outE(), inE() + FROM $root + WHILE $depth <= 4), -- adjust depth as needed + $vertices = (SELECT FROM $walk WHERE @type = 'd' AND class() IN ['SalesInsight','SalesObjectiveInstance','SalesShakeInstance','SalesRep','SalesShakeAssist','Hospital','Physician']), + $edges = (SELECT FROM $walk WHERE @type = 'e'); + + + +The following is about 7700 records + +TRAVERSE out(), in() FROM (SELECT FROM SalesShakeInstance) WHILE $depth <= 7 STRATEGY BREADTH_FIRST + +root/oY9uU2uJ8nD8iY7t + +HTTP: +===== + + +POST + +http://localhost:2480/api/v1/query/local_shakeiq_curonix_poc-app + + +{ + "language": "sql", + "command": "TRAVERSE out(), in() FROM (SELECT FROM SalesShakeInstance) WHILE $depth <= 7 STRATEGY BREADTH_FIRST", + "parameters": {}, + "limit": 10000, + "returnExpanded": true +} + + "command": "SELECT COUNT(*) FROM (TRAVERSE out(), in() FROM (SELECT FROM SalesShakeInstance) WHILE $depth <= 7 STRATEGY BREADTH_FIRST)", + + + +gtime -f "\nElapsed time: %E (%e seconds)" curl -sS --compressed -o /dev/null -w "HTTP elapsed: %{time_total}s size: %{size_download}\n" \ + -u root:oY9uU2uJ8nD8iY7t \ + -H 'Content-Type: application/json' \ + -X POST \ + 'http://localhost:2480/api/v1/command/local_shakeiq_curonix_poc-app' \ + -d '{"language":"sql","command":"TRAVERSE out(), in() FROM (SELECT FROM SalesShakeInstance) WHILE $depth <= 7 STRATEGY BREADTH_FIRST","parameters":{},"limit":10000,"returnExpanded":true}' + + +HTTP elapsed: 0.299217s size: 6,750,599 + + +curl \ + -u root:oY9uU2uJ8nD8iY7t \ + -H 'Content-Type: application/json' \ + -X POST \ + 'http://localhost:2480/api/v1/command/local_shakeiq_curonix_poc-app' \ + -d '{"language":"sql","command":"TRAVERSE out(), in() FROM (SELECT FROM SalesShakeInstance) WHILE $depth <= 7 STRATEGY BREADTH_FIRST","parameters":{},"limit":10000,"returnExpanded":true}' + + +GRPC: +===== + + +{ + "database": "local_shakeiq_curonix_poc-app", + "query": "TRAVERSE out(), in() FROM (SELECT FROM SalesShakeInstance) WHILE $depth <= 7 STRATEGY BREADTH_FIRST", + "parameters": { }, + "credentials": { "username": "root", "password": "oY9uU2uJ8nD8iY7t" }, + "limit": 10000, + "timeoutMs": 30000 +} + +x-arcade-user: root +x-arcade-password: oY9uU2uJ8nD8iY7t +x-arcade-database: local_shakeiq_curonix_poc-app + +grpcurl -plaintext -max-msg-sz 33554432 \ + -H "grpc-accept-encoding: gzip" \ + -H "x-arcade-user: root" \ + -H "x-arcade-password: oY9uU2uJ8nD8iY7t" \ + -H "x-arcade-database: local_shakeiq_curonix_poc-app" \ + -d '{ + "database":"local_shakeiq_curonix_poc-app", + "query":"TRAVERSE out(), in() FROM (SELECT FROM SalesShakeInstance) WHILE $depth <= 7 STRATEGY BREADTH_FIRST", + "parameters":{}, + "credentials":{"username":"root","password":"oY9uU2uJ8nD8iY7t"}, + "limit":10000, + "timeout_ms":30000 + }' \ + localhost:50051 com.arcadedb.grpc.ArcadeDbService/ExecuteQuery + + + + +grpcurl -plaintext -max-msg-sz 33554432 \ + -H "grpc-accept-encoding: gzip" \ + -H "x-arcade-user: root" \ + -H "x-arcade-password: oY9uU2uJ8nD8iY7t" \ + -H "x-arcade-database: local_shakeiq_curonix_poc-app" \ + -d '{"database":"local_shakeiq_curonix_poc-app","query":"TRAVERSE out(), in() FROM (SELECT FROM SalesShakeInstance) WHILE $depth <= 7 STRATEGY BREADTH_FIRST","parameters":{},"credentials":{"username":"root","password":"oY9uU2uJ8nD8iY7t"},"limit":10000,"timeout_ms":30000}' \ + localhost:50051 com.arcadedb.grpc.ArcadeDbService/ExecuteQuery + + + +date +%s%3N; grpcurl -plaintext -max-msg-sz 33554432 \ + -H "grpc-accept-encoding: gzip" \ + -H "x-arcade-user: root" \ + -H "x-arcade-password: oY9uU2uJ8nD8iY7t" \ + -H "x-arcade-database: local_shakeiq_curonix_poc-app" \ + -d '{"database":"local_shakeiq_curonix_poc-app","query":"TRAVERSE out(), in() FROM (SELECT FROM SalesShakeInstance) WHILE $depth <= 7 STRATEGY BREADTH_FIRST","parameters":{},"credentials":{"username":"root","password":"oY9uU2uJ8nD8iY7t"},"limit":10000,"timeout_ms":30000}' \ + localhost:50051 com.arcadedb.grpc.ArcadeDbService/ExecuteQuery \ + >/dev/null; date +%s%3N + + +start=$(date +%s%3N) +grpcurl -plaintext -max-msg-sz 33554432 \ + -H "grpc-accept-encoding: gzip" \ + -H "x-arcade-user: root" \ + -H "x-arcade-password: oY9uU2uJ8nD8iY7t" \ + -H "x-arcade-database: local_shakeiq_curonix_poc-app" \ + -d '{"database":"local_shakeiq_curonix_poc-app","query":"TRAVERSE out(), in() FROM (SELECT FROM SalesShakeInstance) WHILE $depth <= 7 STRATEGY BREADTH_FIRST","parameters":{},"credentials":{"username":"root","password":"oY9uU2uJ8nD8iY7t"},"limit":10000,"timeout_ms":30000}' \ + localhost:50051 com.arcadedb.grpc.ArcadeDbService/ExecuteQuery \ + >/dev/null +end=$(date +%s%3N) +echo "Elapsed time: $((end - start)) ms" + + +cd /Users/ocohen/git/arcadedb/server/src/main/proto + +gtime -f "\nElapsed time: %E (%e seconds)" grpcurl -proto arcadedb-server.proto -plaintext -max-msg-sz 33554432 -H "grpc-accept-encoding: gzip" -H "x-arcade-user: root" -H "x-arcade-password: oY9uU2uJ8nD8iY7t" -H "x-arcade-database: local_shakeiq_curonix_poc-app" -d '{"database":"local_shakeiq_curonix_poc-app","query":"TRAVERSE out(), in() FROM (SELECT FROM SalesShakeInstance) WHILE $depth <= 7 STRATEGY BREADTH_FIRST","parameters":{},"credentials":{"username":"root","password":"oY9uU2uJ8nD8iY7t"},"limit":10000,"timeout_ms":30000}' localhost:50051 com.arcadedb.grpc.ArcadeDbService/ExecuteQuery > /dev/null + + +grpcurl -proto arcadedb-server.proto -plaintext -max-msg-sz 33554432 -H "grpc-accept-encoding: gzip" -H "x-arcade-user: root" -H "x-arcade-password: oY9uU2uJ8nD8iY7t" -H "x-arcade-database: local_shakeiq_curonix_poc-app" -d '{"database":"local_shakeiq_curonix_poc-app","query":"TRAVERSE out(), in() FROM (SELECT FROM SalesShakeInstance) WHILE $depth <= 7 STRATEGY BREADTH_FIRST","parameters":{},"credentials":{"username":"root","password":"oY9uU2uJ8nD8iY7t"},"limit":10000,"timeout_ms":30000}' localhost:50051 com.arcadedb.grpc.ArcadeDbService/ExecuteQuery | wc -c + +8,229,923 + +ghz --enable-compression --insecure \ + --proto arcadedb-server.proto \ + --call com.arcadedb.grpc.ArcadeDbService.ExecuteQuery \ + --metadata '{"grpc-accept-encoding":"gzip","x-arcade-user":"root","x-arcade-password":"oY9uU2uJ8nD8iY7t","x-arcade-database":"local_shakeiq_curonix_poc-app"}' \ + --data '{"database":"local_shakeiq_curonix_poc-app","query":"TRAVERSE out(), in() FROM (SELECT FROM SalesShakeInstance) WHILE $depth <= 7 STRATEGY BREADTH_FIRST","parameters":{},"credentials":{"username":"root","password":"oY9uU2uJ8nD8iY7t"},"limit":10000,"timeout_ms":30000}' \ + --max-send-message-size 33554432 --max-recv-message-size 33554432 \ + --concurrency 5 \ + --total 1 \ + localhost:50051 + + \ No newline at end of file diff --git a/grpc/pom.xml b/grpc/pom.xml new file mode 100644 index 0000000000..537e8717f1 --- /dev/null +++ b/grpc/pom.xml @@ -0,0 +1,140 @@ + + 4.0.0 + + + com.arcadedb + arcadedb-parent + 25.8.1-SNAPSHOT + + + arcadedb-grpc + ArcadeDB gRPC Stubs + jar + + + + + + io.grpc + grpc-netty-shaded + ${grpc.version} + + + io.grpc + grpc-protobuf + ${grpc.version} + + + io.grpc + grpc-stub + ${grpc.version} + + + io.grpc + grpc-services + ${grpc.version} + + + io.grpc + grpc-xds + ${grpc.version} + + + + + com.google.protobuf + protobuf-java + ${protobuf-java.version} + + + com.google.protobuf + protobuf-java-util + ${protobuf-java.version} + + + + + com.google.api.grpc + proto-google-common-protos + ${com.google.api.grpc.version} + + + + + javax.annotation + javax.annotation-api + ${javax.annotation-api.version} + + + + + io.grpc + grpc-testing + ${grpc.version} + test + + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + grpc-interface + ${project.build.directory}/classes + + com/arcadedb/server/grpc/* + + + pack-grpc-client + package + + jar + + + + default-jar + package + + jar + + + + + + + io.github.ascopes + protobuf-maven-plugin + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.6.0 + + + add-source + generate-sources + + add-source + + + + src/generated/protobuf/java + + + + + + + + \ No newline at end of file diff --git a/grpc/src/main/proto/arcadedb-server.proto b/grpc/src/main/proto/arcadedb-server.proto new file mode 100644 index 0000000000..e684fe5453 --- /dev/null +++ b/grpc/src/main/proto/arcadedb-server.proto @@ -0,0 +1,390 @@ +syntax = "proto3"; + +package com.arcadedb.grpc; + +option java_multiple_files = true; +option java_package = "com.arcadedb.server.grpc"; +option java_outer_classname = "ArcadeDbProto"; + +import "google/protobuf/empty.proto"; +import "google/protobuf/struct.proto"; +import "google/protobuf/timestamp.proto"; + +// Main ArcadeDB Service +service ArcadeDbService { + // Database operations + rpc CreateDatabase(CreateDatabaseRequest) returns (CreateDatabaseResponse); + rpc DropDatabase(DropDatabaseRequest) returns (DropDatabaseResponse); + rpc ListDatabases(ListDatabasesRequest) returns (ListDatabasesResponse); + rpc GetDatabaseInfo(GetDatabaseInfoRequest) returns (GetDatabaseInfoResponse); + + // Query operations + rpc ExecuteQuery(ExecuteQueryRequest) returns (ExecuteQueryResponse); + rpc ExecuteCommand(ExecuteCommandRequest) returns (ExecuteCommandResponse); + + // Transaction operations + rpc BeginTransaction(BeginTransactionRequest) returns (BeginTransactionResponse); + rpc CommitTransaction(CommitTransactionRequest) returns (CommitTransactionResponse); + rpc RollbackTransaction(RollbackTransactionRequest) returns (RollbackTransactionResponse); + + // Record operations + rpc CreateRecord(CreateRecordRequest) returns (CreateRecordResponse); + rpc GetRecord(GetRecordRequest) returns (GetRecordResponse); + rpc UpdateRecord(UpdateRecordRequest) returns (UpdateRecordResponse); + rpc DeleteRecord(DeleteRecordRequest) returns (DeleteRecordResponse); + + // Streaming operations + rpc StreamQuery(StreamQueryRequest) returns (stream QueryResult); + + rpc BulkInsert (BulkInsertRequest) returns (InsertSummary); + rpc InsertStream (stream InsertChunk) returns (InsertSummary); + rpc InsertBidirectional (stream InsertRequest) returns (stream InsertResponse); + + // Server operations + rpc GetServerStatus(google.protobuf.Empty) returns (ServerStatusResponse); + rpc Ping(google.protobuf.Empty) returns (PingResponse); +} + +// Common messages +message DatabaseCredentials { + string username = 1; + string password = 2; +} + +message TransactionContext { + string transaction_id = 1; // optional: reuse existing tx or name a new one + string database = 2; // optional if request already carries database + + // NEW (optional) — all default to false + bool begin = 3; // start a tx if not already active for this transaction_id + bool commit = 4; // commit the referenced tx after the RPC work + bool rollback = 5; // rollback the referenced tx after the RPC work + + // Optional QoS (you can add later if you need) + int64 timeout_ms = 6; // server may enforce per-tx timeout + bool read_only = 7; // hint for engines that support RO tx + // enum TxIsolation { DEFAULT=0; READ_COMMITTED=1; REPEATABLE_READ=2; SERIALIZABLE=3; } + // TxIsolation isolation = 8; +} + +// Database operations +message CreateDatabaseRequest { + string database_name = 1; + DatabaseCredentials credentials = 2; + map options = 3; +} + +message CreateDatabaseResponse { + bool success = 1; + string message = 2; + string database_id = 3; +} + +message DropDatabaseRequest { + string database_name = 1; + DatabaseCredentials credentials = 2; +} + +message DropDatabaseResponse { + bool success = 1; + string message = 2; +} + +message ListDatabasesRequest { + DatabaseCredentials credentials = 1; +} + +message ListDatabasesResponse { + repeated DatabaseInfo databases = 1; +} + +message DatabaseInfo { + string name = 1; + int64 size = 2; + string status = 3; + map properties = 4; +} + +message GetDatabaseInfoRequest { + string database_name = 1; + DatabaseCredentials credentials = 2; +} + +message GetDatabaseInfoResponse { + DatabaseInfo info = 1; +} + +// Query operations +message ExecuteQueryRequest { + string database = 1; + string query = 2; + map parameters = 3; + DatabaseCredentials credentials = 4; + TransactionContext transaction = 5; + int32 limit = 6; + int32 timeout_ms = 7; +} + +message ExecuteQueryResponse { + repeated QueryResult results = 1; + int64 execution_time_ms = 2; + string query_plan = 3; +} + +// --- REQUEST --- +message ExecuteCommandRequest { + // existing fields — DO NOT renumber + string database = 1; + string command = 2; + map parameters = 3; + DatabaseCredentials credentials = 4; + TransactionContext transaction = 5; + + // NEW (optional): language of the command (defaults to "sql" on server if empty) + string language = 6; + + // NEW (optional): if true, server may stream/collect rows produced by the command and + // include them in ExecuteCommandResponse.records (subject to max_rows). + bool return_rows = 7; + + // NEW (optional): hard limit on rows returned when return_rows=true. 0 = server default. + int32 max_rows = 8; +} + +// --- RESPONSE --- +message ExecuteCommandResponse { + // existing + bool success = 1; + string message = 2; + int64 affected_records = 3; + int64 execution_time_ms = 4; + + // NEW (optional): result rows if return_rows=true and the command produced rows. + // Use your existing Record message type from the service. + repeated Record records = 5; +} + +message QueryResult { + repeated Record records = 1; + int32 total_records_in_batch = 2; // renamed for clarity + int64 running_total_emitted = 3; // optional; cumulative count so far + bool is_last_batch = 4; +} + +message ColumnMetadata { + string name = 1; + string type = 2; + bool nullable = 3; +} + +// Transaction operations +message BeginTransactionRequest { + string database = 1; + DatabaseCredentials credentials = 2; + TransactionIsolation isolation = 3; +} + +enum TransactionIsolation { + READ_UNCOMMITTED = 0; + READ_COMMITTED = 1; + REPEATABLE_READ = 2; + SERIALIZABLE = 3; +} + +message BeginTransactionResponse { + string transaction_id = 1; + int64 timestamp = 2; +} + +message CommitTransactionRequest { + TransactionContext transaction = 1; + DatabaseCredentials credentials = 2; +} + +message CommitTransactionResponse { + bool success = 1; + string message = 2; + int64 timestamp = 3; +} + +message RollbackTransactionRequest { + TransactionContext transaction = 1; + DatabaseCredentials credentials = 2; +} + +message RollbackTransactionResponse { + bool success = 1; + string message = 2; +} + +// Record operations +message Record { + string rid = 1; + string type = 2; + map properties = 3; + int32 version = 4; +} + +message CreateRecordRequest { + string database = 1; + string type = 2; + map properties = 3; + DatabaseCredentials credentials = 4; + TransactionContext transaction = 5; +} + +message CreateRecordResponse { + Record record = 1; + bool success = 2; + string message = 3; + string identity = 4; +} + +message GetRecordRequest { + string database = 1; + string rid = 2; + DatabaseCredentials credentials = 3; + TransactionContext transaction = 4; +} + +message GetRecordResponse { + Record record = 1; + bool found = 2; +} + +message UpdateRecordRequest { + string database = 1; + string rid = 2; + map properties = 3; + DatabaseCredentials credentials = 4; + TransactionContext transaction = 5; + int32 expected_version = 6; +} + +message UpdateRecordResponse { + Record record = 1; + bool success = 2; + bool updated = 3; + string message = 4; +} + +message DeleteRecordRequest { + string database = 1; + string rid = 2; + DatabaseCredentials credentials = 3; + TransactionContext transaction = 4; +} + +message DeleteRecordResponse { + bool success = 1; + bool deleted = 2; + string message = 3; +} + +// Streaming operations +message StreamQueryRequest { + string database = 1; + string query = 2; + map parameters = 3; + DatabaseCredentials credentials = 4; + int32 batch_size = 5; + + // NEW: how the server should retrieve rows under the hood + enum RetrievalMode { + // Current behavior: run once and iterate results, pushing as you go. + CURSOR = 0; + + // Load all results into memory first, then emit batches. + MATERIALIZE_ALL = 1; + + // Re-issue the query per batch using LIMIT/SKIP (or a server-side cursor/offset) + // so only enough rows for the next batch are fetched. + PAGED = 2; + } + RetrievalMode retrieval_mode = 6; + + // NEW (optional): inline transaction control for streaming + TransactionContext transaction = 7; + + // Optional (if you want to allow non-SQL later) + string language = 8; // default "sql" on server if empty +} + +// Server operations +message ServerStatusResponse { + string version = 1; + string status = 2; + int64 uptime_ms = 3; + int32 active_connections = 4; + map metrics = 5; +} + +message PingResponse { + int64 timestamp = 1; + string message = 2; +} + +enum ConflictMode { CONFLICT_ABORT = 0; CONFLICT_IGNORE = 1; CONFLICT_UPDATE = 2; } +enum TransactionMode { PER_BATCH = 0; PER_STREAM = 1; PER_ROW = 2; } + +message InsertOptions { + string database = 1; + string target_class = 2; // e.g., "Patient" + repeated string key_columns = 3; // for upsert/dedupe + ConflictMode conflict_mode = 4; // default CONFLICT_ABORT + repeated string update_columns_on_conflict = 5; + TransactionMode transaction_mode = 6; // default PER_BATCH + uint32 server_batch_size = 7; // default 1000 + bool validate_only = 8; // dry-run (no commit) + DatabaseCredentials credentials = 9; +} + +message RowError { + uint64 row_index = 1; string code = 2; string message = 3; string field = 4; +} + +message InsertChunk { + string session_id = 1; // optional idempotency key + uint64 chunk_seq = 2; // 1..N + repeated Record rows = 3; // the data +} + +message InsertSummary { + uint64 received = 1; uint64 inserted = 2; uint64 updated = 3; + uint64 ignored = 4; uint64 failed = 5; + repeated RowError errors = 6; // truncate server-side if large + google.protobuf.Timestamp started_at = 7; + google.protobuf.Timestamp finished_at = 8; +} + +message BulkInsertRequest { + InsertOptions options = 1; repeated Record rows = 2; +} + +// message BulkInsertRequest { +// string database = 1; +// string type = 2; +// Record record = 3; +// DatabaseCredentials credentials = 4; +// } + +// message BulkInsertResponse { +// int64 total_inserted = 1; +// int64 total_failed = 2; +// repeated string errors = 3; +// } + +message StartInsert { InsertOptions options = 1; } +message CommitInsert { string session_id = 1; } + +message Started { string session_id = 1; } +message BatchAck { + string session_id = 1; uint64 chunk_seq = 2; + uint64 inserted = 3; uint64 updated = 4; uint64 ignored = 5; uint64 failed = 6; + repeated RowError errors = 7; +} +message Committed { InsertSummary summary = 1; } + +message InsertRequest { oneof msg { StartInsert start = 1; InsertChunk chunk = 2; CommitInsert commit = 3; } } +message InsertResponse { oneof msg { Started started = 1; BatchAck batch_ack = 2; Committed committed = 3; } } + diff --git a/grpcw/pom.xml b/grpcw/pom.xml new file mode 100644 index 0000000000..658032885e --- /dev/null +++ b/grpcw/pom.xml @@ -0,0 +1,64 @@ + + + + 4.0.0 + + + com.arcadedb + arcadedb-parent + 25.8.1-SNAPSHOT + ../pom.xml + + + arcadedb-grpcw + jar + ArcadeDB gRpcW + + + + + + + com.arcadedb + arcadedb-server + ${project.parent.version} + provided + + + com.arcadedb + arcadedb-server + ${project.parent.version} + test + test-jar + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + + diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java new file mode 100644 index 0000000000..8d4f6f9378 --- /dev/null +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java @@ -0,0 +1,1867 @@ +package com.arcadedb.server.grpc; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.arcadedb.database.Database; +import com.arcadedb.database.DatabaseFactory; +import com.arcadedb.database.Document; +import com.arcadedb.database.MutableDocument; +import com.arcadedb.database.RID; +import com.arcadedb.engine.ComponentFile; +import com.arcadedb.graph.Edge; +import com.arcadedb.graph.MutableEdge; +import com.arcadedb.graph.MutableVertex; +import com.arcadedb.graph.Vertex; +import com.arcadedb.query.sql.executor.Result; +import com.arcadedb.query.sql.executor.ResultSet; +import com.arcadedb.schema.DocumentType; +import com.arcadedb.schema.EdgeType; +import com.arcadedb.schema.Schema; +import com.arcadedb.schema.VertexType; +import com.arcadedb.server.ArcadeDBServer; +import com.google.protobuf.Empty; +import com.google.protobuf.Timestamp; +import com.google.protobuf.Value; + +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.grpc.stub.ServerCallStreamObserver; +import io.grpc.stub.StreamObserver; + +/** + * gRPC Service implementation for ArcadeDB + */ +public class ArcadeDbGrpcService extends ArcadeDbServiceGrpc.ArcadeDbServiceImplBase { + + private static final Logger logger = LoggerFactory.getLogger(ArcadeDbGrpcService.class); + + // Transaction management + private final Map activeTransactions = new ConcurrentHashMap<>(); + + // Database connection pool + private final Map databasePool = new ConcurrentHashMap<>(); + + // Track highest acknowledged chunk per session for idempotency + private final Map sessionWatermark = new ConcurrentHashMap<>(); + + // ArcadeDB server reference (optional, for accessing existing databases) + private final ArcadeDBServer arcadeServer; + + // Database directory path + private final String databasePath; + + private static final int DEFAULT_MAX_COMMAND_ROWS = 1000; + + public ArcadeDbGrpcService(String databasePath, ArcadeDBServer server) { + this.databasePath = databasePath; + this.arcadeServer = server; + } + + public void close() { + // Close all open databases + for (Database db : databasePool.values()) { + try { + if (db != null && db.isOpen()) { + db.close(); + } + } + catch (Exception e) { + logger.error("Error closing database", e); + } + } + databasePool.clear(); + + // Clean up transactions + for (Database db : activeTransactions.values()) { + try { + if (db != null && db.isOpen()) { + db.rollback(); + db.close(); + } + } + catch (Exception e) { + logger.error("Error closing transaction database", e); + } + } + activeTransactions.clear(); + } + + @Override + public void createDatabase(CreateDatabaseRequest request, StreamObserver responseObserver) { + + try { + // Validate credentials if needed + validateCredentials(request.getCredentials()); + + try (// Create new database factory for this specific database + DatabaseFactory dbFactory = new DatabaseFactory(databasePath + "/" + request.getDatabaseName())) { + // Create the database + Database database = dbFactory.create(); + database.close(); + } + + CreateDatabaseResponse response = CreateDatabaseResponse.newBuilder().setSuccess(true).setMessage("Database created successfully") + .setDatabaseId(request.getDatabaseName()).build(); + + responseObserver.onNext(response); + responseObserver.onCompleted(); + + } + catch (Exception e) { + logger.error("Error creating database: {}", e.getMessage(), e); + responseObserver.onError(Status.INTERNAL.withDescription("Failed to create database: " + e.getMessage()).asException()); + } + } + + @Override + public void dropDatabase(DropDatabaseRequest req, StreamObserver resp) { + + try { + + final String name = req.getDatabaseName(); + + if (name == null || name.isEmpty()) + throw new IllegalArgumentException("database name is required"); + + try (// Create new database factory for this specific database + DatabaseFactory dbFactory = new DatabaseFactory(databasePath + "/" + req.getDatabaseName())) { + // Create the database + Database database = dbFactory.create(); + database.drop(); + } + + resp.onNext(DropDatabaseResponse.newBuilder().setSuccess(true).build()); + resp.onCompleted(); + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("DropDatabase: " + e.getMessage()).asException()); + } + } + + @Override + public void listDatabases(ListDatabasesRequest req, StreamObserver resp) { + + try { + // TODO implement: mirror GetDatabasesHandler + // java.util.List names = server.getServer().getDatabaseNames(); + // resp.onNext(ListDatabasesResponse.newBuilder().addAllNames(names).build()); + // resp.onCompleted(); + resp.onError(io.grpc.Status.UNIMPLEMENTED.withDescription("ListDatabases not implemented").asException()); + } + catch (Exception e) { + resp.onError(io.grpc.Status.INTERNAL.withDescription("ListDatabases: " + e.getMessage()).asException()); + } + } + + @Override + public void getDatabaseInfo(GetDatabaseInfoRequest req, StreamObserver resp) { + + try { + final String name = req.getDatabaseName(); + Database db = getDatabase(name, req.getCredentials()); + + Schema schema = db.getSchema(); + int typeCount = schema.getTypes().size(); + + DatabaseInfo databaseInfo = DatabaseInfo.newBuilder().setName(name).setStatus("").setSize(typeCount).build(); + + GetDatabaseInfoResponse.Builder b = GetDatabaseInfoResponse.newBuilder().setInfo(databaseInfo); + + resp.onNext(b.build()); + resp.onCompleted(); + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("GetDatabaseInfo: " + e.getMessage()).asException()); + } + } + + @Override + public void executeCommand(ExecuteCommandRequest req, StreamObserver resp) { + final long t0 = System.nanoTime(); + + Database db = null; + boolean beganHere = false; + + try { + // Resolve DB + params + db = getDatabase(req.getDatabase(), req.getCredentials()); + final java.util.Map params = convertParameters(req.getParametersMap()); + + // Language defaults to "sql" when empty + final String language = (req.getLanguage() == null || req.getLanguage().isEmpty()) ? "sql" : req.getLanguage(); + + // Transaction: begin if requested + final boolean hasTx = req.hasTransaction(); + final var tx = hasTx ? req.getTransaction() : null; + if (hasTx && tx.getBegin()) { + db.begin(); + beganHere = true; + } + + long affected = 0L; + final boolean returnRows = req.getReturnRows(); + final int maxRows = req.getMaxRows() > 0 ? req.getMaxRows() : DEFAULT_MAX_COMMAND_ROWS; + + ExecuteCommandResponse.Builder out = ExecuteCommandResponse.newBuilder().setSuccess(true).setMessage("OK"); + + // Execute the command + try (ResultSet rs = db.command(language, req.getCommand(), params)) { + if (returnRows) { + int emitted = 0; + while (rs.hasNext()) { + Result r = rs.next(); + + if (r.isElement()) { + affected++; // count modified/returned records + if (emitted < maxRows) { + out.addRecords(convertToGrpcRecord(r.getElement().get())); + emitted++; + } + } + else { + // Scalar / projection row (e.g., RETURN COUNT) + if (emitted < maxRows) { + com.arcadedb.server.grpc.Record.Builder recB = com.arcadedb.server.grpc.Record.newBuilder(); + for (String p : r.getPropertyNames()) { + recB.putProperties(p, toProtoValue(r.getProperty(p))); + } + out.addRecords(recB.build()); + emitted++; + } + for (String p : r.getPropertyNames()) { + Object v = r.getProperty(p); + if (v instanceof Number n) + affected += n.longValue(); + } + } + } + } + else { + // Not returning rows: still consume to compute 'affected' + while (rs.hasNext()) { + Result r = rs.next(); + if (r.isElement()) { + affected++; + } + else { + for (String p : r.getPropertyNames()) { + Object v = r.getProperty(p); + if (v instanceof Number n) + affected += n.longValue(); + } + } + } + } + } + + // Transaction end — precedence: rollback > commit > begin-only⇒commit + if (hasTx) { + if (tx.getRollback()) { + db.rollback(); + } + else if (tx.getCommit()) { + db.commit(); + } + else if (beganHere) { + // Began but no explicit commit/rollback flag — default to commit (HTTP parity) + db.commit(); + } + } + + final long ms = (System.nanoTime() - t0) / 1_000_000L; + out.setAffectedRecords(affected).setExecutionTimeMs(ms); + + resp.onNext(out.build()); + resp.onCompleted(); + + } + catch (Exception e) { + // Best-effort rollback if we began here and failed + try { + if (beganHere && db != null) + db.rollback(); + } + catch (Exception ignore) { + /* no-op */ } + + final long ms = (System.nanoTime() - t0) / 1_000_000L; + ExecuteCommandResponse err = ExecuteCommandResponse.newBuilder().setSuccess(false) + .setMessage(e.getMessage() == null ? e.toString() : e.getMessage()).setAffectedRecords(0L).setExecutionTimeMs(ms).build(); + + // Prefer returning a structured response so clients always get timing/message + resp.onNext(err); + resp.onCompleted(); + // If you prefer gRPC error codes: comment the two lines above and use: + // resp.onError(Status.INTERNAL.withDescription("ExecuteCommand: " + + // e.getMessage()).asException()); + } + } + + @Override + public void createRecord(CreateRecordRequest req, StreamObserver resp) { + + try { + + Database db = getDatabase(req.getDatabase(), req.getCredentials()); + + final String cls = req.getType(); // or req.getTargetClass() if that’s your proto + if (cls == null || cls.isEmpty()) + throw new IllegalArgumentException("targetClass is required"); + + Schema schema = db.getSchema(); + DocumentType dt = schema.getType(cls); + if (dt == null) + throw new IllegalArgumentException("Class not found: " + cls); + + // All properties from the request (proto map) + final java.util.Map props = req.getPropertiesMap(); + + // --- Vertex --- + if (dt instanceof VertexType) { + com.arcadedb.graph.MutableVertex v = db.newVertex(cls); + // apply properties + props.forEach((k, val) -> v.set(k, toJavaForProperty(db, dt, k, val))); + v.save(); + + CreateRecordResponse.Builder b = CreateRecordResponse.newBuilder(); + + // Adjust to your actual response field name (e.g., setIdentity / setRidStr) + b.setIdentity(v.getIdentity().toString()); + + resp.onNext(b.build()); + resp.onCompleted(); + return; + } + + // --- Edge --- + if (dt instanceof EdgeType) { + // Expect 'out' and 'in' as string RIDs in the properties map + String outStr = null, inStr = null; + if (props.containsKey("out")) { + var pv = props.get("out"); + outStr = pv.hasStringValue() ? pv.getStringValue() : String.valueOf(toJava(pv)); + } + if (props.containsKey("in")) { + var pv = props.get("in"); + inStr = pv.hasStringValue() ? pv.getStringValue() : String.valueOf(toJava(pv)); + } + if (outStr == null || inStr == null) + throw new IllegalArgumentException("Edge requires 'out' and 'in' RIDs"); + + var outEl = db.lookupByRID(new RID(outStr), true); + var inEl = db.lookupByRID(new RID(inStr), true); + if (outEl == null || inEl == null) + throw new IllegalArgumentException("Cannot resolve out/in vertices for edge"); + + // IMPORTANT: use MutableVertex to avoid newEdge(...) ambiguity + com.arcadedb.graph.MutableVertex outV = (MutableVertex) outEl.asVertex(true); + com.arcadedb.graph.MutableVertex inV = (MutableVertex) inEl.asVertex(true); + + com.arcadedb.graph.MutableEdge e = outV.newEdge(cls, inV); + + // apply remaining properties (skip endpoints) + props.forEach((k, val) -> { + if (!"out".equals(k) && !"in".equals(k)) { + e.set(k, toJavaForProperty(db, dt, k, val)); + } + }); + + e.save(); + + CreateRecordResponse.Builder b = CreateRecordResponse.newBuilder(); + // Adjust to your actual response field name + b.setIdentity(e.getIdentity().toString()); + resp.onNext(b.build()); + resp.onCompleted(); + return; + } + + // --- Document --- + MutableDocument d = db.newDocument(cls); + props.forEach((k, val) -> d.set(k, toJavaForProperty(db, dt, k, val))); + d.save(); + + CreateRecordResponse.Builder b = CreateRecordResponse.newBuilder(); + // Adjust to your actual response field name + b.setIdentity(d.getIdentity().toString()); + resp.onNext(b.build()); + resp.onCompleted(); + + } + catch (Exception e) { + resp.onError(Status.INVALID_ARGUMENT.withDescription("CreateRecord: " + e.getMessage()).asException()); + } + } + + @Override + public void getRecord(GetRecordRequest req, StreamObserver resp) { + try { + Database db = getDatabase(req.getDatabase(), req.getCredentials()); + String ridStr = req.getRid(); + if (ridStr == null || ridStr.isEmpty()) + throw new IllegalArgumentException("rid is required"); + + var el = db.lookupByRID(new RID(ridStr), true); + if (el == null) { + resp.onError(Status.NOT_FOUND.withDescription("Record not found: " + ridStr).asException()); + return; + } + resp.onNext(GetRecordResponse.newBuilder().setRecord(convertToGrpcRecord(el.getRecord())).build()); + resp.onCompleted(); + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("GetRecord: " + e.getMessage()).asException()); + } + } + + @Override + public void updateRecord(UpdateRecordRequest req, StreamObserver resp) { + Database db = null; + boolean beganHere = false; + + try { + + db = getDatabase(req.getDatabase(), req.getCredentials()); + + final String ridStr = req.getRid(); + if (ridStr == null || ridStr.isEmpty()) + throw new IllegalArgumentException("rid is required"); + + // Begin transaction if requested + final boolean hasTx = req.hasTransaction(); + final var tx = hasTx ? req.getTransaction() : null; + if (hasTx && tx.getBegin()) { + db.begin(); + beganHere = true; + } + + // Lookup the record by RID + var el = db.lookupByRID(new RID(ridStr), true); + if (el == null) { + resp.onError(Status.NOT_FOUND.withDescription("Record not found: " + ridStr).asException()); + return; + } + + // Get mutable view for updates (works for docs, vertices, edges) + MutableDocument mdoc = (MutableDocument) el.asDocument(true); + + var dtype = db.getSchema().getType(mdoc.getTypeName()); + + final var dbRef = db; + + // Apply updates + req.getPropertiesMap().forEach((k, v) -> mdoc.set(k, toJavaForProperty(dbRef, dtype, k, v))); + + mdoc.save(); + + // Commit/rollback with proper precedence + if (hasTx) { + if (tx.getRollback()) { + db.rollback(); + } + else if (tx.getCommit()) { + db.commit(); + } + else if (beganHere) { + db.commit(); + } + } + + resp.onNext(UpdateRecordResponse.newBuilder().setUpdated(true).setSuccess(true).build()); + resp.onCompleted(); + + } + catch (Exception e) { + try { + if (beganHere && db != null) + db.rollback(); + } + catch (Exception ignore) { + // ignore rollback errors + } + resp.onError(Status.INTERNAL.withDescription("UpdateRecord: " + e.getMessage()).asException()); + } + } + + @Override + public void deleteRecord(DeleteRecordRequest req, StreamObserver resp) { + try { + Database db = getDatabase(req.getDatabase(), req.getCredentials()); + String ridStr = req.getRid(); + if (ridStr == null || ridStr.isEmpty()) + throw new IllegalArgumentException("rid is required"); + + var el = db.lookupByRID(new RID(ridStr), true); + + if (el == null) { + resp.onNext(DeleteRecordResponse.newBuilder().setSuccess(false).setDeleted(false).build()); + resp.onCompleted(); + return; + } + + el.delete(); + resp.onNext(DeleteRecordResponse.newBuilder().setSuccess(true).setDeleted(true).build()); + resp.onCompleted(); + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("DeleteRecord: " + e.getMessage()).asException()); + } + } + + @Override + public void executeQuery(ExecuteQueryRequest request, StreamObserver responseObserver) { + try { + + // Force compression for streaming (usually beneficial) + CompressionAwareService.setResponseCompression(responseObserver, "gzip"); + + Database database = getDatabase(request.getDatabase(), request.getCredentials()); + + // Check if this is part of a transaction + if (request.hasTransaction()) { + database = activeTransactions.get(request.getTransaction().getTransactionId()); + if (database == null) { + throw new IllegalArgumentException("Invalid transaction ID"); + } + } + + // Execute the query + long startTime = System.currentTimeMillis(); + ResultSet resultSet = database.query("sql", request.getQuery(), convertParameters(request.getParametersMap())); + + logger.debug("executeQuery(): to get resultSet = {}", (System.currentTimeMillis() - startTime)); + + // Build response + QueryResult.Builder resultBuilder = QueryResult.newBuilder(); + + // Process results + int count = 0; + while (resultSet.hasNext()) { + Result result = resultSet.next(); + if (result.isElement()) { + com.arcadedb.database.Record dbRecord = result.getElement().get(); + Record grpcRecord = convertToGrpcRecord(dbRecord); + resultBuilder.addRecords(grpcRecord); + count++; + + // Apply limit if specified + if (request.getLimit() > 0 && count >= request.getLimit()) { + break; + } + } + } + + resultBuilder.setTotalRecordsInBatch(count); + + long executionTime = System.currentTimeMillis() - startTime; + + ExecuteQueryResponse response = ExecuteQueryResponse.newBuilder().addResults(resultBuilder.build()).setExecutionTimeMs(executionTime) + .build(); + + logger.debug("executeQuery(): executionTime + response generation = {}", executionTime); + + responseObserver.onNext(response); + responseObserver.onCompleted(); + + } + catch (Exception e) { + logger.error("Error executing query: {}", e.getMessage(), e); + responseObserver.onError(Status.INTERNAL.withDescription("Query execution failed: " + e.getMessage()).asException()); + } + } + + @Override + public void beginTransaction(BeginTransactionRequest request, StreamObserver responseObserver) { + try { + Database database = getDatabase(request.getDatabase(), request.getCredentials()); + + // Begin transaction + database.begin(); + + // Generate transaction ID + String transactionId = generateTransactionId(); + activeTransactions.put(transactionId, database); + + BeginTransactionResponse response = BeginTransactionResponse.newBuilder().setTransactionId(transactionId) + .setTimestamp(System.currentTimeMillis()).build(); + + responseObserver.onNext(response); + responseObserver.onCompleted(); + + } + catch (Exception e) { + logger.error("Error beginning transaction: {}", e.getMessage(), e); + responseObserver.onError(Status.INTERNAL.withDescription("Failed to begin transaction: " + e.getMessage()).asException()); + } + } + + @Override + public void commitTransaction(CommitTransactionRequest request, StreamObserver responseObserver) { + try { + Database database = activeTransactions.remove(request.getTransaction().getTransactionId()); + if (database == null) { + throw new IllegalArgumentException("Invalid transaction ID"); + } + + // Commit transaction + database.commit(); + + CommitTransactionResponse response = CommitTransactionResponse.newBuilder().setSuccess(true) + .setMessage("Transaction committed successfully").setTimestamp(System.currentTimeMillis()).build(); + + responseObserver.onNext(response); + responseObserver.onCompleted(); + + } + catch (Exception e) { + logger.error("Error committing transaction: {}", e.getMessage(), e); + responseObserver.onError(Status.INTERNAL.withDescription("Failed to commit transaction: " + e.getMessage()).asException()); + } + } + + @Override + public void rollbackTransaction(RollbackTransactionRequest request, StreamObserver responseObserver) { + try { + Database database = activeTransactions.remove(request.getTransaction().getTransactionId()); + if (database == null) { + throw new IllegalArgumentException("Invalid transaction ID"); + } + + // Rollback transaction + database.rollback(); + + RollbackTransactionResponse response = RollbackTransactionResponse.newBuilder().setSuccess(true) + .setMessage("Transaction rolled back successfully").build(); + + responseObserver.onNext(response); + responseObserver.onCompleted(); + + } + catch (Exception e) { + logger.error("Error rolling back transaction: {}", e.getMessage(), e); + responseObserver.onError(Status.INTERNAL.withDescription("Failed to rollback transaction: " + e.getMessage()).asException()); + } + } + + @Override + public void streamQuery(StreamQueryRequest request, StreamObserver responseObserver) { + + final ServerCallStreamObserver scso = (ServerCallStreamObserver) responseObserver; + + final java.util.concurrent.atomic.AtomicBoolean cancelled = new java.util.concurrent.atomic.AtomicBoolean(false); + scso.setOnCancelHandler(() -> cancelled.set(true)); + + Database db = null; + boolean beganHere = false; + + try { + db = getDatabase(request.getDatabase(), request.getCredentials()); + final int batchSize = Math.max(1, request.getBatchSize()); + + // --- TX begin if requested --- + final boolean hasTx = request.hasTransaction(); + final var tx = hasTx ? request.getTransaction() : null; + if (hasTx && tx.getBegin()) { + db.begin(); + beganHere = true; + } + + // --- Dispatch on mode (helpers do NOT manage transactions) --- + switch (request.getRetrievalMode()) { + case MATERIALIZE_ALL -> streamMaterialized(db, request, batchSize, scso, cancelled); + case PAGED -> streamPaged(db, request, batchSize, scso, cancelled); + case CURSOR -> streamCursor(db, request, batchSize, scso, cancelled); + default -> streamCursor(db, request, batchSize, scso, cancelled); + } + + // If the client cancelled mid-stream, choose rollback unless caller explicitly asked to commit/rollback. + if (cancelled.get()) { + if (hasTx) { + if (tx.getRollback()) { + db.rollback(); + } else if (tx.getCommit()) { + db.commit(); // caller explicitly wanted commit even if they cancelled + } else if (beganHere) { + db.rollback(); // safe default on cancellation + } + } + return; // don't call onCompleted() + } + + // --- TX end (normal path) — precedence: rollback > commit > begin-only ⇒ commit --- + if (hasTx) { + if (tx.getRollback()) { + db.rollback(); + } else if (tx.getCommit()) { + db.commit(); + } else if (beganHere) { + db.commit(); + } + } + + scso.onCompleted(); + + } catch (Exception e) { + // Best-effort rollback only if we began here and there wasn't an explicit commit/rollback + try { + if (beganHere && db != null) { + if (request.hasTransaction()) { + var tx = request.getTransaction(); + if (!tx.getCommit() && !tx.getRollback()) db.rollback(); + } else { + db.rollback(); + } + } + } catch (Exception ignore) { /* no-op */ } + + if (!cancelled.get()) { + responseObserver.onError(Status.INTERNAL + .withDescription("Stream query failed: " + e.getMessage()) + .asException()); + } + } + } + + /** + * Mode 1 (existing behavior-ish): run once and iterate results, batching as we + * go. + */ + private void streamCursor(Database db, StreamQueryRequest req, int batchSize, ServerCallStreamObserver scso, + AtomicBoolean cancelled) { + long running = 0L; + QueryResult.Builder batch = QueryResult.newBuilder(); + int inBatch = 0; + + try (ResultSet rs = db.query("sql", req.getQuery(), convertParameters(req.getParametersMap()))) { + while (rs.hasNext()) { + if (cancelled.get()) + return; + waitUntilReady(scso, cancelled); + + Result r = rs.next(); + if (!r.isElement()) + continue; + + com.arcadedb.database.Record rec = r.getElement().get(); + batch.addRecords(convertToGrpcRecord(rec)); + inBatch++; + running++; + + if (inBatch >= batchSize) { + safeOnNext(scso, cancelled, + batch.setTotalRecordsInBatch(inBatch).setRunningTotalEmitted(running).setIsLastBatch(false).build()); + batch = QueryResult.newBuilder(); + inBatch = 0; + } + } + + if (!cancelled.get() && inBatch > 0) { + safeOnNext(scso, cancelled, batch.setTotalRecordsInBatch(inBatch).setRunningTotalEmitted(running).setIsLastBatch(true).build()); + } + } + } + + /** Mode 2: materialize everything first (simple, but can be memory-heavy). */ + private void streamMaterialized(Database db, StreamQueryRequest req, int batchSize, ServerCallStreamObserver scso, + AtomicBoolean cancelled) { + final java.util.ArrayList all = new java.util.ArrayList<>(); + try (ResultSet rs = db.query("sql", req.getQuery(), convertParameters(req.getParametersMap()))) { + while (rs.hasNext()) { + if (cancelled.get()) + return; + Result r = rs.next(); + if (!r.isElement()) + continue; + all.add(convertToGrpcRecord(r.getElement().get())); + } + } + + long running = 0L; + for (int i = 0; i < all.size(); i += batchSize) { + if (cancelled.get()) + return; + waitUntilReady(scso, cancelled); + + int end = Math.min(i + batchSize, all.size()); + QueryResult.Builder b = QueryResult.newBuilder(); + for (int j = i; j < end; j++) + b.addRecords(all.get(j)); + + running += (end - i); + safeOnNext(scso, cancelled, + b.setTotalRecordsInBatch(end - i).setRunningTotalEmitted(running).setIsLastBatch(end == all.size()).build()); + } + } + + /** Mode 3: only fetch one page’s worth of rows per emission via LIMIT/SKIP. */ + private void streamPaged(Database db, StreamQueryRequest req, int batchSize, ServerCallStreamObserver scso, + AtomicBoolean cancelled) { + + final String pagedSql = wrapWithSkipLimit(req.getQuery()); // see helper below + int offset = 0; + long running = 0L; + + while (true) { + if (cancelled.get()) + return; + waitUntilReady(scso, cancelled); + + java.util.Map params = new java.util.HashMap<>(convertParameters(req.getParametersMap())); + params.put("_skip", offset); + params.put("_limit", batchSize); + + int count = 0; + QueryResult.Builder b = QueryResult.newBuilder(); + + try (ResultSet rs = db.query("sql", pagedSql, params)) { + while (rs.hasNext()) { + if (cancelled.get()) + return; + Result r = rs.next(); + if (!r.isElement()) + continue; + b.addRecords(convertToGrpcRecord(r.getElement().get())); + count++; + } + } + + if (count == 0) + return; // no more rows + + running += count; + boolean last = count < batchSize; + + safeOnNext(scso, cancelled, b.setTotalRecordsInBatch(count).setRunningTotalEmitted(running).setIsLastBatch(last).build()); + + if (last) + return; + offset += batchSize; + } + } + + /** Wrap arbitrary SQL so we can safely inject LIMIT/SKIP outside. */ + private String wrapWithSkipLimit(String originalSql) { + + // Minimal defensive approach; you can do a real parser if needed. + + // ArcadeDB: SELECT FROM (...) [ORDER BY ...] SKIP :_skip LIMIT :_limit + + return "SELECT FROM (" + originalSql + ") ORDER BY @rid SKIP :_skip LIMIT :_limit"; + } + + private void safeOnNext(ServerCallStreamObserver scso, AtomicBoolean cancelled, QueryResult payload) { + if (cancelled.get()) + return; + try { + scso.onNext(payload); + } + catch (StatusRuntimeException e) { + if (e.getStatus().getCode() == Status.Code.CANCELLED) { + cancelled.set(true); + return; + } + throw e; + } + } + + private void waitUntilReady(ServerCallStreamObserver scso, AtomicBoolean cancelled) { + // Skip if you’re okay with best-effort pushes; otherwise honor transport + // readiness + if (scso.isReady()) + return; + while (!scso.isReady()) { + if (cancelled.get()) + return; + // avoid burning CPU: + try { + Thread.sleep(1); + } + catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + return; + } + } + } + + // --- 1) Unary bulk --- + @Override + public void bulkInsert(BulkInsertRequest req, StreamObserver resp) { + + final InsertOptions opts = defaults(req.getOptions()); // apply defaults (batch size, tx mode, etc.) + final long started = System.currentTimeMillis(); + + try (InsertContext ctx = new InsertContext(opts)) { + + Counts totals = insertRows(ctx, req.getRowsList().iterator()); + + ctx.flushCommit(true); + + resp.onNext(ctx.summary(totals, started)); + resp.onCompleted(); + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("bulkInsert: " + e.getMessage()).asException()); + } + } + + // --- 2) Client-streaming; single summary at end --- + @Override + public StreamObserver insertStream(StreamObserver resp) { + final ServerCallStreamObserver call = (ServerCallStreamObserver) resp; + + // manual pull of inbound request messages (chunks) + call.disableAutoInboundFlowControl(); + + final long startedAt = System.currentTimeMillis(); + + // server defaults; if you want options via first chunk, parse sessionId and + // look them up + final InsertOptions opts = defaults(InsertOptions.newBuilder().build()); + + final InsertContext ctx; + try { + ctx = new InsertContext(opts); // begins tx for PER_STREAM/PER_BATCH if needed + } + catch (Exception e) { + resp.onError(Status.FAILED_PRECONDITION.withDescription(e.getMessage()).asException()); + // return a no-op observer + return new StreamObserver<>() { + public void onNext(InsertChunk c) { + } + + public void onError(Throwable t) { + } + + public void onCompleted() { + } + }; + } + + // accumulate totals across chunks + final Counts totals = new Counts(); + final java.util.concurrent.atomic.AtomicBoolean cancelled = new java.util.concurrent.atomic.AtomicBoolean(false); + call.setOnCancelHandler(() -> cancelled.set(true)); + + // start pulling the first inbound message + call.request(1); + + return new StreamObserver<>() { + @Override + public void onNext(InsertChunk c) { + if (cancelled.get()) + return; + try { + Counts cts = insertRows(ctx, c.getRowsList().iterator()); // returns per-chunk counts + totals.add(cts); + } + catch (Exception e) { + // surface as a failed row group; you can also fail the whole call if you prefer + totals.err(Math.max(0, ctx.received - 1), "DB_ERROR", e.getMessage(), ""); + } + finally { + // ask gRPC to deliver the next chunk + if (!cancelled.get()) + call.request(1); + } + } + + @Override + public void onError(Throwable t) { + // client cancelled or errored mid-stream + ctx.closeQuietly(); + } + + @Override + public void onCompleted() { + try { + ctx.flushCommit(true); // commit if not validate_only + if (!cancelled.get()) { + // build summary FROM totals + startedAt + resp.onNext(ctx.summary(totals, startedAt)); + resp.onCompleted(); + } + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("insertStream: " + e.getMessage()).asException()); + } + finally { + ctx.closeQuietly(); + } + } + }; + } + + // --- 3) Bi-di with per-batch ACKs --- + @Override + public StreamObserver insertBidirectional(StreamObserver resp) { + final ServerCallStreamObserver call = (ServerCallStreamObserver) resp; + call.disableAutoInboundFlowControl(); + + // track one active session InsertContext + final java.util.concurrent.atomic.AtomicReference ref = new java.util.concurrent.atomic.AtomicReference<>(); + final java.util.concurrent.atomic.AtomicBoolean cancelled = new java.util.concurrent.atomic.AtomicBoolean(false); + + call.setOnCancelHandler(() -> { + cancelled.set(true); + InsertContext ctx = ref.get(); + if (ctx != null) { + sessionWatermark.remove(ctx.sessionId); + ctx.closeQuietly(); + } + }); + + // start pulling inbound messages + call.request(1); + + return new StreamObserver<>() { + @Override + public void onNext(InsertRequest req) { + if (cancelled.get()) + return; + + switch (req.getMsgCase()) { + case START -> { + // Build context with defaults+client opts; begin tx as needed + InsertOptions opts = defaults(req.getStart().getOptions()); + InsertContext ctx = new InsertContext(opts); + // init per-session metadata + ctx.startedAt = System.currentTimeMillis(); + ctx.totals = new Counts(); + + ref.set(ctx); + sessionWatermark.put(ctx.sessionId, 0L); + + resp.onNext(InsertResponse.newBuilder().setStarted(Started.newBuilder().setSessionId(ctx.sessionId)).build()); + + call.request(1); + } + + case CHUNK -> { + InsertContext ctx = require(ref.get(), "session not started"); + InsertChunk c = req.getChunk(); + + // Idempotent replay guard + long hi = sessionWatermark.getOrDefault(ctx.sessionId, 0L); + if (c.getChunkSeq() <= hi) { + resp.onNext(InsertResponse.newBuilder() + .setBatchAck(BatchAck.newBuilder().setSessionId(ctx.sessionId).setChunkSeq(c.getChunkSeq())).build()); + call.request(1); + return; + } + + // Process rows and accumulate totals + Counts perChunk = insertRows(ctx, c.getRowsList().iterator()); + ctx.totals.add(perChunk); + + sessionWatermark.put(ctx.sessionId, c.getChunkSeq()); + + // Ack this chunk with per-chunk counts + resp.onNext( + InsertResponse.newBuilder() + .setBatchAck(BatchAck.newBuilder().setSessionId(ctx.sessionId).setChunkSeq(c.getChunkSeq()) + .setInserted(perChunk.inserted).setUpdated(perChunk.updated).setIgnored(perChunk.ignored) + .setFailed(perChunk.failed)) + .build()); + + call.request(1); + } + + case COMMIT -> { + InsertContext ctx = require(ref.get(), "session not started"); + try { + ctx.flushCommit(true); + // final committed summary must use TOTALS + startedAt + InsertSummary sum = ctx.summary(ctx.totals, ctx.startedAt); + resp.onNext(InsertResponse.newBuilder().setCommitted(Committed.newBuilder().setSummary(sum)).build()); + resp.onCompleted(); + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("commit: " + e.getMessage()).asException()); + } + finally { + sessionWatermark.remove(ctx.sessionId); + ctx.closeQuietly(); + } + } + + case MSG_NOT_SET -> { + /* ignore */ } + } + } + + @Override + public void onError(Throwable t) { + InsertContext ctx = ref.get(); + if (ctx != null) { + sessionWatermark.remove(ctx.sessionId); + ctx.closeQuietly(); + } + } + + @Override + public void onCompleted() { + // client half-closed without COMMIT -> choose policy (rollback/commit/ignore) + InsertContext ctx = ref.get(); + if (ctx != null) { + try { + ctx.flushCommit(true); + } + catch (Exception ignore) { + } + sessionWatermark.remove(ctx.sessionId); + ctx.closeQuietly(); + } + } + }; + } + + private boolean tryUpsertVertex(InsertContext ctx, com.arcadedb.graph.MutableVertex incoming) { + var keys = ctx.keyCols; + if (keys.isEmpty()) + return false; + + String where = String.join(" AND ", keys.stream().map(k -> k + " = ?").toList()); + Object[] params = keys.stream() + // read key values from the incoming vertex as a document + .map(k -> ((MutableDocument) incoming).get(k)).toArray(); + + try (var rs = ctx.db.query("sql", "SELECT FROM " + ctx.opts.getTargetClass() + " WHERE " + where, params)) { + if (!rs.hasNext()) + return false; + var res = rs.next(); + if (!res.isElement()) + return false; + + // mutate via MutableDocument view (valid for vertex records) + MutableDocument existingDoc = (MutableDocument) res.getElement().get().asDocument(true); + for (String col : ctx.updateCols) { + existingDoc.set(col, ((MutableDocument) incoming).get(col)); + } + existingDoc.save(); + return true; + } + } + + private boolean tryUpsertDocument(InsertContext ctx, MutableDocument incoming) { + var keys = ctx.keyCols; + if (keys.isEmpty()) + return false; + + String where = String.join(" AND ", keys.stream().map(k -> k + " = ?").toList()); + Object[] params = keys.stream().map(incoming::get).toArray(); + + try (var rs = ctx.db.query("sql", "SELECT FROM " + ctx.opts.getTargetClass() + " WHERE " + where, params)) { + if (!rs.hasNext()) + return false; + var res = rs.next(); + if (!res.isElement()) + return false; + + var existing = (MutableDocument) res.getElement().get().asDocument(true); + for (String col : ctx.updateCols) + existing.set(col, incoming.get(col)); + existing.save(); + return true; + } + } + + // ---------- Core insert plumbing ---------- + + private static final class Counts { + long received, inserted, updated, ignored, failed; + final java.util.List errors = new java.util.ArrayList<>(); + + void add(Counts o) { + received += o.received; + inserted += o.inserted; + updated += o.updated; + ignored += o.ignored; + failed += o.failed; + errors.addAll(o.errors); + } + + void err(long rowIndex, String code, String msg, String field) { + failed++; + errors.add(RowError.newBuilder().setRowIndex(rowIndex).setCode(code).setMessage(msg).setField(field).build()); + } + } + + private InsertSummary toSummary(Counts c, long startedAtMs) { + long now = System.currentTimeMillis(); + return InsertSummary.newBuilder().setReceived(c.received).setInserted(c.inserted).setUpdated(c.updated).setIgnored(c.ignored) + .setFailed(c.failed).addAllErrors(c.errors).setStartedAt(ts(startedAtMs)).setFinishedAt(ts(now)).build(); + } + + private Counts insertRows(InsertContext ctx, java.util.Iterator it) { + Counts c = new Counts(); + int inBatch = 0; + + Schema schema = ctx.db.getSchema(); + DocumentType dt = schema.getType(ctx.opts.getTargetClass()); + boolean isVertex = dt instanceof VertexType; + boolean isEdge = dt instanceof EdgeType; + + while (it.hasNext()) { + Record r = it.next(); + c.received++; + ctx.received++; + + try { + if (ctx.opts.getValidateOnly()) + continue; + + if (isVertex) { + MutableVertex v = ctx.db.newVertex(ctx.opts.getTargetClass()); + applyGrpcRecord(v, r); + if (ctx.opts.getConflictMode() == ConflictMode.CONFLICT_UPDATE && tryUpsertVertex(ctx, v)) { + c.updated++; + } + else { + v.save(); + c.inserted++; + } + } + else if (isEdge) { + + String outRid = getStringProp(r, "out"); // lookup helper you already have + String inRid = getStringProp(r, "in"); + + if (outRid == null || inRid == null) { + c.failed++; + c.errors.add(RowError.newBuilder().setRowIndex(ctx.received - 1).setCode("MISSING_ENDPOINTS") + .setMessage("Edge requires 'out' and 'in'").build()); + } + else { + var outV = ctx.db.lookupByRID(new com.arcadedb.database.RID(outRid), true).asVertex(true); + var inV = ctx.db.lookupByRID(new com.arcadedb.database.RID(inRid), true).asVertex(true); + + // Create edge from the OUT vertex + com.arcadedb.graph.MutableEdge e = outV.newEdge(ctx.opts.getTargetClass(), inV); + applyGrpcRecord(e, r); // sets edge properties + e.save(); + c.inserted++; + } + } + else { + MutableDocument d = ctx.db.newDocument(ctx.opts.getTargetClass()); + applyGrpcRecord(d, r); + if (ctx.opts.getConflictMode() == ConflictMode.CONFLICT_UPDATE && tryUpsertDocument(ctx, d)) { + c.updated++; + } + else { + d.save(); + c.inserted++; + } + } + + } + catch (com.arcadedb.exception.DuplicatedKeyException dup) { + switch (ctx.opts.getConflictMode()) { + case CONFLICT_IGNORE -> c.ignored++; + case CONFLICT_ABORT, UNRECOGNIZED -> c.err(ctx.received - 1, "CONFLICT", dup.getMessage(), ""); + case CONFLICT_UPDATE -> c.err(ctx.received - 1, "CONFLICT", dup.getMessage(), ""); + } + } + catch (Exception e) { + c.err(ctx.received - 1, "DB_ERROR", e.getMessage(), ""); + } + + inBatch++; + if (ctx.opts.getTransactionMode() == TransactionMode.PER_BATCH && inBatch >= serverBatchSize(ctx)) { + ctx.flushCommit(false); + inBatch = 0; + } + else if (ctx.opts.getTransactionMode() == TransactionMode.PER_ROW) { + ctx.flushCommit(false); + inBatch = 0; + } + } + return c; + } + + /** + * Safely extract a String property from a gRPC Record. + * + * @param r the gRPC record + * @param key the property key to lookup + * @return the string value, or null if not present + */ + private static String getStringProp(com.arcadedb.server.grpc.Record r, String key) { + if (r == null || key == null || key.isEmpty()) { + return null; + } + // Adjust to your proto structure: + // If you use getPropertiesMap(): + if (r.getPropertiesMap().containsKey(key)) { + var v = r.getPropertiesMap().get(key); + if (v.hasStringValue()) { + return v.getStringValue(); + } + return v.toString(); // fallback to raw string + } + + // If your proto uses getFieldsMap() instead: + // if (r.getFieldsMap().containsKey(key)) { + // var v = r.getFieldsMap().get(key); + // if (v.hasStringValue()) { + // return v.getStringValue(); + // } + // return v.toString(); + // } + + return null; + } + + private int serverBatchSize(InsertContext ctx) { + return ctx.opts.getServerBatchSize() == 0 ? 1000 : ctx.opts.getServerBatchSize(); + } + + private void applyGrpcRecord(MutableDocument doc, com.arcadedb.server.grpc.Record r) { + r.getPropertiesMap().forEach((k, val) -> doc.set(k, toJava(val))); + } + + private void applyGrpcRecord(MutableVertex vertex, com.arcadedb.server.grpc.Record r) { + r.getPropertiesMap().forEach((k, val) -> vertex.set(k, toJava(val))); + } + + private void applyGrpcRecord(MutableEdge edge, com.arcadedb.server.grpc.Record r) { + r.getPropertiesMap().forEach((k, val) -> edge.set(k, toJava(val))); + } + + private void applyGrpcRecordToDocument(Record r, MutableDocument doc) { + // Example if proto = message Record { map + // properties = 1; } + if (hasMethod(r, "getPropertiesMap")) { + @SuppressWarnings("unchecked") + Map props = (Map) invokeNoArg(r, "getPropertiesMap"); + props.forEach((k, v) -> doc.set(k, toJava(v))); + return; + } + + // Example if proto = message Record { repeated Property properties = 1; message + // Property { string key=1; google.protobuf.Value value=2; } } + if (hasMethod(r, "getPropertiesList")) { + @SuppressWarnings("unchecked") + List list = (List) invokeNoArg(r, "getPropertiesList"); + for (Object p : list) { + String key = (String) invokeNoArg(p, "getKey"); + com.google.protobuf.Value val = (com.google.protobuf.Value) invokeNoArg(p, "getValue"); + doc.set(key, toJava(val)); + } + return; + } + + // Fallback: if your proto DOES have fields map after all + if (hasMethod(r, "getFieldsMap")) { + @SuppressWarnings("unchecked") + Map fields = (Map) invokeNoArg(r, "getFieldsMap"); + fields.forEach((k, v) -> doc.set(k, toJava(v))); + } + } + + // tiny reflection helpers (compile-time safe across proto variants) + private static boolean hasMethod(Object o, String name) { + try { + o.getClass().getMethod(name); + return true; + } + catch (NoSuchMethodException e) { + return false; + } + } + + private static Object invokeNoArg(Object o, String name) { + try { + return o.getClass().getMethod(name).invoke(o); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + private boolean tryUpsert(InsertContext ctx, MutableDocument doc) { + + if (ctx.keyCols.isEmpty()) + return false; + + String where = String.join(" AND ", ctx.keyCols.stream().map(k -> k + " = ?").toList()); + + Object[] params = ctx.keyCols.stream().map(doc::get).toArray(); + + try (ResultSet rs = ctx.db.query("sql", "SELECT FROM " + ctx.opts.getTargetClass() + " WHERE " + where, params)) { + + if (!rs.hasNext()) + return false; + + var res = rs.next(); + + if (!res.isElement()) + return false; + + var existing = res.getElement().get(); + + MutableDocument m = (MutableDocument) existing.asDocument(true); + + for (String col : ctx.updateCols) { + m.set(col, doc.get(col)); + } + + m.save(); + + return true; + } + } + + private static Object toJava(Value v) { + if (v == null) return null; + switch (v.getKindCase()) { + case STRING_VALUE: return v.getStringValue(); + case NUMBER_VALUE: return v.getNumberValue(); // Double + case BOOL_VALUE: return v.getBoolValue(); + case NULL_VALUE: return null; + + case STRUCT_VALUE: { + // Convert each nested Value -> Java recursively + java.util.LinkedHashMap m = new java.util.LinkedHashMap<>(); + v.getStructValue().getFieldsMap().forEach((k, vv) -> m.put(k, toJava(vv))); + return m; // plain Map + } + + case LIST_VALUE: { + java.util.ArrayList list = new java.util.ArrayList<>(v.getListValue().getValuesCount()); + for (com.google.protobuf.Value item : v.getListValue().getValuesList()) { + list.add(toJava(item)); + } + return list; + } + + case KIND_NOT_SET: + default: return null; + } + } + + private InsertOptions defaults(InsertOptions in) { + InsertOptions.Builder b = in.toBuilder(); + if (in.getServerBatchSize() == 0) + b.setServerBatchSize(1000); + if (in.getTransactionMode() == TransactionMode.UNRECOGNIZED) + b.setTransactionMode(TransactionMode.PER_BATCH); + return b.build(); + } + + private static T require(T v, String msg) { + if (v == null) + throw Status.FAILED_PRECONDITION.withDescription(msg).asRuntimeException(); + return v; + } + + private static Timestamp ts(long ms) { + return Timestamp.newBuilder().setSeconds(ms / 1000).setNanos((int) ((ms % 1000) * 1_000_000)).build(); + } + + private final class InsertContext implements AutoCloseable { + protected Counts totals; + final InsertOptions opts; + final Database db; + final java.util.List keyCols; + final java.util.List updateCols; + long startedAt; + + final String sessionId = java.util.UUID.randomUUID().toString(); + long received = 0; + + InsertContext(InsertOptions opts) { + this.opts = opts; + this.db = getDatabase(opts.getDatabase(), opts.getCredentials()); + this.keyCols = opts.getKeyColumnsList(); + this.updateCols = opts.getUpdateColumnsOnConflictList(); + + if (!opts.getValidateOnly()) { + if (opts.getTransactionMode() == TransactionMode.PER_STREAM || opts.getTransactionMode() == TransactionMode.PER_BATCH) { + db.begin(); + } + } + } + + void flushCommit(boolean end) { + if (opts.getValidateOnly()) { + if (end) + db.rollback(); + return; + } + switch (opts.getTransactionMode()) { + case PER_ROW -> db.commit(); + case PER_BATCH -> { + db.commit(); + if (!end) + db.begin(); + } + case PER_STREAM -> { + if (end) + db.commit(); + } + default -> { + } + } + } + + InsertSummary summary(Counts c, long startedAtMs) { + long now = System.currentTimeMillis(); + return InsertSummary.newBuilder().setReceived(c.received).setInserted(c.inserted).setUpdated(c.updated).setIgnored(c.ignored) + .setFailed(c.failed).addAllErrors(c.errors).setStartedAt(ts(startedAtMs)).setFinishedAt(ts(now)).build(); + } + + @Override + public void close() { + } + + void closeQuietly() { + try { + close(); + } + catch (Exception ignore) { + } + } + } + + @Override + public void ping(Empty request, StreamObserver responseObserver) { + PingResponse response = PingResponse.newBuilder().setTimestamp(System.currentTimeMillis()).setMessage("pong").build(); + + responseObserver.onNext(response); + responseObserver.onCompleted(); + } + + @Override + public void getServerStatus(Empty request, StreamObserver responseObserver) { + try { + ServerStatusResponse response = ServerStatusResponse.newBuilder().setVersion("25.8.1-SNAPSHOT").setStatus("RUNNING") + .setUptimeMs(getUptime()).setActiveConnections(getActiveConnections()) + .putMetrics("active_transactions", String.valueOf(activeTransactions.size())).build(); + + responseObserver.onNext(response); + responseObserver.onCompleted(); + + } + catch (Exception e) { + logger.error("Error getting server status: {}", e.getMessage(), e); + responseObserver.onError(Status.INTERNAL.withDescription("Failed to get server status: " + e.getMessage()).asException()); + } + } + + // --- TX helpers -------------------------------------------------------------- + + private static final class TxScope { + final boolean beganHere; + final String txId; + + TxScope(boolean beganHere, String txId) { + this.beganHere = beganHere; + this.txId = txId; + } + } + + /** + * Begin a transaction if requested by TransactionContext. Policy: - If tx.begin + * == true: begin() and mark beganHere=true (we’re not doing per-tx registry in + * this patch). - If tx.timeout_ms > 0 or tx.read_only set: optionally apply + * hints (no-ops here unless your DB supports). Returns a TxScope to be passed + * to endTx(). + */ + private TxScope beginTx(Database db, com.arcadedb.server.grpc.TransactionContext tx) { + if (tx == null) + return new TxScope(false, null); + boolean begin = tx.getBegin(); + String txId = tx.getTransactionId().isEmpty() ? null : tx.getTransactionId(); + + if (begin) { + db.begin(); // You can add isolation/RO if your ArcadeDB build supports it + return new TxScope(true, txId); + } + return new TxScope(false, txId); + } + + /** + * End the transaction according to commit/rollback flags. Precedence: rollback + * > commit > (if we beganHere and neither flag set) commit() + */ + private void endTx(Database db, com.arcadedb.server.grpc.TransactionContext tx, TxScope scope) { + if (tx == null) + return; + try { + if (tx.getRollback()) { + db.rollback(); + return; + } + if (tx.getCommit()) { + db.commit(); + return; + } + if (scope.beganHere) { + // Policy for “begin only”: commit by default (you can change to leave-open if + // you add a tx registry) + db.commit(); + } + } + catch (Exception ignore) { + // swallow – upstream handler will report the original RPC result + } + } + + // Helper methods + + private Database getDatabase(String databaseName, DatabaseCredentials credentials) { + + // Validate credentials + validateCredentials(credentials); + + // Use the same approach as Postgres/Redis plugins + if (arcadeServer != null) { + + // This is how other plugins do it - get the already-open database + Database db = arcadeServer.getDatabase(databaseName); + + logger.debug("getDatabase(): db = {} isOpen = {}", db, db.isOpen()); + + if (db != null) { + return db; + } + } + + // Check if database is already in the pool + String poolKey = databaseName; + Database database = databasePool.get(poolKey); + + if (database != null && database.isOpen()) { + // Return existing open database + return database; + } + + // Create new database connection + synchronized (databasePool) { + // Double-check after acquiring lock + database = databasePool.get(poolKey); + if (database != null && database.isOpen()) { + return database; + } + + // Create database factory for the specific database + DatabaseFactory dbFactory = new DatabaseFactory(databasePath + "/" + databaseName); + + try { + // Open database - ArcadeDB requires MODE parameter + if (dbFactory.exists()) { + // Try READ_ONLY first to avoid conflicts + try { + database = dbFactory.open(ComponentFile.MODE.READ_ONLY); + } + catch (Exception e) { + // If READ_ONLY fails, try READ_WRITE + logger.debug("Opening database in READ_WRITE mode: {}", databaseName); + database = dbFactory.open(ComponentFile.MODE.READ_WRITE); + } + } + else { + // Create if it doesn't exist + database = dbFactory.create(); + } + + // Add to pool + databasePool.put(poolKey, database); + return database; + + } + catch (Exception e) { + logger.error("Failed to open database: {}", databaseName, e); + throw new RuntimeException("Cannot open database: " + databaseName + " - " + e.getMessage(), e); + } + } + } + + private void validateCredentials(DatabaseCredentials credentials) { + // Implement credential validation logic + // This is a placeholder - integrate with ArcadeDB's security system + if (credentials == null || credentials.getUsername().isEmpty()) { + throw new IllegalArgumentException("Invalid credentials"); + } + } + + private Map convertParameters(Map protoParams) { + Map params = new HashMap<>(); + for (Map.Entry entry : protoParams.entrySet()) { + params.put(entry.getKey(), convertFromProtobufValue(entry.getValue())); + } + return params; + } + + private Object convertFromProtobufValue(Value value) { + switch (value.getKindCase()) { + case NULL_VALUE: + return null; + case NUMBER_VALUE: + return value.getNumberValue(); + case STRING_VALUE: + return value.getStringValue(); + case BOOL_VALUE: + return value.getBoolValue(); + case STRUCT_VALUE: + return convertFromProtobufValues(value.getStructValue().getFieldsMap()); + case LIST_VALUE: + return value.getListValue().getValuesList().stream().map(this::convertFromProtobufValue).toArray(); + default: + return null; + } + } + + private Map convertFromProtobufValues(Map values) { + Map result = new HashMap<>(); + for (Map.Entry entry : values.entrySet()) { + result.put(entry.getKey(), convertFromProtobufValue(entry.getValue())); + } + return result; + } + + private Record convertToGrpcRecord(com.arcadedb.database.Record dbRecord) { + Record.Builder builder = Record.newBuilder().setRid(dbRecord.getIdentity().toString()); + + // Handle different record types + if (dbRecord instanceof Document) { + Document doc = (Document) dbRecord; + builder.setType(doc.getTypeName()); + + // Convert properties + Set properties = doc.getPropertyNames(); + for (String property : properties) { + Object value = doc.get(property); + if (value != null) { + builder.putProperties(property, convertToProtobufValue(value)); + } + } + } + else if (dbRecord instanceof Vertex) { + Vertex vertex = (Vertex) dbRecord; + builder.setType(vertex.getTypeName()); + + // Convert properties + Set properties = vertex.getPropertyNames(); + for (String property : properties) { + Object value = vertex.get(property); + if (value != null) { + builder.putProperties(property, convertToProtobufValue(value)); + } + } + } + else if (dbRecord instanceof Edge) { + Edge edge = (Edge) dbRecord; + builder.setType(edge.getTypeName()); + + // Convert properties + Set properties = edge.getPropertyNames(); + for (String property : properties) { + Object value = edge.get(property); + if (value != null) { + builder.putProperties(property, convertToProtobufValue(value)); + } + } + } + + return builder.build(); + } + + private Value convertToProtobufValue(Object value) { + if (value == null) { + return Value.newBuilder().setNullValue(com.google.protobuf.NullValue.NULL_VALUE).build(); + } + else if (value instanceof Boolean) { + return Value.newBuilder().setBoolValue((Boolean) value).build(); + } + else if (value instanceof Number) { + return Value.newBuilder().setNumberValue(((Number) value).doubleValue()).build(); + } + else if (value instanceof String) { + return Value.newBuilder().setStringValue((String) value).build(); + } + else { + // For complex objects, convert to string + return Value.newBuilder().setStringValue(value.toString()).build(); + } + } + + // Proto Value builder from Java (mirror of toJava) + private com.google.protobuf.Value toProtoValue(Object o) { + com.google.protobuf.Value.Builder b = com.google.protobuf.Value.newBuilder(); + if (o == null) + return b.setNullValue(com.google.protobuf.NullValue.NULL_VALUE).build(); + if (o instanceof String s) + return b.setStringValue(s).build(); + if (o instanceof Number n) + return b.setNumberValue(n.doubleValue()).build(); + if (o instanceof Boolean bo) + return b.setBoolValue(bo).build(); + if (o instanceof java.util.Map m) { + var sb = com.google.protobuf.Struct.newBuilder(); + m.forEach((k, v) -> sb.putFields(String.valueOf(k), toProtoValue(v))); + return com.google.protobuf.Value.newBuilder().setStructValue(sb).build(); + } + if (o instanceof java.util.List list) { + var lb = com.google.protobuf.ListValue.newBuilder(); + list.forEach(v -> lb.addValues(toProtoValue(v))); + return com.google.protobuf.Value.newBuilder().setListValue(lb).build(); + } + return b.setStringValue(String.valueOf(o)).build(); + } + + // Apply properties from proto Record to a document/vertex/edge, schema-aware + // for EMBEDDED + private void applyGrpcRecord(MutableDocument d, com.arcadedb.server.grpc.Record r, Database db, String targetClass) { + DocumentType dt = db.getSchema().getType(targetClass); + r.getPropertiesMap().forEach((k, val) -> d.set(k, toJavaForProperty(db, dt, k, val))); + } + + private void applyGrpcRecord(MutableVertex v, com.arcadedb.server.grpc.Record r, Database db) { + DocumentType dt = db.getSchema().getType(v.getTypeName()); + r.getPropertiesMap().forEach((k, val) -> v.set(k, toJavaForProperty(db, dt, k, val))); + } + + private void applyGrpcRecord(MutableEdge e, com.arcadedb.server.grpc.Record r, Database db, String edgeClass) { + DocumentType dt = db.getSchema().getType(edgeClass); + r.getPropertiesMap().forEach((k, val) -> e.set(k, toJavaForProperty(db, dt, k, val))); + } + + // If property type is EMBEDDED and client sent a Struct/Map, create embedded + // document + private Object toJavaForProperty(Database db, DocumentType dt, String key, com.google.protobuf.Value v) { + + var p = (dt != null) ? dt.getPropertyIfExists(key) : null; + + if (p != null && p.getType() == com.arcadedb.schema.Type.EMBEDDED && v.hasStructValue()) { + MutableDocument emb = db.newDocument(dt.getName()); + v.getStructValue().getFieldsMap().forEach((k2, vv) -> emb.set(k2, toJava(vv))); + return emb; + } + return toJava(v); + } + + private String generateTransactionId() { + return "tx_" + System.nanoTime(); + } + + private long getUptime() { + // Implement uptime calculation + return System.currentTimeMillis(); + } + + private int getActiveConnections() { + // Implement active connections count + return 0; + } + +} \ No newline at end of file diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/CompressionAwareService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/CompressionAwareService.java new file mode 100644 index 0000000000..860501d231 --- /dev/null +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/CompressionAwareService.java @@ -0,0 +1,86 @@ +package com.arcadedb.server.grpc; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.grpc.stub.ServerCallStreamObserver; +import io.grpc.stub.StreamObserver; + + +/** + * Enhanced compression-aware service wrapper + */ +class CompressionAwareService { + private static final Logger logger = LoggerFactory.getLogger(CompressionAwareService.class); + + /** + * Force compression for a specific response + */ + public static void setResponseCompression(StreamObserver responseObserver, String compression) { + if (responseObserver instanceof ServerCallStreamObserver) { + ServerCallStreamObserver serverObserver = (ServerCallStreamObserver) responseObserver; + serverObserver.setCompression(compression); + logger.debug("Set response compression to: {}", compression); + } + } + + /** + * Check if current request was compressed (call from service method) + */ + public static boolean isCurrentRequestCompressed() { + GrpcCompressionInterceptor.CompressionInfo info = + GrpcCompressionInterceptor.COMPRESSION_KEY.get(); + return info != null && info.requestCompressed; + } + + /** + * Get current request compression encoding + */ + public static String getCurrentRequestEncoding() { + GrpcCompressionInterceptor.CompressionInfo info = + GrpcCompressionInterceptor.COMPRESSION_KEY.get(); + return info != null ? info.requestEncoding : "identity"; + } + + /** + * Get compression statistics for monitoring + */ + public static class CompressionStats { + private long compressedRequests = 0; + private long uncompressedRequests = 0; + private long compressedResponses = 0; + private long uncompressedResponses = 0; + + public synchronized void recordRequest(boolean compressed) { + if (compressed) { + compressedRequests++; + } else { + uncompressedRequests++; + } + } + + public synchronized void recordResponse(boolean compressed) { + if (compressed) { + compressedResponses++; + } else { + uncompressedResponses++; + } + } + + public synchronized String getStats() { + long totalRequests = compressedRequests + uncompressedRequests; + long totalResponses = compressedResponses + uncompressedResponses; + + double reqCompressionRate = totalRequests > 0 ? + (compressedRequests * 100.0 / totalRequests) : 0; + double respCompressionRate = totalResponses > 0 ? + (compressedResponses * 100.0 / totalResponses) : 0; + + return String.format( + "Compression Stats - Requests: %.1f%% (%d/%d), Responses: %.1f%% (%d/%d)", + reqCompressionRate, compressedRequests, totalRequests, + respCompressionRate, compressedResponses, totalResponses + ); + } + } +} \ No newline at end of file diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcAuthInterceptor.java b/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcAuthInterceptor.java new file mode 100644 index 0000000000..06274dcab1 --- /dev/null +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcAuthInterceptor.java @@ -0,0 +1,142 @@ +package com.arcadedb.server.grpc; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.arcadedb.server.security.ServerSecurity; + +import io.grpc.Context; +import io.grpc.Contexts; +import io.grpc.Metadata; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; +import io.grpc.Status; + +/** + * Authentication interceptor for gRPC requests + */ +class GrpcAuthInterceptor implements ServerInterceptor { + private static final Logger logger = LoggerFactory.getLogger(GrpcAuthInterceptor.class); + + private static final String BEARER_TYPE = "Bearer"; + private static final Metadata.Key AUTHORIZATION_HEADER = + Metadata.Key.of("authorization", Metadata.ASCII_STRING_MARSHALLER); + private static final Metadata.Key USER_HEADER = + Metadata.Key.of("x-arcade-user", Metadata.ASCII_STRING_MARSHALLER); + private static final Metadata.Key PASSWORD_HEADER = + Metadata.Key.of("x-arcade-password", Metadata.ASCII_STRING_MARSHALLER); + private static final Metadata.Key DATABASE_HEADER = + Metadata.Key.of("x-arcade-database", Metadata.ASCII_STRING_MARSHALLER); + + private final ServerSecurity security; + private final boolean securityEnabled; + + public GrpcAuthInterceptor(ServerSecurity security) { + this.security = security; + // Check if security is enabled by checking if it's not null and has users configured + this.securityEnabled = (security != null && security.getUsers() != null && !security.getUsers().isEmpty()); + } + + @Override + public ServerCall.Listener interceptCall( + ServerCall call, + Metadata headers, + ServerCallHandler next) { + + String methodName = call.getMethodDescriptor().getFullMethodName(); + + // Skip auth for health check and reflection + if (methodName.startsWith("grpc.health.") || + methodName.startsWith("grpc.reflection.")) { + return next.startCall(call, headers); + } + + // If security is not enabled, allow all requests + if (!securityEnabled) { + return next.startCall(call, headers); + } + + try { + // Get database name from header (required for authentication) + String database = headers.get(DATABASE_HEADER); + if (database == null || database.isEmpty()) { + database = "default"; // Use default database if not specified + } + + // Try Bearer token authentication first + String authorization = headers.get(AUTHORIZATION_HEADER); + if (authorization != null && authorization.startsWith(BEARER_TYPE)) { + String token = authorization.substring(BEARER_TYPE.length()).trim(); + if (!validateToken(token, database)) { + call.close(Status.UNAUTHENTICATED.withDescription("Invalid token"), new Metadata()); + return new ServerCall.Listener() {}; + } + } else { + // Try basic authentication + String username = headers.get(USER_HEADER); + String password = headers.get(PASSWORD_HEADER); + + if (username == null || password == null) { + // No authentication provided for secured server + call.close(Status.UNAUTHENTICATED.withDescription("Authentication required"), new Metadata()); + return new ServerCall.Listener() {}; + } else { + // Validate credentials + if (!validateCredentials(username, password, database)) { + call.close(Status.UNAUTHENTICATED.withDescription("Invalid credentials"), new Metadata()); + return new ServerCall.Listener() {}; + } + // Add user to context + Context context = Context.current().withValue(USER_CONTEXT_KEY, username); + return Contexts.interceptCall(context, call, headers, next); + } + } + + // Add user context for downstream processing + Context context = Context.current(); + if (authorization != null) { + context = context.withValue(USER_CONTEXT_KEY, extractUserFromToken(authorization)); + } + + return Contexts.interceptCall(context, call, headers, next); + + } catch (Exception e) { + logger.error("Authentication error", e); + call.close(Status.INTERNAL.withDescription("Authentication error"), new Metadata()); + return new ServerCall.Listener() {}; + } + } + + private boolean validateToken(String token, String database) { + // Implement token validation logic + // This could integrate with JWT, OAuth2, or custom token validation + // For now, this is a placeholder + return true; + } + + private boolean validateCredentials(String username, String password, String database) { + if (security == null) { + return true; // No security configured + } + + try { + // ArcadeDB's authenticate method requires database name as well + // Returns a SecurityUser object if authentication succeeds, null otherwise + Object authenticatedUser = security.authenticate(username, password, database); + return authenticatedUser != null; + } catch (Exception e) { + logger.error("Failed to authenticate user: {} for database: {}", username, database, e); + return false; + } + } + + private String extractUserFromToken(String authorization) { + // Extract user information from token + // This is a placeholder - implement based on your token format + return "authenticated-user"; + } + + // Context key for storing authenticated user + public static final Context.Key USER_CONTEXT_KEY = Context.key("user"); +} \ No newline at end of file diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcCompressionInterceptor.java b/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcCompressionInterceptor.java new file mode 100644 index 0000000000..c1221db2f9 --- /dev/null +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcCompressionInterceptor.java @@ -0,0 +1,83 @@ +package com.arcadedb.server.grpc; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.grpc.Context; +import io.grpc.Contexts; +import io.grpc.ForwardingServerCall; +import io.grpc.Metadata; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; + +/** + * Compression interceptor that can force compression based on configuration + */ +class GrpcCompressionInterceptor implements ServerInterceptor { + private static final Logger logger = LoggerFactory.getLogger(GrpcCompressionInterceptor.class); + + // Context key to store compression info + public static final Context.Key COMPRESSION_KEY = Context.key("compression-info"); + + private final boolean forceCompression; + private final String compressionType; + private final int minMessageSizeForCompression; + + public GrpcCompressionInterceptor(boolean forceCompression, String compressionType, int minMessageSizeBytes) { + this.forceCompression = forceCompression; + this.compressionType = compressionType != null ? compressionType : "gzip"; + this.minMessageSizeForCompression = minMessageSizeBytes; + } + + @Override + public ServerCall.Listener interceptCall( + ServerCall call, + Metadata headers, + ServerCallHandler next) { + + String methodName = call.getMethodDescriptor().getFullMethodName(); + + // Check if request is compressed + String requestEncoding = headers.get(Metadata.Key.of("grpc-encoding", Metadata.ASCII_STRING_MARSHALLER)); + boolean requestCompressed = requestEncoding != null && !requestEncoding.equals("identity"); + + // Store compression info in context + CompressionInfo compressionInfo = new CompressionInfo(requestCompressed, requestEncoding); + Context context = Context.current().withValue(COMPRESSION_KEY, compressionInfo); + + // Wrap the call to control compression + ServerCall compressedCall = new ForwardingServerCall.SimpleForwardingServerCall(call) { + private boolean compressionSet = false; + + @Override + public void sendMessage(RespT message) { + if (!compressionSet && forceCompression) { + // Force compression for this response + if (this.getMethodDescriptor().getType().serverSendsOneMessage()) { + // For unary calls, we can set compression + setMessageCompression(true); + compressionSet = true; + logger.debug("Forced {} compression for method: {}", compressionType, methodName); + } + } + super.sendMessage(message); + } + }; + + return Contexts.interceptCall(context, compressedCall, headers, next); + } + + /** + * Helper class to store compression information + */ + public static class CompressionInfo { + public final boolean requestCompressed; + public final String requestEncoding; + + public CompressionInfo(boolean requestCompressed, String requestEncoding) { + this.requestCompressed = requestCompressed; + this.requestEncoding = requestEncoding; + } + } +} \ No newline at end of file diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcLoggingInterceptor.java b/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcLoggingInterceptor.java new file mode 100644 index 0000000000..1dcdc0145f --- /dev/null +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcLoggingInterceptor.java @@ -0,0 +1,81 @@ +package com.arcadedb.server.grpc; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.grpc.ForwardingServerCall; +import io.grpc.Metadata; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; +import io.grpc.Status; + +/** + * Logging interceptor for gRPC requests with compression detection + */ +class GrpcLoggingInterceptor implements ServerInterceptor { + private static final Logger logger = LoggerFactory.getLogger(GrpcLoggingInterceptor.class); + + private static final Metadata.Key GRPC_ENCODING_KEY = + Metadata.Key.of("grpc-encoding", Metadata.ASCII_STRING_MARSHALLER); + private static final Metadata.Key GRPC_ACCEPT_ENCODING_KEY = + Metadata.Key.of("grpc-accept-encoding", Metadata.ASCII_STRING_MARSHALLER); + + @Override + public ServerCall.Listener interceptCall( + ServerCall call, + Metadata headers, + ServerCallHandler next) { + + String methodName = call.getMethodDescriptor().getFullMethodName(); + long startTime = System.currentTimeMillis(); + + // Check if client sent compressed request + String requestEncoding = headers.get(GRPC_ENCODING_KEY); + String acceptEncoding = headers.get(GRPC_ACCEPT_ENCODING_KEY); + boolean requestCompressed = requestEncoding != null && !requestEncoding.equals("identity"); + boolean clientAcceptsCompression = acceptEncoding != null && acceptEncoding.contains("gzip"); + + logger.debug("gRPC call started: {} (request compression: {}, client accepts: {})", + methodName, + requestCompressed ? requestEncoding : "none", + clientAcceptsCompression ? acceptEncoding : "none"); + + ServerCall wrappedCall = new ForwardingServerCall.SimpleForwardingServerCall(call) { + private String responseCompression = "none"; + + @Override + public void sendHeaders(Metadata headers) { + // Check if we're sending compressed response + String encoding = headers.get(GRPC_ENCODING_KEY); + if (encoding != null && !encoding.equals("identity")) { + responseCompression = encoding; + } + super.sendHeaders(headers); + } + + @Override + public void close(Status status, Metadata trailers) { + long duration = System.currentTimeMillis() - startTime; + + // Add compression info to trailers for client visibility + trailers.put(Metadata.Key.of("x-grpc-compression-used", Metadata.ASCII_STRING_MARSHALLER), + responseCompression); + trailers.put(Metadata.Key.of("x-grpc-request-compressed", Metadata.ASCII_STRING_MARSHALLER), + String.valueOf(requestCompressed)); + + if (status.isOk()) { + logger.debug("gRPC call completed: {} ({}ms, req-compression: {}, resp-compression: {})", + methodName, duration, + requestCompressed ? requestEncoding : "none", + responseCompression); + } else { + logger.warn("gRPC call failed: {} - {} ({}ms)", methodName, status, duration); + } + super.close(status, trailers); + } + }; + + return next.startCall(wrappedCall, headers); + } +} \ No newline at end of file diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcMetricsInterceptor.java b/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcMetricsInterceptor.java new file mode 100644 index 0000000000..100a1ee665 --- /dev/null +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcMetricsInterceptor.java @@ -0,0 +1,78 @@ +package com.arcadedb.server.grpc; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.arcadedb.server.ArcadeDBServer; + +import io.grpc.ForwardingServerCall; +import io.grpc.Metadata; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; +import io.grpc.Status; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; + +/** + * Metrics interceptor for gRPC requests using Micrometer + */ +class GrpcMetricsInterceptor implements ServerInterceptor { + + private static final Logger logger = LoggerFactory.getLogger(GrpcMetricsInterceptor.class); + + private final MeterRegistry meterRegistry; + private final Counter requestCounter; + private final Counter errorCounter; + private final Timer requestTimer; + + public GrpcMetricsInterceptor(ArcadeDBServer server) { + // Try to get existing meter registry from server, or create a simple one + this.meterRegistry = new SimpleMeterRegistry(); // In production, integrate with server's meter registry + + this.requestCounter = Counter.builder("grpc.requests.total") + .description("Total number of gRPC requests") + .register(meterRegistry); + + this.errorCounter = Counter.builder("grpc.errors.total") + .description("Total number of gRPC errors") + .register(meterRegistry); + + this.requestTimer = Timer.builder("grpc.request.duration") + .description("gRPC request duration") + .register(meterRegistry); + } + + @Override + public ServerCall.Listener interceptCall( + ServerCall call, + Metadata headers, + ServerCallHandler next) { + + String methodName = call.getMethodDescriptor().getFullMethodName(); + Timer.Sample sample = Timer.start(meterRegistry); + + requestCounter.increment(); + + ServerCall wrappedCall = new ForwardingServerCall.SimpleForwardingServerCall(call) { + @Override + public void close(Status status, Metadata trailers) { + sample.stop(requestTimer); + + if (!status.isOk()) { + errorCounter.increment(); + } + + // Add metrics as trailers for observability + trailers.put(Metadata.Key.of("grpc-metrics-method", Metadata.ASCII_STRING_MARSHALLER), methodName); + trailers.put(Metadata.Key.of("grpc-metrics-status", Metadata.ASCII_STRING_MARSHALLER), status.getCode().toString()); + + super.close(status, trailers); + } + }; + + return next.startCall(wrappedCall, headers); + } +} \ No newline at end of file diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcServerPlugin.java b/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcServerPlugin.java new file mode 100644 index 0000000000..b180acba34 --- /dev/null +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcServerPlugin.java @@ -0,0 +1,386 @@ +package com.arcadedb.server.grpc; + +import java.io.File; +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.arcadedb.ContextConfiguration; +import com.arcadedb.server.ArcadeDBServer; +import com.arcadedb.server.ServerPlugin; +import com.arcadedb.server.security.ServerSecurity; + +import io.grpc.CompressorRegistry; +import io.grpc.DecompressorRegistry; +import io.grpc.Grpc; +import io.grpc.InsecureServerCredentials; +import io.grpc.Server; +import io.grpc.ServerBuilder; +import io.grpc.ServerCredentials; +import io.grpc.TlsServerCredentials; +import io.grpc.health.v1.HealthCheckResponse; +import io.grpc.protobuf.services.HealthStatusManager; +import io.grpc.protobuf.services.ProtoReflectionService; +import io.grpc.xds.XdsServerBuilder; + +/** + * ArcadeDB gRPC Server Plugin + * + * Configuration options: + * - grpc.enabled: Enable/disable gRPC server (default: true) + * - grpc.port: Port for standard gRPC server (default: 50051) + * - grpc.host: Host to bind (default: 0.0.0.0) + * - grpc.mode: Server mode - "standard", "xds", or "both" (default: standard) + * - grpc.xds.port: Port for XDS server (default: 50052) + * - grpc.tls.enabled: Enable TLS (default: false) + * - grpc.tls.cert: Path to TLS certificate + * - grpc.tls.key: Path to TLS private key + * - grpc.maxMessageSize: Max message size in MB (default: 100) + * - grpc.reflection.enabled: Enable gRPC reflection (default: true) + * - grpc.health.enabled: Enable health checking (default: true) + */ +public class GrpcServerPlugin implements ServerPlugin { + + private static final Logger logger = LoggerFactory.getLogger(GrpcServerPlugin.class); + + private ArcadeDBServer arcadeServer; + private Server grpcServer; + private Server xdsServer; + private HealthStatusManager healthManager; + private ArcadeDbGrpcService grpcService; // Keep reference for cleanup + private Thread shutdownHook; + + // Configuration keys as simple strings + private static final String CONFIG_PREFIX = "arcadedb.grpc."; + private static final String CONFIG_ENABLED = CONFIG_PREFIX + "enabled"; + private static final String CONFIG_PORT = CONFIG_PREFIX + "port"; + private static final String CONFIG_HOST = CONFIG_PREFIX + "host"; + private static final String CONFIG_MODE = CONFIG_PREFIX + "mode"; + private static final String CONFIG_XDS_PORT = CONFIG_PREFIX + "xds.port"; + private static final String CONFIG_TLS_ENABLED = CONFIG_PREFIX + "tls.enabled"; + private static final String CONFIG_TLS_CERT = CONFIG_PREFIX + "tls.cert"; + private static final String CONFIG_TLS_KEY = CONFIG_PREFIX + "tls.key"; + private static final String CONFIG_MAX_MESSAGE_SIZE = CONFIG_PREFIX + "maxMessageSize"; + private static final String CONFIG_REFLECTION_ENABLED = CONFIG_PREFIX + "reflection.enabled"; + private static final String CONFIG_HEALTH_ENABLED = CONFIG_PREFIX + "health.enabled"; + private static final String CONFIG_COMPRESSION_ENABLED = CONFIG_PREFIX + "compression.enabled"; + private static final String CONFIG_COMPRESSION_FORCE = CONFIG_PREFIX + "compression.force"; + private static final String CONFIG_COMPRESSION_TYPE = CONFIG_PREFIX + "compression.type"; + + @Override + public void configure(ArcadeDBServer server, ContextConfiguration configuration) { + this.arcadeServer = server; + } + + @Override + public void startService() { + ContextConfiguration config = arcadeServer.getConfiguration(); + + // Get configuration values with defaults + boolean enabled = getConfigBoolean(config, CONFIG_ENABLED, true); + if (!enabled) { + logger.info("gRPC server is disabled"); + return; + } + + String mode = getConfigString(config, CONFIG_MODE, "standard").toLowerCase(); + + try { + switch (mode) { + case "standard": + startStandardServer(config); + break; + case "xds": + startXdsServer(config); + break; + case "both": + startStandardServer(config); + startXdsServer(config); + break; + default: + logger.error("Invalid gRPC mode: {}. Use 'standard', 'xds', or 'both'", mode); + } + + registerShutdownHook(); + + } catch (IOException e) { + logger.error("Failed to start gRPC server", e); + throw new RuntimeException("Failed to start gRPC server", e); + } + } + + private void startStandardServer(ContextConfiguration config) throws IOException { + int port = getConfigInt(config, CONFIG_PORT, 50051); + String host = getConfigString(config, CONFIG_HOST, "0.0.0.0"); + + ServerBuilder serverBuilder; + + // Configure TLS if enabled + if (getConfigBoolean(config, CONFIG_TLS_ENABLED, false)) { + serverBuilder = configureStandardTls(port, config); + } else { + serverBuilder = ServerBuilder.forPort(port); + } + + // Configure the server + configureServer(serverBuilder, config); + + grpcServer = serverBuilder.build().start(); + + // Build status message + StringBuilder status = new StringBuilder(); + status.append("gRPC server started on ").append(host).append(":").append(port); + status.append(" (mode: standard"); + + if (getConfigBoolean(config, CONFIG_TLS_ENABLED, false)) { + status.append(", TLS enabled"); + } + + if (getConfigBoolean(config, CONFIG_COMPRESSION_ENABLED, true)) { + status.append(", compression: "); + if (getConfigBoolean(config, CONFIG_COMPRESSION_FORCE, false)) { + status.append("forced-").append(getConfigString(config, CONFIG_COMPRESSION_TYPE, "gzip")); + } else { + status.append("available"); + } + } + + status.append(")"); + logger.info(status.toString()); + } + + private void startXdsServer(ContextConfiguration config) throws IOException { + int port = getConfigInt(config, CONFIG_XDS_PORT, 50052); + + // XDS server for service mesh integration + // XdsServerBuilder requires ServerCredentials + XdsServerBuilder xdsBuilder = XdsServerBuilder.forPort(port, InsecureServerCredentials.create()); + + // Configure the XDS server as a ServerBuilder + configureServer(xdsBuilder, config); + + xdsServer = xdsBuilder.build().start(); + + logger.info("gRPC XDS server started on port {} (xDS management enabled)", port); + } + + private void configureServer(ServerBuilder serverBuilder, ContextConfiguration config) { + // Get database directory path + String databasePath = arcadeServer.getRootPath() + File.separator + "databases"; + + // Create the main service + ArcadeDbGrpcService mainService = new ArcadeDbGrpcService(databasePath, arcadeServer); + + // Add the main service + serverBuilder.addService(mainService); + + // Add health service if enabled + if (getConfigBoolean(config, CONFIG_HEALTH_ENABLED, true)) { + healthManager = new HealthStatusManager(); + serverBuilder.addService(healthManager.getHealthService()); + + // Set initial health status + healthManager.setStatus( + ArcadeDbGrpcService.class.getName(), + HealthCheckResponse.ServingStatus.SERVING + ); + } + + // Add reflection service if enabled + if (getConfigBoolean(config, CONFIG_REFLECTION_ENABLED, true)) { + serverBuilder.addService(ProtoReflectionService.newInstance()); + } + + serverBuilder.compressorRegistry(CompressorRegistry.getDefaultInstance()).decompressorRegistry(DecompressorRegistry.getDefaultInstance()); + + // Configure max message size + int maxMessageSizeMB = getConfigInt(config, CONFIG_MAX_MESSAGE_SIZE, 100); + serverBuilder.maxInboundMessageSize(maxMessageSizeMB * 1024 * 1024); + + // Add interceptors for logging, metrics, auth, etc. + serverBuilder.intercept(new GrpcLoggingInterceptor()); + serverBuilder.intercept(new GrpcMetricsInterceptor(arcadeServer)); + + // Add compression interceptor if force compression is enabled + if (getConfigBoolean(config, CONFIG_COMPRESSION_FORCE, false)) { + String compressionType = getConfigString(config, CONFIG_COMPRESSION_TYPE, "gzip"); + serverBuilder.intercept(new GrpcCompressionInterceptor(true, compressionType, 1024)); + } + + // Add authentication interceptor if security is configured + ServerSecurity serverSecurity = arcadeServer.getSecurity(); + if (serverSecurity != null) { + serverBuilder.intercept(new GrpcAuthInterceptor(serverSecurity)); + } + } + + private ServerBuilder configureStandardTls(int port, ContextConfiguration config) { + String certPath = getConfigString(config, CONFIG_TLS_CERT, null); + String keyPath = getConfigString(config, CONFIG_TLS_KEY, null); + + if (certPath == null || keyPath == null) { + logger.warn("TLS enabled but certificate or key path not provided. Falling back to insecure."); + return ServerBuilder.forPort(port); + } + + File certFile = new File(certPath); + File keyFile = new File(keyPath); + + if (!certFile.exists() || !keyFile.exists()) { + logger.warn("TLS certificate or key file not found. Falling back to insecure."); + return ServerBuilder.forPort(port); + } + + try { + ServerCredentials credentials = TlsServerCredentials.create(certFile, keyFile); + // Use Grpc.newServerBuilderForPort for TLS + return Grpc.newServerBuilderForPort(port, credentials); + } catch (Exception e) { + logger.error("Failed to configure TLS", e); + return ServerBuilder.forPort(port); + } + } + + private ServerCredentials configureTlsCredentials(ContextConfiguration config) { + String certPath = getConfigString(config, CONFIG_TLS_CERT, null); + String keyPath = getConfigString(config, CONFIG_TLS_KEY, null); + + if (certPath == null || keyPath == null) { + logger.warn("TLS enabled but certificate or key path not provided. Using insecure credentials."); + return InsecureServerCredentials.create(); + } + + File certFile = new File(certPath); + File keyFile = new File(keyPath); + + if (!certFile.exists() || !keyFile.exists()) { + logger.warn("TLS certificate or key file not found. Using insecure credentials."); + return InsecureServerCredentials.create(); + } + + try { + return TlsServerCredentials.create(certFile, keyFile); + } catch (Exception e) { + logger.error("Failed to configure TLS credentials", e); + return InsecureServerCredentials.create(); + } + } + + private void registerShutdownHook() { + shutdownHook = new Thread(() -> { + logger.info("Shutting down gRPC server..."); + stopService(); + }); + Runtime.getRuntime().addShutdownHook(shutdownHook); + } + + @Override + public void stopService() { + try { + // Update health status to NOT_SERVING + if (healthManager != null) { + healthManager.setStatus( + ArcadeDbGrpcService.class.getName(), + HealthCheckResponse.ServingStatus.NOT_SERVING + ); + } + + // Close the gRPC service to release database connections + if (grpcService != null) { + try { + grpcService.close(); + logger.info("gRPC service closed and database connections released"); + } catch (Exception e) { + logger.error("Error closing gRPC service", e); + } + } + + // Shutdown servers gracefully + if (grpcServer != null) { + grpcServer.shutdown(); + if (!grpcServer.awaitTermination(30, TimeUnit.SECONDS)) { + grpcServer.shutdownNow(); + grpcServer.awaitTermination(5, TimeUnit.SECONDS); + } + logger.info("Standard gRPC server stopped"); + } + + if (xdsServer != null) { + xdsServer.shutdown(); + if (!xdsServer.awaitTermination(30, TimeUnit.SECONDS)) { + xdsServer.shutdownNow(); + xdsServer.awaitTermination(5, TimeUnit.SECONDS); + } + logger.info("XDS gRPC server stopped"); + } + + // Remove shutdown hook if it exists + if (shutdownHook != null) { + try { + Runtime.getRuntime().removeShutdownHook(shutdownHook); + } catch (IllegalStateException e) { + // Already shutting down + } + } + + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.error("Interrupted while shutting down gRPC server", e); + } + } + + /** + * Get the status of the gRPC servers + */ + public ServerStatus getStatus() { + return new ServerStatus( + grpcServer != null && !grpcServer.isShutdown(), + xdsServer != null && !xdsServer.isShutdown(), + grpcServer != null ? grpcServer.getPort() : -1, + xdsServer != null ? xdsServer.getPort() : -1 + ); + } + + // Configuration helper methods + private String getConfigString(ContextConfiguration config, String key, String defaultValue) { + + return config.getValueAsString(key, defaultValue); + } + + private int getConfigInt(ContextConfiguration config, String key, int defaultValue) { + String value = getConfigString(config, key, null); + if (value != null) { + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + logger.warn("Invalid integer value for {}: {}", key, value); + } + } + return defaultValue; + } + + private boolean getConfigBoolean(ContextConfiguration config, String key, boolean defaultValue) { + String value = getConfigString(config, key, null); + if (value != null) { + return Boolean.parseBoolean(value); + } + return defaultValue; + } + + public static class ServerStatus { + public final boolean standardServerRunning; + public final boolean xdsServerRunning; + public final int standardPort; + public final int xdsPort; + + public ServerStatus(boolean standardServerRunning, boolean xdsServerRunning, + int standardPort, int xdsPort) { + this.standardServerRunning = standardServerRunning; + this.xdsServerRunning = xdsServerRunning; + this.standardPort = standardPort; + this.xdsPort = xdsPort; + } + } +} \ No newline at end of file diff --git a/grpcw/src/test/resources/arcadedb-log.properties b/grpcw/src/test/resources/arcadedb-log.properties new file mode 100644 index 0000000000..536f217457 --- /dev/null +++ b/grpcw/src/test/resources/arcadedb-log.properties @@ -0,0 +1,31 @@ +# +# Copyright 2021-present Arcade Data Ltd +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Specify the handlers to create in the root logger +# (all loggers are children of the root logger) +# The following creates two handlers +handlers=java.util.logging.ConsoleHandler +# Set the default logging level for the root logger +.level=WARNING +com.arcadedb.level=WARNING +# Set the default logging level for new ConsoleHandler instances +java.util.logging.ConsoleHandler.level=WARNING +# Set the default formatter for new ConsoleHandler instances +java.util.logging.ConsoleHandler.formatter=com.arcadedb.utility.AnsiLogFormatter diff --git a/network/pom.xml b/network/pom.xml index 56aadbf55f..facea09538 100644 --- a/network/pom.xml +++ b/network/pom.xml @@ -32,18 +32,22 @@ arcadedb-network jar ArcadeDB Network + 5.15.2 + + com.arcadedb arcadedb-engine ${project.parent.version} compile + - + diff --git a/network/src/main/java/com/arcadedb/remote/RemoteDatabase.java b/network/src/main/java/com/arcadedb/remote/RemoteDatabase.java index eb3e8a7f81..3b0ed48448 100644 --- a/network/src/main/java/com/arcadedb/remote/RemoteDatabase.java +++ b/network/src/main/java/com/arcadedb/remote/RemoteDatabase.java @@ -505,7 +505,7 @@ String getSessionId() { return sessionId; } - void setSessionId(String sessionId) { + protected void setSessionId(String sessionId) { this.sessionId = sessionId; } @@ -556,7 +556,7 @@ protected Record json2Record(final JSONObject result) { return null; } - RID saveRecord(final MutableDocument record) { + protected RID saveRecord(final MutableDocument record) { stats.createRecord.incrementAndGet(); RID rid = record.getIdentity(); @@ -569,7 +569,7 @@ RID saveRecord(final MutableDocument record) { return rid; } - RID saveRecord(final MutableDocument record, final String bucketName) { + protected RID saveRecord(final MutableDocument record, final String bucketName) { stats.createRecord.incrementAndGet(); RID rid = record.getIdentity(); diff --git a/network/src/main/java/com/arcadedb/remote/RemoteDocumentType.java b/network/src/main/java/com/arcadedb/remote/RemoteDocumentType.java index 5116926829..b1a9517568 100644 --- a/network/src/main/java/com/arcadedb/remote/RemoteDocumentType.java +++ b/network/src/main/java/com/arcadedb/remote/RemoteDocumentType.java @@ -122,7 +122,7 @@ public String getName() { return name; } - @Override +// @Override public void rename(final String newName) { remoteDatabase.command("sql", "alter type `" + name + "` name `" + newName + "`"); remoteDatabase.getSchema().reload(); @@ -254,12 +254,12 @@ public DocumentType removeSuperType(final DocumentType superType) { return this; } - @Override +// @Override public Set getAliases() { return aliases; } - @Override +// @Override public DocumentType setAliases(final Set aliases) { final String aliasesAsString = aliases.stream().map(a -> "`" + a + "`").collect(Collectors.joining(",")); remoteDatabase.command("sql", "alter type `" + name + "` aliases " + aliasesAsString); diff --git a/network/src/main/java/com/arcadedb/remote/RemoteImmutableDocument.java b/network/src/main/java/com/arcadedb/remote/RemoteImmutableDocument.java index 20e8cf13f8..213170d06b 100644 --- a/network/src/main/java/com/arcadedb/remote/RemoteImmutableDocument.java +++ b/network/src/main/java/com/arcadedb/remote/RemoteImmutableDocument.java @@ -41,7 +41,7 @@ public class RemoteImmutableDocument extends ImmutableDocument { protected final RemoteDatabase remoteDatabase; protected final Map map; - protected RemoteImmutableDocument(final RemoteDatabase remoteDatabase, final Map attributes) { + public RemoteImmutableDocument(final RemoteDatabase remoteDatabase, final Map attributes) { super(null, remoteDatabase.getSchema().getType((String) attributes.get(Property.TYPE_PROPERTY)), null, null); this.remoteDatabase = remoteDatabase; diff --git a/network/src/main/java/com/arcadedb/remote/RemoteImmutableVertex.java b/network/src/main/java/com/arcadedb/remote/RemoteImmutableVertex.java index 17da2fa0e8..569be119ee 100644 --- a/network/src/main/java/com/arcadedb/remote/RemoteImmutableVertex.java +++ b/network/src/main/java/com/arcadedb/remote/RemoteImmutableVertex.java @@ -37,7 +37,7 @@ public class RemoteImmutableVertex extends RemoteImmutableDocument implements Vertex { private final RemoteVertex internal; - protected RemoteImmutableVertex(final RemoteDatabase database, final Map properties) { + public RemoteImmutableVertex(final RemoteDatabase database, final Map properties) { super(database, properties); this.internal = new RemoteVertex(this, database); } diff --git a/network/src/main/proto/arcadedb-server.proto b/network/src/main/proto/arcadedb-server.proto new file mode 100644 index 0000000000..456c8fb35e --- /dev/null +++ b/network/src/main/proto/arcadedb-server.proto @@ -0,0 +1,281 @@ +syntax = "proto3"; + +package com.arcadedb.grpc; + +option java_multiple_files = true; +option java_package = "com.arcadedb.server.grpc"; +option java_outer_classname = "ArcadeDbProto"; + +import "google/protobuf/empty.proto"; +import "google/protobuf/struct.proto"; + +// Main ArcadeDB Service +service ArcadeDbService { + // Database operations + rpc CreateDatabase(CreateDatabaseRequest) returns (CreateDatabaseResponse); + rpc DropDatabase(DropDatabaseRequest) returns (DropDatabaseResponse); + rpc ListDatabases(ListDatabasesRequest) returns (ListDatabasesResponse); + rpc GetDatabaseInfo(GetDatabaseInfoRequest) returns (GetDatabaseInfoResponse); + + // Query operations + rpc ExecuteQuery(ExecuteQueryRequest) returns (ExecuteQueryResponse); + rpc ExecuteCommand(ExecuteCommandRequest) returns (ExecuteCommandResponse); + + // Transaction operations + rpc BeginTransaction(BeginTransactionRequest) returns (BeginTransactionResponse); + rpc CommitTransaction(CommitTransactionRequest) returns (CommitTransactionResponse); + rpc RollbackTransaction(RollbackTransactionRequest) returns (RollbackTransactionResponse); + + // Record operations + rpc CreateRecord(CreateRecordRequest) returns (CreateRecordResponse); + rpc GetRecord(GetRecordRequest) returns (GetRecordResponse); + rpc UpdateRecord(UpdateRecordRequest) returns (UpdateRecordResponse); + rpc DeleteRecord(DeleteRecordRequest) returns (DeleteRecordResponse); + + // Streaming operations + rpc StreamQuery(StreamQueryRequest) returns (stream QueryResult); + rpc BulkInsert(stream BulkInsertRequest) returns (BulkInsertResponse); + + // Server operations + rpc GetServerStatus(google.protobuf.Empty) returns (ServerStatusResponse); + rpc Ping(google.protobuf.Empty) returns (PingResponse); +} + +// Common messages +message DatabaseCredentials { + string username = 1; + string password = 2; +} + +message TransactionContext { + string transaction_id = 1; + string database = 2; +} + +// Database operations +message CreateDatabaseRequest { + string database_name = 1; + DatabaseCredentials credentials = 2; + map options = 3; +} + +message CreateDatabaseResponse { + bool success = 1; + string message = 2; + string database_id = 3; +} + +message DropDatabaseRequest { + string database_name = 1; + DatabaseCredentials credentials = 2; +} + +message DropDatabaseResponse { + bool success = 1; + string message = 2; +} + +message ListDatabasesRequest { + DatabaseCredentials credentials = 1; +} + +message ListDatabasesResponse { + repeated DatabaseInfo databases = 1; +} + +message DatabaseInfo { + string name = 1; + int64 size = 2; + string status = 3; + map properties = 4; +} + +message GetDatabaseInfoRequest { + string database_name = 1; + DatabaseCredentials credentials = 2; +} + +message GetDatabaseInfoResponse { + DatabaseInfo info = 1; +} + +// Query operations +message ExecuteQueryRequest { + string database = 1; + string query = 2; + map parameters = 3; + DatabaseCredentials credentials = 4; + TransactionContext transaction = 5; + int32 limit = 6; + int32 timeout_ms = 7; +} + +message ExecuteQueryResponse { + repeated QueryResult results = 1; + int64 execution_time_ms = 2; + string query_plan = 3; +} + +message ExecuteCommandRequest { + string database = 1; + string command = 2; + map parameters = 3; + DatabaseCredentials credentials = 4; + TransactionContext transaction = 5; +} + +message ExecuteCommandResponse { + bool success = 1; + string message = 2; + int64 affected_records = 3; + int64 execution_time_ms = 4; +} + +message QueryResult { + repeated Record records = 1; + repeated ColumnMetadata columns = 2; + int64 total_records = 3; +} + +message ColumnMetadata { + string name = 1; + string type = 2; + bool nullable = 3; +} + +// Transaction operations +message BeginTransactionRequest { + string database = 1; + DatabaseCredentials credentials = 2; + TransactionIsolation isolation = 3; +} + +enum TransactionIsolation { + READ_UNCOMMITTED = 0; + READ_COMMITTED = 1; + REPEATABLE_READ = 2; + SERIALIZABLE = 3; +} + +message BeginTransactionResponse { + string transaction_id = 1; + int64 timestamp = 2; +} + +message CommitTransactionRequest { + TransactionContext transaction = 1; + DatabaseCredentials credentials = 2; +} + +message CommitTransactionResponse { + bool success = 1; + string message = 2; + int64 timestamp = 3; +} + +message RollbackTransactionRequest { + TransactionContext transaction = 1; + DatabaseCredentials credentials = 2; +} + +message RollbackTransactionResponse { + bool success = 1; + string message = 2; +} + +// Record operations +message Record { + string rid = 1; + string type = 2; + map properties = 3; + int32 version = 4; +} + +message CreateRecordRequest { + string database = 1; + string type = 2; + map properties = 3; + DatabaseCredentials credentials = 4; + TransactionContext transaction = 5; +} + +message CreateRecordResponse { + Record record = 1; + bool success = 2; + string message = 3; +} + +message GetRecordRequest { + string database = 1; + string rid = 2; + DatabaseCredentials credentials = 3; + TransactionContext transaction = 4; +} + +message GetRecordResponse { + Record record = 1; + bool found = 2; +} + +message UpdateRecordRequest { + string database = 1; + string rid = 2; + map properties = 3; + DatabaseCredentials credentials = 4; + TransactionContext transaction = 5; + int32 expected_version = 6; +} + +message UpdateRecordResponse { + Record record = 1; + bool success = 2; + string message = 3; +} + +message DeleteRecordRequest { + string database = 1; + string rid = 2; + DatabaseCredentials credentials = 3; + TransactionContext transaction = 4; +} + +message DeleteRecordResponse { + bool success = 1; + string message = 2; +} + +// Streaming operations +message StreamQueryRequest { + string database = 1; + string query = 2; + map parameters = 3; + DatabaseCredentials credentials = 4; + int32 batch_size = 5; +} + +message BulkInsertRequest { + string database = 1; + string type = 2; + Record record = 3; + DatabaseCredentials credentials = 4; +} + +message BulkInsertResponse { + int64 total_inserted = 1; + int64 total_failed = 2; + repeated string errors = 3; +} + +// Server operations +message ServerStatusResponse { + string version = 1; + string status = 2; + int64 uptime_ms = 3; + int32 active_connections = 4; + map metrics = 5; +} + +message PingResponse { + int64 timestamp = 1; + string message = 2; +} \ No newline at end of file diff --git a/package/pom.xml b/package/pom.xml index 9e98840727..943eef1687 100644 --- a/package/pom.xml +++ b/package/pom.xml @@ -172,6 +172,18 @@ + + com.arcadedb + arcadedb-grpcw + ${project.parent.version} + shaded + + + * + * + + + com.arcadedb arcadedb-graphql diff --git a/pom.xml b/pom.xml index f47455ce79..8466fcecb7 100644 --- a/pom.xml +++ b/pom.xml @@ -88,6 +88,19 @@ 5.0.0 5.19.0 3.1.4 + + + + 3.8.0 + + 1.75.0 + + 4.32.0 + 4.32.0 + + 2.26.0 + 1.3.2 + @@ -111,6 +124,7 @@ + grpc engine network server @@ -122,11 +136,13 @@ mongodbw redisw postgresw + grpcw coverage studio package e2e e2e-perf + grpc-client @@ -350,6 +366,47 @@ + + + + io.github.ascopes + protobuf-maven-plugin + ${protobuf-maven-plugin.version} + + ${protoc.version} + true + + + io.grpc + protoc-gen-grpc-java + ${grpc.version} + + + + ${project.basedir}/src/main/proto + ${project.basedir}/src/generated/proto + + src/generated/protobuf/java + + *.proto + **/*.proto + + + + + + generate + + + + ${project.basedir}/src/generated/proto + ${project.basedir}/src/main/proto + + + + + + diff --git a/server/pom.xml b/server/pom.xml index e67e8d3b3e..c8dff53e2a 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -48,21 +48,6 @@ - - - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - com.arcadedb @@ -76,6 +61,12 @@ ${project.parent.version} compile + + com.arcadedb + arcadedb-grpc + ${project.parent.version} + compile + com.arcadedb arcadedb-integration @@ -103,6 +94,5 @@ micrometer-core ${micrometer.version} - diff --git a/server/src/main/java/com/arcadedb/server/monitor/ServerMonitor.java b/server/src/main/java/com/arcadedb/server/monitor/ServerMonitor.java index b4467b2da0..88d91626f7 100644 --- a/server/src/main/java/com/arcadedb/server/monitor/ServerMonitor.java +++ b/server/src/main/java/com/arcadedb/server/monitor/ServerMonitor.java @@ -23,8 +23,15 @@ import com.arcadedb.server.ArcadeDBServer; import com.arcadedb.server.event.ServerEventLog; -import java.io.*; -import java.util.concurrent.atomic.*; +import javax.management.MBeanServer; +import javax.management.ObjectName; +import java.io.File; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryUsage; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Level; +import java.util.logging.Logger; /** * Monitor ArcadeDB's server health. @@ -32,110 +39,242 @@ * @author Luca Garulli (l.garulli@arcadedata.com) */ public class ServerMonitor { - private static final int INTERVAL_TIME = 10_000; - private static final int MINS_30 = 30 * 60 * 1_000; - private static final int HOURS_24 = 24 * 60 * 60 * 1_000; - private final ArcadeDBServer server; - private volatile Thread checker; - private AtomicBoolean running = new AtomicBoolean(false); - private long lastHotspotSafepointTime = 0L; - private long lastHotspotSafepointCount = 0L; - private long lastHeapWarningReported = 0L; - private long lastDiskSpaceWarningReported = 0L; - - public ServerMonitor(final ArcadeDBServer server) { - this.server = server; - } - - public void start() { - running.set(true); - checker = new Thread(() -> monitor()); - checker.start(); - } - - private void monitor() { - while (running.get()) { - try { - checkDiskSpace(); - checkHeapRAM(); - checkJVMHotSpot(); - - Thread.sleep(INTERVAL_TIME); - } catch (InterruptedException e) { - break; - } catch (Exception e) { - // IGNORE IT - e.printStackTrace(); - } - } - } - - private void checkDiskSpace() { - if (System.currentTimeMillis() - lastDiskSpaceWarningReported < HOURS_24) - // REPORT ONLY EVERY 24H FROM THE LAST WARNING - return; - - final long freeSpace = new File(".").getFreeSpace(); - final long totalSpace = new File(".").getTotalSpace(); - final float freeSpacePerc = freeSpace * 100F / totalSpace; - if (freeSpacePerc < 20) { - // REPORT THE SPIKE - server.getEventLog().reportEvent(ServerEventLog.EVENT_TYPE.WARNING, "JVM", null, "Available space on disk is only " + ((int) freeSpacePerc) + "%"); - lastDiskSpaceWarningReported = System.currentTimeMillis(); - } - } - - private void checkHeapRAM() { - if (System.currentTimeMillis() - lastHeapWarningReported < MINS_30) - // REPORT ONLY EVERY 30 MINS FROM THE LAST WARNING - return; - - final Runtime runtime = Runtime.getRuntime(); - final long heapUsed = runtime.totalMemory() - runtime.freeMemory(); - final long heapMax = runtime.maxMemory(); - final float heapAvailablePerc = (heapMax - heapUsed) * 100F / heapMax; - - if (heapAvailablePerc < 20) { - // REPORT THE SPIKE - server.getEventLog() - .reportEvent(ServerEventLog.EVENT_TYPE.WARNING, "JVM", null, "Server overloaded: available heap RAM is only " + ((int) heapAvailablePerc) + "%"); - lastHeapWarningReported = System.currentTimeMillis(); - } - } - - private void checkJVMHotSpot() { - final sun.management.HotspotRuntimeMBean runtime = sun.management.ManagementFactoryHelper.getHotspotRuntimeMBean(); - - final long hotspotSafepointTime = runtime.getTotalSafepointTime(); - final long hotspotSafepointCount = runtime.getSafepointCount(); - - if (lastHotspotSafepointCount > 0) { - final float lastAvgSafepointTime = lastHotspotSafepointTime / (float) lastHotspotSafepointCount; - final float avgSafepointTime = hotspotSafepointTime / (float) hotspotSafepointCount; - final float deltaPerc = (avgSafepointTime - lastAvgSafepointTime) * 100 / lastAvgSafepointTime; - - if (deltaPerc > 20) { - // REPORT THE SPIKE - server.getEventLog().reportEvent(ServerEventLog.EVENT_TYPE.WARNING, "JVM", null, - "Server overloaded: JVM Safepoint spiked up " + ((int) deltaPerc) + "% from the last sampling"); - } - } - - lastHotspotSafepointTime = hotspotSafepointTime; - lastHotspotSafepointCount = hotspotSafepointCount; - } - - public void stop() { - running.set(false); - - if (checker != null) { - try { - checker.interrupt(); - checker.join(INTERVAL_TIME + 100); - } catch (Exception e) { - // IGNORE IT, ASSUMING THE THREAD STOPPED - } - } - } - -} + private static final Logger LOGGER = Logger.getLogger(ServerMonitor.class.getName()); + private static final int INTERVAL_TIME = 10_000; + private static final int MINS_30 = 30 * 60 * 1_000; + private static final int HOURS_24 = 24 * 60 * 60 * 1_000; + private final ArcadeDBServer server; + private volatile Thread checker; + private AtomicBoolean running = new AtomicBoolean(false); + private long lastHotspotSafepointTime = 0L; + private long lastHotspotSafepointCount = 0L; + private long lastHeapWarningReported = 0L; + private long lastDiskSpaceWarningReported = 0L; + + // JMX related fields + private MBeanServer mBeanServer; + private ObjectName hotspotRuntimeMBean; + private boolean safepointMonitoringAvailable = false; + private MemoryMXBean memoryMXBean; + + public ServerMonitor(final ArcadeDBServer server) { + this.server = server; + initializeJMXMonitoring(); + } + + private void initializeJMXMonitoring() { + try { + mBeanServer = ManagementFactory.getPlatformMBeanServer(); + memoryMXBean = ManagementFactory.getMemoryMXBean(); + + // Try to access HotSpot runtime MBean through standard JMX + try { + hotspotRuntimeMBean = new ObjectName("sun.management:type=HotspotRuntime"); + if (mBeanServer.isRegistered(hotspotRuntimeMBean)) { + safepointMonitoringAvailable = true; + LOGGER.log(Level.FINE, "HotSpot safepoint monitoring is available"); + } + } + catch (Exception e) { + LOGGER.log(Level.FINE, "HotSpot safepoint monitoring is not available: " + e.getMessage()); + } + } + catch (Exception e) { + LOGGER.log(Level.WARNING, "Failed to initialize JMX monitoring: " + e.getMessage(), e); + } + } + + public void start() { + running.set(true); + checker = new Thread(() -> monitor(), "ArcadeDB-ServerMonitor"); + checker.setDaemon(true); + checker.start(); + } + + private void monitor() { + while (running.get()) { + try { + checkDiskSpace(); + checkHeapRAM(); + checkJVMHotSpot(); + + Thread.sleep(INTERVAL_TIME); + } + catch (InterruptedException e) { + // Expected when stopping the monitor + Thread.currentThread().interrupt(); + break; + } + catch (Exception e) { + // Log the error but continue monitoring + LOGGER.log(Level.WARNING, "Error during server monitoring", e); + } + } + } + + private void checkDiskSpace() { + if (System.currentTimeMillis() - lastDiskSpaceWarningReported < HOURS_24) { + // REPORT ONLY EVERY 24H FROM THE LAST WARNING + return; + } + + try { + final File currentDir = new File("."); + final long freeSpace = currentDir.getUsableSpace(); // Better than getFreeSpace() + final long totalSpace = currentDir.getTotalSpace(); + + if (totalSpace > 0) { + final float freeSpacePerc = freeSpace * 100F / totalSpace; + if (freeSpacePerc < 20) { + // REPORT THE SPIKE + server.getEventLog().reportEvent(ServerEventLog.EVENT_TYPE.WARNING, "JVM", null, + String.format("Available space on disk is only %.1f%% (%.2f GB free of %.2f GB total)", freeSpacePerc, + freeSpace / (1024.0 * 1024.0 * 1024.0), totalSpace / (1024.0 * 1024.0 * 1024.0))); + lastDiskSpaceWarningReported = System.currentTimeMillis(); + } + } + } + catch (SecurityException e) { + LOGGER.log(Level.FINE, "Cannot check disk space due to security restrictions", e); + } + } + + private void checkHeapRAM() { + if (System.currentTimeMillis() - lastHeapWarningReported < MINS_30) { + // REPORT ONLY EVERY 30 MINS FROM THE LAST WARNING + return; + } + + try { + // Use MemoryMXBean for more accurate memory information + MemoryUsage heapUsage = memoryMXBean.getHeapMemoryUsage(); + + final long heapUsed = heapUsage.getUsed(); + final long heapMax = heapUsage.getMax(); + + if (heapMax > 0) { + final float heapAvailablePerc = (heapMax - heapUsed) * 100F / heapMax; + + if (heapAvailablePerc < 20) { + // REPORT THE SPIKE + server.getEventLog().reportEvent(ServerEventLog.EVENT_TYPE.WARNING, "JVM", null, + String.format("Server overloaded: available heap RAM is only %.1f%% (%.2f GB used of %.2f GB max)", + heapAvailablePerc, heapUsed / (1024.0 * 1024.0 * 1024.0), heapMax / (1024.0 * 1024.0 * 1024.0))); + lastHeapWarningReported = System.currentTimeMillis(); + + // Optionally suggest GC if memory is critically low + if (heapAvailablePerc < 10) { + LOGGER.log(Level.INFO, "Suggesting garbage collection due to low memory"); + System.gc(); + } + } + } + } + catch (Exception e) { + LOGGER.log(Level.FINE, "Error checking heap memory", e); + } + } + + private void checkJVMHotSpot() { + if (!safepointMonitoringAvailable) { + // Safepoint monitoring not available, skip this check + return; + } + + try { + // Access HotSpot metrics through JMX MBean + Long hotspotSafepointTime = (Long) mBeanServer.getAttribute(hotspotRuntimeMBean, "TotalSafepointTime"); + Long hotspotSafepointCount = (Long) mBeanServer.getAttribute(hotspotRuntimeMBean, "SafepointCount"); + + if (hotspotSafepointTime != null && hotspotSafepointCount != null && hotspotSafepointCount > 0) { + if (lastHotspotSafepointCount > 0) { + final float lastAvgSafepointTime = lastHotspotSafepointTime / (float) lastHotspotSafepointCount; + final float avgSafepointTime = hotspotSafepointTime / (float) hotspotSafepointCount; + + if (lastAvgSafepointTime > 0) { + final float deltaPerc = (avgSafepointTime - lastAvgSafepointTime) * 100 / lastAvgSafepointTime; + + if (deltaPerc > 20) { + // REPORT THE SPIKE + server.getEventLog().reportEvent(ServerEventLog.EVENT_TYPE.WARNING, "JVM", null, String.format( + "Server overloaded: JVM Safepoint spiked up %.1f%% from the last sampling (avg time: %.2fms -> %.2fms)", + deltaPerc, lastAvgSafepointTime, avgSafepointTime)); + } + } + } + + lastHotspotSafepointTime = hotspotSafepointTime; + lastHotspotSafepointCount = hotspotSafepointCount; + } + } + catch (Exception e) { + // If we can't access safepoint metrics, disable future attempts + safepointMonitoringAvailable = false; + LOGGER.log(Level.FINE, "Cannot access HotSpot safepoint metrics, disabling this monitoring", e); + } + } + + /** + * Alternative monitoring using GC metrics if safepoint monitoring is not + * available + */ + private void checkGCMetrics() { + try { + var gcBeans = ManagementFactory.getGarbageCollectorMXBeans(); + long totalGCTime = 0; + long totalGCCount = 0; + + for (var gcBean : gcBeans) { + totalGCTime += gcBean.getCollectionTime(); + totalGCCount += gcBean.getCollectionCount(); + } + + // You can add logic here to track and report on GC spikes + // similar to safepoint monitoring + + } + catch (Exception e) { + LOGGER.log(Level.FINE, "Error checking GC metrics", e); + } + } + + public void stop() { + running.set(false); + + if (checker != null) { + try { + checker.interrupt(); + checker.join(INTERVAL_TIME + 100); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOGGER.log(Level.FINE, "Interrupted while stopping monitor thread", e); + } + } + } + + /** + * Get current monitoring status + */ + public MonitoringStatus getStatus() { + return new MonitoringStatus(running.get(), safepointMonitoringAvailable, System.currentTimeMillis() - lastHeapWarningReported < MINS_30, + System.currentTimeMillis() - lastDiskSpaceWarningReported < HOURS_24); + } + + /** + * Status class for monitoring information + */ + public static class MonitoringStatus { + public final boolean isRunning; + public final boolean safepointMonitoringAvailable; + public final boolean recentHeapWarning; + public final boolean recentDiskWarning; + + public MonitoringStatus(boolean isRunning, boolean safepointMonitoringAvailable, boolean recentHeapWarning, boolean recentDiskWarning) { + this.isRunning = isRunning; + this.safepointMonitoringAvailable = safepointMonitoringAvailable; + this.recentHeapWarning = recentHeapWarning; + this.recentDiskWarning = recentDiskWarning; + } + } +} \ No newline at end of file diff --git a/server/src/main/proto/arcadedb-server.proto b/server/src/main/proto/arcadedb-server.proto new file mode 100644 index 0000000000..456c8fb35e --- /dev/null +++ b/server/src/main/proto/arcadedb-server.proto @@ -0,0 +1,281 @@ +syntax = "proto3"; + +package com.arcadedb.grpc; + +option java_multiple_files = true; +option java_package = "com.arcadedb.server.grpc"; +option java_outer_classname = "ArcadeDbProto"; + +import "google/protobuf/empty.proto"; +import "google/protobuf/struct.proto"; + +// Main ArcadeDB Service +service ArcadeDbService { + // Database operations + rpc CreateDatabase(CreateDatabaseRequest) returns (CreateDatabaseResponse); + rpc DropDatabase(DropDatabaseRequest) returns (DropDatabaseResponse); + rpc ListDatabases(ListDatabasesRequest) returns (ListDatabasesResponse); + rpc GetDatabaseInfo(GetDatabaseInfoRequest) returns (GetDatabaseInfoResponse); + + // Query operations + rpc ExecuteQuery(ExecuteQueryRequest) returns (ExecuteQueryResponse); + rpc ExecuteCommand(ExecuteCommandRequest) returns (ExecuteCommandResponse); + + // Transaction operations + rpc BeginTransaction(BeginTransactionRequest) returns (BeginTransactionResponse); + rpc CommitTransaction(CommitTransactionRequest) returns (CommitTransactionResponse); + rpc RollbackTransaction(RollbackTransactionRequest) returns (RollbackTransactionResponse); + + // Record operations + rpc CreateRecord(CreateRecordRequest) returns (CreateRecordResponse); + rpc GetRecord(GetRecordRequest) returns (GetRecordResponse); + rpc UpdateRecord(UpdateRecordRequest) returns (UpdateRecordResponse); + rpc DeleteRecord(DeleteRecordRequest) returns (DeleteRecordResponse); + + // Streaming operations + rpc StreamQuery(StreamQueryRequest) returns (stream QueryResult); + rpc BulkInsert(stream BulkInsertRequest) returns (BulkInsertResponse); + + // Server operations + rpc GetServerStatus(google.protobuf.Empty) returns (ServerStatusResponse); + rpc Ping(google.protobuf.Empty) returns (PingResponse); +} + +// Common messages +message DatabaseCredentials { + string username = 1; + string password = 2; +} + +message TransactionContext { + string transaction_id = 1; + string database = 2; +} + +// Database operations +message CreateDatabaseRequest { + string database_name = 1; + DatabaseCredentials credentials = 2; + map options = 3; +} + +message CreateDatabaseResponse { + bool success = 1; + string message = 2; + string database_id = 3; +} + +message DropDatabaseRequest { + string database_name = 1; + DatabaseCredentials credentials = 2; +} + +message DropDatabaseResponse { + bool success = 1; + string message = 2; +} + +message ListDatabasesRequest { + DatabaseCredentials credentials = 1; +} + +message ListDatabasesResponse { + repeated DatabaseInfo databases = 1; +} + +message DatabaseInfo { + string name = 1; + int64 size = 2; + string status = 3; + map properties = 4; +} + +message GetDatabaseInfoRequest { + string database_name = 1; + DatabaseCredentials credentials = 2; +} + +message GetDatabaseInfoResponse { + DatabaseInfo info = 1; +} + +// Query operations +message ExecuteQueryRequest { + string database = 1; + string query = 2; + map parameters = 3; + DatabaseCredentials credentials = 4; + TransactionContext transaction = 5; + int32 limit = 6; + int32 timeout_ms = 7; +} + +message ExecuteQueryResponse { + repeated QueryResult results = 1; + int64 execution_time_ms = 2; + string query_plan = 3; +} + +message ExecuteCommandRequest { + string database = 1; + string command = 2; + map parameters = 3; + DatabaseCredentials credentials = 4; + TransactionContext transaction = 5; +} + +message ExecuteCommandResponse { + bool success = 1; + string message = 2; + int64 affected_records = 3; + int64 execution_time_ms = 4; +} + +message QueryResult { + repeated Record records = 1; + repeated ColumnMetadata columns = 2; + int64 total_records = 3; +} + +message ColumnMetadata { + string name = 1; + string type = 2; + bool nullable = 3; +} + +// Transaction operations +message BeginTransactionRequest { + string database = 1; + DatabaseCredentials credentials = 2; + TransactionIsolation isolation = 3; +} + +enum TransactionIsolation { + READ_UNCOMMITTED = 0; + READ_COMMITTED = 1; + REPEATABLE_READ = 2; + SERIALIZABLE = 3; +} + +message BeginTransactionResponse { + string transaction_id = 1; + int64 timestamp = 2; +} + +message CommitTransactionRequest { + TransactionContext transaction = 1; + DatabaseCredentials credentials = 2; +} + +message CommitTransactionResponse { + bool success = 1; + string message = 2; + int64 timestamp = 3; +} + +message RollbackTransactionRequest { + TransactionContext transaction = 1; + DatabaseCredentials credentials = 2; +} + +message RollbackTransactionResponse { + bool success = 1; + string message = 2; +} + +// Record operations +message Record { + string rid = 1; + string type = 2; + map properties = 3; + int32 version = 4; +} + +message CreateRecordRequest { + string database = 1; + string type = 2; + map properties = 3; + DatabaseCredentials credentials = 4; + TransactionContext transaction = 5; +} + +message CreateRecordResponse { + Record record = 1; + bool success = 2; + string message = 3; +} + +message GetRecordRequest { + string database = 1; + string rid = 2; + DatabaseCredentials credentials = 3; + TransactionContext transaction = 4; +} + +message GetRecordResponse { + Record record = 1; + bool found = 2; +} + +message UpdateRecordRequest { + string database = 1; + string rid = 2; + map properties = 3; + DatabaseCredentials credentials = 4; + TransactionContext transaction = 5; + int32 expected_version = 6; +} + +message UpdateRecordResponse { + Record record = 1; + bool success = 2; + string message = 3; +} + +message DeleteRecordRequest { + string database = 1; + string rid = 2; + DatabaseCredentials credentials = 3; + TransactionContext transaction = 4; +} + +message DeleteRecordResponse { + bool success = 1; + string message = 2; +} + +// Streaming operations +message StreamQueryRequest { + string database = 1; + string query = 2; + map parameters = 3; + DatabaseCredentials credentials = 4; + int32 batch_size = 5; +} + +message BulkInsertRequest { + string database = 1; + string type = 2; + Record record = 3; + DatabaseCredentials credentials = 4; +} + +message BulkInsertResponse { + int64 total_inserted = 1; + int64 total_failed = 2; + repeated string errors = 3; +} + +// Server operations +message ServerStatusResponse { + string version = 1; + string status = 2; + int64 uptime_ms = 3; + int32 active_connections = 4; + map metrics = 5; +} + +message PingResponse { + int64 timestamp = 1; + string message = 2; +} \ No newline at end of file From 39ad39463be5022cc52e72f302b751c907e188be Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Sun, 31 Aug 2025 18:12:20 -0700 Subject: [PATCH 02/35] Checkpoint Commit --- grpc-client/docs/Readme.txt | 14 + grpc-client/pom.xml | 9 +- .../remote/grpc/RemoteGrpcDatabase.java | 18 +- .../RemoteGrpcDatabaseWithCompression.java | 62 +- .../remote/grpc/RemoteGrpcServer.java | 120 ++ .../remote/grpc/ArcadeDbHTTPTvsGRPCBench.java | 299 +++++ .../server/grpc/ArcadeDbGrpcService.java | 49 +- .../com/arcadedb/remote/RemoteServer.java | 11 +- server/TMP/grpc/ArcadeDbGrpcService.java | 1082 +++++++++++++++++ server/TMP/grpc/CompressionAwareService.java | 86 ++ server/TMP/grpc/GrpcAuthInterceptor.java | 142 +++ .../TMP/grpc/GrpcCompressionInterceptor.java | 83 ++ server/TMP/grpc/GrpcLoggingInterceptor.java | 81 ++ server/TMP/grpc/GrpcMetricsInterceptor.java | 78 ++ server/TMP/grpc/GrpcServerPlugin.java | 386 ++++++ 15 files changed, 2459 insertions(+), 61 deletions(-) create mode 100644 grpc-client/docs/Readme.txt create mode 100644 grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcServer.java create mode 100644 grpc-client/src/test/java/com/arcadedb/remote/grpc/ArcadeDbHTTPTvsGRPCBench.java create mode 100644 server/TMP/grpc/ArcadeDbGrpcService.java create mode 100644 server/TMP/grpc/CompressionAwareService.java create mode 100644 server/TMP/grpc/GrpcAuthInterceptor.java create mode 100644 server/TMP/grpc/GrpcCompressionInterceptor.java create mode 100644 server/TMP/grpc/GrpcLoggingInterceptor.java create mode 100644 server/TMP/grpc/GrpcMetricsInterceptor.java create mode 100644 server/TMP/grpc/GrpcServerPlugin.java diff --git a/grpc-client/docs/Readme.txt b/grpc-client/docs/Readme.txt new file mode 100644 index 0000000000..051f316d04 --- /dev/null +++ b/grpc-client/docs/Readme.txt @@ -0,0 +1,14 @@ + +PWD: /Users/ocohen/git/Verdance/ArcadeDB-GRPC/arcadedb-25.8.1-SNAPSHOT + + +./bin/server.sh -Darcadedb.server.rootPassword=root1234 -Darcadedb.server.name=Arcade_GRPC_Test -Darcadedb.dumpConfigAtStartup=true -Darcadedb.server.mode=development -Darcadedb.server.rootPath=../var/arcadedb -Darcadedb.server.plugins=GRPC:com.arcadedb.server.grpc.GrpcServerPlugin -Xms512M -Xmx4096M -XX:InitialRAMPercentage=50.0 -XX:MaxRAMPercentage=75.0 -Darcadedb.server.httpIncomingPort=2489 -Darcadedb.grpc.enabled=true -Darcadedb.grpc.port=50059 -Darcadedb.grpc.mode=standard -Darcadedb.grpc.reflection.enabled=true -Darcadedb.grpc.health.enabled=true + + +Ports: + +HTTP: 2489 +GTPC: 50059 + +root +root1234 diff --git a/grpc-client/pom.xml b/grpc-client/pom.xml index eca96e0662..cf6f2196af 100644 --- a/grpc-client/pom.xml +++ b/grpc-client/pom.xml @@ -121,7 +121,14 @@ ${grpc.version} test - + + + org.slf4j + slf4j-simple + 2.0.17 + runtime + + diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java index b2a2a09edf..b5ead6ebec 100644 --- a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java @@ -105,14 +105,14 @@ public class RemoteGrpcDatabase extends RemoteDatabase { private String databaseName; private RemoteTransactionExplicitLock explicitLock; - public RemoteGrpcDatabase(final String server, final int port, final String databaseName, final String userName, final String userPassword) { - this(server, port, databaseName, userName, userPassword, new ContextConfiguration()); + public RemoteGrpcDatabase(final String server, final int grpcPort, final int httpPort, final String databaseName, final String userName, final String userPassword) { + this(server, grpcPort, httpPort, databaseName, userName, userPassword, new ContextConfiguration()); } - public RemoteGrpcDatabase(final String server, final int port, final String databaseName, final String userName, final String userPassword, + public RemoteGrpcDatabase(final String server, final int grpcPort, final int httpPort, final String databaseName, final String userName, final String userPassword, final ContextConfiguration configuration) { - super(server, 2480, databaseName, userName, userPassword, configuration); + super(server, httpPort, databaseName, userName, userPassword, configuration); this.userName = userName; this.userPassword = userPassword; @@ -120,7 +120,7 @@ public RemoteGrpcDatabase(final String server, final int port, final String data this.databaseName = databaseName; // Create gRPC channel - this.channel = createChannel(server, port); + this.channel = createChannel(server, grpcPort); // Create stubs with authentication CallCredentials credentials = createCallCredentials(userName, userPassword); @@ -184,7 +184,7 @@ public void thisUsesUnstableApi() { } }; } - + /** * Override this method to customize blocking stub creation */ @@ -1542,7 +1542,7 @@ private com.google.protobuf.Value toProtoValue(Object v) { // gRPC-specific helper methods - private DatabaseCredentials buildCredentials() { + public DatabaseCredentials buildCredentials() { return DatabaseCredentials.newBuilder().setUsername(getUserName()).setPassword(getUserPassword()).build(); } @@ -1774,9 +1774,9 @@ private void handleGrpcException(StatusRuntimeException e) { */ public static class RemoteGrpcDatabaseWithCompression extends RemoteGrpcDatabase { - public RemoteGrpcDatabaseWithCompression(String server, int port, String databaseName, String userName, String userPassword, + public RemoteGrpcDatabaseWithCompression(String server, int grpcPort, int httpPort, String databaseName, String userName, String userPassword, ContextConfiguration configuration) { - super(server, port, databaseName, userName, userPassword, configuration); + super(server, grpcPort, httpPort, databaseName, userName, userPassword, configuration); } @Override diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseWithCompression.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseWithCompression.java index 3965b6f205..bae6c1d49b 100644 --- a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseWithCompression.java +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseWithCompression.java @@ -12,40 +12,30 @@ * Extended RemoteGrpcDatabase with compression support */ public class RemoteGrpcDatabaseWithCompression extends RemoteGrpcDatabase { - - public RemoteGrpcDatabaseWithCompression(String server, int port, - String databaseName, - String userName, - String userPassword, - ContextConfiguration configuration) { - super(server, port, databaseName, userName, userPassword, configuration); - } - - @Override - protected ManagedChannel createChannel(String server, int port) { - - return ManagedChannelBuilder.forAddress(server, port) - .usePlaintext() // No TLS/SSL - .maxInboundMessageSize(100 * 1024 * 1024) // 100MB max message size - .keepAliveTime(30, TimeUnit.SECONDS) // Keep-alive configuration - .keepAliveTimeout(10, TimeUnit.SECONDS) - .keepAliveWithoutCalls(true) - .build(); - } - - @Override - protected ArcadeDbServiceGrpc.ArcadeDbServiceBlockingStub createBlockingStub(ManagedChannel channel) { - return ArcadeDbServiceGrpc.newBlockingStub(channel) - .withCompression("gzip") // Enable GZIP compression - .withCallCredentials(createCredentials()) - .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS); - } - - @Override - protected ArcadeDbServiceGrpc.ArcadeDbServiceStub createAsyncStub(ManagedChannel channel) { - return ArcadeDbServiceGrpc.newStub(channel) - .withCompression("gzip") // Enable GZIP compression - .withCallCredentials(createCredentials()) - .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS); - } + + public RemoteGrpcDatabaseWithCompression(String server, int grpcPort, int httpPort, String databaseName, String userName, + String userPassword, ContextConfiguration configuration) { + super(server, grpcPort, httpPort, databaseName, userName, userPassword, configuration); + } + + @Override + protected ManagedChannel createChannel(String server, int port) { + + return ManagedChannelBuilder.forAddress(server, port).usePlaintext() // No TLS/SSL + .maxInboundMessageSize(100 * 1024 * 1024) // 100MB max message size + .keepAliveTime(30, TimeUnit.SECONDS) // Keep-alive configuration + .keepAliveTimeout(10, TimeUnit.SECONDS).keepAliveWithoutCalls(true).build(); + } + + @Override + protected ArcadeDbServiceGrpc.ArcadeDbServiceBlockingStub createBlockingStub(ManagedChannel channel) { + return ArcadeDbServiceGrpc.newBlockingStub(channel).withCompression("gzip") // Enable GZIP compression + .withCallCredentials(createCredentials()).withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS); + } + + @Override + protected ArcadeDbServiceGrpc.ArcadeDbServiceStub createAsyncStub(ManagedChannel channel) { + return ArcadeDbServiceGrpc.newStub(channel).withCompression("gzip") // Enable GZIP compression + .withCallCredentials(createCredentials()).withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS); + } } diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcServer.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcServer.java new file mode 100644 index 0000000000..661b7e144d --- /dev/null +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcServer.java @@ -0,0 +1,120 @@ +package com.arcadedb.remote.grpc; + +import com.arcadedb.server.grpc.ArcadeDbServiceGrpc; +import com.arcadedb.server.grpc.CreateDatabaseRequest; +import com.arcadedb.server.grpc.DatabaseCredentials; +import com.arcadedb.server.grpc.DropDatabaseRequest; +import com.arcadedb.server.grpc.ListDatabasesRequest; +import com.arcadedb.server.grpc.ListDatabasesResponse; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.stub.AbstractStub; + +import java.util.List; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +/** + * Minimal server-scope gRPC wrapper (HTTP RemoteServer equivalent), implemented + * ONLY with RPCs present in your current proto. + * + * Features: - listDatabases() - existsDatabase(name) via listDatabases() - + * createDatabase(name, type) - createDatabaseIfMissing(name, type) - + * dropDatabase(name, force?) // 'force' only if defined in your proto; + * otherwise ignored + * + * Add more methods later when you extend the proto (ping, serverInfo, user + * mgmt, etc.). + */ +public class RemoteGrpcServer implements AutoCloseable { + + private final String host; + private final int port; + private final String user; + private final String pass; + + private final long defaultTimeoutMs; + + private final ManagedChannel channel; + private final ArcadeDbServiceGrpc.ArcadeDbServiceBlockingStub blocking; + + public RemoteGrpcServer(final String host, final int port, final String user, final String pass) { + this(host, port, user, pass, 30_000); + } + + public RemoteGrpcServer(final String host, final int port, final String user, final String pass, final long defaultTimeoutMs) { + this.host = Objects.requireNonNull(host, "host"); + this.port = port; + this.user = Objects.requireNonNull(user, "user"); + this.pass = Objects.requireNonNull(pass, "pass"); + this.defaultTimeoutMs = defaultTimeoutMs > 0 ? defaultTimeoutMs : 30_000; + + this.channel = ManagedChannelBuilder.forAddress(this.host, this.port).usePlaintext() // switch to TLS if enabled server-side + .build(); + + this.blocking = ArcadeDbServiceGrpc.newBlockingStub(channel); + } + + @Override + public void close() { + channel.shutdown(); + try { + channel.awaitTermination(5, TimeUnit.SECONDS); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + private > S withDeadline(S stub, long timeoutMs) { + long t = (timeoutMs > 0) ? timeoutMs : defaultTimeoutMs; + return stub.withDeadlineAfter(t, TimeUnit.MILLISECONDS); + } + + public DatabaseCredentials buildCredentials() { + return DatabaseCredentials.newBuilder().setUsername(user == null ? "" : user).setPassword(pass == null ? "" : pass).build(); + } + + /** Returns the list of database names from the server. */ + public List listDatabases() { + + ListDatabasesResponse resp = withDeadline(blocking, defaultTimeoutMs) + .listDatabases(ListDatabasesRequest.newBuilder().setCredentials(buildCredentials()).build()); + + return resp.getDatabasesList().stream().map(dbInfo -> dbInfo.getName()).toList(); + } + + /** Checks existence by listing (no ExistsDatabaseRequest needed). */ + public boolean existsDatabase(final String database) { + return listDatabases().stream().anyMatch(n -> n.equalsIgnoreCase(database)); + } + + /** Creates a database with type "graph" or "document". */ + public void createDatabase(final String database) { + withDeadline(blocking, defaultTimeoutMs).createDatabase(CreateDatabaseRequest.newBuilder() + .setDatabaseName(database) + .setCredentials(buildCredentials()).build()); + } + + /** No-op if already present; creates otherwise. */ + public void createDatabaseIfMissing(final String database) { + if (!existsDatabase(database)) { + createDatabase(database); + } + } + + /** Drops a database. If your proto supports 'force', add it here. */ + public void dropDatabase(final String database) { + withDeadline(blocking, defaultTimeoutMs) + .dropDatabase(DropDatabaseRequest.newBuilder().setDatabaseName(database).setCredentials(buildCredentials()).build()); + } + + public String endpoint() { + return host + ":" + port; + } + + @Override + public String toString() { + return "RemoteGrpcServer{endpoint=" + endpoint() + ", user='" + user + "'}"; + } +} \ No newline at end of file diff --git a/grpc-client/src/test/java/com/arcadedb/remote/grpc/ArcadeDbHTTPTvsGRPCBench.java b/grpc-client/src/test/java/com/arcadedb/remote/grpc/ArcadeDbHTTPTvsGRPCBench.java new file mode 100644 index 0000000000..987fc40617 --- /dev/null +++ b/grpc-client/src/test/java/com/arcadedb/remote/grpc/ArcadeDbHTTPTvsGRPCBench.java @@ -0,0 +1,299 @@ +package com.arcadedb.remote.grpc; + +import java.io.File; +import java.nio.file.Files; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import com.arcadedb.remote.RemoteDatabase; // HTTP client +import com.arcadedb.server.grpc.InsertOptions; +import com.arcadedb.server.grpc.InsertSummary; +import com.arcadedb.server.grpc.StreamQueryRequest; + +public class ArcadeDbHTTPTvsGRPCBench { + + // ---- Config (edit or pass via env/args) ---- + static String DB_NAME = System.getenv().getOrDefault("ARCADE_DB", "ArcadeDB"); + static String HTTP_HOST = System.getenv().getOrDefault("ARCADE_HTTP_HOST", "127.0.0.1"); + static int HTTP_PORT = Integer.parseInt(System.getenv().getOrDefault("ARCADE_HTTP_PORT", "2489")); + static String GRPC_HOST = System.getenv().getOrDefault("ARCADE_GRPC_HOST", "127.0.0.1"); + static int GRPC_PORT = Integer.parseInt(System.getenv().getOrDefault("ARCADE_GRPC_PORT", "50059")); + + static String USER = System.getenv().getOrDefault("ARCADE_USER", "root"); + static String PASS = System.getenv().getOrDefault("ARCADE_PASS", "root1234"); + + public static void main(String[] args) throws Exception { + + // override via args if provided: host/ports/db/user/pass + for (String a : args) { + if (a.startsWith("--db=")) + DB_NAME = a.substring(5); + else if (a.startsWith("--http=")) { + var hp = a.substring(7).split(":"); + HTTP_HOST = hp[0]; + HTTP_PORT = Integer.parseInt(hp[1]); + } + else if (a.startsWith("--grpc=")) { + var hp = a.substring(7).split(":"); + GRPC_HOST = hp[0]; + GRPC_PORT = Integer.parseInt(hp[1]); + } + else if (a.startsWith("--user=")) + USER = a.substring(7); + else if (a.startsWith("--pass=")) + PASS = a.substring(7); + } + + System.out.printf("Benchmarking DB='%s' HTTP=%s:%d gRPC=%s:%d as %s%n", DB_NAME, HTTP_HOST, HTTP_PORT, GRPC_HOST, GRPC_PORT, USER); + + RemoteGrpcServer grpcServer = new RemoteGrpcServer(GRPC_HOST, GRPC_PORT, USER, PASS); + + if (grpcServer.existsDatabase(DB_NAME)) { + + System.out.println("Database exists"); + } + else { + + grpcServer.createDatabase(DB_NAME); + } + + // ---------- Open both clients ---------- + try (RemoteDatabase http = new RemoteDatabase(HTTP_HOST, HTTP_PORT, DB_NAME, USER, PASS); + RemoteGrpcDatabase grpc = new RemoteGrpcDatabase(GRPC_HOST, GRPC_PORT, HTTP_PORT, DB_NAME, USER, PASS)) { + + // ensure schema aligned + prepareSchemaHTTP(http); + prepareSchemaGRPC(grpc); + + // --- apples-to-apples data set (clear prior run for repeatability) --- + cleanupHTTP(http); + cleanupGRPC(grpc); + + // ---------- BULK INSERT (100 rows) ---------- + List> rows = buildFeedbackRows(100); + time("HTTP bulkInsert(100)", () -> { + http.begin(); + // Replace with your HTTP bulk insert (if you don’t have one, fallback to looped + // insert) + rows.forEach(m -> http.command("sql", "INSERT INTO UserFeedback CONTENT ?", Map.of("param1", m))); + http.commit(); + }); + + time("gRPC bulkInsert(100)", () -> { + grpc.begin(); + InsertOptions opts = defaultInsertOptions(grpc, "UserFeedback", List.of("id")); // id unique -> idempotent + InsertSummary s = grpc.insertBulkAsListOfMaps(opts, rows, 120_000); + grpc.commit(); + System.out.printf(" gRPC summary: received=%d, inserted=%d, updated=%d, ignored=%d, failed=%d, errors=%d%n", s.getReceived(), + s.getInserted(), s.getUpdated(), s.getIgnored(), s.getFailed(), s.getErrorsCount()); + }); + + // ---------- Single insert ---------- + Map one = uf("UF-SINGLE", "tenantZ", "UI", "FEATURE", "Single insert via API", null, Instant.now()); + time("HTTP insert(1)", () -> { + http.begin(); + http.command("sql", "INSERT INTO UserFeedback CONTENT :m", Map.of("m", one)); + http.commit(); + }); + time("gRPC insert(1)", () -> { + grpc.begin(); + InsertOptions opts = defaultInsertOptions(grpc, "UserFeedback", List.of("id")); + grpc.insertBulkAsListOfMaps(opts, List.of(one), 60_000); + grpc.commit(); + }); + + // ---------- Update ---------- + time("HTTP update by id", () -> { + http.begin(); + http.command("sql", "UPDATE UserFeedback SET feedback = 'UPDATED_HTTP' WHERE id = 'UF-0001'"); + http.commit(); + }); + time("gRPC update by id", () -> { + grpc.begin(); + grpc.executeCommand(DB_NAME, "UPDATE UserFeedback SET feedback = 'UPDATED_GRPC' WHERE id = 'UF-0001'", Map.of(), "sql", false, 0, + /* tx */ null, 60_000); + grpc.commit(); + }); + + // ---------- Point read by index ---------- + time("HTTP point read (id)", () -> { + var rs = http.query("sql", "SELECT FROM UserFeedback WHERE id = 'UF-0001'"); + int count = 0; + while (rs.hasNext()) { + rs.next(); + count++; + } + System.out.println(" HTTP rows=" + count); + }); + time("gRPC point read (id)", () -> { + var it = grpc.queryStream(DB_NAME, "SELECT FROM UserFeedback WHERE id = 'UF-0001'", Map.of(), 50, + StreamQueryRequest.RetrievalMode.CURSOR, /* tx */ null, 60_000); + int count = 0; + while (it.hasNext()) { + it.next(); + count++; + } + System.out.println(" gRPC rows=" + count); + }); + + // ---------- Streaming query (3 modes) ---------- + runStreamBench(grpc, "SELECT FROM UserFeedback", 200, StreamQueryRequest.RetrievalMode.CURSOR); + runStreamBench(grpc, "SELECT FROM UserFeedback", 200, StreamQueryRequest.RetrievalMode.MATERIALIZE_ALL); + runStreamBench(grpc, "SELECT FROM UserFeedback", 200, StreamQueryRequest.RetrievalMode.PAGED); + + // ---------- Delete subset ---------- + time("HTTP delete subset", () -> { + http.begin(); + http.command("sql", "DELETE FROM UserFeedback WHERE id LIKE 'UF-00%'"); + http.commit(); + }); + time("gRPC delete subset", () -> { + grpc.begin(); + grpc.executeCommand(DB_NAME, "DELETE FROM UserFeedback WHERE id LIKE 'UF-00%'", Map.of(), "sql", false, 0, null, 60_000); + grpc.commit(); + }); + + } // try-with-resources + } + + // ------------------------------------------------------ + // Schema prep + // ------------------------------------------------------ + + private static void prepareSchemaHTTP(RemoteDatabase http) { + // Create vertex type + index if missing (id unique) + http.command("sql", "CREATE VERTEX TYPE UserFeedback IF NOT EXISTS"); + http.command("sql", "CREATE PROPERTY UserFeedback.id STRING IF NOT EXISTS"); + http.command("sql", "CREATE INDEX UserFeedback.id IF NOT EXISTS UNIQUE"); + http.command("sql", "CREATE PROPERTY UserFeedback.applicationArea STRING IF NOT EXISTS"); + http.command("sql", "CREATE PROPERTY UserFeedback.empowerTenantId STRING IF NOT EXISTS"); + http.command("sql", "CREATE PROPERTY UserFeedback.empowerType STRING IF NOT EXISTS"); + http.command("sql", "CREATE PROPERTY UserFeedback.feedback STRING IF NOT EXISTS"); + http.command("sql", "CREATE PROPERTY UserFeedback.timestamp DATETIME IF NOT EXISTS"); + http.command("sql", "CREATE PROPERTY UserFeedback.image EMBEDDED IF NOT EXISTS"); + } + + private static void prepareSchemaGRPC(RemoteGrpcDatabase grpc) { + // Mirror the same via command API for apples-to-apples + grpc.executeCommand(DB_NAME, "CREATE VERTEX TYPE UserFeedback IF NOT EXISTS", Map.of(), "sql", false, 0, null, 60_000); + grpc.executeCommand(DB_NAME, "CREATE PROPERTY UserFeedback.id STRING IF NOT EXISTS", Map.of(), "sql", false, 0, null, 60_000); + grpc.executeCommand(DB_NAME, "CREATE INDEX UserFeedback.id IF NOT EXISTS UNIQUE", Map.of(), "sql", false, 0, null, 60_000); + + grpc.executeCommand(DB_NAME, "CREATE PROPERTY UserFeedback.applicationArea STRING IF NOT EXISTS", Map.of(), "sql", false, 0, null, + 60_000); + grpc.executeCommand(DB_NAME, "CREATE PROPERTY UserFeedback.empowerTenantId STRING IF NOT EXISTS", Map.of(), "sql", false, 0, null, + 60_000); + grpc.executeCommand(DB_NAME, "CREATE PROPERTY UserFeedback.empowerType STRING IF NOT EXISTS", Map.of(), "sql", false, 0, null, 60_000); + grpc.executeCommand(DB_NAME, "CREATE PROPERTY UserFeedback.feedback STRING IF NOT EXISTS", Map.of(), "sql", false, 0, null, 60_000); + grpc.executeCommand(DB_NAME, "CREATE PROPERTY UserFeedback.timestamp DATETIME IF NOT EXISTS", Map.of(), "sql", false, 0, null, 60_000); + grpc.executeCommand(DB_NAME, "CREATE PROPERTY UserFeedback.image EMBEDDED IF NOT EXISTS", Map.of(), "sql", false, 0, null, 60_000); + } + + private static void cleanupHTTP(RemoteDatabase http) { + http.command("sql", "DELETE FROM UserFeedback"); + } + + private static void cleanupGRPC(RemoteGrpcDatabase grpc) { + grpc.executeCommand(DB_NAME, "DELETE FROM UserFeedback", Map.of(), "sql", false, 0, null, 60_000); + } + + // ------------------------------------------------------ + // Workloads + // ------------------------------------------------------ + + private static void runStreamBench(RemoteGrpcDatabase grpc, String sql, int batchSize, StreamQueryRequest.RetrievalMode mode) { + time("gRPC stream " + mode + " (batchSize=" + batchSize + ")", () -> { + var it = grpc.queryStream(DB_NAME, sql, Map.of(), batchSize, mode, /* tx */ null, 120_000); + var total = new AtomicInteger(0); + while (it.hasNext()) { + it.next(); + total.incrementAndGet(); + } + System.out.println(" rows=" + total.get()); + }); + } + + // ------------------------------------------------------ + // Data builders + // ------------------------------------------------------ + + private static List> buildFeedbackRows(int n) throws Exception { + // optional: shared embedded image payload + byte[] png = loadDefaultPNG(); + String b64 = (png != null) ? Base64.getEncoder().encodeToString(png) : null; + Map embeddedImage = (b64 != null) ? Map.of("mime", "image/png", "data", b64) : null; + + List> out = new ArrayList<>(n); + for (int i = 1; i <= n; i++) { + out.add(uf(String.format("UF-%04d", i), "tenantA", "UI", "USER_FEEDBACK", "Bulk feedback #" + i, embeddedImage, Instant.now())); + } + return out; + } + + private static Map uf(String id, String tenant, String area, String type, String feedback, Map image, + Instant ts) { + Map m = new LinkedHashMap<>(); + m.put("id", id); + m.put("empowerTenantId", tenant); + m.put("applicationArea", area); + m.put("empowerType", type); + m.put("feedback", feedback); + if (image != null) + m.put("image", image); // EMBEDDED map (server coerces to embedded doc) + m.put("timestamp", Date.from(ts)); + return m; + } + + private static byte[] loadDefaultPNG() { + try { + File f = new File("./data/Curonix/Logos/default.png"); + if (!f.exists()) + return null; + return Files.readAllBytes(f.toPath()); + } + catch (Exception ignore) { + return null; + } + } + + private static InsertOptions defaultInsertOptions(RemoteGrpcDatabase grpc, String targetClass, List keyCols) { + // Minimal options: target class + key columns for idempotency; update + // last-write-wins + return InsertOptions.newBuilder().setDatabase(DB_NAME).setTargetClass(targetClass).addAllKeyColumns(keyCols) + .setConflictMode(com.arcadedb.server.grpc.ConflictMode.CONFLICT_UPDATE) + .addAllUpdateColumnsOnConflict(Arrays.asList("feedback", "image", "timestamp", "applicationArea", "empowerType")) + .setTransactionMode(com.arcadedb.server.grpc.TransactionMode.PER_BATCH).setServerBatchSize(500) + .setCredentials(grpc.buildCredentials()) // if your server validates per-request creds + .build(); + } + + // ------------------------------------------------------ + // Timing helper + // ------------------------------------------------------ + + @FunctionalInterface + interface ThrowingRunnable { + void run() throws Exception; + } + + private static void time(String label, ThrowingRunnable r) { + long t0 = System.currentTimeMillis(); + try { + r.run(); + long ms = System.currentTimeMillis() - t0; + System.out.printf("%-35s %8d ms%n", label + ":", ms); + } + catch (Exception e) { + long ms = System.currentTimeMillis() - t0; + System.out.printf("%-35s %8d ms (ERROR: %s)%n", label + ":", ms, e.getMessage()); + throw new RuntimeException(e); + } + } +} \ No newline at end of file diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java index 8d4f6f9378..9aaa0c2b4a 100644 --- a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java @@ -1,5 +1,6 @@ package com.arcadedb.server.grpc; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -148,19 +149,47 @@ public void dropDatabase(DropDatabaseRequest req, StreamObserver resp) { - + try { - // TODO implement: mirror GetDatabasesHandler - // java.util.List names = server.getServer().getDatabaseNames(); - // resp.onNext(ListDatabasesResponse.newBuilder().addAllNames(names).build()); - // resp.onCompleted(); - resp.onError(io.grpc.Status.UNIMPLEMENTED.withDescription("ListDatabases not implemented").asException()); - } + + validateCredentials(req.getCredentials()); + + java.util.Collection names = arcadeServer.getDatabaseNames(); + java.util.ArrayList out = new java.util.ArrayList<>(names); + + java.util.Collections.sort(out, String.CASE_INSENSITIVE_ORDER); + + List allDatabaseInfos = new ArrayList<>(); + + for (String dbName : out) { + + //ServerDatabase db = arcadeServer.getDatabase(dbName); + + DatabaseInfo dbInfo = DatabaseInfo.newBuilder().setName(dbName).build(); + + allDatabaseInfos.add(dbInfo); + } + + // 3) Build and send response + ListDatabasesResponse respMsg = ListDatabasesResponse.newBuilder() + .addAllDatabases(allDatabaseInfos) + .build(); + + resp.onNext(respMsg); + resp.onCompleted(); + } + catch (SecurityException se) { + resp.onError(io.grpc.Status.PERMISSION_DENIED + .withDescription("ListDatabases: " + se.getMessage()) + .asException()); + } catch (Exception e) { - resp.onError(io.grpc.Status.INTERNAL.withDescription("ListDatabases: " + e.getMessage()).asException()); - } + resp.onError(io.grpc.Status.INTERNAL + .withDescription("ListDatabases: " + e.getMessage()) + .asException()); + } } - + @Override public void getDatabaseInfo(GetDatabaseInfoRequest req, StreamObserver resp) { diff --git a/network/src/main/java/com/arcadedb/remote/RemoteServer.java b/network/src/main/java/com/arcadedb/remote/RemoteServer.java index a8f0477939..0403484164 100644 --- a/network/src/main/java/com/arcadedb/remote/RemoteServer.java +++ b/network/src/main/java/com/arcadedb/remote/RemoteServer.java @@ -18,15 +18,16 @@ */ package com.arcadedb.remote; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import com.arcadedb.ContextConfiguration; import com.arcadedb.exception.DatabaseOperationException; import com.arcadedb.serializer.json.JSONObject; -import java.net.*; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.util.*; - /** * Remote Database implementation. It's not thread safe. For multi-thread usage create one instance of RemoteDatabase per thread. * diff --git a/server/TMP/grpc/ArcadeDbGrpcService.java b/server/TMP/grpc/ArcadeDbGrpcService.java new file mode 100644 index 0000000000..1a48868c03 --- /dev/null +++ b/server/TMP/grpc/ArcadeDbGrpcService.java @@ -0,0 +1,1082 @@ +package com.arcadedb.server.grpc; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.arcadedb.database.Database; +import com.arcadedb.database.DatabaseFactory; +import com.arcadedb.database.Document; +import com.arcadedb.database.MutableDocument; +import com.arcadedb.engine.ComponentFile; +import com.arcadedb.exception.DuplicatedKeyException; +import com.arcadedb.exception.ValidationException; +import com.arcadedb.graph.Edge; +import com.arcadedb.graph.Vertex; +import com.arcadedb.query.sql.executor.Result; +import com.arcadedb.query.sql.executor.ResultSet; +import com.arcadedb.server.ArcadeDBServer; +import com.google.protobuf.Empty; +import com.google.protobuf.Timestamp; +import com.google.protobuf.Value; + +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.grpc.stub.ServerCallStreamObserver; +import io.grpc.stub.StreamObserver; + +/** + * gRPC Service implementation for ArcadeDB + */ +public class ArcadeDbGrpcService extends ArcadeDbServiceGrpc.ArcadeDbServiceImplBase { + + private static final Logger logger = LoggerFactory.getLogger(ArcadeDbGrpcService.class); + + // Transaction management + private final Map activeTransactions = new ConcurrentHashMap<>(); + + // Database connection pool + private final Map databasePool = new ConcurrentHashMap<>(); + + // Track highest acknowledged chunk per session for idempotency + private final Map sessionWatermark = new ConcurrentHashMap<>(); + + + // ArcadeDB server reference (optional, for accessing existing databases) + private final ArcadeDBServer arcadeServer; + + // Database directory path + private final String databasePath; + + public ArcadeDbGrpcService(String databasePath, ArcadeDBServer server) { + this.databasePath = databasePath; + this.arcadeServer = server; + } + + public void close() { + // Close all open databases + for (Database db : databasePool.values()) { + try { + if (db != null && db.isOpen()) { + db.close(); + } + } catch (Exception e) { + logger.error("Error closing database", e); + } + } + databasePool.clear(); + + // Clean up transactions + for (Database db : activeTransactions.values()) { + try { + if (db != null && db.isOpen()) { + db.rollback(); + db.close(); + } + } catch (Exception e) { + logger.error("Error closing transaction database", e); + } + } + activeTransactions.clear(); + } + + @Override + public void createDatabase(CreateDatabaseRequest request, + StreamObserver responseObserver) { + try { + // Validate credentials if needed + validateCredentials(request.getCredentials()); + + // Create new database factory for this specific database + DatabaseFactory dbFactory = new DatabaseFactory(databasePath + "/" + request.getDatabaseName()); + + // Create the database + Database database = dbFactory.create(); + database.close(); + + CreateDatabaseResponse response = CreateDatabaseResponse.newBuilder() + .setSuccess(true) + .setMessage("Database created successfully") + .setDatabaseId(request.getDatabaseName()) + .build(); + + responseObserver.onNext(response); + responseObserver.onCompleted(); + + } catch (Exception e) { + logger.error("Error creating database: {}", e.getMessage(), e); + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to create database: " + e.getMessage()) + .asException()); + } + } + + @Override + public void executeQuery(ExecuteQueryRequest request, + StreamObserver responseObserver) { + try { + + // Force compression for streaming (usually beneficial) + CompressionAwareService.setResponseCompression(responseObserver, "gzip"); + + Database database = getDatabase(request.getDatabase(), request.getCredentials()); + + // Check if this is part of a transaction + if (request.hasTransaction()) { + database = activeTransactions.get(request.getTransaction().getTransactionId()); + if (database == null) { + throw new IllegalArgumentException("Invalid transaction ID"); + } + } + + // Execute the query + long startTime = System.currentTimeMillis(); + ResultSet resultSet = database.query("sql", request.getQuery(), + convertParameters(request.getParametersMap())); + + logger.debug("executeQuery(): to get resultSet = {}", (System.currentTimeMillis() - startTime)); + + // Build response + QueryResult.Builder resultBuilder = QueryResult.newBuilder(); + + // Process results + int count = 0; + while (resultSet.hasNext()) { + Result result = resultSet.next(); + if (result.isElement()) { + com.arcadedb.database.Record dbRecord = result.getElement().get(); + Record grpcRecord = convertToGrpcRecord(dbRecord); + resultBuilder.addRecords(grpcRecord); + count++; + + // Apply limit if specified + if (request.getLimit() > 0 && count >= request.getLimit()) { + break; + } + } + } + + resultBuilder.setTotalRecordsInBatch(count); + + long executionTime = System.currentTimeMillis() - startTime; + + ExecuteQueryResponse response = ExecuteQueryResponse.newBuilder() + .addResults(resultBuilder.build()) + .setExecutionTimeMs(executionTime) + .build(); + + logger.debug("executeQuery(): executionTime + response generation = {}", executionTime); + + responseObserver.onNext(response); + responseObserver.onCompleted(); + + } catch (Exception e) { + logger.error("Error executing query: {}", e.getMessage(), e); + responseObserver.onError(Status.INTERNAL + .withDescription("Query execution failed: " + e.getMessage()) + .asException()); + } + } + + @Override + public void beginTransaction(BeginTransactionRequest request, + StreamObserver responseObserver) { + try { + Database database = getDatabase(request.getDatabase(), request.getCredentials()); + + // Begin transaction + database.begin(); + + // Generate transaction ID + String transactionId = generateTransactionId(); + activeTransactions.put(transactionId, database); + + BeginTransactionResponse response = BeginTransactionResponse.newBuilder() + .setTransactionId(transactionId) + .setTimestamp(System.currentTimeMillis()) + .build(); + + responseObserver.onNext(response); + responseObserver.onCompleted(); + + } catch (Exception e) { + logger.error("Error beginning transaction: {}", e.getMessage(), e); + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to begin transaction: " + e.getMessage()) + .asException()); + } + } + + @Override + public void commitTransaction(CommitTransactionRequest request, + StreamObserver responseObserver) { + try { + Database database = activeTransactions.remove(request.getTransaction().getTransactionId()); + if (database == null) { + throw new IllegalArgumentException("Invalid transaction ID"); + } + + // Commit transaction + database.commit(); + + CommitTransactionResponse response = CommitTransactionResponse.newBuilder() + .setSuccess(true) + .setMessage("Transaction committed successfully") + .setTimestamp(System.currentTimeMillis()) + .build(); + + responseObserver.onNext(response); + responseObserver.onCompleted(); + + } catch (Exception e) { + logger.error("Error committing transaction: {}", e.getMessage(), e); + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to commit transaction: " + e.getMessage()) + .asException()); + } + } + + @Override + public void rollbackTransaction(RollbackTransactionRequest request, + StreamObserver responseObserver) { + try { + Database database = activeTransactions.remove(request.getTransaction().getTransactionId()); + if (database == null) { + throw new IllegalArgumentException("Invalid transaction ID"); + } + + // Rollback transaction + database.rollback(); + + RollbackTransactionResponse response = RollbackTransactionResponse.newBuilder() + .setSuccess(true) + .setMessage("Transaction rolled back successfully") + .build(); + + responseObserver.onNext(response); + responseObserver.onCompleted(); + + } catch (Exception e) { + logger.error("Error rolling back transaction: {}", e.getMessage(), e); + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to rollback transaction: " + e.getMessage()) + .asException()); + } + } + + @Override + public void streamQuery(StreamQueryRequest request, StreamObserver responseObserver) { + final ServerCallStreamObserver scso = + (ServerCallStreamObserver) responseObserver; + + final AtomicBoolean cancelled = new AtomicBoolean(false); + scso.setOnCancelHandler(() -> cancelled.set(true)); + + try { + Database database = getDatabase(request.getDatabase(), request.getCredentials()); + final int batchSize = Math.max(1, request.getBatchSize()); + + // Dispatch on mode + switch (request.getRetrievalMode()) { + case MATERIALIZE_ALL -> streamMaterialized(database, request, batchSize, scso, cancelled); + case PAGED -> streamPaged(database, request, batchSize, scso, cancelled); + case CURSOR -> streamCursor(database, request, batchSize, scso, cancelled); + default -> streamCursor(database, request, batchSize, scso, cancelled); + } + + if (!cancelled.get()) scso.onCompleted(); + } catch (Exception e) { + if (!cancelled.get()) { + responseObserver.onError( + Status.INTERNAL.withDescription("Stream query failed: " + e.getMessage()).asException()); + } + } + } + + /** Mode 1 (existing behavior-ish): run once and iterate results, batching as we go. */ + private void streamCursor(Database db, StreamQueryRequest req, int batchSize, ServerCallStreamObserver scso, + AtomicBoolean cancelled) { + long running = 0L; + QueryResult.Builder batch = QueryResult.newBuilder(); + int inBatch = 0; + + try (ResultSet rs = db.query("sql", req.getQuery(), convertParameters(req.getParametersMap()))) { + while (rs.hasNext()) { + if (cancelled.get()) + return; + waitUntilReady(scso, cancelled); + + Result r = rs.next(); + if (!r.isElement()) + continue; + + com.arcadedb.database.Record rec = r.getElement().get(); + batch.addRecords(convertToGrpcRecord(rec)); + inBatch++; + running++; + + if (inBatch >= batchSize) { + safeOnNext(scso, cancelled, + batch.setTotalRecordsInBatch(inBatch).setRunningTotalEmitted(running).setIsLastBatch(false).build()); + batch = QueryResult.newBuilder(); + inBatch = 0; + } + } + + if (!cancelled.get() && inBatch > 0) { + safeOnNext(scso, cancelled, batch.setTotalRecordsInBatch(inBatch).setRunningTotalEmitted(running).setIsLastBatch(true).build()); + } + } + } + + /** Mode 2: materialize everything first (simple, but can be memory-heavy). */ + private void streamMaterialized(Database db, StreamQueryRequest req, int batchSize, ServerCallStreamObserver scso, + AtomicBoolean cancelled) { + final java.util.ArrayList all = new java.util.ArrayList<>(); + try (ResultSet rs = db.query("sql", req.getQuery(), convertParameters(req.getParametersMap()))) { + while (rs.hasNext()) { + if (cancelled.get()) + return; + Result r = rs.next(); + if (!r.isElement()) + continue; + all.add(convertToGrpcRecord(r.getElement().get())); + } + } + + long running = 0L; + for (int i = 0; i < all.size(); i += batchSize) { + if (cancelled.get()) + return; + waitUntilReady(scso, cancelled); + + int end = Math.min(i + batchSize, all.size()); + QueryResult.Builder b = QueryResult.newBuilder(); + for (int j = i; j < end; j++) + b.addRecords(all.get(j)); + + running += (end - i); + safeOnNext(scso, cancelled, + b.setTotalRecordsInBatch(end - i).setRunningTotalEmitted(running).setIsLastBatch(end == all.size()).build()); + } + } + + /** Mode 3: only fetch one page’s worth of rows per emission via LIMIT/SKIP. */ + private void streamPaged(Database db, StreamQueryRequest req, int batchSize, ServerCallStreamObserver scso, + AtomicBoolean cancelled) { + + final String pagedSql = wrapWithSkipLimit(req.getQuery()); // see helper below + int offset = 0; + long running = 0L; + + while (true) { + if (cancelled.get()) + return; + waitUntilReady(scso, cancelled); + + java.util.Map params = new java.util.HashMap<>(convertParameters(req.getParametersMap())); + params.put("_skip", offset); + params.put("_limit", batchSize); + + int count = 0; + QueryResult.Builder b = QueryResult.newBuilder(); + + try (ResultSet rs = db.query("sql", pagedSql, params)) { + while (rs.hasNext()) { + if (cancelled.get()) + return; + Result r = rs.next(); + if (!r.isElement()) + continue; + b.addRecords(convertToGrpcRecord(r.getElement().get())); + count++; + } + } + + if (count == 0) + return; // no more rows + + running += count; + boolean last = count < batchSize; + + safeOnNext(scso, cancelled, b.setTotalRecordsInBatch(count).setRunningTotalEmitted(running).setIsLastBatch(last).build()); + + if (last) + return; + offset += batchSize; + } + } + + /** Wrap arbitrary SQL so we can safely inject LIMIT/SKIP outside. */ + private String wrapWithSkipLimit(String originalSql) { + + // Minimal defensive approach; you can do a real parser if needed. + + // ArcadeDB: SELECT FROM (...) [ORDER BY ...] SKIP :_skip LIMIT :_limit + + return "SELECT FROM (" + originalSql + ") ORDER BY @rid SKIP :_skip LIMIT :_limit"; + } + + private void safeOnNext(ServerCallStreamObserver scso, AtomicBoolean cancelled, QueryResult payload) { + if (cancelled.get()) + return; + try { + scso.onNext(payload); + } + catch (StatusRuntimeException e) { + if (e.getStatus().getCode() == Status.Code.CANCELLED) { + cancelled.set(true); + return; + } + throw e; + } + } + + private void waitUntilReady(ServerCallStreamObserver scso, AtomicBoolean cancelled) { + // Skip if you’re okay with best-effort pushes; otherwise honor transport readiness + if (scso.isReady()) + return; + while (!scso.isReady()) { + if (cancelled.get()) + return; + // avoid burning CPU: + try { + Thread.sleep(1); + } + catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + return; + } + } + } + + // --- 1) Unary bulk --- + @Override + public void bulkInsert(BulkInsertRequest req, StreamObserver resp) { + final InsertOptions opts = defaults(req.getOptions()); + try (InsertContext ctx = new InsertContext(opts)) { + insertRows(ctx, req.getRowsList().iterator()); + ctx.flushCommit(true); + resp.onNext(ctx.summary()); + resp.onCompleted(); + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("bulkInsert: " + e.getMessage()).asException()); + } + } + + // --- 2) Client-streaming; single summary at end --- + @Override + public StreamObserver insertStream(StreamObserver resp) { + final ServerCallStreamObserver call = (ServerCallStreamObserver) resp; + call.disableAutoInboundFlowControl(); + + // You can pass options via the first chunk's session_id, or set server + // defaults; here: server defaults + final InsertOptions opts = defaults(InsertOptions.newBuilder().build()); + final InsertContext ctx; + try { + ctx = new InsertContext(opts); + } + catch (Exception e) { + resp.onError(Status.FAILED_PRECONDITION.withDescription(e.getMessage()).asException()); + return new StreamObserver<>() { + public void onNext(InsertChunk c) { + } + + public void onError(Throwable t) { + } + + public void onCompleted() { + } + }; + } + + return new StreamObserver<>() { + @Override + public void onNext(InsertChunk c) { + insertRows(ctx, c.getRowsList().iterator()); + call.request(1); + } + + @Override + public void onError(Throwable t) { + ctx.closeQuietly(); + } + + @Override + public void onCompleted() { + try { + ctx.flushCommit(true); + resp.onNext(ctx.summary()); + resp.onCompleted(); + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("insertStream: " + e.getMessage()).asException()); + } + finally { + ctx.closeQuietly(); + } + } + }; + } + + // --- 3) Bi-di with per-batch ACKs --- + @Override + public StreamObserver insertBidirectional(StreamObserver resp) { + final ServerCallStreamObserver call = (ServerCallStreamObserver) resp; + call.disableAutoInboundFlowControl(); + + final AtomicReference ref = new AtomicReference<>(); + return new StreamObserver<>() { + @Override + public void onNext(InsertRequest req) { + switch (req.getMsgCase()) { + case START -> { + InsertOptions opts = defaults(req.getStart().getOptions()); + InsertContext ctx = new InsertContext(opts); + ref.set(ctx); + sessionWatermark.put(ctx.sessionId, 0L); + resp.onNext(InsertResponse.newBuilder().setStarted(Started.newBuilder().setSessionId(ctx.sessionId).build()).build()); + call.request(1); + } + case CHUNK -> { + InsertContext ctx = require(ref.get(), "session not started"); + InsertChunk c = req.getChunk(); + + // Idempotent replay guard + long hi = sessionWatermark.getOrDefault(ctx.sessionId, 0L); + if (c.getChunkSeq() <= hi) { + resp.onNext(InsertResponse.newBuilder() + .setBatchAck(BatchAck.newBuilder().setSessionId(ctx.sessionId).setChunkSeq(c.getChunkSeq()).build()).build()); + call.request(1); + return; + } + + BatchCounts counts = insertRows(ctx, c.getRowsList().iterator()); + sessionWatermark.put(ctx.sessionId, c.getChunkSeq()); + + resp.onNext(InsertResponse.newBuilder() + .setBatchAck(BatchAck.newBuilder().setSessionId(ctx.sessionId).setChunkSeq(c.getChunkSeq()) + .setInserted(counts.inserted).setUpdated(counts.updated).setIgnored(counts.ignored).setFailed(counts.failed) + .addAllErrors(counts.errors)) + .build()); + call.request(1); + } + case COMMIT -> { + InsertContext ctx = require(ref.get(), "session not started"); + try { + ctx.flushCommit(true); + resp.onNext(InsertResponse.newBuilder().setCommitted(Committed.newBuilder().setSummary(ctx.summary()).build()).build()); + resp.onCompleted(); + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("commit: " + e.getMessage()).asException()); + } + finally { + sessionWatermark.remove(ctx.sessionId); + ctx.closeQuietly(); + } + } + case MSG_NOT_SET -> { + /* ignore */ } + } + } + + @Override + public void onError(Throwable t) { + InsertContext ctx = ref.get(); + if (ctx != null) + ctx.closeQuietly(); + } + + @Override + public void onCompleted() { + /* if client half-closes without COMMIT, choose policy */ } + }; + } + + // ---------- Core insert plumbing ---------- + + private static final class BatchCounts { + long inserted, updated, ignored, failed; + final List errors = new ArrayList<>(); + + void addError(long rowIndex, String code, String msg, String field) { + failed++; + errors.add(RowError.newBuilder().setRowIndex(rowIndex).setCode(code).setMessage(msg).setField(field).build()); + } + } + + private BatchCounts insertRows(InsertContext ctx, Iterator it) { + + BatchCounts c = new BatchCounts(); + + int inBatch = 0; + + while (it.hasNext()) { + Record r = it.next(); + ctx.received++; + try { + + MutableDocument doc = ctx.db.newDocument(ctx.opts.getTargetClass()); + + // map fields: + applyGrpcRecordToDocument(r, doc); + + switch (ctx.opts.getConflictMode()) { + case CONFLICT_ABORT -> { + doc.save(); + c.inserted++; + } + case CONFLICT_IGNORE -> { + try { + doc.save(); + c.inserted++; + } + catch (Exception dup) { + c.ignored++; + } + } + case CONFLICT_UPDATE -> { + if (tryUpsert(ctx, doc)) + c.updated++; + else { + doc.save(); + c.inserted++; + } + } + } + + } + catch (ValidationException ve) { + c.addError(ctx.received - 1, "VALIDATION", ve.getMessage(), ""); + } + catch (DuplicatedKeyException dke) { + if (ctx.opts.getConflictMode() == ConflictMode.CONFLICT_ABORT) { + c.addError(ctx.received - 1, "CONFLICT", dke.getMessage(), ""); + } + else { + c.ignored++; + } + } + catch (Exception e) { + c.addError(ctx.received - 1, "DB_ERROR", e.getMessage(), ""); + } + + inBatch++; + if (ctx.opts.getTransactionMode() == TransactionMode.PER_BATCH && inBatch >= ctx.serverBatchSize()) { + ctx.flushCommit(false); + inBatch = 0; + } + else if (ctx.opts.getTransactionMode() == TransactionMode.PER_ROW) { + ctx.flushCommit(false); + inBatch = 0; + } + } + + return c; + } + + private void applyGrpcRecordToDocument(Record r, MutableDocument doc) { + // Example if proto = message Record { map + // properties = 1; } + if (hasMethod(r, "getPropertiesMap")) { + @SuppressWarnings("unchecked") + Map props = (Map) invokeNoArg(r, "getPropertiesMap"); + props.forEach((k, v) -> doc.set(k, toJava(v))); + return; + } + + // Example if proto = message Record { repeated Property properties = 1; message + // Property { string key=1; google.protobuf.Value value=2; } } + if (hasMethod(r, "getPropertiesList")) { + @SuppressWarnings("unchecked") + List list = (List) invokeNoArg(r, "getPropertiesList"); + for (Object p : list) { + String key = (String) invokeNoArg(p, "getKey"); + com.google.protobuf.Value val = (com.google.protobuf.Value) invokeNoArg(p, "getValue"); + doc.set(key, toJava(val)); + } + return; + } + + // Fallback: if your proto DOES have fields map after all + if (hasMethod(r, "getFieldsMap")) { + @SuppressWarnings("unchecked") + Map fields = (Map) invokeNoArg(r, "getFieldsMap"); + fields.forEach((k, v) -> doc.set(k, toJava(v))); + } + } + + // tiny reflection helpers (compile-time safe across proto variants) + private static boolean hasMethod(Object o, String name) { + try { + o.getClass().getMethod(name); + return true; + } + catch (NoSuchMethodException e) { + return false; + } + } + + private static Object invokeNoArg(Object o, String name) { + try { + return o.getClass().getMethod(name).invoke(o); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + private boolean tryUpsert(InsertContext ctx, MutableDocument doc) { + + if (ctx.keyCols.isEmpty()) + return false; + + String where = String.join(" AND ", ctx.keyCols.stream().map(k -> k + " = ?").toList()); + + Object[] params = ctx.keyCols.stream().map(doc::get).toArray(); + + try (ResultSet rs = ctx.db.query("sql", "SELECT FROM " + ctx.opts.getTargetClass() + " WHERE " + where, params)) { + + if (!rs.hasNext()) + return false; + + var res = rs.next(); + + if (!res.isElement()) + return false; + + var existing = res.getElement().get(); + + MutableDocument m = (MutableDocument) existing.asDocument(true); + + for (String col : ctx.updateCols) { + m.set(col, doc.get(col)); + } + + m.save(); + + return true; + } + } + + private static Object toJava(Value v) { + return switch (v.getKindCase()) { + case STRING_VALUE -> v.getStringValue(); + case NUMBER_VALUE -> v.getNumberValue(); + case BOOL_VALUE -> v.getBoolValue(); + case NULL_VALUE -> null; + case STRUCT_VALUE -> new HashMap<>(v.getStructValue().getFieldsMap()); // or nested doc + case LIST_VALUE -> v.getListValue().getValuesList().stream().map(ArcadeDbGrpcService::toJava).toList(); + case KIND_NOT_SET -> null; + }; + } + + private InsertOptions defaults(InsertOptions in) { + InsertOptions.Builder b = in.toBuilder(); + if (in.getServerBatchSize() == 0) + b.setServerBatchSize(1000); + if (in.getTransactionMode() == TransactionMode.UNRECOGNIZED) + b.setTransactionMode(TransactionMode.PER_BATCH); + return b.build(); + } + + private static T require(T v, String msg) { + if (v == null) + throw Status.FAILED_PRECONDITION.withDescription(msg).asRuntimeException(); + return v; + } + + private static Timestamp ts(long ms) { + return Timestamp.newBuilder().setSeconds(ms / 1000).setNanos((int) ((ms % 1000) * 1_000_000)).build(); + } + + private final class InsertContext implements AutoCloseable { + + final InsertOptions opts; + final Database db; + final String sessionId = UUID.randomUUID().toString(); + final List keyCols; + final List updateCols; + long received = 0; + final long started = System.currentTimeMillis(); + + InsertContext(InsertOptions opts) { + this.opts = opts; + this.db = getDatabase(opts.getDatabase(), opts.getCredentials()); // your existing helper + this.keyCols = opts.getKeyColumnsList(); + this.updateCols = opts.getUpdateColumnsOnConflictList(); + if (opts.getTransactionMode() == TransactionMode.PER_STREAM) + db.begin(); + } + + int serverBatchSize() { + return opts.getServerBatchSize(); + } + + void flushCommit(boolean end) { + if (opts.getValidateOnly()) { + if (end && opts.getTransactionMode() == TransactionMode.PER_STREAM) + db.rollback(); + return; + } + if (opts.getTransactionMode() == TransactionMode.PER_BATCH) { + db.commit(); + db.begin(); + } + else if (end && opts.getTransactionMode() == TransactionMode.PER_STREAM) + db.commit(); + } + + InsertSummary summary() { + return InsertSummary.newBuilder().setReceived(received) // set by caller + // inserted/updated/ignored/failed are tracked in BatchCounts -> you can total + // them here if you prefer + .setStartedAt(ts(started)).setFinishedAt(ts(System.currentTimeMillis())).build(); + } + + @Override + public void close() { + } + + void closeQuietly() { + try { + close(); + } + catch (Exception ignore) { + } + } + } + + @Override + public void ping(Empty request, StreamObserver responseObserver) { + PingResponse response = PingResponse.newBuilder() + .setTimestamp(System.currentTimeMillis()) + .setMessage("pong") + .build(); + + responseObserver.onNext(response); + responseObserver.onCompleted(); + } + + @Override + public void getServerStatus(Empty request, StreamObserver responseObserver) { + try { + ServerStatusResponse response = ServerStatusResponse.newBuilder() + .setVersion("25.8.1-SNAPSHOT") + .setStatus("RUNNING") + .setUptimeMs(getUptime()) + .setActiveConnections(getActiveConnections()) + .putMetrics("active_transactions", String.valueOf(activeTransactions.size())) + .build(); + + responseObserver.onNext(response); + responseObserver.onCompleted(); + + } catch (Exception e) { + logger.error("Error getting server status: {}", e.getMessage(), e); + responseObserver.onError(Status.INTERNAL + .withDescription("Failed to get server status: " + e.getMessage()) + .asException()); + } + } + + // Helper methods + + private Database getDatabase(String databaseName, DatabaseCredentials credentials) { + + // Validate credentials + validateCredentials(credentials); + + // Use the same approach as Postgres/Redis plugins + if (arcadeServer != null) { + + // This is how other plugins do it - get the already-open database + Database db = arcadeServer.getDatabase(databaseName); + + logger.debug("getDatabase(): db = {} isOpen = {}", db, db.isOpen()); + + if (db != null) { + return db; + } + } + + // Check if database is already in the pool + String poolKey = databaseName; + Database database = databasePool.get(poolKey); + + if (database != null && database.isOpen()) { + // Return existing open database + return database; + } + + // Create new database connection + synchronized (databasePool) { + // Double-check after acquiring lock + database = databasePool.get(poolKey); + if (database != null && database.isOpen()) { + return database; + } + + // Create database factory for the specific database + DatabaseFactory dbFactory = new DatabaseFactory(databasePath + "/" + databaseName); + + try { + // Open database - ArcadeDB requires MODE parameter + if (dbFactory.exists()) { + // Try READ_ONLY first to avoid conflicts + try { + database = dbFactory.open(ComponentFile.MODE.READ_ONLY); + } catch (Exception e) { + // If READ_ONLY fails, try READ_WRITE + logger.debug("Opening database in READ_WRITE mode: {}", databaseName); + database = dbFactory.open(ComponentFile.MODE.READ_WRITE); + } + } else { + // Create if it doesn't exist + database = dbFactory.create(); + } + + // Add to pool + databasePool.put(poolKey, database); + return database; + + } catch (Exception e) { + logger.error("Failed to open database: {}", databaseName, e); + throw new RuntimeException("Cannot open database: " + databaseName + " - " + e.getMessage(), e); + } + } + } + + private void validateCredentials(DatabaseCredentials credentials) { + // Implement credential validation logic + // This is a placeholder - integrate with ArcadeDB's security system + if (credentials == null || credentials.getUsername().isEmpty()) { + throw new IllegalArgumentException("Invalid credentials"); + } + } + + private Map convertParameters(Map protoParams) { + Map params = new HashMap<>(); + for (Map.Entry entry : protoParams.entrySet()) { + params.put(entry.getKey(), convertFromProtobufValue(entry.getValue())); + } + return params; + } + + private Object convertFromProtobufValue(Value value) { + switch (value.getKindCase()) { + case NULL_VALUE: + return null; + case NUMBER_VALUE: + return value.getNumberValue(); + case STRING_VALUE: + return value.getStringValue(); + case BOOL_VALUE: + return value.getBoolValue(); + case STRUCT_VALUE: + return convertFromProtobufValues(value.getStructValue().getFieldsMap()); + case LIST_VALUE: + return value.getListValue().getValuesList().stream() + .map(this::convertFromProtobufValue) + .toArray(); + default: + return null; + } + } + + private Map convertFromProtobufValues(Map values) { + Map result = new HashMap<>(); + for (Map.Entry entry : values.entrySet()) { + result.put(entry.getKey(), convertFromProtobufValue(entry.getValue())); + } + return result; + } + + private Record convertToGrpcRecord(com.arcadedb.database.Record dbRecord) { + Record.Builder builder = Record.newBuilder() + .setRid(dbRecord.getIdentity().toString()); + + // Handle different record types + if (dbRecord instanceof Document) { + Document doc = (Document) dbRecord; + builder.setType(doc.getTypeName()); + + // Convert properties + Set properties = doc.getPropertyNames(); + for (String property : properties) { + Object value = doc.get(property); + if (value != null) { + builder.putProperties(property, convertToProtobufValue(value)); + } + } + } else if (dbRecord instanceof Vertex) { + Vertex vertex = (Vertex) dbRecord; + builder.setType(vertex.getTypeName()); + + // Convert properties + Set properties = vertex.getPropertyNames(); + for (String property : properties) { + Object value = vertex.get(property); + if (value != null) { + builder.putProperties(property, convertToProtobufValue(value)); + } + } + } else if (dbRecord instanceof Edge) { + Edge edge = (Edge) dbRecord; + builder.setType(edge.getTypeName()); + + // Convert properties + Set properties = edge.getPropertyNames(); + for (String property : properties) { + Object value = edge.get(property); + if (value != null) { + builder.putProperties(property, convertToProtobufValue(value)); + } + } + } + + return builder.build(); + } + + private Value convertToProtobufValue(Object value) { + if (value == null) { + return Value.newBuilder().setNullValue(com.google.protobuf.NullValue.NULL_VALUE).build(); + } else if (value instanceof Boolean) { + return Value.newBuilder().setBoolValue((Boolean) value).build(); + } else if (value instanceof Number) { + return Value.newBuilder().setNumberValue(((Number) value).doubleValue()).build(); + } else if (value instanceof String) { + return Value.newBuilder().setStringValue((String) value).build(); + } else { + // For complex objects, convert to string + return Value.newBuilder().setStringValue(value.toString()).build(); + } + } + + private String generateTransactionId() { + return "tx_" + System.nanoTime(); + } + + private long getUptime() { + // Implement uptime calculation + return System.currentTimeMillis(); + } + + private int getActiveConnections() { + // Implement active connections count + return 0; + } +} \ No newline at end of file diff --git a/server/TMP/grpc/CompressionAwareService.java b/server/TMP/grpc/CompressionAwareService.java new file mode 100644 index 0000000000..860501d231 --- /dev/null +++ b/server/TMP/grpc/CompressionAwareService.java @@ -0,0 +1,86 @@ +package com.arcadedb.server.grpc; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.grpc.stub.ServerCallStreamObserver; +import io.grpc.stub.StreamObserver; + + +/** + * Enhanced compression-aware service wrapper + */ +class CompressionAwareService { + private static final Logger logger = LoggerFactory.getLogger(CompressionAwareService.class); + + /** + * Force compression for a specific response + */ + public static void setResponseCompression(StreamObserver responseObserver, String compression) { + if (responseObserver instanceof ServerCallStreamObserver) { + ServerCallStreamObserver serverObserver = (ServerCallStreamObserver) responseObserver; + serverObserver.setCompression(compression); + logger.debug("Set response compression to: {}", compression); + } + } + + /** + * Check if current request was compressed (call from service method) + */ + public static boolean isCurrentRequestCompressed() { + GrpcCompressionInterceptor.CompressionInfo info = + GrpcCompressionInterceptor.COMPRESSION_KEY.get(); + return info != null && info.requestCompressed; + } + + /** + * Get current request compression encoding + */ + public static String getCurrentRequestEncoding() { + GrpcCompressionInterceptor.CompressionInfo info = + GrpcCompressionInterceptor.COMPRESSION_KEY.get(); + return info != null ? info.requestEncoding : "identity"; + } + + /** + * Get compression statistics for monitoring + */ + public static class CompressionStats { + private long compressedRequests = 0; + private long uncompressedRequests = 0; + private long compressedResponses = 0; + private long uncompressedResponses = 0; + + public synchronized void recordRequest(boolean compressed) { + if (compressed) { + compressedRequests++; + } else { + uncompressedRequests++; + } + } + + public synchronized void recordResponse(boolean compressed) { + if (compressed) { + compressedResponses++; + } else { + uncompressedResponses++; + } + } + + public synchronized String getStats() { + long totalRequests = compressedRequests + uncompressedRequests; + long totalResponses = compressedResponses + uncompressedResponses; + + double reqCompressionRate = totalRequests > 0 ? + (compressedRequests * 100.0 / totalRequests) : 0; + double respCompressionRate = totalResponses > 0 ? + (compressedResponses * 100.0 / totalResponses) : 0; + + return String.format( + "Compression Stats - Requests: %.1f%% (%d/%d), Responses: %.1f%% (%d/%d)", + reqCompressionRate, compressedRequests, totalRequests, + respCompressionRate, compressedResponses, totalResponses + ); + } + } +} \ No newline at end of file diff --git a/server/TMP/grpc/GrpcAuthInterceptor.java b/server/TMP/grpc/GrpcAuthInterceptor.java new file mode 100644 index 0000000000..06274dcab1 --- /dev/null +++ b/server/TMP/grpc/GrpcAuthInterceptor.java @@ -0,0 +1,142 @@ +package com.arcadedb.server.grpc; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.arcadedb.server.security.ServerSecurity; + +import io.grpc.Context; +import io.grpc.Contexts; +import io.grpc.Metadata; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; +import io.grpc.Status; + +/** + * Authentication interceptor for gRPC requests + */ +class GrpcAuthInterceptor implements ServerInterceptor { + private static final Logger logger = LoggerFactory.getLogger(GrpcAuthInterceptor.class); + + private static final String BEARER_TYPE = "Bearer"; + private static final Metadata.Key AUTHORIZATION_HEADER = + Metadata.Key.of("authorization", Metadata.ASCII_STRING_MARSHALLER); + private static final Metadata.Key USER_HEADER = + Metadata.Key.of("x-arcade-user", Metadata.ASCII_STRING_MARSHALLER); + private static final Metadata.Key PASSWORD_HEADER = + Metadata.Key.of("x-arcade-password", Metadata.ASCII_STRING_MARSHALLER); + private static final Metadata.Key DATABASE_HEADER = + Metadata.Key.of("x-arcade-database", Metadata.ASCII_STRING_MARSHALLER); + + private final ServerSecurity security; + private final boolean securityEnabled; + + public GrpcAuthInterceptor(ServerSecurity security) { + this.security = security; + // Check if security is enabled by checking if it's not null and has users configured + this.securityEnabled = (security != null && security.getUsers() != null && !security.getUsers().isEmpty()); + } + + @Override + public ServerCall.Listener interceptCall( + ServerCall call, + Metadata headers, + ServerCallHandler next) { + + String methodName = call.getMethodDescriptor().getFullMethodName(); + + // Skip auth for health check and reflection + if (methodName.startsWith("grpc.health.") || + methodName.startsWith("grpc.reflection.")) { + return next.startCall(call, headers); + } + + // If security is not enabled, allow all requests + if (!securityEnabled) { + return next.startCall(call, headers); + } + + try { + // Get database name from header (required for authentication) + String database = headers.get(DATABASE_HEADER); + if (database == null || database.isEmpty()) { + database = "default"; // Use default database if not specified + } + + // Try Bearer token authentication first + String authorization = headers.get(AUTHORIZATION_HEADER); + if (authorization != null && authorization.startsWith(BEARER_TYPE)) { + String token = authorization.substring(BEARER_TYPE.length()).trim(); + if (!validateToken(token, database)) { + call.close(Status.UNAUTHENTICATED.withDescription("Invalid token"), new Metadata()); + return new ServerCall.Listener() {}; + } + } else { + // Try basic authentication + String username = headers.get(USER_HEADER); + String password = headers.get(PASSWORD_HEADER); + + if (username == null || password == null) { + // No authentication provided for secured server + call.close(Status.UNAUTHENTICATED.withDescription("Authentication required"), new Metadata()); + return new ServerCall.Listener() {}; + } else { + // Validate credentials + if (!validateCredentials(username, password, database)) { + call.close(Status.UNAUTHENTICATED.withDescription("Invalid credentials"), new Metadata()); + return new ServerCall.Listener() {}; + } + // Add user to context + Context context = Context.current().withValue(USER_CONTEXT_KEY, username); + return Contexts.interceptCall(context, call, headers, next); + } + } + + // Add user context for downstream processing + Context context = Context.current(); + if (authorization != null) { + context = context.withValue(USER_CONTEXT_KEY, extractUserFromToken(authorization)); + } + + return Contexts.interceptCall(context, call, headers, next); + + } catch (Exception e) { + logger.error("Authentication error", e); + call.close(Status.INTERNAL.withDescription("Authentication error"), new Metadata()); + return new ServerCall.Listener() {}; + } + } + + private boolean validateToken(String token, String database) { + // Implement token validation logic + // This could integrate with JWT, OAuth2, or custom token validation + // For now, this is a placeholder + return true; + } + + private boolean validateCredentials(String username, String password, String database) { + if (security == null) { + return true; // No security configured + } + + try { + // ArcadeDB's authenticate method requires database name as well + // Returns a SecurityUser object if authentication succeeds, null otherwise + Object authenticatedUser = security.authenticate(username, password, database); + return authenticatedUser != null; + } catch (Exception e) { + logger.error("Failed to authenticate user: {} for database: {}", username, database, e); + return false; + } + } + + private String extractUserFromToken(String authorization) { + // Extract user information from token + // This is a placeholder - implement based on your token format + return "authenticated-user"; + } + + // Context key for storing authenticated user + public static final Context.Key USER_CONTEXT_KEY = Context.key("user"); +} \ No newline at end of file diff --git a/server/TMP/grpc/GrpcCompressionInterceptor.java b/server/TMP/grpc/GrpcCompressionInterceptor.java new file mode 100644 index 0000000000..c1221db2f9 --- /dev/null +++ b/server/TMP/grpc/GrpcCompressionInterceptor.java @@ -0,0 +1,83 @@ +package com.arcadedb.server.grpc; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.grpc.Context; +import io.grpc.Contexts; +import io.grpc.ForwardingServerCall; +import io.grpc.Metadata; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; + +/** + * Compression interceptor that can force compression based on configuration + */ +class GrpcCompressionInterceptor implements ServerInterceptor { + private static final Logger logger = LoggerFactory.getLogger(GrpcCompressionInterceptor.class); + + // Context key to store compression info + public static final Context.Key COMPRESSION_KEY = Context.key("compression-info"); + + private final boolean forceCompression; + private final String compressionType; + private final int minMessageSizeForCompression; + + public GrpcCompressionInterceptor(boolean forceCompression, String compressionType, int minMessageSizeBytes) { + this.forceCompression = forceCompression; + this.compressionType = compressionType != null ? compressionType : "gzip"; + this.minMessageSizeForCompression = minMessageSizeBytes; + } + + @Override + public ServerCall.Listener interceptCall( + ServerCall call, + Metadata headers, + ServerCallHandler next) { + + String methodName = call.getMethodDescriptor().getFullMethodName(); + + // Check if request is compressed + String requestEncoding = headers.get(Metadata.Key.of("grpc-encoding", Metadata.ASCII_STRING_MARSHALLER)); + boolean requestCompressed = requestEncoding != null && !requestEncoding.equals("identity"); + + // Store compression info in context + CompressionInfo compressionInfo = new CompressionInfo(requestCompressed, requestEncoding); + Context context = Context.current().withValue(COMPRESSION_KEY, compressionInfo); + + // Wrap the call to control compression + ServerCall compressedCall = new ForwardingServerCall.SimpleForwardingServerCall(call) { + private boolean compressionSet = false; + + @Override + public void sendMessage(RespT message) { + if (!compressionSet && forceCompression) { + // Force compression for this response + if (this.getMethodDescriptor().getType().serverSendsOneMessage()) { + // For unary calls, we can set compression + setMessageCompression(true); + compressionSet = true; + logger.debug("Forced {} compression for method: {}", compressionType, methodName); + } + } + super.sendMessage(message); + } + }; + + return Contexts.interceptCall(context, compressedCall, headers, next); + } + + /** + * Helper class to store compression information + */ + public static class CompressionInfo { + public final boolean requestCompressed; + public final String requestEncoding; + + public CompressionInfo(boolean requestCompressed, String requestEncoding) { + this.requestCompressed = requestCompressed; + this.requestEncoding = requestEncoding; + } + } +} \ No newline at end of file diff --git a/server/TMP/grpc/GrpcLoggingInterceptor.java b/server/TMP/grpc/GrpcLoggingInterceptor.java new file mode 100644 index 0000000000..1dcdc0145f --- /dev/null +++ b/server/TMP/grpc/GrpcLoggingInterceptor.java @@ -0,0 +1,81 @@ +package com.arcadedb.server.grpc; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.grpc.ForwardingServerCall; +import io.grpc.Metadata; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; +import io.grpc.Status; + +/** + * Logging interceptor for gRPC requests with compression detection + */ +class GrpcLoggingInterceptor implements ServerInterceptor { + private static final Logger logger = LoggerFactory.getLogger(GrpcLoggingInterceptor.class); + + private static final Metadata.Key GRPC_ENCODING_KEY = + Metadata.Key.of("grpc-encoding", Metadata.ASCII_STRING_MARSHALLER); + private static final Metadata.Key GRPC_ACCEPT_ENCODING_KEY = + Metadata.Key.of("grpc-accept-encoding", Metadata.ASCII_STRING_MARSHALLER); + + @Override + public ServerCall.Listener interceptCall( + ServerCall call, + Metadata headers, + ServerCallHandler next) { + + String methodName = call.getMethodDescriptor().getFullMethodName(); + long startTime = System.currentTimeMillis(); + + // Check if client sent compressed request + String requestEncoding = headers.get(GRPC_ENCODING_KEY); + String acceptEncoding = headers.get(GRPC_ACCEPT_ENCODING_KEY); + boolean requestCompressed = requestEncoding != null && !requestEncoding.equals("identity"); + boolean clientAcceptsCompression = acceptEncoding != null && acceptEncoding.contains("gzip"); + + logger.debug("gRPC call started: {} (request compression: {}, client accepts: {})", + methodName, + requestCompressed ? requestEncoding : "none", + clientAcceptsCompression ? acceptEncoding : "none"); + + ServerCall wrappedCall = new ForwardingServerCall.SimpleForwardingServerCall(call) { + private String responseCompression = "none"; + + @Override + public void sendHeaders(Metadata headers) { + // Check if we're sending compressed response + String encoding = headers.get(GRPC_ENCODING_KEY); + if (encoding != null && !encoding.equals("identity")) { + responseCompression = encoding; + } + super.sendHeaders(headers); + } + + @Override + public void close(Status status, Metadata trailers) { + long duration = System.currentTimeMillis() - startTime; + + // Add compression info to trailers for client visibility + trailers.put(Metadata.Key.of("x-grpc-compression-used", Metadata.ASCII_STRING_MARSHALLER), + responseCompression); + trailers.put(Metadata.Key.of("x-grpc-request-compressed", Metadata.ASCII_STRING_MARSHALLER), + String.valueOf(requestCompressed)); + + if (status.isOk()) { + logger.debug("gRPC call completed: {} ({}ms, req-compression: {}, resp-compression: {})", + methodName, duration, + requestCompressed ? requestEncoding : "none", + responseCompression); + } else { + logger.warn("gRPC call failed: {} - {} ({}ms)", methodName, status, duration); + } + super.close(status, trailers); + } + }; + + return next.startCall(wrappedCall, headers); + } +} \ No newline at end of file diff --git a/server/TMP/grpc/GrpcMetricsInterceptor.java b/server/TMP/grpc/GrpcMetricsInterceptor.java new file mode 100644 index 0000000000..100a1ee665 --- /dev/null +++ b/server/TMP/grpc/GrpcMetricsInterceptor.java @@ -0,0 +1,78 @@ +package com.arcadedb.server.grpc; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.arcadedb.server.ArcadeDBServer; + +import io.grpc.ForwardingServerCall; +import io.grpc.Metadata; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; +import io.grpc.Status; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; + +/** + * Metrics interceptor for gRPC requests using Micrometer + */ +class GrpcMetricsInterceptor implements ServerInterceptor { + + private static final Logger logger = LoggerFactory.getLogger(GrpcMetricsInterceptor.class); + + private final MeterRegistry meterRegistry; + private final Counter requestCounter; + private final Counter errorCounter; + private final Timer requestTimer; + + public GrpcMetricsInterceptor(ArcadeDBServer server) { + // Try to get existing meter registry from server, or create a simple one + this.meterRegistry = new SimpleMeterRegistry(); // In production, integrate with server's meter registry + + this.requestCounter = Counter.builder("grpc.requests.total") + .description("Total number of gRPC requests") + .register(meterRegistry); + + this.errorCounter = Counter.builder("grpc.errors.total") + .description("Total number of gRPC errors") + .register(meterRegistry); + + this.requestTimer = Timer.builder("grpc.request.duration") + .description("gRPC request duration") + .register(meterRegistry); + } + + @Override + public ServerCall.Listener interceptCall( + ServerCall call, + Metadata headers, + ServerCallHandler next) { + + String methodName = call.getMethodDescriptor().getFullMethodName(); + Timer.Sample sample = Timer.start(meterRegistry); + + requestCounter.increment(); + + ServerCall wrappedCall = new ForwardingServerCall.SimpleForwardingServerCall(call) { + @Override + public void close(Status status, Metadata trailers) { + sample.stop(requestTimer); + + if (!status.isOk()) { + errorCounter.increment(); + } + + // Add metrics as trailers for observability + trailers.put(Metadata.Key.of("grpc-metrics-method", Metadata.ASCII_STRING_MARSHALLER), methodName); + trailers.put(Metadata.Key.of("grpc-metrics-status", Metadata.ASCII_STRING_MARSHALLER), status.getCode().toString()); + + super.close(status, trailers); + } + }; + + return next.startCall(wrappedCall, headers); + } +} \ No newline at end of file diff --git a/server/TMP/grpc/GrpcServerPlugin.java b/server/TMP/grpc/GrpcServerPlugin.java new file mode 100644 index 0000000000..b180acba34 --- /dev/null +++ b/server/TMP/grpc/GrpcServerPlugin.java @@ -0,0 +1,386 @@ +package com.arcadedb.server.grpc; + +import java.io.File; +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.arcadedb.ContextConfiguration; +import com.arcadedb.server.ArcadeDBServer; +import com.arcadedb.server.ServerPlugin; +import com.arcadedb.server.security.ServerSecurity; + +import io.grpc.CompressorRegistry; +import io.grpc.DecompressorRegistry; +import io.grpc.Grpc; +import io.grpc.InsecureServerCredentials; +import io.grpc.Server; +import io.grpc.ServerBuilder; +import io.grpc.ServerCredentials; +import io.grpc.TlsServerCredentials; +import io.grpc.health.v1.HealthCheckResponse; +import io.grpc.protobuf.services.HealthStatusManager; +import io.grpc.protobuf.services.ProtoReflectionService; +import io.grpc.xds.XdsServerBuilder; + +/** + * ArcadeDB gRPC Server Plugin + * + * Configuration options: + * - grpc.enabled: Enable/disable gRPC server (default: true) + * - grpc.port: Port for standard gRPC server (default: 50051) + * - grpc.host: Host to bind (default: 0.0.0.0) + * - grpc.mode: Server mode - "standard", "xds", or "both" (default: standard) + * - grpc.xds.port: Port for XDS server (default: 50052) + * - grpc.tls.enabled: Enable TLS (default: false) + * - grpc.tls.cert: Path to TLS certificate + * - grpc.tls.key: Path to TLS private key + * - grpc.maxMessageSize: Max message size in MB (default: 100) + * - grpc.reflection.enabled: Enable gRPC reflection (default: true) + * - grpc.health.enabled: Enable health checking (default: true) + */ +public class GrpcServerPlugin implements ServerPlugin { + + private static final Logger logger = LoggerFactory.getLogger(GrpcServerPlugin.class); + + private ArcadeDBServer arcadeServer; + private Server grpcServer; + private Server xdsServer; + private HealthStatusManager healthManager; + private ArcadeDbGrpcService grpcService; // Keep reference for cleanup + private Thread shutdownHook; + + // Configuration keys as simple strings + private static final String CONFIG_PREFIX = "arcadedb.grpc."; + private static final String CONFIG_ENABLED = CONFIG_PREFIX + "enabled"; + private static final String CONFIG_PORT = CONFIG_PREFIX + "port"; + private static final String CONFIG_HOST = CONFIG_PREFIX + "host"; + private static final String CONFIG_MODE = CONFIG_PREFIX + "mode"; + private static final String CONFIG_XDS_PORT = CONFIG_PREFIX + "xds.port"; + private static final String CONFIG_TLS_ENABLED = CONFIG_PREFIX + "tls.enabled"; + private static final String CONFIG_TLS_CERT = CONFIG_PREFIX + "tls.cert"; + private static final String CONFIG_TLS_KEY = CONFIG_PREFIX + "tls.key"; + private static final String CONFIG_MAX_MESSAGE_SIZE = CONFIG_PREFIX + "maxMessageSize"; + private static final String CONFIG_REFLECTION_ENABLED = CONFIG_PREFIX + "reflection.enabled"; + private static final String CONFIG_HEALTH_ENABLED = CONFIG_PREFIX + "health.enabled"; + private static final String CONFIG_COMPRESSION_ENABLED = CONFIG_PREFIX + "compression.enabled"; + private static final String CONFIG_COMPRESSION_FORCE = CONFIG_PREFIX + "compression.force"; + private static final String CONFIG_COMPRESSION_TYPE = CONFIG_PREFIX + "compression.type"; + + @Override + public void configure(ArcadeDBServer server, ContextConfiguration configuration) { + this.arcadeServer = server; + } + + @Override + public void startService() { + ContextConfiguration config = arcadeServer.getConfiguration(); + + // Get configuration values with defaults + boolean enabled = getConfigBoolean(config, CONFIG_ENABLED, true); + if (!enabled) { + logger.info("gRPC server is disabled"); + return; + } + + String mode = getConfigString(config, CONFIG_MODE, "standard").toLowerCase(); + + try { + switch (mode) { + case "standard": + startStandardServer(config); + break; + case "xds": + startXdsServer(config); + break; + case "both": + startStandardServer(config); + startXdsServer(config); + break; + default: + logger.error("Invalid gRPC mode: {}. Use 'standard', 'xds', or 'both'", mode); + } + + registerShutdownHook(); + + } catch (IOException e) { + logger.error("Failed to start gRPC server", e); + throw new RuntimeException("Failed to start gRPC server", e); + } + } + + private void startStandardServer(ContextConfiguration config) throws IOException { + int port = getConfigInt(config, CONFIG_PORT, 50051); + String host = getConfigString(config, CONFIG_HOST, "0.0.0.0"); + + ServerBuilder serverBuilder; + + // Configure TLS if enabled + if (getConfigBoolean(config, CONFIG_TLS_ENABLED, false)) { + serverBuilder = configureStandardTls(port, config); + } else { + serverBuilder = ServerBuilder.forPort(port); + } + + // Configure the server + configureServer(serverBuilder, config); + + grpcServer = serverBuilder.build().start(); + + // Build status message + StringBuilder status = new StringBuilder(); + status.append("gRPC server started on ").append(host).append(":").append(port); + status.append(" (mode: standard"); + + if (getConfigBoolean(config, CONFIG_TLS_ENABLED, false)) { + status.append(", TLS enabled"); + } + + if (getConfigBoolean(config, CONFIG_COMPRESSION_ENABLED, true)) { + status.append(", compression: "); + if (getConfigBoolean(config, CONFIG_COMPRESSION_FORCE, false)) { + status.append("forced-").append(getConfigString(config, CONFIG_COMPRESSION_TYPE, "gzip")); + } else { + status.append("available"); + } + } + + status.append(")"); + logger.info(status.toString()); + } + + private void startXdsServer(ContextConfiguration config) throws IOException { + int port = getConfigInt(config, CONFIG_XDS_PORT, 50052); + + // XDS server for service mesh integration + // XdsServerBuilder requires ServerCredentials + XdsServerBuilder xdsBuilder = XdsServerBuilder.forPort(port, InsecureServerCredentials.create()); + + // Configure the XDS server as a ServerBuilder + configureServer(xdsBuilder, config); + + xdsServer = xdsBuilder.build().start(); + + logger.info("gRPC XDS server started on port {} (xDS management enabled)", port); + } + + private void configureServer(ServerBuilder serverBuilder, ContextConfiguration config) { + // Get database directory path + String databasePath = arcadeServer.getRootPath() + File.separator + "databases"; + + // Create the main service + ArcadeDbGrpcService mainService = new ArcadeDbGrpcService(databasePath, arcadeServer); + + // Add the main service + serverBuilder.addService(mainService); + + // Add health service if enabled + if (getConfigBoolean(config, CONFIG_HEALTH_ENABLED, true)) { + healthManager = new HealthStatusManager(); + serverBuilder.addService(healthManager.getHealthService()); + + // Set initial health status + healthManager.setStatus( + ArcadeDbGrpcService.class.getName(), + HealthCheckResponse.ServingStatus.SERVING + ); + } + + // Add reflection service if enabled + if (getConfigBoolean(config, CONFIG_REFLECTION_ENABLED, true)) { + serverBuilder.addService(ProtoReflectionService.newInstance()); + } + + serverBuilder.compressorRegistry(CompressorRegistry.getDefaultInstance()).decompressorRegistry(DecompressorRegistry.getDefaultInstance()); + + // Configure max message size + int maxMessageSizeMB = getConfigInt(config, CONFIG_MAX_MESSAGE_SIZE, 100); + serverBuilder.maxInboundMessageSize(maxMessageSizeMB * 1024 * 1024); + + // Add interceptors for logging, metrics, auth, etc. + serverBuilder.intercept(new GrpcLoggingInterceptor()); + serverBuilder.intercept(new GrpcMetricsInterceptor(arcadeServer)); + + // Add compression interceptor if force compression is enabled + if (getConfigBoolean(config, CONFIG_COMPRESSION_FORCE, false)) { + String compressionType = getConfigString(config, CONFIG_COMPRESSION_TYPE, "gzip"); + serverBuilder.intercept(new GrpcCompressionInterceptor(true, compressionType, 1024)); + } + + // Add authentication interceptor if security is configured + ServerSecurity serverSecurity = arcadeServer.getSecurity(); + if (serverSecurity != null) { + serverBuilder.intercept(new GrpcAuthInterceptor(serverSecurity)); + } + } + + private ServerBuilder configureStandardTls(int port, ContextConfiguration config) { + String certPath = getConfigString(config, CONFIG_TLS_CERT, null); + String keyPath = getConfigString(config, CONFIG_TLS_KEY, null); + + if (certPath == null || keyPath == null) { + logger.warn("TLS enabled but certificate or key path not provided. Falling back to insecure."); + return ServerBuilder.forPort(port); + } + + File certFile = new File(certPath); + File keyFile = new File(keyPath); + + if (!certFile.exists() || !keyFile.exists()) { + logger.warn("TLS certificate or key file not found. Falling back to insecure."); + return ServerBuilder.forPort(port); + } + + try { + ServerCredentials credentials = TlsServerCredentials.create(certFile, keyFile); + // Use Grpc.newServerBuilderForPort for TLS + return Grpc.newServerBuilderForPort(port, credentials); + } catch (Exception e) { + logger.error("Failed to configure TLS", e); + return ServerBuilder.forPort(port); + } + } + + private ServerCredentials configureTlsCredentials(ContextConfiguration config) { + String certPath = getConfigString(config, CONFIG_TLS_CERT, null); + String keyPath = getConfigString(config, CONFIG_TLS_KEY, null); + + if (certPath == null || keyPath == null) { + logger.warn("TLS enabled but certificate or key path not provided. Using insecure credentials."); + return InsecureServerCredentials.create(); + } + + File certFile = new File(certPath); + File keyFile = new File(keyPath); + + if (!certFile.exists() || !keyFile.exists()) { + logger.warn("TLS certificate or key file not found. Using insecure credentials."); + return InsecureServerCredentials.create(); + } + + try { + return TlsServerCredentials.create(certFile, keyFile); + } catch (Exception e) { + logger.error("Failed to configure TLS credentials", e); + return InsecureServerCredentials.create(); + } + } + + private void registerShutdownHook() { + shutdownHook = new Thread(() -> { + logger.info("Shutting down gRPC server..."); + stopService(); + }); + Runtime.getRuntime().addShutdownHook(shutdownHook); + } + + @Override + public void stopService() { + try { + // Update health status to NOT_SERVING + if (healthManager != null) { + healthManager.setStatus( + ArcadeDbGrpcService.class.getName(), + HealthCheckResponse.ServingStatus.NOT_SERVING + ); + } + + // Close the gRPC service to release database connections + if (grpcService != null) { + try { + grpcService.close(); + logger.info("gRPC service closed and database connections released"); + } catch (Exception e) { + logger.error("Error closing gRPC service", e); + } + } + + // Shutdown servers gracefully + if (grpcServer != null) { + grpcServer.shutdown(); + if (!grpcServer.awaitTermination(30, TimeUnit.SECONDS)) { + grpcServer.shutdownNow(); + grpcServer.awaitTermination(5, TimeUnit.SECONDS); + } + logger.info("Standard gRPC server stopped"); + } + + if (xdsServer != null) { + xdsServer.shutdown(); + if (!xdsServer.awaitTermination(30, TimeUnit.SECONDS)) { + xdsServer.shutdownNow(); + xdsServer.awaitTermination(5, TimeUnit.SECONDS); + } + logger.info("XDS gRPC server stopped"); + } + + // Remove shutdown hook if it exists + if (shutdownHook != null) { + try { + Runtime.getRuntime().removeShutdownHook(shutdownHook); + } catch (IllegalStateException e) { + // Already shutting down + } + } + + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.error("Interrupted while shutting down gRPC server", e); + } + } + + /** + * Get the status of the gRPC servers + */ + public ServerStatus getStatus() { + return new ServerStatus( + grpcServer != null && !grpcServer.isShutdown(), + xdsServer != null && !xdsServer.isShutdown(), + grpcServer != null ? grpcServer.getPort() : -1, + xdsServer != null ? xdsServer.getPort() : -1 + ); + } + + // Configuration helper methods + private String getConfigString(ContextConfiguration config, String key, String defaultValue) { + + return config.getValueAsString(key, defaultValue); + } + + private int getConfigInt(ContextConfiguration config, String key, int defaultValue) { + String value = getConfigString(config, key, null); + if (value != null) { + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + logger.warn("Invalid integer value for {}: {}", key, value); + } + } + return defaultValue; + } + + private boolean getConfigBoolean(ContextConfiguration config, String key, boolean defaultValue) { + String value = getConfigString(config, key, null); + if (value != null) { + return Boolean.parseBoolean(value); + } + return defaultValue; + } + + public static class ServerStatus { + public final boolean standardServerRunning; + public final boolean xdsServerRunning; + public final int standardPort; + public final int xdsPort; + + public ServerStatus(boolean standardServerRunning, boolean xdsServerRunning, + int standardPort, int xdsPort) { + this.standardServerRunning = standardServerRunning; + this.xdsServerRunning = xdsServerRunning; + this.standardPort = standardPort; + this.xdsPort = xdsPort; + } + } +} \ No newline at end of file From b81da83f2ef449802c3a600ed23cdd6ccd2d8324 Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Sun, 31 Aug 2025 18:33:31 -0700 Subject: [PATCH 03/35] Checkpoint Commit --- .../remote/grpc/RemoteGrpcServer.java | 69 ++++++++++++++++--- .../remote/grpc/ArcadeDbHTTPTvsGRPCBench.java | 20 +++--- .../server/grpc/ArcadeDbGrpcService.java | 4 ++ 3 files changed, 77 insertions(+), 16 deletions(-) diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcServer.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcServer.java index 661b7e144d..1e21669dcd 100644 --- a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcServer.java +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcServer.java @@ -6,12 +6,18 @@ import com.arcadedb.server.grpc.DropDatabaseRequest; import com.arcadedb.server.grpc.ListDatabasesRequest; import com.arcadedb.server.grpc.ListDatabasesResponse; + +import io.grpc.CallCredentials; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; +import io.grpc.Metadata; +import io.grpc.CallCredentials.MetadataApplier; +import io.grpc.CallCredentials.RequestInfo; import io.grpc.stub.AbstractStub; import java.util.List; import java.util.Objects; +import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; /** @@ -30,8 +36,8 @@ public class RemoteGrpcServer implements AutoCloseable { private final String host; private final int port; - private final String user; - private final String pass; + private final String userName; + private final String userPassword; private final long defaultTimeoutMs; @@ -43,16 +49,21 @@ public RemoteGrpcServer(final String host, final int port, final String user, fi } public RemoteGrpcServer(final String host, final int port, final String user, final String pass, final long defaultTimeoutMs) { + this.host = Objects.requireNonNull(host, "host"); + this.port = port; - this.user = Objects.requireNonNull(user, "user"); - this.pass = Objects.requireNonNull(pass, "pass"); + + this.userName = Objects.requireNonNull(user, "user"); + + this.userPassword = Objects.requireNonNull(pass, "pass"); + this.defaultTimeoutMs = defaultTimeoutMs > 0 ? defaultTimeoutMs : 30_000; this.channel = ManagedChannelBuilder.forAddress(this.host, this.port).usePlaintext() // switch to TLS if enabled server-side .build(); - this.blocking = ArcadeDbServiceGrpc.newBlockingStub(channel); + this.blocking = ArcadeDbServiceGrpc.newBlockingStub(channel).withCallCredentials(createCallCredentials(userName, userPassword)); } @Override @@ -71,8 +82,8 @@ private > S withDeadline(S stub, long timeoutMs) { return stub.withDeadlineAfter(t, TimeUnit.MILLISECONDS); } - public DatabaseCredentials buildCredentials() { - return DatabaseCredentials.newBuilder().setUsername(user == null ? "" : user).setPassword(pass == null ? "" : pass).build(); + public DatabaseCredentials buildCredentials() { + return DatabaseCredentials.newBuilder().setUsername(userName == null ? "" : userName).setPassword(userPassword == null ? "" : userPassword).build(); } /** Returns the list of database names from the server. */ @@ -115,6 +126,48 @@ public String endpoint() { @Override public String toString() { - return "RemoteGrpcServer{endpoint=" + endpoint() + ", user='" + user + "'}"; + return "RemoteGrpcServer{endpoint=" + endpoint() + ", userName='" + userName + "'}"; } + + /** + * Creates call credentials for authentication + */ + protected CallCredentials createCallCredentials(String userName, String userPassword) { + return new CallCredentials() { + @Override + public void applyRequestMetadata(RequestInfo requestInfo, Executor appExecutor, MetadataApplier applier) { + Metadata headers = new Metadata(); + headers.put(Metadata.Key.of("username", Metadata.ASCII_STRING_MARSHALLER), userName); + headers.put(Metadata.Key.of("password", Metadata.ASCII_STRING_MARSHALLER), userPassword); + headers.put(Metadata.Key.of("x-arcade-user", Metadata.ASCII_STRING_MARSHALLER), userName); + headers.put(Metadata.Key.of("x-arcade-password", Metadata.ASCII_STRING_MARSHALLER), userPassword); + applier.apply(headers); + } + + @Override + public void thisUsesUnstableApi() { + // Required by the interface + } + }; + } + + protected CallCredentials createCredentials() { + return new CallCredentials() { + @Override + public void applyRequestMetadata(RequestInfo requestInfo, Executor appExecutor, MetadataApplier applier) { + Metadata headers = new Metadata(); + headers.put(Metadata.Key.of("username", Metadata.ASCII_STRING_MARSHALLER), userName); + headers.put(Metadata.Key.of("password", Metadata.ASCII_STRING_MARSHALLER), userPassword); + headers.put(Metadata.Key.of("x-arcade-user", Metadata.ASCII_STRING_MARSHALLER), userName); + headers.put(Metadata.Key.of("x-arcade-password", Metadata.ASCII_STRING_MARSHALLER), userPassword); + applier.apply(headers); + } + + // x-arcade-user: root" -H "x-arcade-password: oY9uU2uJ8nD8iY7t" -H "x-arcade-database: local_shakeiq_curonix_poc-app" + @Override + public void thisUsesUnstableApi() { + // Required by the interface + } + }; + } } \ No newline at end of file diff --git a/grpc-client/src/test/java/com/arcadedb/remote/grpc/ArcadeDbHTTPTvsGRPCBench.java b/grpc-client/src/test/java/com/arcadedb/remote/grpc/ArcadeDbHTTPTvsGRPCBench.java index 987fc40617..a808bda5be 100644 --- a/grpc-client/src/test/java/com/arcadedb/remote/grpc/ArcadeDbHTTPTvsGRPCBench.java +++ b/grpc-client/src/test/java/com/arcadedb/remote/grpc/ArcadeDbHTTPTvsGRPCBench.java @@ -53,16 +53,20 @@ else if (a.startsWith("--pass=")) System.out.printf("Benchmarking DB='%s' HTTP=%s:%d gRPC=%s:%d as %s%n", DB_NAME, HTTP_HOST, HTTP_PORT, GRPC_HOST, GRPC_PORT, USER); - RemoteGrpcServer grpcServer = new RemoteGrpcServer(GRPC_HOST, GRPC_PORT, USER, PASS); - - if (grpcServer.existsDatabase(DB_NAME)) { - - System.out.println("Database exists"); - } - else { + try (RemoteGrpcServer grpcServer = new RemoteGrpcServer(GRPC_HOST, GRPC_PORT, USER, PASS)) { - grpcServer.createDatabase(DB_NAME); + if (grpcServer.existsDatabase(DB_NAME)) { + + System.out.printf("Database %s exists%n", DB_NAME); + } + else { + + System.out.printf("Creating database %s%n", DB_NAME); + + grpcServer.createDatabase(DB_NAME); + } } + // ---------- Open both clients ---------- try (RemoteDatabase http = new RemoteDatabase(HTTP_HOST, HTTP_PORT, DB_NAME, USER, PASS); diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java index 9aaa0c2b4a..a59f06daf3 100644 --- a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java @@ -150,6 +150,8 @@ public void dropDatabase(DropDatabaseRequest req, StreamObserver resp) { + logger.info("listDatabases(): Entry ..."); + try { validateCredentials(req.getCredentials()); @@ -1716,8 +1718,10 @@ private Database getDatabase(String databaseName, DatabaseCredentials credential } private void validateCredentials(DatabaseCredentials credentials) { + // Implement credential validation logic // This is a placeholder - integrate with ArcadeDB's security system + if (credentials == null || credentials.getUsername().isEmpty()) { throw new IllegalArgumentException("Invalid credentials"); } From af1fb5e4761b8d97bead9f75c1a255e894ed49a7 Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Sun, 31 Aug 2025 21:22:10 -0700 Subject: [PATCH 04/35] Checkpoint Commit --- .../remote/grpc/ArcadeDbHTTPTvsGRPCBench.java | 70 ++++++++++--------- .../server/grpc/ArcadeDbGrpcService.java | 13 ++-- 2 files changed, 42 insertions(+), 41 deletions(-) diff --git a/grpc-client/src/test/java/com/arcadedb/remote/grpc/ArcadeDbHTTPTvsGRPCBench.java b/grpc-client/src/test/java/com/arcadedb/remote/grpc/ArcadeDbHTTPTvsGRPCBench.java index a808bda5be..9199fa1beb 100644 --- a/grpc-client/src/test/java/com/arcadedb/remote/grpc/ArcadeDbHTTPTvsGRPCBench.java +++ b/grpc-client/src/test/java/com/arcadedb/remote/grpc/ArcadeDbHTTPTvsGRPCBench.java @@ -30,7 +30,7 @@ public class ArcadeDbHTTPTvsGRPCBench { static String PASS = System.getenv().getOrDefault("ARCADE_PASS", "root1234"); public static void main(String[] args) throws Exception { - + // override via args if provided: host/ports/db/user/pass for (String a : args) { if (a.startsWith("--db=")) @@ -54,24 +54,23 @@ else if (a.startsWith("--pass=")) System.out.printf("Benchmarking DB='%s' HTTP=%s:%d gRPC=%s:%d as %s%n", DB_NAME, HTTP_HOST, HTTP_PORT, GRPC_HOST, GRPC_PORT, USER); try (RemoteGrpcServer grpcServer = new RemoteGrpcServer(GRPC_HOST, GRPC_PORT, USER, PASS)) { - + if (grpcServer.existsDatabase(DB_NAME)) { - + System.out.printf("Database %s exists%n", DB_NAME); } else { - + System.out.printf("Creating database %s%n", DB_NAME); - + grpcServer.createDatabase(DB_NAME); } } - - + // ---------- Open both clients ---------- try (RemoteDatabase http = new RemoteDatabase(HTTP_HOST, HTTP_PORT, DB_NAME, USER, PASS); RemoteGrpcDatabase grpc = new RemoteGrpcDatabase(GRPC_HOST, GRPC_PORT, HTTP_PORT, DB_NAME, USER, PASS)) { - + // ensure schema aligned prepareSchemaHTTP(http); prepareSchemaGRPC(grpc); @@ -86,7 +85,7 @@ else if (a.startsWith("--pass=")) http.begin(); // Replace with your HTTP bulk insert (if you don’t have one, fallback to looped // insert) - rows.forEach(m -> http.command("sql", "INSERT INTO UserFeedback CONTENT ?", Map.of("param1", m))); + rows.forEach(m -> http.command("sql", "INSERT INTO UserFeedback :m", Map.of("m", m))); http.commit(); }); @@ -168,36 +167,43 @@ else if (a.startsWith("--pass=")) } // ------------------------------------------------------ - // Schema prep + // Schema prep - HTTP // ------------------------------------------------------ - private static void prepareSchemaHTTP(RemoteDatabase http) { - // Create vertex type + index if missing (id unique) + // Create vertex type http.command("sql", "CREATE VERTEX TYPE UserFeedback IF NOT EXISTS"); - http.command("sql", "CREATE PROPERTY UserFeedback.id STRING IF NOT EXISTS"); - http.command("sql", "CREATE INDEX UserFeedback.id IF NOT EXISTS UNIQUE"); - http.command("sql", "CREATE PROPERTY UserFeedback.applicationArea STRING IF NOT EXISTS"); - http.command("sql", "CREATE PROPERTY UserFeedback.empowerTenantId STRING IF NOT EXISTS"); - http.command("sql", "CREATE PROPERTY UserFeedback.empowerType STRING IF NOT EXISTS"); - http.command("sql", "CREATE PROPERTY UserFeedback.feedback STRING IF NOT EXISTS"); - http.command("sql", "CREATE PROPERTY UserFeedback.timestamp DATETIME IF NOT EXISTS"); - http.command("sql", "CREATE PROPERTY UserFeedback.image EMBEDDED IF NOT EXISTS"); + + // Create properties + http.command("sql", "CREATE PROPERTY UserFeedback.id IF NOT EXISTS STRING"); + http.command("sql", "CREATE PROPERTY UserFeedback.applicationArea IF NOT EXISTS STRING"); + http.command("sql", "CREATE PROPERTY UserFeedback.empowerTenantId IF NOT EXISTS STRING"); + http.command("sql", "CREATE PROPERTY UserFeedback.empowerType IF NOT EXISTS STRING"); + http.command("sql", "CREATE PROPERTY UserFeedback.feedback IF NOT EXISTS STRING"); + http.command("sql", "CREATE PROPERTY UserFeedback.timestamp IF NOT EXISTS DATETIME"); + http.command("sql", "CREATE PROPERTY UserFeedback.image IF NOT EXISTS EMBEDDED"); + + // Create index + http.command("sql", "CREATE INDEX IF NOT EXISTS ON UserFeedback (id) UNIQUE"); } + // ------------------------------------------------------ + // Schema prep - gRPC + // ------------------------------------------------------ private static void prepareSchemaGRPC(RemoteGrpcDatabase grpc) { - // Mirror the same via command API for apples-to-apples + // Create vertex type grpc.executeCommand(DB_NAME, "CREATE VERTEX TYPE UserFeedback IF NOT EXISTS", Map.of(), "sql", false, 0, null, 60_000); - grpc.executeCommand(DB_NAME, "CREATE PROPERTY UserFeedback.id STRING IF NOT EXISTS", Map.of(), "sql", false, 0, null, 60_000); - grpc.executeCommand(DB_NAME, "CREATE INDEX UserFeedback.id IF NOT EXISTS UNIQUE", Map.of(), "sql", false, 0, null, 60_000); - - grpc.executeCommand(DB_NAME, "CREATE PROPERTY UserFeedback.applicationArea STRING IF NOT EXISTS", Map.of(), "sql", false, 0, null, - 60_000); - grpc.executeCommand(DB_NAME, "CREATE PROPERTY UserFeedback.empowerTenantId STRING IF NOT EXISTS", Map.of(), "sql", false, 0, null, - 60_000); - grpc.executeCommand(DB_NAME, "CREATE PROPERTY UserFeedback.empowerType STRING IF NOT EXISTS", Map.of(), "sql", false, 0, null, 60_000); - grpc.executeCommand(DB_NAME, "CREATE PROPERTY UserFeedback.feedback STRING IF NOT EXISTS", Map.of(), "sql", false, 0, null, 60_000); - grpc.executeCommand(DB_NAME, "CREATE PROPERTY UserFeedback.timestamp DATETIME IF NOT EXISTS", Map.of(), "sql", false, 0, null, 60_000); - grpc.executeCommand(DB_NAME, "CREATE PROPERTY UserFeedback.image EMBEDDED IF NOT EXISTS", Map.of(), "sql", false, 0, null, 60_000); + + // Create properties + grpc.executeCommand(DB_NAME, "CREATE PROPERTY UserFeedback.id IF NOT EXISTS STRING", Map.of(), "sql", false, 0, null, 60_000); + grpc.executeCommand(DB_NAME, "CREATE PROPERTY UserFeedback.applicationArea IF NOT EXISTS STRING", Map.of(), "sql", false, 0, null, 60_000); + grpc.executeCommand(DB_NAME, "CREATE PROPERTY UserFeedback.empowerTenantId IF NOT EXISTS STRING", Map.of(), "sql", false, 0, null, 60_000); + grpc.executeCommand(DB_NAME, "CREATE PROPERTY UserFeedback.empowerType IF NOT EXISTS STRING", Map.of(), "sql", false, 0, null, 60_000); + grpc.executeCommand(DB_NAME, "CREATE PROPERTY UserFeedback.feedback IF NOT EXISTS STRING", Map.of(), "sql", false, 0, null, 60_000); + grpc.executeCommand(DB_NAME, "CREATE PROPERTY UserFeedback.timestamp IF NOT EXISTS DATETIME", Map.of(), "sql", false, 0, null, 60_000); + grpc.executeCommand(DB_NAME, "CREATE PROPERTY UserFeedback.image IF NOT EXISTS EMBEDDED", Map.of(), "sql", false, 0, null, 60_000); + + // Create index + grpc.executeCommand(DB_NAME, "CREATE INDEX IF NOT EXISTS ON UserFeedback (id) UNIQUE", Map.of(), "sql", false, 0, null, 60_000); } private static void cleanupHTTP(RemoteDatabase http) { diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java index a59f06daf3..3f0dfa31e0 100644 --- a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java @@ -17,6 +17,7 @@ import com.arcadedb.database.MutableDocument; import com.arcadedb.database.RID; import com.arcadedb.engine.ComponentFile; +import com.arcadedb.engine.ComponentFile.MODE; import com.arcadedb.graph.Edge; import com.arcadedb.graph.MutableEdge; import com.arcadedb.graph.MutableVertex; @@ -28,6 +29,7 @@ import com.arcadedb.schema.Schema; import com.arcadedb.schema.VertexType; import com.arcadedb.server.ArcadeDBServer; +import com.arcadedb.server.ServerDatabase; import com.google.protobuf.Empty; import com.google.protobuf.Timestamp; import com.google.protobuf.Value; @@ -102,13 +104,8 @@ public void createDatabase(CreateDatabaseRequest request, StreamObserver resp) { - logger.info("listDatabases(): Entry ..."); - try { validateCredentials(req.getCredentials()); From ab64b7b3ccfc001810df1232032e56d8003364dd Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Sun, 31 Aug 2025 21:50:38 -0700 Subject: [PATCH 05/35] Checkpoint Commit --- grpc-client/pom.xml | 7 +++ .../remote/grpc/RemoteGrpcDatabase.java | 35 ++++++++++- .../remote/grpc/ArcadeDbHTTPTvsGRPCBench.java | 60 +++++++++++-------- 3 files changed, 75 insertions(+), 27 deletions(-) diff --git a/grpc-client/pom.xml b/grpc-client/pom.xml index cf6f2196af..9962e3412b 100644 --- a/grpc-client/pom.xml +++ b/grpc-client/pom.xml @@ -129,6 +129,13 @@ runtime + + com.fasterxml.jackson.core + jackson-databind + 2.19.2 + runtime + + diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java index b5ead6ebec..b84804ed2e 100644 --- a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java @@ -465,14 +465,45 @@ public ResultSet command(final String language, final String command, final Map< } public com.arcadedb.server.grpc.ExecuteCommandResponse execSql(String db, String sql, Map params, long timeoutMs) { - return executeCommand(db, sql, params, "sql", /*returnRows*/ false, /*maxRows*/ 0, txBeginCommit(), timeoutMs); + return executeCommand(db, "sql", sql, params, /*returnRows*/ false, /*maxRows*/ 0, txBeginCommit(), timeoutMs); + } + + public com.arcadedb.server.grpc.ExecuteCommandResponse execSql(String sql, Map params, long timeoutMs) { + return executeCommand(databaseName, "sql", sql, params, /*returnRows*/ false, /*maxRows*/ 0, txBeginCommit(), timeoutMs); } public com.arcadedb.server.grpc.ExecuteCommandResponse executeCommand( - String database, + String language, String command, Map params, + boolean returnRows, + int maxRows, + com.arcadedb.server.grpc.TransactionContext tx, + long timeoutMs) { + + var reqB = com.arcadedb.server.grpc.ExecuteCommandRequest.newBuilder() + .setDatabase(databaseName) + .setCommand(command) + .putAllParameters(convertParamsToProto(params)) + .setLanguage(langOrDefault(language)) + .setReturnRows(returnRows) + .setMaxRows(maxRows > 0 ? maxRows : 0); + + if (tx != null) reqB.setTransaction(tx); + // credentials: if your stub builds creds implicitly, set here if required + reqB.setCredentials(buildCredentials()); + + return blockingStub + .withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS) + .executeCommand(reqB.build()); + } + + + public com.arcadedb.server.grpc.ExecuteCommandResponse executeCommand( + String database, String language, + String command, + Map params, boolean returnRows, int maxRows, com.arcadedb.server.grpc.TransactionContext tx, diff --git a/grpc-client/src/test/java/com/arcadedb/remote/grpc/ArcadeDbHTTPTvsGRPCBench.java b/grpc-client/src/test/java/com/arcadedb/remote/grpc/ArcadeDbHTTPTvsGRPCBench.java index 9199fa1beb..9435766316 100644 --- a/grpc-client/src/test/java/com/arcadedb/remote/grpc/ArcadeDbHTTPTvsGRPCBench.java +++ b/grpc-client/src/test/java/com/arcadedb/remote/grpc/ArcadeDbHTTPTvsGRPCBench.java @@ -16,6 +16,7 @@ import com.arcadedb.server.grpc.InsertOptions; import com.arcadedb.server.grpc.InsertSummary; import com.arcadedb.server.grpc.StreamQueryRequest; +import com.fasterxml.jackson.databind.ObjectMapper; public class ArcadeDbHTTPTvsGRPCBench { @@ -28,6 +29,8 @@ public class ArcadeDbHTTPTvsGRPCBench { static String USER = System.getenv().getOrDefault("ARCADE_USER", "root"); static String PASS = System.getenv().getOrDefault("ARCADE_PASS", "root1234"); + + static ObjectMapper objectMapper = new ObjectMapper(); public static void main(String[] args) throws Exception { @@ -81,11 +84,17 @@ else if (a.startsWith("--pass=")) // ---------- BULK INSERT (100 rows) ---------- List> rows = buildFeedbackRows(100); + + String jsonContent = objectMapper.writeValueAsString(rows); + + System.out.printf("JSON:%n%s%n", jsonContent); + time("HTTP bulkInsert(100)", () -> { + http.begin(); - // Replace with your HTTP bulk insert (if you don’t have one, fallback to looped - // insert) - rows.forEach(m -> http.command("sql", "INSERT INTO UserFeedback :m", Map.of("m", m))); + + http.command("sqlscript", "INSERT INTO UserFeedback CONTENT " + jsonContent); + http.commit(); }); @@ -102,7 +111,7 @@ else if (a.startsWith("--pass=")) Map one = uf("UF-SINGLE", "tenantZ", "UI", "FEATURE", "Single insert via API", null, Instant.now()); time("HTTP insert(1)", () -> { http.begin(); - http.command("sql", "INSERT INTO UserFeedback CONTENT :m", Map.of("m", one)); + http.command("sqlscript", "INSERT INTO UserFeedback CONTENT :m", Map.of("m", one)); http.commit(); }); time("gRPC insert(1)", () -> { @@ -115,12 +124,12 @@ else if (a.startsWith("--pass=")) // ---------- Update ---------- time("HTTP update by id", () -> { http.begin(); - http.command("sql", "UPDATE UserFeedback SET feedback = 'UPDATED_HTTP' WHERE id = 'UF-0001'"); + http.command("sqlscript", "UPDATE UserFeedback SET feedback = 'UPDATED_HTTP' WHERE id = 'UF-0001'"); http.commit(); }); time("gRPC update by id", () -> { grpc.begin(); - grpc.executeCommand(DB_NAME, "UPDATE UserFeedback SET feedback = 'UPDATED_GRPC' WHERE id = 'UF-0001'", Map.of(), "sql", false, 0, + grpc.executeCommand("sql", "UPDATE UserFeedback SET feedback = 'UPDATED_GRPC' WHERE id = 'UF-0001'", Map.of(), false, 0, /* tx */ null, 60_000); grpc.commit(); }); @@ -159,7 +168,7 @@ else if (a.startsWith("--pass=")) }); time("gRPC delete subset", () -> { grpc.begin(); - grpc.executeCommand(DB_NAME, "DELETE FROM UserFeedback WHERE id LIKE 'UF-00%'", Map.of(), "sql", false, 0, null, 60_000); + grpc.executeCommand("sql", "DELETE FROM UserFeedback WHERE id LIKE 'UF-00%'", Map.of(), false, 0, null, 60_000); grpc.commit(); }); @@ -174,36 +183,37 @@ private static void prepareSchemaHTTP(RemoteDatabase http) { http.command("sql", "CREATE VERTEX TYPE UserFeedback IF NOT EXISTS"); // Create properties - http.command("sql", "CREATE PROPERTY UserFeedback.id IF NOT EXISTS STRING"); - http.command("sql", "CREATE PROPERTY UserFeedback.applicationArea IF NOT EXISTS STRING"); - http.command("sql", "CREATE PROPERTY UserFeedback.empowerTenantId IF NOT EXISTS STRING"); - http.command("sql", "CREATE PROPERTY UserFeedback.empowerType IF NOT EXISTS STRING"); - http.command("sql", "CREATE PROPERTY UserFeedback.feedback IF NOT EXISTS STRING"); - http.command("sql", "CREATE PROPERTY UserFeedback.timestamp IF NOT EXISTS DATETIME"); - http.command("sql", "CREATE PROPERTY UserFeedback.image IF NOT EXISTS EMBEDDED"); + http.command("sqlscript", "CREATE PROPERTY UserFeedback.id IF NOT EXISTS STRING"); + http.command("sqlscript", "CREATE PROPERTY UserFeedback.applicationArea IF NOT EXISTS STRING"); + http.command("sqlscript", "CREATE PROPERTY UserFeedback.empowerTenantId IF NOT EXISTS STRING"); + http.command("sqlscript", "CREATE PROPERTY UserFeedback.empowerType IF NOT EXISTS STRING"); + http.command("sqlscript", "CREATE PROPERTY UserFeedback.feedback IF NOT EXISTS STRING"); + http.command("sqlscript", "CREATE PROPERTY UserFeedback.timestamp IF NOT EXISTS DATETIME"); + http.command("sqlscript", "CREATE PROPERTY UserFeedback.image IF NOT EXISTS EMBEDDED"); // Create index - http.command("sql", "CREATE INDEX IF NOT EXISTS ON UserFeedback (id) UNIQUE"); + http.command("sqlscript", "CREATE INDEX IF NOT EXISTS ON UserFeedback (id) UNIQUE"); } // ------------------------------------------------------ // Schema prep - gRPC // ------------------------------------------------------ private static void prepareSchemaGRPC(RemoteGrpcDatabase grpc) { + // Create vertex type - grpc.executeCommand(DB_NAME, "CREATE VERTEX TYPE UserFeedback IF NOT EXISTS", Map.of(), "sql", false, 0, null, 60_000); + grpc.executeCommand("sqlscript", "CREATE VERTEX TYPE UserFeedback IF NOT EXISTS", Map.of(), false, 0, null, 60_000); // Create properties - grpc.executeCommand(DB_NAME, "CREATE PROPERTY UserFeedback.id IF NOT EXISTS STRING", Map.of(), "sql", false, 0, null, 60_000); - grpc.executeCommand(DB_NAME, "CREATE PROPERTY UserFeedback.applicationArea IF NOT EXISTS STRING", Map.of(), "sql", false, 0, null, 60_000); - grpc.executeCommand(DB_NAME, "CREATE PROPERTY UserFeedback.empowerTenantId IF NOT EXISTS STRING", Map.of(), "sql", false, 0, null, 60_000); - grpc.executeCommand(DB_NAME, "CREATE PROPERTY UserFeedback.empowerType IF NOT EXISTS STRING", Map.of(), "sql", false, 0, null, 60_000); - grpc.executeCommand(DB_NAME, "CREATE PROPERTY UserFeedback.feedback IF NOT EXISTS STRING", Map.of(), "sql", false, 0, null, 60_000); - grpc.executeCommand(DB_NAME, "CREATE PROPERTY UserFeedback.timestamp IF NOT EXISTS DATETIME", Map.of(), "sql", false, 0, null, 60_000); - grpc.executeCommand(DB_NAME, "CREATE PROPERTY UserFeedback.image IF NOT EXISTS EMBEDDED", Map.of(), "sql", false, 0, null, 60_000); + grpc.executeCommand("sqlscript", "CREATE PROPERTY UserFeedback.id IF NOT EXISTS STRING", Map.of(), false, 0, null, 60_000); + grpc.executeCommand("sqlscript", "CREATE PROPERTY UserFeedback.applicationArea IF NOT EXISTS STRING", Map.of(), false, 0, null, 60_000); + grpc.executeCommand("sqlscript", "CREATE PROPERTY UserFeedback.empowerTenantId IF NOT EXISTS STRING", Map.of(), false, 0, null, 60_000); + grpc.executeCommand("sqlscript", "CREATE PROPERTY UserFeedback.empowerType IF NOT EXISTS STRING", Map.of(), false, 0, null, 60_000); + grpc.executeCommand("sqlscript", "CREATE PROPERTY UserFeedback.feedback IF NOT EXISTS STRING", Map.of(), false, 0, null, 60_000); + grpc.executeCommand("sqlscript", "CREATE PROPERTY UserFeedback.timestamp IF NOT EXISTS DATETIME", Map.of(), false, 0, null, 60_000); + grpc.executeCommand("sqlscript", "CREATE PROPERTY UserFeedback.image IF NOT EXISTS EMBEDDED", Map.of(), false, 0, null, 60_000); // Create index - grpc.executeCommand(DB_NAME, "CREATE INDEX IF NOT EXISTS ON UserFeedback (id) UNIQUE", Map.of(), "sql", false, 0, null, 60_000); + grpc.executeCommand("sqlscript", "CREATE INDEX IF NOT EXISTS ON UserFeedback (id) UNIQUE", Map.of(), false, 0, null, 60_000); } private static void cleanupHTTP(RemoteDatabase http) { @@ -211,7 +221,7 @@ private static void cleanupHTTP(RemoteDatabase http) { } private static void cleanupGRPC(RemoteGrpcDatabase grpc) { - grpc.executeCommand(DB_NAME, "DELETE FROM UserFeedback", Map.of(), "sql", false, 0, null, 60_000); + grpc.executeCommand("sql", "DELETE FROM UserFeedback", Map.of(), false, 0, null, 60_000); } // ------------------------------------------------------ From 871eb971e170814d9f2a7e9f17f8bb16c020d33a Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Mon, 1 Sep 2025 08:03:51 -0700 Subject: [PATCH 06/35] Checkpoint Commit --- grpc/src/main/proto/arcadedb-server.proto | 120 +++- .../server/grpc/ArcadeDbGrpcAdminService.java | 535 ++++++++++++++++++ .../server/grpc/ArcadeDbGrpcService.java | 28 +- 3 files changed, 633 insertions(+), 50 deletions(-) create mode 100644 grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcAdminService.java diff --git a/grpc/src/main/proto/arcadedb-server.proto b/grpc/src/main/proto/arcadedb-server.proto index e684fe5453..01e08fb795 100644 --- a/grpc/src/main/proto/arcadedb-server.proto +++ b/grpc/src/main/proto/arcadedb-server.proto @@ -39,10 +39,24 @@ service ArcadeDbService { rpc BulkInsert (BulkInsertRequest) returns (InsertSummary); rpc InsertStream (stream InsertChunk) returns (InsertSummary); rpc InsertBidirectional (stream InsertRequest) returns (stream InsertResponse); - - // Server operations +} + +service ArcadeDbAdminService { + + rpc Ping(PingRequest) returns (PingResponse); + rpc GetServerInfo(GetServerInfoRequest) returns (GetServerInfoResponse); rpc GetServerStatus(google.protobuf.Empty) returns (ServerStatusResponse); - rpc Ping(google.protobuf.Empty) returns (PingResponse); + + rpc ListDatabases(ListDatabasesRequest) returns (ListDatabasesResponse); + rpc ExistsDatabase(ExistsDatabaseRequest) returns (ExistsDatabaseResponse); + rpc CreateDatabase(CreateDatabaseRequest) returns (CreateDatabaseResponse); + rpc DropDatabase(DropDatabaseRequest) returns (DropDatabaseResponse); + + rpc GetDatabaseInfo(GetDatabaseInfoRequest) returns (GetDatabaseInfoResponse); + + // Optional user management + rpc CreateUser(CreateUserRequest) returns (CreateUserResponse); + rpc DeleteUser(DeleteUserRequest) returns (DeleteUserResponse); } // Common messages @@ -70,8 +84,9 @@ message TransactionContext { // Database operations message CreateDatabaseRequest { string database_name = 1; - DatabaseCredentials credentials = 2; - map options = 3; + string database_type = 2; + DatabaseCredentials credentials = 3; + map options = 4; } message CreateDatabaseResponse { @@ -99,12 +114,16 @@ message ListDatabasesResponse { } message DatabaseInfo { - string name = 1; - int64 size = 2; - string status = 3; - map properties = 4; + string name = 1; + string type = 2; // "graph" / "document" + int64 records = 3; // total records (approx or exact based on engine) + int32 classes = 4; + int32 indexes = 5; + int64 size_bytes = 6; // if available + map properties = 7; // extra info, free-form } +// Database-level info summary message GetDatabaseInfoRequest { string database_name = 1; DatabaseCredentials credentials = 2; @@ -310,20 +329,6 @@ message StreamQueryRequest { string language = 8; // default "sql" on server if empty } -// Server operations -message ServerStatusResponse { - string version = 1; - string status = 2; - int64 uptime_ms = 3; - int32 active_connections = 4; - map metrics = 5; -} - -message PingResponse { - int64 timestamp = 1; - string message = 2; -} - enum ConflictMode { CONFLICT_ABORT = 0; CONFLICT_IGNORE = 1; CONFLICT_UPDATE = 2; } enum TransactionMode { PER_BATCH = 0; PER_STREAM = 1; PER_ROW = 2; } @@ -388,3 +393,72 @@ message Committed { InsertSummary summary = 1; } message InsertRequest { oneof msg { StartInsert start = 1; InsertChunk chunk = 2; CommitInsert commit = 3; } } message InsertResponse { oneof msg { Started started = 1; BatchAck batch_ack = 2; Committed committed = 3; } } +message PingRequest { + // Optional: include credentials if your server enforces auth on ping + DatabaseCredentials credentials = 1; +} + +message PingResponse { + bool ok = 1; + int64 server_time_ms = 2; // epoch millis + int64 timestamp = 3; + string message = 4; +} + +// Server operations +message ServerStatusResponse { + string version = 1; + string status = 2; + int64 uptime_ms = 3; + int32 active_connections = 4; + map metrics = 5; +} + +// Lightweight server info; extend as needed +message GetServerInfoRequest { + DatabaseCredentials credentials = 1; +} + +message GetServerInfoResponse { + string version = 1; // e.g. "25.8.1-SNAPSHOT" + string edition = 2; // "CE"/"EE" (if applicable) + int64 start_time_ms = 3; // epoch millis + int64 uptime_ms = 4; + int32 http_port = 5; + int32 grpc_port = 6; + int32 binary_port = 7; + int32 databases_count = 8; + map features = 9; // arbitrary flags: {"ssl":"true",...} +} + +// If you don't already have this pair: +message ExistsDatabaseRequest { + string database = 1; + DatabaseCredentials credentials = 2; +} +message ExistsDatabaseResponse { + bool exists = 1; +} + +// Optional user management (server-scope) +message CreateUserRequest { + DatabaseCredentials credentials = 1; // admin credentials + string user = 2; + string password = 3; + string role = 4; // or permissions string; adjust to your security model +} + +message CreateUserResponse { + bool success = 1; + string message = 2; +} + +message DeleteUserRequest { + DatabaseCredentials credentials = 1; // admin credentials + string user = 2; +} + +message DeleteUserResponse { + bool success = 1; + string message = 2; +} \ No newline at end of file diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcAdminService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcAdminService.java new file mode 100644 index 0000000000..3cf79d45d1 --- /dev/null +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcAdminService.java @@ -0,0 +1,535 @@ +package com.arcadedb.server.grpc; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import com.arcadedb.database.Database; +import com.arcadedb.engine.ComponentFile; +import com.arcadedb.index.Index; +import com.arcadedb.schema.DocumentType; +import com.arcadedb.schema.Schema; +import com.arcadedb.schema.VertexType; + +import io.grpc.Status; +import io.grpc.stub.StreamObserver; + +/** + * Consolidated, compile-clean admin service for the new ArcadeDbAdminService. - + * No usage of APIs that were missing in your build - Safe fallbacks for + * version/ports/start time if not exposed by server - Array .length for indexes + * - Inference of db kind (graph/document) via vertex types + */ +public class ArcadeDbGrpcAdminService extends ArcadeDbAdminServiceGrpc.ArcadeDbAdminServiceImplBase { + + // Replace with your concrete server class if needed + private final Object server; + private final CredentialsValidator credentialsValidator; + + public ArcadeDbGrpcAdminService(final Object server, final CredentialsValidator credentialsValidator) { + this.server = Objects.requireNonNull(server, "server"); + this.credentialsValidator = Objects.requireNonNull(credentialsValidator, "credentialsValidator"); + } + + // ------------------------------------------------------------------------------------ + // RPCs + // ------------------------------------------------------------------------------------ + + @Override + public void ping(PingRequest req, StreamObserver resp) { + try { + // If you want ping to be open, comment out the next line + authenticate(req.getCredentials()); + + PingResponse out = PingResponse.newBuilder().setOk(true).setServerTimeMs(System.currentTimeMillis()).build(); + resp.onNext(out); + resp.onCompleted(); + } + catch (SecurityException se) { + resp.onError(Status.UNAUTHENTICATED.withDescription(se.getMessage()).asException()); + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("ping: " + e.getMessage()).asException()); + } + } + + @Override + public void getServerInfo(GetServerInfoRequest req, StreamObserver resp) { + try { + authenticate(req.getCredentials()); + + // Safer fallbacks (adjust these 3 helpers to your server if you have getters) + final String version = safeServerVersion(); + final long startMs = safeServerStartMs(); + final long uptime = (startMs > 0) ? Math.max(0, System.currentTimeMillis() - startMs) : 0L; + + final int httpPort = safeHttpPort(); + final int grpcPort = safeGrpcPort(); + final int binaryPort = safeBinaryPort(); + + final List dbNames = new ArrayList<>(getDatabaseNames()); + int dbCount = dbNames.size(); + + GetServerInfoResponse out = GetServerInfoResponse.newBuilder().setVersion(version).setEdition("CE") // adjust if you expose edition + .setStartTimeMs(startMs).setUptimeMs(uptime).setHttpPort(httpPort).setGrpcPort(grpcPort).setBinaryPort(binaryPort) + .setDatabasesCount(dbCount).build(); + + resp.onNext(out); + resp.onCompleted(); + } + catch (SecurityException se) { + resp.onError(Status.UNAUTHENTICATED.withDescription(se.getMessage()).asException()); + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("getServerInfo: " + e.getMessage()).asException()); + } + } + + @Override + public void listDatabases(ListDatabasesRequest req, StreamObserver resp) { + + try { + + authenticate(req.getCredentials()); + + ArrayList names = new ArrayList<>(getDatabaseNames()); + names.sort(String.CASE_INSENSITIVE_ORDER); + + List allDatabaseInfos = new ArrayList<>(); + + for (String dbName : names) { + + //ServerDatabase db = arcadeServer.getDatabase(dbName); + + DatabaseInfo dbInfo = DatabaseInfo.newBuilder().setName(dbName).build(); + + allDatabaseInfos.add(dbInfo); + } + + resp.onNext(ListDatabasesResponse.newBuilder().addAllDatabases(allDatabaseInfos).build()); + resp.onCompleted(); + } + catch (SecurityException se) { + resp.onError(Status.UNAUTHENTICATED.withDescription(se.getMessage()).asException()); + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("listDatabases: " + e.getMessage()).asException()); + } + } + + @Override + public void existsDatabase(ExistsDatabaseRequest req, StreamObserver resp) { + + try { + + authenticate(req.getCredentials()); + + final String name = req.getDatabase(); // proto should define 'name' for the DB + + boolean exists = containsDatabaseIgnoreCase(name); + + resp.onNext(ExistsDatabaseResponse.newBuilder().setExists(exists).build()); + resp.onCompleted(); + } + catch (SecurityException se) { + resp.onError(Status.UNAUTHENTICATED.withDescription(se.getMessage()).asException()); + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("existsDatabase: " + e.getMessage()).asException()); + } + } + + @Override + public void createDatabase(CreateDatabaseRequest req, StreamObserver resp) { + + try { + authenticate(req.getCredentials()); + + final String name = req.getDatabaseName(); // DB name in proto + final String type = req.getDatabaseType(); // "graph" or "document" (logical) + + if (containsDatabaseIgnoreCase(name)) { + + resp.onNext(CreateDatabaseResponse.newBuilder().build()); + resp.onCompleted(); + return; + } + + // Physical creation (READ_WRITE is the common default) + createDatabasePhysical(name); + + // Optional: if requested 'graph', initialize default graph types + if ("graph".equalsIgnoreCase(type)) { + try (Database db = openDatabase(name)) { + Schema s = db.getSchema(); + if (!existsVertexType(s, "V")) + s.createVertexType("V"); + if (!existsEdgeType(s, "E")) + s.createEdgeType("E"); + } + } + + resp.onNext(CreateDatabaseResponse.newBuilder().build()); + resp.onCompleted(); + } + catch (SecurityException se) { + resp.onError(Status.UNAUTHENTICATED.withDescription(se.getMessage()).asException()); + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("createDatabase: " + e.getMessage()).asException()); + } + } + + @Override + public void dropDatabase(DropDatabaseRequest req, StreamObserver resp) { + + try { + + authenticate(req.getCredentials()); + + final String name = req.getDatabaseName(); + + if (!containsDatabaseIgnoreCase(name)) { + resp.onNext(DropDatabaseResponse.newBuilder().build()); + resp.onCompleted(); + return; + } + + dropDatabasePhysical(name); + + resp.onNext(DropDatabaseResponse.newBuilder().build()); + resp.onCompleted(); + } + catch (SecurityException se) { + resp.onError(Status.UNAUTHENTICATED.withDescription(se.getMessage()).asException()); + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("dropDatabase: " + e.getMessage()).asException()); + } + } + + @Override + public void getDatabaseInfo(GetDatabaseInfoRequest req, StreamObserver resp) { + + try { + + authenticate(req.getCredentials()); + + final String name = req.getDatabaseName(); + + if (!containsDatabaseIgnoreCase(name)) { + resp.onError(Status.NOT_FOUND.withDescription("Database not found: " + name).asException()); + return; + } + + try (Database db = openDatabase(name)) { + + Schema schema = db.getSchema(); + + // Count classes + int classes = 0; + try { + classes = schema.getTypes().size(); + } + catch (Throwable ignore) { + } + + // Count indexes (Index[] in your build) + int indexes = 0; + try { + Index[] idx = schema.getIndexes(); + indexes = (idx != null) ? idx.length : 0; + } + catch (Throwable ignore) { + } + + // Approximate record count (fast-ish; adjust to your needs) + long records = approximateRecordCount(db); + + // Infer db kind: "graph" if any vertex type exists + String kind = "document"; + + try { + + var vIter = schema.getTypes().stream().filter(t -> t instanceof VertexType); + + if (vIter != null && vIter.iterator().hasNext()) + kind = "graph"; + } + catch (Throwable ignore) { + } + + DatabaseInfo dbInfo = DatabaseInfo.newBuilder().setName(name).setClasses(classes).setIndexes(indexes).setRecords(records).build(); + + GetDatabaseInfoResponse out = GetDatabaseInfoResponse.newBuilder().setInfo(dbInfo).build(); + + resp.onNext(out); + resp.onCompleted(); + } + } + catch (SecurityException se) { + resp.onError(Status.UNAUTHENTICATED.withDescription(se.getMessage()).asException()); + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("getDatabaseInfo: " + e.getMessage()).asException()); + } + } + + @Override + public void createUser(CreateUserRequest req, StreamObserver resp) { + try { + authenticate(req.getCredentials()); + final String user = req.getUser(); + final String pwd = req.getPassword(); + final String role = req.getRole(); + + if (user == null || user.isBlank()) + throw new IllegalArgumentException("user is required"); + if (pwd == null || pwd.isBlank()) + throw new IllegalArgumentException("password is required"); + + // TODO: wire into your security backend here + // server.getSecurity().createUser(user, pwd, role); + + resp.onNext(CreateUserResponse.newBuilder().setSuccess(true).setMessage("OK").build()); + resp.onCompleted(); + } + catch (SecurityException se) { + resp.onError(Status.PERMISSION_DENIED.withDescription(se.getMessage()).asException()); + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("createUser: " + e.getMessage()).asException()); + } + } + + @Override + public void deleteUser(DeleteUserRequest req, StreamObserver resp) { + try { + authenticate(req.getCredentials()); + final String user = req.getUser(); + if (user == null || user.isBlank()) + throw new IllegalArgumentException("user is required"); + + // TODO: wire into your security backend here + // server.getSecurity().deleteUser(user); + + resp.onNext(DeleteUserResponse.newBuilder().setSuccess(true).setMessage("OK").build()); + resp.onCompleted(); + } + catch (SecurityException se) { + resp.onError(Status.PERMISSION_DENIED.withDescription(se.getMessage()).asException()); + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("deleteUser: " + e.getMessage()).asException()); + } + } + + // ------------------------------------------------------------------------------------ + // Helpers + // ------------------------------------------------------------------------------------ + + private void authenticate(DatabaseCredentials creds) { + if (creds == null) + throw new SecurityException("Authentication required"); + final String user = creds.getUsername(); // matches your proto (not getUser()) + final String pass = creds.getPassword(); + if (user == null || user.isBlank()) + throw new SecurityException("Authentication required"); + + if (!credentialsValidator.authenticateServer(user, pass)) { + throw new SecurityException("Invalid credentials"); + } + } + + /** + * Get DB names from the server. Adjust if your server exposes a different + * accessor. + */ + @SuppressWarnings("unchecked") + private Collection getDatabaseNames() { + // Replace with your server API: + // e.g., ((ArcadeDBServer)server).getDatabaseNames() + try { + var m = server.getClass().getMethod("getDatabaseNames"); + Object res = m.invoke(server); + if (res instanceof Collection c) { + return (Collection) c; + } + else if (res instanceof String[] arr) { + return Arrays.asList(arr); + } + } + catch (Throwable ignore) { + } + return Collections.emptyList(); + } + + private boolean containsDatabaseIgnoreCase(String name) { + for (String n : getDatabaseNames()) { + if (n.equalsIgnoreCase(name)) + return true; + } + return false; + } + + /** + * Create DB physically with READ_WRITE mode. Adjust to your server signature. + */ + private void createDatabasePhysical(String name) throws Exception { + // Typical signature: createDatabase(String, ComponentFile.MODE) + var m = server.getClass().getMethod("createDatabase", String.class, ComponentFile.MODE.class); + m.invoke(server, name, ComponentFile.MODE.READ_WRITE); + } + + /** Drop DB physically. Prefer API with 'removeFiles' boolean if available. */ + private void dropDatabasePhysical(String name) throws Exception { + try { + var m = server.getClass().getMethod("dropDatabase", String.class, boolean.class); + m.invoke(server, name, Boolean.TRUE); + } + catch (NoSuchMethodException nsme) { + // Fallback: dropDatabase(String) if present + var m2 = server.getClass().getMethod("dropDatabase", String.class); + m2.invoke(server, name); + } + } + + /** Open database for read ops. Adjust to your server's open/get method. */ + private Database openDatabase(String name) throws Exception { + // Commonly: server.getDatabase(name) or server.openDatabase(name) + try { + var m = server.getClass().getMethod("getDatabase", String.class); + Object db = m.invoke(server, name); + return (Database) db; + } + catch (NoSuchMethodException nsme) { + var m2 = server.getClass().getMethod("openDatabase", String.class); + Object db = m2.invoke(server, name); + return (Database) db; + } + } + + /** Approximate record count with a quick pass across types. */ + private long approximateRecordCount(Database db) { + long total = 0L; + try { + for (DocumentType t : db.getSchema().getTypes()) { + try { + // exact=false when supported; otherwise this counts exactly + total += db.countType(t.getName(), false); + } + catch (Throwable ignore) { + } + } + } + catch (Throwable ignore) { + } + return total; + } + + private boolean existsVertexType(Schema s, String name) { + + try { + return s.existsType(name); + } + catch (Throwable t) { + return false; + } + } + + private boolean existsEdgeType(Schema s, String name) { + + try { + + return s.existsType(name); + } + catch (Throwable t) { + return false; + } + } + + // ---------- safe server info fallbacks (optional; return sentinel values if + // not exposed) ---------- + + private String safeServerVersion() { + try { + var m = server.getClass().getMethod("getProductVersion"); + Object v = m.invoke(server); + return (v != null) ? v.toString() : "unknown"; + } + catch (Throwable t) { + return "unknown"; + } + } + + private long safeServerStartMs() { + try { + var m = server.getClass().getMethod("getStartTime"); + Object v = m.invoke(server); + if (v instanceof Number n) + return n.longValue(); + } + catch (Throwable t) { + // ignore + } + return 0L; + } + + private int safeHttpPort() { + try { + var m = server.getClass().getMethod("getHttpServer"); + Object http = m.invoke(server); + if (http != null) { + var pm = http.getClass().getMethod("getPort"); + Object p = pm.invoke(http); + if (p instanceof Number n) + return n.intValue(); + } + } + catch (Throwable ignore) { + } + return -1; + } + + private int safeGrpcPort() { + try { + var m = server.getClass().getMethod("getGrpcServer"); + Object g = m.invoke(server); + if (g != null) { + var pm = g.getClass().getMethod("getPort"); + Object p = pm.invoke(g); + if (p instanceof Number n) + return n.intValue(); + } + } + catch (Throwable ignore) { + } + return -1; + } + + private int safeBinaryPort() { + try { + var m = server.getClass().getMethod("getBinaryServer"); + Object b = m.invoke(server); + if (b != null) { + var pm = b.getClass().getMethod("getPort"); + Object p = pm.invoke(b); + if (p instanceof Number n) + return n.intValue(); + } + } + catch (Throwable ignore) { + } + return -1; + } + + // ------------------------------------------------------------------------------------ + // Auth hook + // ------------------------------------------------------------------------------------ + public interface CredentialsValidator { + boolean authenticateServer(String user, String password); + } +} \ No newline at end of file diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java index 3f0dfa31e0..a01e4d5c83 100644 --- a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java @@ -30,7 +30,6 @@ import com.arcadedb.schema.VertexType; import com.arcadedb.server.ArcadeDBServer; import com.arcadedb.server.ServerDatabase; -import com.google.protobuf.Empty; import com.google.protobuf.Timestamp; import com.google.protobuf.Value; @@ -197,7 +196,7 @@ public void getDatabaseInfo(GetDatabaseInfoRequest req, StreamObserver responseObserver) { - PingResponse response = PingResponse.newBuilder().setTimestamp(System.currentTimeMillis()).setMessage("pong").build(); - - responseObserver.onNext(response); - responseObserver.onCompleted(); - } - - @Override - public void getServerStatus(Empty request, StreamObserver responseObserver) { - try { - ServerStatusResponse response = ServerStatusResponse.newBuilder().setVersion("25.8.1-SNAPSHOT").setStatus("RUNNING") - .setUptimeMs(getUptime()).setActiveConnections(getActiveConnections()) - .putMetrics("active_transactions", String.valueOf(activeTransactions.size())).build(); - - responseObserver.onNext(response); - responseObserver.onCompleted(); - - } - catch (Exception e) { - logger.error("Error getting server status: {}", e.getMessage(), e); - responseObserver.onError(Status.INTERNAL.withDescription("Failed to get server status: " + e.getMessage()).asException()); - } - } - // --- TX helpers -------------------------------------------------------------- private static final class TxScope { From 485038684e8ad92eda8cd791fb7357a9d520784b Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Mon, 1 Sep 2025 08:17:09 -0700 Subject: [PATCH 07/35] Checkpoint Commit --- grpc-client/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/grpc-client/pom.xml b/grpc-client/pom.xml index 9962e3412b..ee245b649b 100644 --- a/grpc-client/pom.xml +++ b/grpc-client/pom.xml @@ -126,14 +126,14 @@ org.slf4j slf4j-simple 2.0.17 - runtime + test com.fasterxml.jackson.core jackson-databind 2.19.2 - runtime + test From 97ae92f34ad0b164895822651b9b5ca33937fd0f Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Mon, 1 Sep 2025 08:30:56 -0700 Subject: [PATCH 08/35] provided as scope --- grpc-client/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/grpc-client/pom.xml b/grpc-client/pom.xml index ee245b649b..f95c91a141 100644 --- a/grpc-client/pom.xml +++ b/grpc-client/pom.xml @@ -126,14 +126,14 @@ org.slf4j slf4j-simple 2.0.17 - test + provided com.fasterxml.jackson.core jackson-databind 2.19.2 - test + provided From 44dcb657f480d6f6be8faa2335ff3bf35e51b3b2 Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Mon, 1 Sep 2025 13:59:46 -0700 Subject: [PATCH 09/35] Checkpoint Commit --- .../remote/grpc/RemoteGrpcDatabase.java | 1946 +++++++++-------- .../RemoteGrpcDatabaseWithCompression.java | 4 +- .../remote/grpc/RemoteGrpcServer.java | 55 +- .../remote/grpc/ArcadeDbHTTPTvsGRPCBench.java | 6 +- grpc/src/main/proto/arcadedb-server.proto | 639 +++--- .../server/grpc/ArcadeDbGrpcAdminService.java | 37 +- .../server/grpc/ArcadeDbGrpcService.java | 755 ++++--- .../server/grpc/GrpcServerPlugin.java | 10 +- 8 files changed, 1827 insertions(+), 1625 deletions(-) diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java index b84804ed2e..5a91f5fe8b 100644 --- a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java @@ -39,36 +39,30 @@ import com.arcadedb.server.grpc.BatchAck; import com.arcadedb.server.grpc.BeginTransactionRequest; import com.arcadedb.server.grpc.BeginTransactionResponse; -import com.arcadedb.server.grpc.CommitInsert; +import com.arcadedb.server.grpc.Commit; import com.arcadedb.server.grpc.CommitTransactionRequest; import com.arcadedb.server.grpc.CommitTransactionResponse; -import com.arcadedb.server.grpc.CreateRecordRequest; -import com.arcadedb.server.grpc.CreateRecordResponse; import com.arcadedb.server.grpc.DatabaseCredentials; import com.arcadedb.server.grpc.DeleteRecordRequest; import com.arcadedb.server.grpc.DeleteRecordResponse; -import com.arcadedb.server.grpc.DropDatabaseRequest; -import com.arcadedb.server.grpc.DropDatabaseResponse; import com.arcadedb.server.grpc.ExecuteCommandRequest; import com.arcadedb.server.grpc.ExecuteCommandResponse; import com.arcadedb.server.grpc.ExecuteQueryRequest; import com.arcadedb.server.grpc.ExecuteQueryResponse; -import com.arcadedb.server.grpc.GetRecordRequest; -import com.arcadedb.server.grpc.GetRecordResponse; import com.arcadedb.server.grpc.InsertChunk; import com.arcadedb.server.grpc.InsertOptions; import com.arcadedb.server.grpc.InsertRequest; import com.arcadedb.server.grpc.InsertResponse; import com.arcadedb.server.grpc.InsertSummary; +import com.arcadedb.server.grpc.LookupByRidRequest; +import com.arcadedb.server.grpc.LookupByRidResponse; import com.arcadedb.server.grpc.QueryResult; import com.arcadedb.server.grpc.RollbackTransactionRequest; import com.arcadedb.server.grpc.RollbackTransactionResponse; -import com.arcadedb.server.grpc.StartInsert; +import com.arcadedb.server.grpc.Start; import com.arcadedb.server.grpc.StreamQueryRequest; import com.arcadedb.server.grpc.TransactionContext; import com.arcadedb.server.grpc.TransactionIsolation; -import com.arcadedb.server.grpc.UpdateRecordRequest; -import com.arcadedb.server.grpc.UpdateRecordResponse; import com.google.protobuf.Struct; import com.google.protobuf.Value; @@ -79,7 +73,9 @@ import io.grpc.ManagedChannelBuilder; import io.grpc.Metadata; import io.grpc.Status; +import io.grpc.StatusException; import io.grpc.StatusRuntimeException; +import io.grpc.stub.BlockingClientCall; /** * Remote Database implementation using gRPC protocol instead of HTTP. Extends @@ -90,14 +86,14 @@ * @author Oleg Cohen (oleg.cohen@gmail.com) */ public class RemoteGrpcDatabase extends RemoteDatabase { - + private final ManagedChannel channel; - - private final ArcadeDbServiceGrpc.ArcadeDbServiceBlockingStub blockingStub; + + private final ArcadeDbServiceGrpc.ArcadeDbServiceBlockingV2Stub blockingStub; private final ArcadeDbServiceGrpc.ArcadeDbServiceStub asyncStub; - + private String transactionId; - + private final RemoteSchema schema; private final String userName; @@ -105,18 +101,19 @@ public class RemoteGrpcDatabase extends RemoteDatabase { private String databaseName; private RemoteTransactionExplicitLock explicitLock; - public RemoteGrpcDatabase(final String server, final int grpcPort, final int httpPort, final String databaseName, final String userName, final String userPassword) { + public RemoteGrpcDatabase(final String server, final int grpcPort, final int httpPort, final String databaseName, final String userName, + final String userPassword) { this(server, grpcPort, httpPort, databaseName, userName, userPassword, new ContextConfiguration()); } - public RemoteGrpcDatabase(final String server, final int grpcPort, final int httpPort, final String databaseName, final String userName, final String userPassword, - final ContextConfiguration configuration) { + public RemoteGrpcDatabase(final String server, final int grpcPort, final int httpPort, final String databaseName, final String userName, + final String userPassword, final ContextConfiguration configuration) { super(server, httpPort, databaseName, userName, userPassword, configuration); this.userName = userName; this.userPassword = userPassword; - + this.databaseName = databaseName; // Create gRPC channel @@ -132,7 +129,7 @@ public RemoteGrpcDatabase(final String server, final int grpcPort, final int htt // Create gRPC-specific schema this.schema = new RemoteSchema(this); } - + /** * Creates the gRPC channel. Override this for custom channel configuration. */ @@ -177,19 +174,20 @@ public void applyRequestMetadata(RequestInfo requestInfo, Executor appExecutor, applier.apply(headers); } - // x-arcade-user: root" -H "x-arcade-password: oY9uU2uJ8nD8iY7t" -H "x-arcade-database: local_shakeiq_curonix_poc-app" + // x-arcade-user: root" -H "x-arcade-password: oY9uU2uJ8nD8iY7t" -H + // "x-arcade-database: local_shakeiq_curonix_poc-app" @Override public void thisUsesUnstableApi() { // Required by the interface } }; } - + /** * Override this method to customize blocking stub creation */ - protected ArcadeDbServiceGrpc.ArcadeDbServiceBlockingStub createBlockingStub(ManagedChannel channel) { - return ArcadeDbServiceGrpc.newBlockingStub(channel).withCallCredentials(createCredentials()); + protected ArcadeDbServiceGrpc.ArcadeDbServiceBlockingV2Stub createBlockingStub(ManagedChannel channel) { + return ArcadeDbServiceGrpc.newBlockingV2Stub(channel).withCallCredentials(createCredentials()); } protected ArcadeDbServiceGrpc.ArcadeDbServiceStub createAsyncStub(ManagedChannel channel) { @@ -225,27 +223,6 @@ public void close() { } } - @Override - public void drop() { - checkDatabaseIsOpen(); - - DropDatabaseRequest request = DropDatabaseRequest.newBuilder().setDatabaseName(getName()).setCredentials(buildCredentials()).build(); - - try { - DropDatabaseResponse response = blockingStub - .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) - .dropDatabase(request); - if (!response.getSuccess()) { - throw new DatabaseOperationException("Failed to drop database: " + response.getMessage()); - } - } - catch (StatusRuntimeException e) { - throw new DatabaseOperationException("Error on deleting database", e); - } - - close(); - } - @Override public void begin(final Database.TRANSACTION_ISOLATION_LEVEL isolationLevel) { checkDatabaseIsOpen(); @@ -256,9 +233,7 @@ public void begin(final Database.TRANSACTION_ISOLATION_LEVEL isolationLevel) { .setIsolation(mapIsolationLevel(isolationLevel)).build(); try { - BeginTransactionResponse response = blockingStub - .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) - .beginTransaction(request); + BeginTransactionResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).beginTransaction(request); transactionId = response.getTransactionId(); // Store transaction ID in parent class session management setSessionId(transactionId); @@ -266,6 +241,9 @@ public void begin(final Database.TRANSACTION_ISOLATION_LEVEL isolationLevel) { catch (StatusRuntimeException e) { throw new TransactionException("Error on transaction begin", e); } + catch (StatusException e) { + throw new TransactionException("Error on transaction begin", e); + } } @Override @@ -281,9 +259,7 @@ public void commit() { .setCredentials(buildCredentials()).build(); try { - CommitTransactionResponse response = blockingStub - .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) - .commitTransaction(request); + CommitTransactionResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).commitTransaction(request); if (!response.getSuccess()) { throw new TransactionException("Failed to commit transaction: " + response.getMessage()); } @@ -291,6 +267,9 @@ public void commit() { catch (StatusRuntimeException e) { handleGrpcException(e); } + catch (StatusException e) { + handleGrpcException(e); + } finally { transactionId = null; setSessionId(null); @@ -310,8 +289,7 @@ public void rollback() { .setCredentials(buildCredentials()).build(); try { - RollbackTransactionResponse response = blockingStub - .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + RollbackTransactionResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) .rollbackTransaction(request); if (!response.getSuccess()) { throw new TransactionException("Failed to rollback transaction: " + response.getMessage()); @@ -320,6 +298,9 @@ public void rollback() { catch (StatusRuntimeException e) { throw new TransactionException("Error on transaction rollback", e); } + catch (StatusException e) { + throw new TransactionException("Error on transaction begin", e); + } finally { transactionId = null; setSessionId(null); @@ -328,19 +309,21 @@ public void rollback() { @Override public Record lookupByRID(final RID rid, final boolean loadContent) { + checkDatabaseIsOpen(); + stats.readRecord.incrementAndGet(); if (rid == null) throw new IllegalArgumentException("Record is null"); - GetRecordRequest request = GetRecordRequest.newBuilder().setDatabase(getName()).setRid(rid.toString()).setCredentials(buildCredentials()) - .build(); + LookupByRidRequest request = LookupByRidRequest.newBuilder().setDatabase(getName()).setRid(rid.toString()) + .setCredentials(buildCredentials()).build(); try { - GetRecordResponse response = blockingStub - .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) - .getRecord(request); + + LookupByRidResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).lookupByRid(request); + if (!response.getFound()) { throw new RecordNotFoundException("Record " + rid + " not found", rid); } @@ -350,10 +333,15 @@ public Record lookupByRID(final RID rid, final boolean loadContent) { handleGrpcException(e); return null; } + catch (StatusException e) { + handleGrpcException(e); + return null; + } } @Override public void deleteRecord(final Record record) { + checkDatabaseIsOpen(); stats.deleteRecord.incrementAndGet(); @@ -364,9 +352,7 @@ public void deleteRecord(final Record record) { .setCredentials(buildCredentials()).build(); try { - DeleteRecordResponse response = blockingStub - .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) - .deleteRecord(request); + DeleteRecordResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).deleteRecord(request); if (!response.getSuccess()) { throw new DatabaseOperationException("Failed to delete record: " + response.getMessage()); } @@ -374,6 +360,9 @@ public void deleteRecord(final Record record) { catch (StatusRuntimeException e) { handleGrpcException(e); } + catch (StatusException e) { + handleGrpcException(e); + } } @Override @@ -406,8 +395,7 @@ public ResultSet query(final String language, final String query, final Map params = mapArgs(args); - + return query(language, query, params); } - + @Override public ResultSet command(final String language, final String command, final Map params) { checkDatabaseIsOpen(); @@ -444,8 +436,7 @@ public ResultSet command(final String language, final String command, final Map< } try { - ExecuteCommandResponse response = blockingStub - .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + ExecuteCommandResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) .executeCommand(requestBuilder.build()); // Create result set with command execution info @@ -462,117 +453,137 @@ public ResultSet command(final String language, final String command, final Map< handleGrpcException(e); return new InternalResultSet(); } + catch (StatusException e) { + handleGrpcException(e); + return new InternalResultSet(); + } + } + + public com.arcadedb.server.grpc.ExecuteCommandResponse execSql(String db, String sql, Map params, long timeoutMs) { + return executeCommand(db, "sql", sql, params, /* returnRows */ false, /* maxRows */ 0, txBeginCommit(), timeoutMs); + } + + public com.arcadedb.server.grpc.ExecuteCommandResponse execSql(String sql, Map params, long timeoutMs) { + return executeCommand(databaseName, "sql", sql, params, /* returnRows */ false, /* maxRows */ 0, txBeginCommit(), timeoutMs); + } + + public com.arcadedb.server.grpc.ExecuteCommandResponse executeCommand(String language, String command, Map params, + boolean returnRows, int maxRows, com.arcadedb.server.grpc.TransactionContext tx, long timeoutMs) { + + var reqB = com.arcadedb.server.grpc.ExecuteCommandRequest.newBuilder().setDatabase(databaseName).setCommand(command) + .putAllParameters(convertParamsToProto(params)).setLanguage(langOrDefault(language)).setReturnRows(returnRows) + .setMaxRows(maxRows > 0 ? maxRows : 0); + + if (tx != null) + reqB.setTransaction(tx); + // credentials: if your stub builds creds implicitly, set here if required + reqB.setCredentials(buildCredentials()); + + try { + + return blockingStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS).executeCommand(reqB.build()); + } + catch (StatusException e) { + + throw new RuntimeException("Failed to execute command: " + e.getMessage(), e); + } + } + + public com.arcadedb.server.grpc.ExecuteCommandResponse executeCommand(String database, String language, String command, + Map params, boolean returnRows, int maxRows, com.arcadedb.server.grpc.TransactionContext tx, long timeoutMs) { + + var reqB = com.arcadedb.server.grpc.ExecuteCommandRequest.newBuilder().setDatabase(database).setCommand(command) + .putAllParameters(convertParamsToProto(params)).setLanguage(langOrDefault(language)).setReturnRows(returnRows) + .setMaxRows(maxRows > 0 ? maxRows : 0); + + if (tx != null) + reqB.setTransaction(tx); + // credentials: if your stub builds creds implicitly, set here if required + reqB.setCredentials(buildCredentials()); + + try { + return blockingStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS).executeCommand(reqB.build()); + } + catch (StatusException e) { + + throw new RuntimeException("Failed to execute command: " + e.getMessage(), e); + } } - public com.arcadedb.server.grpc.ExecuteCommandResponse execSql(String db, String sql, Map params, long timeoutMs) { - return executeCommand(db, "sql", sql, params, /*returnRows*/ false, /*maxRows*/ 0, txBeginCommit(), timeoutMs); - } - - public com.arcadedb.server.grpc.ExecuteCommandResponse execSql(String sql, Map params, long timeoutMs) { - return executeCommand(databaseName, "sql", sql, params, /*returnRows*/ false, /*maxRows*/ 0, txBeginCommit(), timeoutMs); - } - - public com.arcadedb.server.grpc.ExecuteCommandResponse executeCommand( - String language, - String command, - Map params, - boolean returnRows, - int maxRows, - com.arcadedb.server.grpc.TransactionContext tx, - long timeoutMs) { - - var reqB = com.arcadedb.server.grpc.ExecuteCommandRequest.newBuilder() - .setDatabase(databaseName) - .setCommand(command) - .putAllParameters(convertParamsToProto(params)) - .setLanguage(langOrDefault(language)) - .setReturnRows(returnRows) - .setMaxRows(maxRows > 0 ? maxRows : 0); - - if (tx != null) reqB.setTransaction(tx); - // credentials: if your stub builds creds implicitly, set here if required - reqB.setCredentials(buildCredentials()); - - return blockingStub - .withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS) - .executeCommand(reqB.build()); - } - - - public com.arcadedb.server.grpc.ExecuteCommandResponse executeCommand( - String database, - String language, - String command, - Map params, - boolean returnRows, - int maxRows, - com.arcadedb.server.grpc.TransactionContext tx, - long timeoutMs) { - - var reqB = com.arcadedb.server.grpc.ExecuteCommandRequest.newBuilder() - .setDatabase(database) - .setCommand(command) - .putAllParameters(convertParamsToProto(params)) - .setLanguage(langOrDefault(language)) - .setReturnRows(returnRows) - .setMaxRows(maxRows > 0 ? maxRows : 0); - - if (tx != null) reqB.setTransaction(tx); - // credentials: if your stub builds creds implicitly, set here if required - reqB.setCredentials(buildCredentials()); - - return blockingStub - .withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS) - .executeCommand(reqB.build()); - } - @Override protected RID saveRecord(final MutableDocument record) { stats.createRecord.incrementAndGet(); - RID rid = record.getIdentity(); + final RID rid = record.getIdentity(); + if (rid != null) { - // Update existing record - UpdateRecordRequest request = UpdateRecordRequest.newBuilder().setDatabase(getName()).setRid(rid.toString()) - .putAllProperties(convertParamsToProto(record.toMap(false))).setCredentials(buildCredentials()).build(); + + // -------- UPDATE (partial) -------- + // Build the nested proto Record for the "partial" oneof + com.arcadedb.server.grpc.Record partial = com.arcadedb.server.grpc.Record.newBuilder() + .putAllProperties(convertParamsToProto(record.toMap(false))).build(); + + com.arcadedb.server.grpc.UpdateRecordRequest request = com.arcadedb.server.grpc.UpdateRecordRequest.newBuilder() + .setDatabase(getName()).setRid(rid.toString()).setRecord(partial).setDatabase(databaseName) + .setCredentials(buildCredentials()).build(); try { - UpdateRecordResponse response = blockingStub - .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) - .updateRecord(request); - - if (!response.getSuccess()) { - throw new DatabaseOperationException("Failed to update record"); - } + com.arcadedb.server.grpc.UpdateRecordResponse response = blockingStub + .withDeadlineAfter(getTimeout(), java.util.concurrent.TimeUnit.MILLISECONDS).updateRecord(request); + + // If your proto has flags, you can check response.getSuccess()/getUpdated() + // Otherwise, treat non-exception as success. return rid; } - catch (StatusRuntimeException e) { + catch (io.grpc.StatusRuntimeException e) { + handleGrpcException(e); + return null; + } + catch (io.grpc.StatusException e) { handleGrpcException(e); return null; } } else { - // Create new record - CreateRecordRequest request = CreateRecordRequest.newBuilder().setDatabase(getName()).setType(record.getTypeName()) - .putAllProperties(convertParamsToProto(record.toMap(false))).setCredentials(buildCredentials()).build(); + // -------- CREATE -------- + com.arcadedb.server.grpc.Record recMsg = com.arcadedb.server.grpc.Record.newBuilder() + .putAllProperties(convertParamsToProto(record.toMap(false))).build(); + + com.arcadedb.server.grpc.CreateRecordRequest request = com.arcadedb.server.grpc.CreateRecordRequest.newBuilder() + .setDatabase(getName()).setType(record.getTypeName()).setRecord(recMsg) // nested Record payload + .setCredentials(buildCredentials()).build(); try { - CreateRecordResponse response = blockingStub - .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) - .createRecord(request); - - if (!response.getSuccess()) { - throw new DatabaseOperationException("Failed to create record"); + com.arcadedb.server.grpc.CreateRecordResponse response = blockingStub + .withDeadlineAfter(getTimeout(), java.util.concurrent.TimeUnit.MILLISECONDS).createRecord(request); + + // Proto returns the newly created RID as a string + final String ridStr = response.getRid(); + if (ridStr == null || ridStr.isEmpty()) { + throw new com.arcadedb.exception.DatabaseOperationException("Failed to create record (empty RID)"); + } + + // Construct a RID from the returned string + // Prefer the single-arg ctor if available; otherwise use the (Database, String) + // ctor. + try { + return new com.arcadedb.database.RID(ridStr); + } + catch (NoSuchMethodError | IllegalArgumentException ex) { + // Fallback for older APIs expecting (Database, String) + return new com.arcadedb.database.RID(this, ridStr); } - return new RID(this, response.getRecord().getRid()); } - catch (StatusRuntimeException e) { + catch (io.grpc.StatusRuntimeException e) { + handleGrpcException(e); + return null; + } + catch (io.grpc.StatusException e) { handleGrpcException(e); return null; } } } - // /** // * Executes a streaming query that returns results in batches. // * Useful for processing large result sets without loading everything into memory. @@ -678,616 +689,709 @@ protected RID saveRecord(final MutableDocument record) { // public Iterator queryStream(final String language, final String query) { // return queryStream(language, query, 100); // } - + /* -StreamQueryRequest req = StreamQueryRequest.newBuilder() - .setDatabase(db) - .setQuery(sql) - .setBatchSize(500) - .setRetrievalMode(StreamQueryRequest.RetrievalMode.PAGED) - .setTransaction(TransactionContext.newBuilder() - .setBegin(true) - .setCommit(true) // or setRollback(true) - .build()) - .build(); + * StreamQueryRequest req = StreamQueryRequest.newBuilder() .setDatabase(db) + * .setQuery(sql) .setBatchSize(500) + * .setRetrievalMode(StreamQueryRequest.RetrievalMode.PAGED) + * .setTransaction(TransactionContext.newBuilder() .setBegin(true) + * .setCommit(true) // or setRollback(true) .build()) .build(); */ - + // Convenience: default batch size stays 100, default mode = CURSOR - public Iterator queryStream(final String language, - final String query) { - return queryStream(language, query, /*batchSize*/100, - StreamQueryRequest.RetrievalMode.CURSOR); + public Iterator queryStream(final String language, final String query) { + return queryStream(language, query, /* batchSize */100, StreamQueryRequest.RetrievalMode.CURSOR); } - public Iterator queryStream(final String language, - final String query, - final int batchSize) { - return queryStream(language, query, batchSize, - StreamQueryRequest.RetrievalMode.CURSOR); + public Iterator queryStream(final String language, final String query, final int batchSize) { + return queryStream(language, query, batchSize, StreamQueryRequest.RetrievalMode.CURSOR); } // NEW: choose retrieval mode - public Iterator queryStream(final String language, - final String query, - final int batchSize, - final StreamQueryRequest.RetrievalMode mode) { - checkDatabaseIsOpen(); - stats.queries.incrementAndGet(); - - StreamQueryRequest request = StreamQueryRequest.newBuilder() - .setDatabase(getName()) - .setQuery(query) - .setCredentials(buildCredentials()) - .setBatchSize(batchSize > 0 ? batchSize : 100) - .setRetrievalMode(mode) // <--- NEW - .build(); - - Iterator responseIterator = blockingStub - .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) - .streamQuery(request); - - return new Iterator() { - private Iterator currentBatch = Collections.emptyIterator(); - - @Override - public boolean hasNext() { - if (currentBatch.hasNext()) return true; - - while (responseIterator.hasNext()) { - QueryResult result = responseIterator.next(); - - // Defensive: skip any empty batches the server might send - if (result.getRecordsCount() == 0) { - if (result.getIsLastBatch()) return false; - continue; - } - - currentBatch = result.getRecordsList().iterator(); - return true; - } - return false; - } - - @Override - public Record next() { - if (!hasNext()) throw new NoSuchElementException(); - return grpcRecordToRecord(currentBatch.next()); - } - }; + public Iterator queryStream(final String language, final String query, final int batchSize, + final StreamQueryRequest.RetrievalMode mode) { + checkDatabaseIsOpen(); + stats.queries.incrementAndGet(); + + StreamQueryRequest request = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(query).setCredentials(buildCredentials()) + .setBatchSize(batchSize > 0 ? batchSize : 100).setRetrievalMode(mode) // <--- NEW + .build(); + + BlockingClientCall responseIterator = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .streamQuery(request); + + return new Iterator() { + + private Iterator currentBatch = Collections.emptyIterator(); + + @Override + public boolean hasNext() { + if (currentBatch.hasNext()) + return true; + + try { + while (responseIterator.hasNext()) { + + QueryResult result = responseIterator.read(); + + // Defensive: skip any empty batches the server might send + if (result.getRecordsCount() == 0) { + if (result.getIsLastBatch()) + return false; + continue; + } + + currentBatch = result.getRecordsList().iterator(); + return true; + } + } + catch (InterruptedException e) { + + throw new RuntimeException("queryStream: interrupted", e); + } + catch (StatusException e) { + + throw new RuntimeException("queryStream: error", e); + } + + return false; + } + + @Override + public Record next() { + if (!hasNext()) + throw new NoSuchElementException(); + return grpcRecordToRecord(currentBatch.next()); + } + }; } // PARAMETERIZED variant with retrieval mode - public Iterator queryStream(final String language, - final String query, - final Map params, - final int batchSize, - final StreamQueryRequest.RetrievalMode mode) { - checkDatabaseIsOpen(); - stats.queries.incrementAndGet(); - - StreamQueryRequest.Builder b = StreamQueryRequest.newBuilder() - .setDatabase(getName()) - .setQuery(query) - .setCredentials(buildCredentials()) - .setBatchSize(batchSize > 0 ? batchSize : 100) - .setRetrievalMode(mode); // <--- NEW - - if (params != null && !params.isEmpty()) { - b.putAllParameters(convertParamsToProto(params)); - } - - Iterator responseIterator = blockingStub - .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) - .streamQuery(b.build()); - - return new Iterator() { - private Iterator currentBatch = Collections.emptyIterator(); - - @Override - public boolean hasNext() { - if (currentBatch.hasNext()) return true; - - while (responseIterator.hasNext()) { - QueryResult result = responseIterator.next(); - - if (result.getRecordsCount() == 0) { - if (result.getIsLastBatch()) return false; - continue; - } - - currentBatch = result.getRecordsList().iterator(); - return true; - } - return false; - } - - @Override - public Record next() { - if (!hasNext()) throw new NoSuchElementException(); - return grpcRecordToRecord(currentBatch.next()); - } - }; + public Iterator queryStream(final String language, final String query, final Map params, final int batchSize, + final StreamQueryRequest.RetrievalMode mode) { + checkDatabaseIsOpen(); + stats.queries.incrementAndGet(); + + StreamQueryRequest.Builder b = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(query).setCredentials(buildCredentials()) + .setBatchSize(batchSize > 0 ? batchSize : 100).setRetrievalMode(mode); // <--- NEW + + if (params != null && !params.isEmpty()) { + b.putAllParameters(convertParamsToProto(params)); + } + + BlockingClientCall responseIterator = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .streamQuery(b.build()); + + return new Iterator() { + private Iterator currentBatch = Collections.emptyIterator(); + + @Override + public boolean hasNext() { + if (currentBatch.hasNext()) + return true; + + try { + while (responseIterator.hasNext()) { + QueryResult result = responseIterator.read(); + + if (result.getRecordsCount() == 0) { + if (result.getIsLastBatch()) + return false; + continue; + } + + currentBatch = result.getRecordsList().iterator(); + return true; + } + } + catch (InterruptedException e) { + + throw new RuntimeException("queryStream: interrupted", e); + } + catch (StatusException e) { + + throw new RuntimeException("queryStream: error", e); + } + + return false; + } + + @Override + public Record next() { + if (!hasNext()) + throw new NoSuchElementException(); + return grpcRecordToRecord(currentBatch.next()); + } + }; } // Keep the old signature working (defaults to CURSOR) - public Iterator queryStream(final String language, - final String query, - final Map params, - final int batchSize) { - return queryStream(language, query, params, batchSize, - StreamQueryRequest.RetrievalMode.CURSOR); - } - + public Iterator queryStream(final String language, final String query, final Map params, final int batchSize) { + return queryStream(language, query, params, batchSize, StreamQueryRequest.RetrievalMode.CURSOR); + } + public static final class QueryBatch { - private final List records; - private final int totalInBatch; - private final long runningTotal; - private final boolean lastBatch; - - public QueryBatch(List list, int totalInBatch, long runningTotal, boolean lastBatch) { - this.records = list; - this.totalInBatch = totalInBatch; - this.runningTotal = runningTotal; - this.lastBatch = lastBatch; - } - public List records() { return records; } - public int totalInBatch() { return totalInBatch; } - public long runningTotal() { return runningTotal; } - public boolean isLastBatch() { return lastBatch; } - } - - public Iterator queryStreamBatches(final String language, - final String query, - final Map params, - final int batchSize, - final StreamQueryRequest.RetrievalMode mode) { - checkDatabaseIsOpen(); - stats.queries.incrementAndGet(); - - StreamQueryRequest.Builder b = StreamQueryRequest.newBuilder() - .setDatabase(getName()) - .setQuery(query) - .setCredentials(buildCredentials()) - .setBatchSize(batchSize > 0 ? batchSize : 100) - .setRetrievalMode(mode); - - if (params != null && !params.isEmpty()) { - b.putAllParameters(convertParamsToProto(params)); - } - - Iterator responseIterator = blockingStub - .withWaitForReady() // optional, improves robustness - .withDeadlineAfter(/* e.g. */ 10, TimeUnit.MINUTES) - .streamQuery(b.build()); - - return new Iterator() { - private QueryBatch nextBatch = null; - private boolean drained = false; - - @Override - public boolean hasNext() { - if (nextBatch != null) return true; - if (drained) return false; - - while (responseIterator.hasNext()) { - QueryResult qr = responseIterator.next(); - int n = qr.getTotalRecordsInBatch(); // server-populated - // Guard: some servers could omit this; fallback to list size. - if (n == 0) n = qr.getRecordsCount(); - - if (qr.getRecordsCount() == 0 && !qr.getIsLastBatch()) { - // skip empty non-terminal batch - continue; - } - - List converted = new ArrayList<>(qr.getRecordsCount()); - for (com.arcadedb.server.grpc.Record gr : qr.getRecordsList()) { - converted.add(grpcRecordToRecord(gr)); - } - - nextBatch = new QueryBatch( - converted, - n, - qr.getRunningTotalEmitted(), - qr.getIsLastBatch() - ); - - if (qr.getIsLastBatch()) { - drained = true; // no more after this (even if server sent empty terminal) - } - return true; - } - - drained = true; - return false; - } - - @Override - public QueryBatch next() { - if (!hasNext()) throw new NoSuchElementException(); - QueryBatch out = nextBatch; - nextBatch = null; - return out; - } - }; - } - - public Iterator queryStream( - String database, - String sql, - Map params, - int batchSize, - com.arcadedb.server.grpc.StreamQueryRequest.RetrievalMode mode, - com.arcadedb.server.grpc.TransactionContext tx, - long timeoutMs) { - - var reqB = com.arcadedb.server.grpc.StreamQueryRequest.newBuilder() - .setDatabase(database) - .setQuery(sql) - .putAllParameters(convertParamsToProto(params)) - .setCredentials(buildCredentials()) - .setBatchSize(batchSize > 0 ? batchSize : 100) - .setRetrievalMode(mode); - - if (tx != null) reqB.setTransaction(tx); - - Iterator it = blockingStub - .withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS) - .streamQuery(reqB.build()); - - return new Iterator<>() { - private Iterator curr = java.util.Collections.emptyIterator(); - public boolean hasNext() { - if (curr.hasNext()) return true; - if (it.hasNext()) { curr = it.next().getRecordsList().iterator(); return hasNext(); } - return false; - } - public com.arcadedb.server.grpc.Record next() { - if (!hasNext()) throw new java.util.NoSuchElementException(); - return curr.next(); - } - }; - } - - public Iterator queryStreamBatches( - String passLabel, - String sql, - Map params, - int batchSize, - com.arcadedb.server.grpc.StreamQueryRequest.RetrievalMode mode, - com.arcadedb.server.grpc.TransactionContext tx, - long timeoutMs) { - - var reqB = com.arcadedb.server.grpc.StreamQueryRequest.newBuilder() - .setDatabase(getName()) - .setQuery(sql) - .putAllParameters(convertParamsToProto(params)) - .setCredentials(buildCredentials()) - .setBatchSize(batchSize > 0 ? batchSize : 100) - .setRetrievalMode(mode); - - if (tx != null) reqB.setTransaction(tx); - - Iterator respIter = blockingStub - .withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS) - .streamQuery(reqB.build()); - - return new Iterator<>() { - public boolean hasNext() { return respIter.hasNext(); } - public QueryBatch next() { - - var qr = respIter.next(); - - - List converted = new ArrayList<>(qr.getRecordsCount()); - for (com.arcadedb.server.grpc.Record gr : qr.getRecordsList()) { - converted.add(grpcRecordToRecord(gr)); - } - - return new QueryBatch( - converted, - qr.getTotalRecordsInBatch(), // int totalInBatch - qr.getRunningTotalEmitted(), // long runningTotal - qr.getIsLastBatch() // boolean lastBatch - ); - } - }; - } - - // public QueryBatch(List records, int totalInBatch, long runningTotal, boolean lastBatch) { - - public String createVertex(String cls, Map props, long timeoutMs) { - var req = com.arcadedb.server.grpc.CreateRecordRequest.newBuilder() - .setDatabase(getName()) - .setType(cls) - .putAllProperties(convertParamsToProto(props)) - .setCredentials(buildCredentials()) - .build(); - var res = blockingStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS).createRecord(req); - // adjust to your response field name - return res.getIdentity(); // e.g., "#12:0" - } - - public boolean updateRecord(String rid, Map props, long timeoutMs) { - var req = com.arcadedb.server.grpc.UpdateRecordRequest.newBuilder() - .setDatabase(getName()) - .setRid(rid) - .putAllProperties(convertParamsToProto(props)) - .setCredentials(buildCredentials()) - // .setTransaction(txBeginCommit()) // if you add tx to this RPC - .build(); - var res = blockingStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS).updateRecord(req); - return res.getSuccess(); + private final List records; + private final int totalInBatch; + private final long runningTotal; + private final boolean lastBatch; + + public QueryBatch(List list, int totalInBatch, long runningTotal, boolean lastBatch) { + this.records = list; + this.totalInBatch = totalInBatch; + this.runningTotal = runningTotal; + this.lastBatch = lastBatch; + } + + public List records() { + return records; + } + + public int totalInBatch() { + return totalInBatch; + } + + public long runningTotal() { + return runningTotal; + } + + public boolean isLastBatch() { + return lastBatch; + } + } + + public Iterator queryStreamBatches(final String language, final String query, final Map params, + final int batchSize, final StreamQueryRequest.RetrievalMode mode) { + checkDatabaseIsOpen(); + stats.queries.incrementAndGet(); + + StreamQueryRequest.Builder b = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(query).setCredentials(buildCredentials()) + .setBatchSize(batchSize > 0 ? batchSize : 100).setRetrievalMode(mode); + + if (params != null && !params.isEmpty()) { + b.putAllParameters(convertParamsToProto(params)); + } + + BlockingClientCall responseIterator = blockingStub.withWaitForReady() // optional, improves robustness + .withDeadlineAfter(/* e.g. */ 10, TimeUnit.MINUTES).streamQuery(b.build()); + + return new Iterator() { + private QueryBatch nextBatch = null; + private boolean drained = false; + + @Override + public boolean hasNext() { + if (nextBatch != null) + return true; + if (drained) + return false; + + try { + while (responseIterator.hasNext()) { + QueryResult qr = responseIterator.read(); + int n = qr.getTotalRecordsInBatch(); // server-populated + // Guard: some servers could omit this; fallback to list size. + if (n == 0) + n = qr.getRecordsCount(); + + if (qr.getRecordsCount() == 0 && !qr.getIsLastBatch()) { + // skip empty non-terminal batch + continue; + } + + List converted = new ArrayList<>(qr.getRecordsCount()); + for (com.arcadedb.server.grpc.Record gr : qr.getRecordsList()) { + converted.add(grpcRecordToRecord(gr)); + } + + nextBatch = new QueryBatch(converted, n, qr.getRunningTotalEmitted(), qr.getIsLastBatch()); + + if (qr.getIsLastBatch()) { + drained = true; // no more after this (even if server sent empty terminal) + } + return true; + } + } + catch (InterruptedException e) { + + throw new RuntimeException("queryStreamBatches: interrupted", e); + } + catch (StatusException e) { + + throw new RuntimeException("queryStreamBatches: error", e); + } + + drained = true; + return false; + } + + @Override + public QueryBatch next() { + if (!hasNext()) + throw new NoSuchElementException(); + QueryBatch out = nextBatch; + nextBatch = null; + return out; + } + }; + } + + public Iterator queryStream(String database, String sql, Map params, int batchSize, + com.arcadedb.server.grpc.StreamQueryRequest.RetrievalMode mode, com.arcadedb.server.grpc.TransactionContext tx, long timeoutMs) { + + var reqB = com.arcadedb.server.grpc.StreamQueryRequest.newBuilder().setDatabase(database).setQuery(sql) + .putAllParameters(convertParamsToProto(params)).setCredentials(buildCredentials()).setBatchSize(batchSize > 0 ? batchSize : 100) + .setRetrievalMode(mode); + + if (tx != null) + reqB.setTransaction(tx); + + BlockingClientCall it = blockingStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS) + .streamQuery(reqB.build()); + + return new Iterator<>() { + + private Iterator curr = java.util.Collections.emptyIterator(); + + public boolean hasNext() { + + if (curr.hasNext()) + return true; + + try { + if (it.hasNext()) { + curr = it.read().getRecordsList().iterator(); + return hasNext(); + } + } + catch (InterruptedException e) { + + throw new RuntimeException("queryStream: interrupted", e); + } + catch (StatusException e) { + + throw new RuntimeException("queryStream: error", e); + } + + return false; + } + + public com.arcadedb.server.grpc.Record next() { + if (!hasNext()) + throw new java.util.NoSuchElementException(); + return curr.next(); + } + }; + } + + public Iterator queryStreamBatches(String passLabel, String sql, Map params, int batchSize, + com.arcadedb.server.grpc.StreamQueryRequest.RetrievalMode mode, com.arcadedb.server.grpc.TransactionContext tx, long timeoutMs) { + + var reqB = com.arcadedb.server.grpc.StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(sql) + .putAllParameters(convertParamsToProto(params)).setCredentials(buildCredentials()).setBatchSize(batchSize > 0 ? batchSize : 100) + .setRetrievalMode(mode); + + if (tx != null) + reqB.setTransaction(tx); + + BlockingClientCall respIter = blockingStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS) + .streamQuery(reqB.build()); + + return new Iterator<>() { + + public boolean hasNext() { + + try { + return respIter.hasNext(); + } + catch (InterruptedException e) { + + throw new RuntimeException("queryStreamBatches: interrupted", e); + } + catch (StatusException e) { + + throw new RuntimeException("queryStreamBatches: error", e); + } + } + + public QueryBatch next() { + + QueryResult qr; + + try { + qr = respIter.read(); + List converted = new ArrayList<>(qr.getRecordsCount()); + for (com.arcadedb.server.grpc.Record gr : qr.getRecordsList()) { + converted.add(grpcRecordToRecord(gr)); + } + + return new QueryBatch(converted, qr.getTotalRecordsInBatch(), // int totalInBatch + qr.getRunningTotalEmitted(), // long runningTotal + qr.getIsLastBatch() // boolean lastBatch + ); + } + catch (InterruptedException e) { + + throw new RuntimeException("queryStreamBatches: interrupted", e); + } + catch (StatusException e) { + + throw new RuntimeException("queryStreamBatches: error", e); + } + } + }; + } + + public String createVertex(String cls, Map props, long timeoutMs) { + // Build the nested proto Record payload + com.arcadedb.server.grpc.Record recMsg = com.arcadedb.server.grpc.Record.newBuilder().putAllProperties(convertParamsToProto(props)) + .build(); + + // Build the Create request + com.arcadedb.server.grpc.CreateRecordRequest req = com.arcadedb.server.grpc.CreateRecordRequest.newBuilder().setDatabase(getName()) + .setType(cls).setRecord(recMsg) // <<<< NESTED RECORD (not top-level properties) + .setCredentials(buildCredentials()).build(); + + // Call RPC + com.arcadedb.server.grpc.CreateRecordResponse res; + + try { + + res = blockingStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS).createRecord(req); + + // Response carries new RID string + return res.getRid(); // e.g., "#12:0" + } + catch (StatusException e) { + + throw new RuntimeException("Failed to create vertex", e); + } + } + + public boolean updateRecord(String rid, Map props, long timeoutMs) { + + // Build the nested proto Record payload for a PATCH + com.arcadedb.server.grpc.Record record = com.arcadedb.server.grpc.Record.newBuilder().putAllProperties(convertParamsToProto(props)) + .build(); + + // Build the Update request (use setRecord(...) for full replace instead) + com.arcadedb.server.grpc.UpdateRecordRequest req = com.arcadedb.server.grpc.UpdateRecordRequest.newBuilder().setDatabase(getName()) + .setRid(rid).setRecord(record) // <<<< oneof {record|partial} + .setCredentials(buildCredentials()).build(); + + // Call RPC + com.arcadedb.server.grpc.UpdateRecordResponse res; + + try { + res = blockingStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS).updateRecord(req); + + // Choose the flag your proto defines. Most builds expose getSuccess(). + return res.getSuccess(); + // If your generated class exposes getUpdated() instead, use: + // return res.getUpdated(); + } + catch (StatusException e) { + + throw new RuntimeException("Failed to update record", e); + } } public boolean deleteRecord(String rid, long timeoutMs) { - var req = com.arcadedb.server.grpc.DeleteRecordRequest.newBuilder() - .setDatabase(getName()) - .setRid(rid) - .setCredentials(buildCredentials()) - .build(); - var res = blockingStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS).deleteRecord(req); - return res.getDeleted(); - } - + var req = com.arcadedb.server.grpc.DeleteRecordRequest.newBuilder().setDatabase(getName()).setRid(rid).setCredentials(buildCredentials()) + .build(); + DeleteRecordResponse res; + + try { + + res = blockingStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS).deleteRecord(req); + return res.getDeleted(); + } + catch (StatusException e) { + + throw new RuntimeException("Failed to delete record", e); + } + } + @Override public RemoteTransactionExplicitLock acquireLock() { - // Need gRPC-specific implementation - if (explicitLock == null) - explicitLock = new RemoteGrpcTransactionExplicitLock(this); - - return explicitLock; - } - + // Need gRPC-specific implementation + if (explicitLock == null) + explicitLock = new RemoteGrpcTransactionExplicitLock(this); + + return explicitLock; + } + @Override public long countBucket(final String bucketName) { - checkDatabaseIsOpen(); - stats.countBucket.incrementAndGet(); - ResultSet result = query("sql", "select count(*) as count from bucket:" + bucketName); - if (result.hasNext()) { - Number count = result.next().getProperty("count"); - return count != null ? count.longValue() : 0; - } - return 0; - } - + checkDatabaseIsOpen(); + stats.countBucket.incrementAndGet(); + ResultSet result = query("sql", "select count(*) as count from bucket:" + bucketName); + if (result.hasNext()) { + Number count = result.next().getProperty("count"); + return count != null ? count.longValue() : 0; + } + return 0; + } + @Override public long countType(final String typeName, final boolean polymorphic) { - checkDatabaseIsOpen(); - stats.countType.incrementAndGet(); - final String appendix = polymorphic ? "" : " where @type = '" + typeName + "'"; - ResultSet result = query("sql", "select count(*) as count from " + typeName + appendix); - if (result.hasNext()) { - Number count = result.next().getProperty("count"); - return count != null ? count.longValue() : 0; - } - return 0; - } - + checkDatabaseIsOpen(); + stats.countType.incrementAndGet(); + final String appendix = polymorphic ? "" : " where @type = '" + typeName + "'"; + ResultSet result = query("sql", "select count(*) as count from " + typeName + appendix); + if (result.hasNext()) { + Number count = result.next().getProperty("count"); + return count != null ? count.longValue() : 0; + } + return 0; + } + @Override public Record lookupByRID(final RID rid) { - return lookupByRID(rid, true); + return lookupByRID(rid, true); } @Override public boolean existsRecord(RID rid) { - stats.existsRecord.incrementAndGet(); - if (rid == null) - throw new IllegalArgumentException("Record is null"); - try { - return lookupByRID(rid, false) != null; - } catch (RecordNotFoundException e) { - return false; - } - } - - - public com.arcadedb.server.grpc.InsertSummary insertBulk( - final com.arcadedb.server.grpc.InsertOptions options, - final java.util.List protoRows, - final long timeoutMs) { - - // Ensure options carry DB + credentials as the server expects - com.arcadedb.server.grpc.InsertOptions.Builder ob = options.toBuilder(); - - if (options.getDatabase() == null || options.getDatabase().isEmpty()) { - ob.setDatabase(getName()); // your wrapper's DB name - } - - boolean missingCreds = - !options.hasCredentials() - || options.getCredentials().getUsername().isEmpty(); - if (missingCreds) { - ob.setCredentials(buildCredentials()); // your existing helper used by queries - } - - InsertOptions newOptions = ob.build(); - - com.arcadedb.server.grpc.BulkInsertRequest req = - com.arcadedb.server.grpc.BulkInsertRequest.newBuilder() - .setOptions(newOptions) - .addAllRows(protoRows) - .build(); - - return blockingStub - .withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS) - .bulkInsert(req); + stats.existsRecord.incrementAndGet(); + if (rid == null) + throw new IllegalArgumentException("Record is null"); + try { + return lookupByRID(rid, false) != null; + } + catch (RecordNotFoundException e) { + return false; + } + } + + public com.arcadedb.server.grpc.InsertSummary insertBulk(final com.arcadedb.server.grpc.InsertOptions options, + final java.util.List protoRows, final long timeoutMs) { + + // Ensure options carry DB + credentials as the server expects + com.arcadedb.server.grpc.InsertOptions.Builder ob = options.toBuilder(); + + if (options.getDatabase() == null || options.getDatabase().isEmpty()) { + ob.setDatabase(getName()); // your wrapper's DB name + } + + boolean missingCreds = !options.hasCredentials() || options.getCredentials().getUsername().isEmpty(); + if (missingCreds) { + ob.setCredentials(buildCredentials()); // your existing helper used by queries + } + + InsertOptions newOptions = ob.build(); + + com.arcadedb.server.grpc.BulkInsertRequest req = com.arcadedb.server.grpc.BulkInsertRequest.newBuilder().setOptions(newOptions) + .addAllRows(protoRows).build(); + + try { + return blockingStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS).bulkInsert(req); + } + catch (StatusException e) { + + throw new RuntimeException("insertBulk() -> failed: " + e.getStatus()); + } } // Convenience overload that accepts domain rows (convert first) - public com.arcadedb.server.grpc.InsertSummary insertBulkAsListOfMaps( - final com.arcadedb.server.grpc.InsertOptions options, - final java.util.List> rows, - final long timeoutMs) { - - java.util.List protoRows = - rows.stream() - .map(this::toProtoRecordFromMap) // your converter - .collect(java.util.stream.Collectors.toList()); - - return insertBulk(options, protoRows, timeoutMs); - } - - public com.arcadedb.server.grpc.InsertSummary ingestStream( - final com.arcadedb.server.grpc.InsertOptions options, // note: InsertStream RPC does not carry options - final java.util.List protoRows, - final int chunkSize, - final long timeoutMs) throws InterruptedException { - - final java.util.concurrent.CountDownLatch done = new java.util.concurrent.CountDownLatch(1); - final java.util.concurrent.atomic.AtomicReference summaryRef = - new java.util.concurrent.atomic.AtomicReference<>(); - - // Ensure options carry DB + credentials as the server expects - com.arcadedb.server.grpc.InsertOptions.Builder ob = options.toBuilder(); - - if (options.getDatabase() == null || options.getDatabase().isEmpty()) { - ob.setDatabase(getName()); // your wrapper's DB name - } - - boolean missingCreds = - !options.hasCredentials() - || options.getCredentials().getUsername().isEmpty(); - if (missingCreds) { - ob.setCredentials(buildCredentials()); // your existing helper used by queries - } - - io.grpc.stub.StreamObserver resp = - new io.grpc.stub.StreamObserver<>() { - @Override public void onNext(com.arcadedb.server.grpc.InsertSummary value) { summaryRef.set(value); } - @Override public void onError(Throwable t) { done.countDown(); } - @Override public void onCompleted() { done.countDown(); } - }; - - // Use the write service async stub, per-call deadline - var stub = - this.asyncStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS); - - final io.grpc.stub.StreamObserver req = stub.insertStream(resp); - - // Stream chunks - final String sessionId = "sess-" + System.nanoTime(); - long seq = 1; - for (int i = 0; i < protoRows.size(); i += chunkSize) { - final int end = Math.min(i + chunkSize, protoRows.size()); - final com.arcadedb.server.grpc.InsertChunk chunk = - com.arcadedb.server.grpc.InsertChunk.newBuilder() - .setSessionId(sessionId) - .setChunkSeq(seq++) - .addAllRows(protoRows.subList(i, end)) - .build(); - req.onNext(chunk); - } - req.onCompleted(); - - done.await(timeoutMs + 5_000, java.util.concurrent.TimeUnit.MILLISECONDS); - - final com.arcadedb.server.grpc.InsertSummary s = summaryRef.get(); - return (s != null) - ? s - : com.arcadedb.server.grpc.InsertSummary.newBuilder().setReceived(protoRows.size()).build(); - } - + public com.arcadedb.server.grpc.InsertSummary insertBulkAsListOfMaps(final com.arcadedb.server.grpc.InsertOptions options, + final java.util.List> rows, final long timeoutMs) { + + java.util.List protoRows = rows.stream().map(this::toProtoRecordFromMap) // your converter + .collect(java.util.stream.Collectors.toList()); + + return insertBulk(options, protoRows, timeoutMs); + } + + public com.arcadedb.server.grpc.InsertSummary ingestStream(final com.arcadedb.server.grpc.InsertOptions options, + final java.util.List protoRows, final int chunkSize, final long timeoutMs) + throws InterruptedException { + + final java.util.concurrent.CountDownLatch done = new java.util.concurrent.CountDownLatch(1); + final java.util.concurrent.atomic.AtomicReference summaryRef = new java.util.concurrent.atomic.AtomicReference<>(); + + // Ensure options carry DB + credentials as the server expects + com.arcadedb.server.grpc.InsertOptions.Builder ob = options.toBuilder(); + + if (options.getDatabase() == null || options.getDatabase().isEmpty()) { + ob.setDatabase(getName()); // your wrapper's DB name + } + + boolean missingCreds = !options.hasCredentials() || options.getCredentials().getUsername().isEmpty(); + + if (missingCreds) { + ob.setCredentials(buildCredentials()); // your existing helper used by queries + } + + io.grpc.stub.StreamObserver resp = new io.grpc.stub.StreamObserver<>() { + @Override + public void onNext(com.arcadedb.server.grpc.InsertSummary value) { + summaryRef.set(value); + } + + @Override + public void onError(Throwable t) { + done.countDown(); + } + + @Override + public void onCompleted() { + done.countDown(); + } + }; + + // Use the write service async stub, per-call deadline + var stub = this.asyncStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS); + + final io.grpc.stub.StreamObserver req = stub.insertStream(resp); + + // Stream chunks + final String sessionId = "sess-" + System.nanoTime(); + + long seq = 1; + + for (int i = 0; i < protoRows.size(); i += chunkSize) { + + final int end = Math.min(i + chunkSize, protoRows.size()); + + final com.arcadedb.server.grpc.InsertChunk chunk = com.arcadedb.server.grpc.InsertChunk.newBuilder().setSessionId(sessionId) + .setOptions(ob.build()).setChunkSeq(seq++).addAllRows(protoRows.subList(i, end)).build(); + + req.onNext(chunk); + } + + req.onCompleted(); + + done.await(timeoutMs + 5_000, java.util.concurrent.TimeUnit.MILLISECONDS); + + final com.arcadedb.server.grpc.InsertSummary s = summaryRef.get(); + return (s != null) ? s : com.arcadedb.server.grpc.InsertSummary.newBuilder().setReceived(protoRows.size()).build(); + } + // Convenience overload - public com.arcadedb.server.grpc.InsertSummary ingestStreamAsListOfMaps( - final com.arcadedb.server.grpc.InsertOptions options, - final java.util.List> rows, - final int chunkSize, - final long timeoutMs) throws InterruptedException { - - java.util.List protoRows = - rows.stream().map(this::toProtoRecordFromMap).collect(java.util.stream.Collectors.toList()); - return ingestStream(options, protoRows, chunkSize, timeoutMs); - } - - /** Push records using InsertBidirectional with per-batch ACKs. */ - public InsertSummary ingestBidi(List rows, InsertOptions opts, int chunkSize, int maxInflight) throws InterruptedException { - - final String sessionId = UUID.randomUUID().toString(); - - // Ensure options carry DB + credentials as the server expects - com.arcadedb.server.grpc.InsertOptions.Builder ob = opts.toBuilder(); - - if (opts.getDatabase() == null || opts.getDatabase().isEmpty()) { - ob.setDatabase(getName()); // your wrapper's DB name - } - - boolean missingCreds = - !opts.hasCredentials() - || opts.getCredentials().getUsername().isEmpty(); - if (missingCreds) { - ob.setCredentials(buildCredentials()); // your existing helper used by queries - } - + public com.arcadedb.server.grpc.InsertSummary ingestStreamAsListOfMaps(final com.arcadedb.server.grpc.InsertOptions options, + final java.util.List> rows, final int chunkSize, final long timeoutMs) throws InterruptedException { + + java.util.List protoRows = rows.stream().map(this::toProtoRecordFromMap) + .collect(java.util.stream.Collectors.toList()); + + return ingestStream(options, protoRows, chunkSize, timeoutMs); + } + + /** + * Core implementation of InsertBidirectional ingest: - Sends a START once with + * effective InsertOptions (db/creds enforced) - Streams chunks of mapped rows + * (using chunkSize) while respecting backpressure (maxInflight) - Collects + * per-batch ACKs and returns COMMITTED summary if present, else aggregates ACKs + * + * @param row type (e.g., com.arcadedb.database.Record or + * Map) + * @param rows source rows + * @param options base InsertOptions; db/credentials will be injected if + * missing + * @param chunkSize max rows per chunk + * @param maxInflight max number of unacknowledged chunks allowed + * @param timeoutMs overall client-side timeout + * @param mapper maps T → grpc Record (must be pure & fast) + * @return InsertSummary as returned by COMMITTED, or aggregated from ACKs if + * COMMITTED missing + * @throws InterruptedException if the await is interrupted + */ + private com.arcadedb.server.grpc.InsertSummary ingestBidiCore(final java.util.List rows, + final com.arcadedb.server.grpc.InsertOptions options, final int chunkSize, final int maxInflight, final long timeoutMs, + final java.util.function.Function mapper) throws InterruptedException { + + // 1) Ensure options carry DB + credentials the server expects + final com.arcadedb.server.grpc.InsertOptions.Builder ob = options.toBuilder(); + if (options.getDatabase() == null || options.getDatabase().isEmpty()) { + ob.setDatabase(getName()); + } + final boolean missingCreds = !options.hasCredentials() || options.getCredentials().getUsername().isEmpty(); + if (missingCreds) { + ob.setCredentials(buildCredentials()); + } + final com.arcadedb.server.grpc.InsertOptions effectiveOpts = ob.build(); + + // 2) Pre-map rows → proto once (outside the observer) + final java.util.List protoRows = rows.stream().map(mapper) + .collect(java.util.stream.Collectors.toList()); + + // 3) Streaming state + final String sessionId = "sess-" + System.nanoTime(); final java.util.concurrent.CountDownLatch done = new java.util.concurrent.CountDownLatch(1); final java.util.concurrent.atomic.AtomicLong seq = new java.util.concurrent.atomic.AtomicLong(1); + final java.util.concurrent.atomic.AtomicInteger cursor = new java.util.concurrent.atomic.AtomicInteger(0); final java.util.concurrent.atomic.AtomicInteger sent = new java.util.concurrent.atomic.AtomicInteger(0); final java.util.concurrent.atomic.AtomicInteger acked = new java.util.concurrent.atomic.AtomicInteger(0); - final java.util.List acks = java.util.Collections.synchronizedList(new java.util.ArrayList<>()); - final java.util.concurrent.atomic.AtomicReference committed = new java.util.concurrent.atomic.AtomicReference<>(); + final java.util.concurrent.atomic.AtomicReference committed = new java.util.concurrent.atomic.AtomicReference<>(); + final java.util.List acks = java.util.Collections.synchronizedList(new java.util.ArrayList<>()); - io.grpc.stub.ClientResponseObserver obs = new io.grpc.stub.ClientResponseObserver<>() { - - io.grpc.stub.ClientCallStreamObserver req; + final io.grpc.stub.ClientResponseObserver observer = new io.grpc.stub.ClientResponseObserver<>() { + + io.grpc.stub.ClientCallStreamObserver req; + volatile boolean started = false; @Override - public void beforeStart(io.grpc.stub.ClientCallStreamObserver r) { + public void beforeStart(io.grpc.stub.ClientCallStreamObserver r) { this.req = r; r.disableAutoInboundFlowControl(); r.setOnReadyHandler(this::drain); } - private int cursor = 0; - private void drain() { if (!req.isReady()) return; - if (seq.get() == 1) { // send START once - req.onNext(InsertRequest.newBuilder().setStart(StartInsert.newBuilder().setOptions(ob.build())).build()); + + // Send START exactly once + if (!started) { + req.onNext(com.arcadedb.server.grpc.InsertRequest.newBuilder() + .setStart(com.arcadedb.server.grpc.Start.newBuilder().setOptions(effectiveOpts)).build()); + started = true; } + + // While there is capacity (backpressure) and in-flight < maxInflight → send + // chunks while (req.isReady() && (sent.get() - acked.get()) < maxInflight) { - - if (cursor >= rows.size()) + final int start = cursor.get(); + if (start >= protoRows.size()) break; - - int end = Math.min(cursor + chunkSize, rows.size()); - - List protoRows = rows.stream() - .map(RemoteGrpcDatabase.this::toProtoRecordFromDbRecord) - .toList(); - - InsertChunk chunk = InsertChunk.newBuilder() - .setSessionId(sessionId) - .setChunkSeq(seq.getAndIncrement()) - .addAllRows(protoRows) - .build(); - - req.onNext(InsertRequest.newBuilder().setChunk(chunk).build()); - cursor = end; + + final int end = Math.min(start + chunkSize, protoRows.size()); + final java.util.List slice = protoRows.subList(start, end); + + final com.arcadedb.server.grpc.InsertChunk chunk = com.arcadedb.server.grpc.InsertChunk.newBuilder().setSessionId(sessionId) + .setChunkSeq(seq.getAndIncrement()).addAllRows(slice).build(); + + req.onNext(com.arcadedb.server.grpc.InsertRequest.newBuilder().setChunk(chunk).build()); + + cursor.set(end); sent.incrementAndGet(); } - req.request(1); - if (cursor >= rows.size() && acked.get() >= sent.get()) { - req.onNext(InsertRequest.newBuilder().setCommit(CommitInsert.newBuilder().setSessionId(sessionId)).build()); + + // If we sent everything and all ACKed → send COMMIT and half-close + if (cursor.get() >= protoRows.size() && acked.get() >= sent.get()) { + req.onNext(com.arcadedb.server.grpc.InsertRequest.newBuilder() + .setCommit(com.arcadedb.server.grpc.Commit.newBuilder().setSessionId(sessionId)).build()); req.onCompleted(); } } @Override - public void onNext(InsertResponse v) { + public void onNext(com.arcadedb.server.grpc.InsertResponse v) { switch (v.getMsgCase()) { case STARTED -> { /* ok */ } case BATCH_ACK -> { acks.add(v.getBatchAck()); acked.incrementAndGet(); + // Now that capacity freed up, try to send more drain(); } case COMMITTED -> committed.set(v.getCommitted().getSummary()); case MSG_NOT_SET -> { + /* ignore */ } } - } - req.request(1); + // NOTE: no req.request(1) here; responses are server-pushed } @Override @@ -1301,255 +1405,190 @@ public void onCompleted() { } }; - asyncStub.insertBidirectional(obs); - - done.await(5, java.util.concurrent.TimeUnit.MINUTES); - - InsertSummary s = committed.get(); - if (s != null) - return s; - - long ins = acks.stream().mapToLong(BatchAck::getInserted).sum(); - long upd = acks.stream().mapToLong(BatchAck::getUpdated).sum(); - long ign = acks.stream().mapToLong(BatchAck::getIgnored).sum(); - long fail = acks.stream().mapToLong(BatchAck::getFailed).sum(); - - return InsertSummary.newBuilder().setReceived(rows.size()).setInserted(ins).setUpdated(upd).setIgnored(ign).setFailed(fail).build(); - } - + // 4) Kick off the bidi call with deadline + this.asyncStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS).insertBidirectional(observer); + + // 5) Wait for completion + done.await(timeoutMs + 5_000, java.util.concurrent.TimeUnit.MILLISECONDS); + + // 6) Prefer COMMITTED summary if present; otherwise aggregate ACKs + final com.arcadedb.server.grpc.InsertSummary finalSummary = committed.get(); + if (finalSummary != null) + return finalSummary; + + final long ins = acks.stream().mapToLong(com.arcadedb.server.grpc.BatchAck::getInserted).sum(); + final long upd = acks.stream().mapToLong(com.arcadedb.server.grpc.BatchAck::getUpdated).sum(); + final long ign = acks.stream().mapToLong(com.arcadedb.server.grpc.BatchAck::getIgnored).sum(); + final long fail = acks.stream().mapToLong(com.arcadedb.server.grpc.BatchAck::getFailed).sum(); + + return com.arcadedb.server.grpc.InsertSummary.newBuilder().setReceived(protoRows.size()).setInserted(ins).setUpdated(upd).setIgnored(ign) + .setFailed(fail).build(); + } + /** - * Bidirectional streaming ingest with per-batch ACKs. + * Pushes domain {@code com.arcadedb.database.Record} rows via + * InsertBidirectional with per-batch ACKs. * - * @param options InsertOptions (database, target class, conflict mode, tx mode, etc.) - * @param rowsProto list of PROTO Records (com.arcadedb.server.grpc.Record) - * @param chunkSize rows per chunk - * @param maxInflight max unacked chunks - * @param timeoutMs overall timeout for the ingest (client-side) + *

+ * Behavior: + *

    + *
  • Sends a {@code START} with effective {@link InsertOptions} + * (DB/credentials auto-injected if missing).
    • + *
  • Streams chunks of rows (size = {@code chunkSize}) while respecting + * backpressure (at most {@code maxInflight} unacknowledged chunks).
    • + *
  • Each server {@code BATCH_ACK} reduces in-flight count and triggers + * sending more.
    • + *
  • When all chunks are acked, sends {@code COMMIT} and returns the final + * {@code InsertSummary} from the server if available; otherwise aggregates ACKs + * to form a summary.
    • + *
+ * + * @param rows domain records to send + * @param opts base InsertOptions (target class, conflict mode, tx mode). + * DB/credentials will be filled if missing + * @param chunkSize number of rows per chunk (must be > 0) + * @param maxInflight max unacknowledged chunks allowed (backpressure window) + * @return InsertSummary from COMMITTED or aggregated from ACKs + * @throws InterruptedException if waiting for completion is interrupted */ - public com.arcadedb.server.grpc.InsertSummary ingestBidi( - final com.arcadedb.server.grpc.InsertOptions options, - final java.util.List> rows, - final int chunkSize, - final int maxInflight, - final long timeoutMs) throws InterruptedException { - - // Ensure options carry DB + credentials as the server expects - com.arcadedb.server.grpc.InsertOptions.Builder ob = options.toBuilder(); - - if (options.getDatabase() == null || options.getDatabase().isEmpty()) { - ob.setDatabase(getName()); // your wrapper's DB name - } - - boolean missingCreds = - !options.hasCredentials() - || options.getCredentials().getUsername().isEmpty(); - if (missingCreds) { - ob.setCredentials(buildCredentials()); // your existing helper used by queries - } - - // Pre-convert to PROTO records (outside the observer!) - final java.util.List protoRows = - rows.stream() - .map(RemoteGrpcDatabase.this::toProtoRecordFromMap) - .collect(java.util.stream.Collectors.toList()); - - final String sessionId = "sess-" + System.nanoTime(); - final java.util.concurrent.CountDownLatch done = new java.util.concurrent.CountDownLatch(1); - final java.util.concurrent.atomic.AtomicLong seq = new java.util.concurrent.atomic.AtomicLong(1); - final java.util.concurrent.atomic.AtomicInteger cursor = new java.util.concurrent.atomic.AtomicInteger(0); - final java.util.concurrent.atomic.AtomicInteger sent = new java.util.concurrent.atomic.AtomicInteger(0); - final java.util.concurrent.atomic.AtomicInteger acked = new java.util.concurrent.atomic.AtomicInteger(0); - final java.util.List acks = - java.util.Collections.synchronizedList(new java.util.ArrayList<>()); - final java.util.concurrent.atomic.AtomicReference committed = - new java.util.concurrent.atomic.AtomicReference<>(); - - io.grpc.stub.ClientResponseObserver observer = - new io.grpc.stub.ClientResponseObserver<>() { - io.grpc.stub.ClientCallStreamObserver req; - - @Override - public void beforeStart(io.grpc.stub.ClientCallStreamObserver r) { - this.req = r; - r.disableAutoInboundFlowControl(); - r.setOnReadyHandler(this::drain); - } - - private void drain() { - if (!req.isReady()) return; - - if (seq.get() == 1) { - req.onNext(com.arcadedb.server.grpc.InsertRequest.newBuilder() - .setStart(com.arcadedb.server.grpc.StartInsert.newBuilder().setOptions(ob.build())) - .build()); - } - - while (req.isReady() && (sent.get() - acked.get()) < maxInflight) { - int start = cursor.get(); - if (start >= protoRows.size()) break; - int end = Math.min(start + chunkSize, protoRows.size()); - - com.arcadedb.server.grpc.InsertChunk chunk = - com.arcadedb.server.grpc.InsertChunk.newBuilder() - .setSessionId(sessionId) - .setChunkSeq(seq.getAndIncrement()) - .addAllRows(protoRows.subList(start, end)) - .build(); - - req.onNext(com.arcadedb.server.grpc.InsertRequest.newBuilder().setChunk(chunk).build()); - cursor.set(end); - sent.incrementAndGet(); - } - - req.request(1); - - if (cursor.get() >= protoRows.size() && acked.get() >= sent.get()) { - req.onNext(com.arcadedb.server.grpc.InsertRequest.newBuilder() - .setCommit(com.arcadedb.server.grpc.CommitInsert.newBuilder().setSessionId(sessionId)) - .build()); - req.onCompleted(); - } - } - - @Override - public void onNext(com.arcadedb.server.grpc.InsertResponse v) { - switch (v.getMsgCase()) { - case STARTED -> { /* ok */ } - case BATCH_ACK -> { - acks.add(v.getBatchAck()); - acked.incrementAndGet(); - drain(); - } - case COMMITTED -> committed.set(v.getCommitted().getSummary()); - case MSG_NOT_SET -> {} - } - req.request(1); - } - - @Override public void onError(Throwable t) { done.countDown(); } - @Override public void onCompleted() { done.countDown(); } - }; - - var stub = this.asyncStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS); - stub.insertBidirectional(observer); - - done.await(timeoutMs + 5_000, java.util.concurrent.TimeUnit.MILLISECONDS); - - var summary = committed.get(); - if (summary != null) return summary; - - long ins = acks.stream().mapToLong(com.arcadedb.server.grpc.BatchAck::getInserted).sum(); - long upd = acks.stream().mapToLong(com.arcadedb.server.grpc.BatchAck::getUpdated).sum(); - long ign = acks.stream().mapToLong(com.arcadedb.server.grpc.BatchAck::getIgnored).sum(); - long fail = acks.stream().mapToLong(com.arcadedb.server.grpc.BatchAck::getFailed).sum(); - - return com.arcadedb.server.grpc.InsertSummary.newBuilder() - .setReceived(protoRows.size()) - .setInserted(ins) - .setUpdated(upd) - .setIgnored(ign) - .setFailed(fail) - .build(); + public com.arcadedb.server.grpc.InsertSummary ingestBidi(final java.util.List rows, + final com.arcadedb.server.grpc.InsertOptions opts, final int chunkSize, final int maxInflight) throws InterruptedException { + return ingestBidiCore(rows, opts, chunkSize, maxInflight, /* timeoutMs */ 5 * 60_000L, // or pass a parameter if you prefer + (Object o) -> toProtoRecordFromDbRecord((com.arcadedb.database.Record) o)); + } + + /** + * Pushes map-shaped rows (property map per row) via InsertBidirectional with + * per-batch ACKs. + * + *

+ * Behavior is identical to {@link #ingestBidi(List, InsertOptions, int, int)} + * but accepts pre-materialized property maps. This is useful when ingesting + * from JSON/CSV or an HTTP pipeline. + * + * @param options base InsertOptions (target class, conflict mode, tx mode). + * DB/credentials will be filled if missing + * @param rows list of maps (one per record) to insert/update + * @param chunkSize rows per chunk + * @param maxInflight max unacknowledged chunks allowed + * @param timeoutMs overall client-side timeout + * @return InsertSummary from COMMITTED or aggregated from ACKs + * @throws InterruptedException if the await is interrupted + */ + public com.arcadedb.server.grpc.InsertSummary ingestBidi(final com.arcadedb.server.grpc.InsertOptions options, + final java.util.List> rows, final int chunkSize, final int maxInflight, final long timeoutMs) + throws InterruptedException { + return ingestBidiCore(rows, options, chunkSize, maxInflight, timeoutMs, + (Object o) -> toProtoRecordFromMap((java.util.Map) o)); } // Map -> PROTO Value private com.arcadedb.server.grpc.Record toProtoRecord(Map row) { - com.arcadedb.server.grpc.Record.Builder b = com.arcadedb.server.grpc.Record.newBuilder(); - // Example if your proto uses a map fields/properties - row.forEach((k, v) -> b.putProperties(k, toProtoValue(v))); - return b.build(); + com.arcadedb.server.grpc.Record.Builder b = com.arcadedb.server.grpc.Record.newBuilder(); + // Example if your proto uses a map + // fields/properties + row.forEach((k, v) -> b.putProperties(k, toProtoValue(v))); + return b.build(); } // Map -> PROTO Record private com.arcadedb.server.grpc.Record toProtoRecordFromMap(java.util.Map row) { - com.arcadedb.server.grpc.Record.Builder b = com.arcadedb.server.grpc.Record.newBuilder(); - row.forEach((k, v) -> b.putProperties(k, toProtoValue(v))); - return b.build(); + com.arcadedb.server.grpc.Record.Builder b = com.arcadedb.server.grpc.Record.newBuilder(); + row.forEach((k, v) -> b.putProperties(k, toProtoValue(v))); + return b.build(); } - + // Domain Record (storage) -> PROTO Record - private com.arcadedb.server.grpc.Record toProtoRecordFromDbRecord( - com.arcadedb.database.Record rec) { - - // Promote to a read-only Document; null if this record isn't a document - com.arcadedb.database.Document doc = rec.asDocument(); - if (doc == null) { - return com.arcadedb.server.grpc.Record.getDefaultInstance(); - } - - com.arcadedb.server.grpc.Record.Builder b = com.arcadedb.server.grpc.Record.newBuilder(); - - // Iterate document properties - for (String name : doc.getPropertyNames()) { - // Depending on your ArcadeDB version, use ONE of these: - Object v = doc.get(name); // <-- common in ArcadeDB - // Object v = doc.getProperty(name); // <-- use this if doc.get(...) doesn't exist in your API - b.putProperties(name, toProtoValue(v)); // or putFields(...) if your proto uses "fields" - } - return b.build(); - } - + private com.arcadedb.server.grpc.Record toProtoRecordFromDbRecord(com.arcadedb.database.Record rec) { + + // Promote to a read-only Document; null if this record isn't a document + com.arcadedb.database.Document doc = rec.asDocument(); + if (doc == null) { + return com.arcadedb.server.grpc.Record.getDefaultInstance(); + } + + com.arcadedb.server.grpc.Record.Builder b = com.arcadedb.server.grpc.Record.newBuilder(); + + // Iterate document properties + for (String name : doc.getPropertyNames()) { + // Depending on your ArcadeDB version, use ONE of these: + Object v = doc.get(name); // <-- common in ArcadeDB + // Object v = doc.getProperty(name); // <-- use this if doc.get(...) doesn't + // exist in your API + b.putProperties(name, toProtoValue(v)); // or putFields(...) if your proto uses "fields" + } + return b.build(); + } // google.protobuf.Value -> Java (mirror of your toProtoValue) private static Object fromProtoValue(com.google.protobuf.Value v) { - if (v == null) return null; - switch (v.getKindCase()) { - case NULL_VALUE: return null; - case BOOL_VALUE: return v.getBoolValue(); - case NUMBER_VALUE: return v.getNumberValue(); - case STRING_VALUE: return v.getStringValue(); - case LIST_VALUE: { - java.util.List out = new java.util.ArrayList<>(); - for (com.google.protobuf.Value item : v.getListValue().getValuesList()) { - out.add(fromProtoValue(item)); - } - return out; - } - case STRUCT_VALUE: { - java.util.Map m = new java.util.LinkedHashMap<>(); - v.getStructValue().getFieldsMap().forEach((k,vv) -> m.put(k, fromProtoValue(vv))); - return m; // server-side schema can later coerce EMBEDDED if needed - } - case KIND_NOT_SET: - default: - return null; - } - } - + if (v == null) + return null; + switch (v.getKindCase()) { + case NULL_VALUE: + return null; + case BOOL_VALUE: + return v.getBoolValue(); + case NUMBER_VALUE: + return v.getNumberValue(); + case STRING_VALUE: + return v.getStringValue(); + case LIST_VALUE: { + java.util.List out = new java.util.ArrayList<>(); + for (com.google.protobuf.Value item : v.getListValue().getValuesList()) { + out.add(fromProtoValue(item)); + } + return out; + } + case STRUCT_VALUE: { + java.util.Map m = new java.util.LinkedHashMap<>(); + v.getStructValue().getFieldsMap().forEach((k, vv) -> m.put(k, fromProtoValue(vv))); + return m; // server-side schema can later coerce EMBEDDED if needed + } + case KIND_NOT_SET: + default: + return null; + } + } + // Query Result -> PROTO Record private com.arcadedb.server.grpc.Record toProtoRecordFromResult(Result res) { - com.arcadedb.server.grpc.Record.Builder b = com.arcadedb.server.grpc.Record.newBuilder(); - for (String name : res.getPropertyNames()) { - Object v = res.getProperty(name); - b.putProperties(name, toProtoValue(v)); // or putFields(...) - } - return b.build(); - } + com.arcadedb.server.grpc.Record.Builder b = com.arcadedb.server.grpc.Record.newBuilder(); + for (String name : res.getPropertyNames()) { + Object v = res.getProperty(name); + b.putProperties(name, toProtoValue(v)); // or putFields(...) + } + return b.build(); + } // Convert Java object -> protobuf Value (extend as needed) private com.google.protobuf.Value toProtoValue(Object v) { - com.google.protobuf.Value.Builder vb = com.google.protobuf.Value.newBuilder(); - if (v == null) return vb.setNullValue(com.google.protobuf.NullValue.NULL_VALUE).build(); - if (v instanceof String s) return vb.setStringValue(s).build(); - if (v instanceof Boolean b) return vb.setBoolValue(b).build(); - if (v instanceof Number n) return vb.setNumberValue(n.doubleValue()).build(); - if (v instanceof java.time.Instant t) - return vb.setStringValue(t.toString()).build(); // or a Struct for Timestamp if you prefer - if (v instanceof java.util.Map m) { - var sb = com.google.protobuf.Struct.newBuilder(); - for (var e : m.entrySet()) - sb.putFields(String.valueOf(e.getKey()), toProtoValue(e.getValue())); - return vb.setStructValue(sb.build()).build(); - } - if (v instanceof java.util.List list) { - var lb = com.google.protobuf.ListValue.newBuilder(); - for (Object e : list) lb.addValues(toProtoValue(e)); - return vb.setListValue(lb.build()).build(); - } - return vb.setStringValue(String.valueOf(v)).build(); - } - + com.google.protobuf.Value.Builder vb = com.google.protobuf.Value.newBuilder(); + if (v == null) + return vb.setNullValue(com.google.protobuf.NullValue.NULL_VALUE).build(); + if (v instanceof String s) + return vb.setStringValue(s).build(); + if (v instanceof Boolean b) + return vb.setBoolValue(b).build(); + if (v instanceof Number n) + return vb.setNumberValue(n.doubleValue()).build(); + if (v instanceof java.time.Instant t) + return vb.setStringValue(t.toString()).build(); // or a Struct for Timestamp if you prefer + if (v instanceof java.util.Map m) { + var sb = com.google.protobuf.Struct.newBuilder(); + for (var e : m.entrySet()) + sb.putFields(String.valueOf(e.getKey()), toProtoValue(e.getValue())); + return vb.setStructValue(sb.build()).build(); + } + if (v instanceof java.util.List list) { + var lb = com.google.protobuf.ListValue.newBuilder(); + for (Object e : list) + lb.addValues(toProtoValue(e)); + return vb.setListValue(lb.build()).build(); + } + return vb.setStringValue(String.valueOf(v)).build(); + } + // Override HTTP-specific methods to prevent usage // @Override // HttpRequest.Builder createRequestBuilder(String httpMethod, String url) { @@ -1591,27 +1630,27 @@ private TransactionIsolation mapIsolationLevel(Database.TRANSACTION_ISOLATION_LE return TransactionIsolation.READ_COMMITTED; } } - + // ---- TX helpers ------------------------------------------------------------- private static com.arcadedb.server.grpc.TransactionContext txBeginCommit() { - return com.arcadedb.server.grpc.TransactionContext.newBuilder() - .setBegin(true).setCommit(true).build(); + return com.arcadedb.server.grpc.TransactionContext.newBuilder().setBegin(true).setCommit(true).build(); } + private static com.arcadedb.server.grpc.TransactionContext txNone() { - return com.arcadedb.server.grpc.TransactionContext.getDefaultInstance(); + return com.arcadedb.server.grpc.TransactionContext.getDefaultInstance(); } // Optional: language defaulting (server defaults to "sql" too) private static String langOrDefault(String language) { - return (language == null || language.isEmpty()) ? "sql" : language; - } + return (language == null || language.isEmpty()) ? "sql" : language; + } private Iterator streamQuery(final String query) { + StreamQueryRequest request = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(query).setCredentials(buildCredentials()) .setBatchSize(100).build(); - Iterator responseIterator = blockingStub - .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + final BlockingClientCall blockingClientCall = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) .streamQuery(request); return new Iterator() { @@ -1619,14 +1658,28 @@ private Iterator streamQuery(final String query) { @Override public boolean hasNext() { + if (currentBatch.hasNext()) { return true; } - if (responseIterator.hasNext()) { - QueryResult result = responseIterator.next(); - currentBatch = result.getRecordsList().iterator(); - return currentBatch.hasNext(); + + try { + + if (blockingClientCall.hasNext()) { + QueryResult result = blockingClientCall.read(); + currentBatch = result.getRecordsList().iterator(); + return currentBatch.hasNext(); + } + } + catch (InterruptedException e) { + + throw new RuntimeException("streamQuery: Interrupted", e); + } + catch (StatusException e) { + + throw new RuntimeException("streamQuery: error occured", e); } + return false; } @@ -1800,13 +1853,33 @@ private void handleGrpcException(StatusRuntimeException e) { } } + private void handleGrpcException(StatusException e) { + Status status = e.getStatus(); + switch (status.getCode()) { + case NOT_FOUND: + throw new RecordNotFoundException(status.getDescription(), null); + case ALREADY_EXISTS: + throw new DuplicatedKeyException("", "", null); + case ABORTED: + throw new ConcurrentModificationException(status.getDescription()); + case DEADLINE_EXCEEDED: + throw new TimeoutException(status.getDescription()); + case PERMISSION_DENIED: + throw new SecurityException(status.getDescription()); + case UNAVAILABLE: + throw new NeedRetryException(status.getDescription()); + default: + throw new RemoteException("gRPC error: " + status.getDescription(), e); + } + } + /** * Extended version with compression support */ public static class RemoteGrpcDatabaseWithCompression extends RemoteGrpcDatabase { - public RemoteGrpcDatabaseWithCompression(String server, int grpcPort, int httpPort, String databaseName, String userName, String userPassword, - ContextConfiguration configuration) { + public RemoteGrpcDatabaseWithCompression(String server, int grpcPort, int httpPort, String databaseName, String userName, + String userPassword, ContextConfiguration configuration) { super(server, grpcPort, httpPort, databaseName, userName, userPassword, configuration); } @@ -1814,8 +1887,7 @@ public RemoteGrpcDatabaseWithCompression(String server, int grpcPort, int httpPo protected ManagedChannel createChannel(String server, int port) { return ManagedChannelBuilder.forAddress(server, port).usePlaintext() // No TLS/SSL - .compressorRegistry(CompressorRegistry.getDefaultInstance()) - .decompressorRegistry(DecompressorRegistry.getDefaultInstance()) + .compressorRegistry(CompressorRegistry.getDefaultInstance()).decompressorRegistry(DecompressorRegistry.getDefaultInstance()) .maxInboundMessageSize(100 * 1024 * 1024) // 100MB max message size .keepAliveTime(30, TimeUnit.SECONDS) // Keep-alive configuration .keepAliveTimeout(10, TimeUnit.SECONDS).keepAliveWithoutCalls(true).build(); diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseWithCompression.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseWithCompression.java index bae6c1d49b..220d0c637e 100644 --- a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseWithCompression.java +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseWithCompression.java @@ -28,8 +28,8 @@ protected ManagedChannel createChannel(String server, int port) { } @Override - protected ArcadeDbServiceGrpc.ArcadeDbServiceBlockingStub createBlockingStub(ManagedChannel channel) { - return ArcadeDbServiceGrpc.newBlockingStub(channel).withCompression("gzip") // Enable GZIP compression + protected ArcadeDbServiceGrpc.ArcadeDbServiceBlockingV2Stub createBlockingStub(ManagedChannel channel) { + return ArcadeDbServiceGrpc.newBlockingV2Stub(channel).withCompression("gzip") // Enable GZIP compression .withCallCredentials(createCredentials()).withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS); } diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcServer.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcServer.java index 1e21669dcd..f5093dea7c 100644 --- a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcServer.java +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcServer.java @@ -1,6 +1,11 @@ package com.arcadedb.remote.grpc; -import com.arcadedb.server.grpc.ArcadeDbServiceGrpc; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; + +import com.arcadedb.server.grpc.ArcadeDbAdminServiceGrpc; import com.arcadedb.server.grpc.CreateDatabaseRequest; import com.arcadedb.server.grpc.DatabaseCredentials; import com.arcadedb.server.grpc.DropDatabaseRequest; @@ -11,15 +16,9 @@ import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import io.grpc.Metadata; -import io.grpc.CallCredentials.MetadataApplier; -import io.grpc.CallCredentials.RequestInfo; +import io.grpc.StatusException; import io.grpc.stub.AbstractStub; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.Executor; -import java.util.concurrent.TimeUnit; - /** * Minimal server-scope gRPC wrapper (HTTP RemoteServer equivalent), implemented * ONLY with RPCs present in your current proto. @@ -42,7 +41,7 @@ public class RemoteGrpcServer implements AutoCloseable { private final long defaultTimeoutMs; private final ManagedChannel channel; - private final ArcadeDbServiceGrpc.ArcadeDbServiceBlockingStub blocking; + private final ArcadeDbAdminServiceGrpc.ArcadeDbAdminServiceBlockingV2Stub blockingV2Stub; public RemoteGrpcServer(final String host, final int port, final String user, final String pass) { this(host, port, user, pass, 30_000); @@ -63,7 +62,7 @@ public RemoteGrpcServer(final String host, final int port, final String user, fi this.channel = ManagedChannelBuilder.forAddress(this.host, this.port).usePlaintext() // switch to TLS if enabled server-side .build(); - this.blocking = ArcadeDbServiceGrpc.newBlockingStub(channel).withCallCredentials(createCallCredentials(userName, userPassword)); + this.blockingV2Stub = ArcadeDbAdminServiceGrpc.newBlockingV2Stub(channel).withCallCredentials(createCallCredentials(userName, userPassword)); } @Override @@ -89,10 +88,18 @@ public DatabaseCredentials buildCredentials() { /** Returns the list of database names from the server. */ public List listDatabases() { - ListDatabasesResponse resp = withDeadline(blocking, defaultTimeoutMs) - .listDatabases(ListDatabasesRequest.newBuilder().setCredentials(buildCredentials()).build()); + ListDatabasesResponse resp = null; - return resp.getDatabasesList().stream().map(dbInfo -> dbInfo.getName()).toList(); + try { + + resp = withDeadline(blockingV2Stub, defaultTimeoutMs) + .listDatabases(ListDatabasesRequest.newBuilder().setCredentials(buildCredentials()).build()); + return resp.getDatabasesList(); + } + catch (StatusException e) { + + throw new RuntimeException("Failed to list databases: " + e.getMessage(), e); + } } /** Checks existence by listing (no ExistsDatabaseRequest needed). */ @@ -102,9 +109,15 @@ public boolean existsDatabase(final String database) { /** Creates a database with type "graph" or "document". */ public void createDatabase(final String database) { - withDeadline(blocking, defaultTimeoutMs).createDatabase(CreateDatabaseRequest.newBuilder() - .setDatabaseName(database) - .setCredentials(buildCredentials()).build()); + + try { + withDeadline(blockingV2Stub, defaultTimeoutMs).createDatabase(CreateDatabaseRequest.newBuilder() + .setName(database) + .setCredentials(buildCredentials()).build()); + } + catch (StatusException e) { + throw new RuntimeException("Failed to create database: " + e.getMessage(), e); + } } /** No-op if already present; creates otherwise. */ @@ -116,8 +129,14 @@ public void createDatabaseIfMissing(final String database) { /** Drops a database. If your proto supports 'force', add it here. */ public void dropDatabase(final String database) { - withDeadline(blocking, defaultTimeoutMs) - .dropDatabase(DropDatabaseRequest.newBuilder().setDatabaseName(database).setCredentials(buildCredentials()).build()); + + try { + withDeadline(blockingV2Stub, defaultTimeoutMs) + .dropDatabase(DropDatabaseRequest.newBuilder().setName(database).setCredentials(buildCredentials()).build()); + } + catch (StatusException e) { + throw new RuntimeException("Failed to drop database: " + e.getMessage(), e); + } } public String endpoint() { diff --git a/grpc-client/src/test/java/com/arcadedb/remote/grpc/ArcadeDbHTTPTvsGRPCBench.java b/grpc-client/src/test/java/com/arcadedb/remote/grpc/ArcadeDbHTTPTvsGRPCBench.java index 9435766316..f0f767b6e7 100644 --- a/grpc-client/src/test/java/com/arcadedb/remote/grpc/ArcadeDbHTTPTvsGRPCBench.java +++ b/grpc-client/src/test/java/com/arcadedb/remote/grpc/ArcadeDbHTTPTvsGRPCBench.java @@ -14,6 +14,8 @@ import com.arcadedb.remote.RemoteDatabase; // HTTP client import com.arcadedb.server.grpc.InsertOptions; +import com.arcadedb.server.grpc.InsertOptions.ConflictMode; +import com.arcadedb.server.grpc.InsertOptions.TransactionMode; import com.arcadedb.server.grpc.InsertSummary; import com.arcadedb.server.grpc.StreamQueryRequest; import com.fasterxml.jackson.databind.ObjectMapper; @@ -287,9 +289,9 @@ private static InsertOptions defaultInsertOptions(RemoteGrpcDatabase grpc, Strin // Minimal options: target class + key columns for idempotency; update // last-write-wins return InsertOptions.newBuilder().setDatabase(DB_NAME).setTargetClass(targetClass).addAllKeyColumns(keyCols) - .setConflictMode(com.arcadedb.server.grpc.ConflictMode.CONFLICT_UPDATE) + .setConflictMode(ConflictMode.CONFLICT_UPDATE) .addAllUpdateColumnsOnConflict(Arrays.asList("feedback", "image", "timestamp", "applicationArea", "empowerType")) - .setTransactionMode(com.arcadedb.server.grpc.TransactionMode.PER_BATCH).setServerBatchSize(500) + .setTransactionMode(TransactionMode.PER_BATCH).setServerBatchSize(500) .setCredentials(grpc.buildCredentials()) // if your server validates per-request creds .build(); } diff --git a/grpc/src/main/proto/arcadedb-server.proto b/grpc/src/main/proto/arcadedb-server.proto index 01e08fb795..0a013a85b2 100644 --- a/grpc/src/main/proto/arcadedb-server.proto +++ b/grpc/src/main/proto/arcadedb-server.proto @@ -10,193 +10,95 @@ import "google/protobuf/empty.proto"; import "google/protobuf/struct.proto"; import "google/protobuf/timestamp.proto"; -// Main ArcadeDB Service -service ArcadeDbService { - // Database operations - rpc CreateDatabase(CreateDatabaseRequest) returns (CreateDatabaseResponse); - rpc DropDatabase(DropDatabaseRequest) returns (DropDatabaseResponse); - rpc ListDatabases(ListDatabasesRequest) returns (ListDatabasesResponse); - rpc GetDatabaseInfo(GetDatabaseInfoRequest) returns (GetDatabaseInfoResponse); - - // Query operations - rpc ExecuteQuery(ExecuteQueryRequest) returns (ExecuteQueryResponse); - rpc ExecuteCommand(ExecuteCommandRequest) returns (ExecuteCommandResponse); - - // Transaction operations - rpc BeginTransaction(BeginTransactionRequest) returns (BeginTransactionResponse); - rpc CommitTransaction(CommitTransactionRequest) returns (CommitTransactionResponse); - rpc RollbackTransaction(RollbackTransactionRequest) returns (RollbackTransactionResponse); - - // Record operations - rpc CreateRecord(CreateRecordRequest) returns (CreateRecordResponse); - rpc GetRecord(GetRecordRequest) returns (GetRecordResponse); - rpc UpdateRecord(UpdateRecordRequest) returns (UpdateRecordResponse); - rpc DeleteRecord(DeleteRecordRequest) returns (DeleteRecordResponse); - - // Streaming operations - rpc StreamQuery(StreamQueryRequest) returns (stream QueryResult); - - rpc BulkInsert (BulkInsertRequest) returns (InsertSummary); - rpc InsertStream (stream InsertChunk) returns (InsertSummary); - rpc InsertBidirectional (stream InsertRequest) returns (stream InsertResponse); -} - -service ArcadeDbAdminService { - - rpc Ping(PingRequest) returns (PingResponse); - rpc GetServerInfo(GetServerInfoRequest) returns (GetServerInfoResponse); - rpc GetServerStatus(google.protobuf.Empty) returns (ServerStatusResponse); - - rpc ListDatabases(ListDatabasesRequest) returns (ListDatabasesResponse); - rpc ExistsDatabase(ExistsDatabaseRequest) returns (ExistsDatabaseResponse); - rpc CreateDatabase(CreateDatabaseRequest) returns (CreateDatabaseResponse); - rpc DropDatabase(DropDatabaseRequest) returns (DropDatabaseResponse); - - rpc GetDatabaseInfo(GetDatabaseInfoRequest) returns (GetDatabaseInfoResponse); +// ----------------------------------------------------------------------------- +// Common types +// ----------------------------------------------------------------------------- - // Optional user management - rpc CreateUser(CreateUserRequest) returns (CreateUserResponse); - rpc DeleteUser(DeleteUserRequest) returns (DeleteUserResponse); -} - -// Common messages message DatabaseCredentials { string username = 1; string password = 2; } message TransactionContext { - string transaction_id = 1; // optional: reuse existing tx or name a new one - string database = 2; // optional if request already carries database - - // NEW (optional) — all default to false - bool begin = 3; // start a tx if not already active for this transaction_id - bool commit = 4; // commit the referenced tx after the RPC work - bool rollback = 5; // rollback the referenced tx after the RPC work - - // Optional QoS (you can add later if you need) - int64 timeout_ms = 6; // server may enforce per-tx timeout - bool read_only = 7; // hint for engines that support RO tx - // enum TxIsolation { DEFAULT=0; READ_COMMITTED=1; REPEATABLE_READ=2; SERIALIZABLE=3; } - // TxIsolation isolation = 8; -} - -// Database operations -message CreateDatabaseRequest { - string database_name = 1; - string database_type = 2; - DatabaseCredentials credentials = 3; - map options = 4; + string transaction_id = 1; // optional correlation + string database = 2; // optional if request already has database + bool begin = 3; // start a tx if not already active + bool commit = 4; // commit at end of call/stream + bool rollback = 5; // rollback at end of call/stream + int64 timeout_ms = 6; // optional per-tx timeout + bool read_only = 7; // optional hint } -message CreateDatabaseResponse { - bool success = 1; - string message = 2; - string database_id = 3; -} - -message DropDatabaseRequest { - string database_name = 1; - DatabaseCredentials credentials = 2; -} - -message DropDatabaseResponse { - bool success = 1; - string message = 2; -} - -message ListDatabasesRequest { - DatabaseCredentials credentials = 1; -} - -message ListDatabasesResponse { - repeated DatabaseInfo databases = 1; -} - -message DatabaseInfo { - string name = 1; - string type = 2; // "graph" / "document" - int64 records = 3; // total records (approx or exact based on engine) - int32 classes = 4; - int32 indexes = 5; - int64 size_bytes = 6; // if available - map properties = 7; // extra info, free-form +message RowError { + uint64 row_index = 1; string code = 2; string message = 3; string field = 4; } -// Database-level info summary -message GetDatabaseInfoRequest { - string database_name = 1; - DatabaseCredentials credentials = 2; +// Generic record payload (document/vertex/edge) +message Record { + string rid = 1; + string type = 2; // optional logical type/class name + map properties = 3; } -message GetDatabaseInfoResponse { - DatabaseInfo info = 1; -} +// ----------------------------------------------------------------------------- +// Query (streaming) API +// ----------------------------------------------------------------------------- -// Query operations -message ExecuteQueryRequest { +message StreamQueryRequest { string database = 1; string query = 2; map parameters = 3; DatabaseCredentials credentials = 4; - TransactionContext transaction = 5; - int32 limit = 6; - int32 timeout_ms = 7; -} + int32 batch_size = 5; -message ExecuteQueryResponse { - repeated QueryResult results = 1; - int64 execution_time_ms = 2; - string query_plan = 3; -} + enum RetrievalMode { + CURSOR = 0; // run once, stream as you iterate + MATERIALIZE_ALL = 1; // load all then emit in batches + PAGED = 2; // re-issue with LIMIT/SKIP per batch + } + RetrievalMode retrieval_mode = 6; -// --- REQUEST --- -message ExecuteCommandRequest { - // existing fields — DO NOT renumber - string database = 1; - string command = 2; - map parameters = 3; - DatabaseCredentials credentials = 4; - TransactionContext transaction = 5; + string language = 7; // e.g., "sql" (default if empty) + TransactionContext transaction = 8; // optional +} - // NEW (optional): language of the command (defaults to "sql" on server if empty) - string language = 6; +message QueryResult { + repeated Record records = 1; + int32 total_records_in_batch = 2; + int64 running_total_emitted = 3; + bool is_last_batch = 4; +} - // NEW (optional): if true, server may stream/collect rows produced by the command and - // include them in ExecuteCommandResponse.records (subject to max_rows). - bool return_rows = 7; +service ArcadeDbService { - // NEW (optional): hard limit on rows returned when return_rows=true. 0 = server default. - int32 max_rows = 8; -} + // Streaming queries + rpc StreamQuery (StreamQueryRequest) returns (stream QueryResult); -// --- RESPONSE --- -message ExecuteCommandResponse { - // existing - bool success = 1; - string message = 2; - int64 affected_records = 3; - int64 execution_time_ms = 4; + // Execute arbitrary command (INSERT/UPDATE/DDL/etc.) + rpc ExecuteCommand (ExecuteCommandRequest) returns (ExecuteCommandResponse); + rpc ExecuteQuery(ExecuteQueryRequest) returns (ExecuteQueryResponse); - // NEW (optional): result rows if return_rows=true and the command produced rows. - // Use your existing Record message type from the service. - repeated Record records = 5; -} + // Single-record helpers + rpc CreateRecord (CreateRecordRequest) returns (CreateRecordResponse); + rpc UpdateRecord (UpdateRecordRequest) returns (UpdateRecordResponse); + rpc LookupByRid (LookupByRidRequest) returns (LookupByRidResponse); + rpc DeleteRecord(DeleteRecordRequest) returns (DeleteRecordResponse); -message QueryResult { - repeated Record records = 1; - int32 total_records_in_batch = 2; // renamed for clarity - int64 running_total_emitted = 3; // optional; cumulative count so far - bool is_last_batch = 4; -} + // Insert APIs (unary, client-streaming, and bidi) + rpc BulkInsert (BulkInsertRequest) returns (InsertSummary); + rpc InsertStream (stream InsertChunk) returns (InsertSummary); + rpc InsertBidirectional (stream InsertRequest) returns (stream InsertResponse); -message ColumnMetadata { - string name = 1; - string type = 2; - bool nullable = 3; + // Transaction operations + rpc BeginTransaction(BeginTransactionRequest) returns (BeginTransactionResponse); + rpc CommitTransaction(CommitTransactionRequest) returns (CommitTransactionResponse); + rpc RollbackTransaction(RollbackTransactionRequest) returns (RollbackTransactionResponse); } +// ----------------------------------------------------------------------------- // Transaction operations +// ----------------------------------------------------------------------------- + message BeginTransactionRequest { string database = 1; DatabaseCredentials credentials = 2; @@ -236,228 +138,367 @@ message RollbackTransactionResponse { string message = 2; } -// Record operations -message Record { - string rid = 1; - string type = 2; - map properties = 3; - int32 version = 4; +// ----------------------------------------------------------------------------- +// Delete Record +// ----------------------------------------------------------------------------- + +message DeleteRecordRequest { + string database = 1; + string rid = 2; + DatabaseCredentials credentials = 3; + TransactionContext transaction = 4; // optional begin/commit/rollback flags } -message CreateRecordRequest { +message DeleteRecordResponse { + bool success = 1; // call succeeded + bool deleted = 2; // record actually removed + string message = 3; // optional info ("not found", error text if success=false, etc.) +} + +// ----------------------------------------------------------------------------- +// Execute Command +// ----------------------------------------------------------------------------- + +message ExecuteCommandRequest { string database = 1; - string type = 2; - map properties = 3; + string command = 2; + map parameters = 3; DatabaseCredentials credentials = 4; - TransactionContext transaction = 5; + TransactionContext transaction = 5; + + string language = 6; // "sql" if empty + + bool return_rows = 7; // include rows in response + int32 max_rows = 8; // cap returned rows (server may still count all) } -message CreateRecordResponse { - Record record = 1; - bool success = 2; - string message = 3; - string identity = 4; +message ExecuteCommandResponse { + bool success = 1; + string message = 2; + + int64 affected_records = 3; // server-computed (elements + numeric scalars) + int64 execution_time_ms = 4; + + repeated Record records = 5; // optional row payload if return_rows=true } -message GetRecordRequest { +// ----------------------------------------------------------------------------- +// Execute Query +// ----------------------------------------------------------------------------- + +message ExecuteQueryRequest { string database = 1; - string rid = 2; - DatabaseCredentials credentials = 3; - TransactionContext transaction = 4; + string query = 2; + map parameters = 3; + DatabaseCredentials credentials = 4; + TransactionContext transaction = 5; + int32 limit = 6; + int32 timeout_ms = 7; } -message GetRecordResponse { - Record record = 1; - bool found = 2; +message ExecuteQueryResponse { + repeated QueryResult results = 1; + int64 execution_time_ms = 2; + string query_plan = 3; } -message UpdateRecordRequest { +// ----------------------------------------------------------------------------- +// Record CRUD helpers +// ----------------------------------------------------------------------------- + +message CreateRecordRequest { string database = 1; - string rid = 2; - map properties = 3; - DatabaseCredentials credentials = 4; + DatabaseCredentials credentials = 2; + string type = 3; // target class/type + Record record = 4; // payload TransactionContext transaction = 5; - int32 expected_version = 6; +} +message CreateRecordResponse { + string rid = 1; } -message UpdateRecordResponse { - Record record = 1; - bool success = 2; - bool updated = 3; - string message = 4; +// Helper wrapper so we can use a map with oneof +message PropertiesUpdate { + map properties = 1; } -message DeleteRecordRequest { +message UpdateRecordRequest { string database = 1; - string rid = 2; - DatabaseCredentials credentials = 3; - TransactionContext transaction = 4; + DatabaseCredentials credentials = 2; + string rid = 3; + + oneof payload { + Record record = 4; // full replacement + PropertiesUpdate partial = 5; // partial update (map wrapped) + } + + TransactionContext transaction = 6; } -message DeleteRecordResponse { +message UpdateRecordResponse { bool success = 1; - bool deleted = 2; - string message = 3; + bool updated = 2; } -// Streaming operations -message StreamQueryRequest { +message LookupByRidRequest { string database = 1; - string query = 2; - map parameters = 3; - DatabaseCredentials credentials = 4; - int32 batch_size = 5; + DatabaseCredentials credentials = 2; + string rid = 3; +} +message LookupByRidResponse { + bool found = 1; + Record record = 2; // present if found +} - // NEW: how the server should retrieve rows under the hood - enum RetrievalMode { - // Current behavior: run once and iterate results, pushing as you go. - CURSOR = 0; +// ----------------------------------------------------------------------------- +// Insert API (shared types) +// ----------------------------------------------------------------------------- - // Load all results into memory first, then emit batches. - MATERIALIZE_ALL = 1; +message InsertError { + string message = 1; + int64 row_index = 2; // index within the chunk (or -1 if not applicable) + string code = 3; // optional machine-usable code + string field = 4; +} - // Re-issue the query per batch using LIMIT/SKIP (or a server-side cursor/offset) - // so only enough rows for the next batch are fetched. - PAGED = 2; +message InsertSummary { + int64 received = 1; + int64 inserted = 2; + int64 updated = 3; + int64 ignored = 4; + int64 failed = 5; + repeated InsertError errors = 6; + int64 execution_time_ms = 7; + google.protobuf.Timestamp started_at = 8; + google.protobuf.Timestamp finished_at = 9; +} + +message InsertOptions { + + string target_class = 1; // required + repeated string key_columns = 2; + + enum ConflictMode { + CONFLICT_ERROR = 0; // default (fail on dup) + CONFLICT_UPDATE = 1; // upsert/update + CONFLICT_IGNORE = 2; // drop dup + CONFLICT_ABORT = 3; } - RetrievalMode retrieval_mode = 6; - // NEW (optional): inline transaction control for streaming - TransactionContext transaction = 7; + ConflictMode conflict_mode = 3; - // Optional (if you want to allow non-SQL later) - string language = 8; // default "sql" on server if empty -} + repeated string update_columns_on_conflict = 4; -enum ConflictMode { CONFLICT_ABORT = 0; CONFLICT_IGNORE = 1; CONFLICT_UPDATE = 2; } -enum TransactionMode { PER_BATCH = 0; PER_STREAM = 1; PER_ROW = 2; } + enum TransactionMode { + PER_REQUEST = 0; // one tx for entire RPC + PER_BATCH = 1; // commit per streamed chunk + PER_STREAM = 2; // commit per streamed chunk + PER_ROW = 3; // one tx for entire RPC + NONE = 4; // caller manages externally + } -message InsertOptions { - string database = 1; - string target_class = 2; // e.g., "Patient" - repeated string key_columns = 3; // for upsert/dedupe - ConflictMode conflict_mode = 4; // default CONFLICT_ABORT - repeated string update_columns_on_conflict = 5; - TransactionMode transaction_mode = 6; // default PER_BATCH - uint32 server_batch_size = 7; // default 1000 - bool validate_only = 8; // dry-run (no commit) + TransactionMode transaction_mode = 5; + + int32 server_batch_size = 6; // hint; 0 => server chooses + + bool validate_only = 7; // dry-run (no commit) + + string database = 8; DatabaseCredentials credentials = 9; } -message RowError { - uint64 row_index = 1; string code = 2; string message = 3; string field = 4; +// ----------------------------------------------------------------------------- +// Insert: Unary Bulk +// ----------------------------------------------------------------------------- + +message BulkInsertRequest { + string database = 1; + DatabaseCredentials credentials = 2; + + InsertOptions options = 3; + repeated Record rows = 4; + + TransactionContext transaction = 5; // optional } +// ----------------------------------------------------------------------------- +// Insert: Client-Streaming +// ----------------------------------------------------------------------------- + message InsertChunk { - string session_id = 1; // optional idempotency key - uint64 chunk_seq = 2; // 1..N - repeated Record rows = 3; // the data + // REQUIRED on the first chunk; optional on subsequent chunks (server caches) + string database = 1; + DatabaseCredentials credentials = 2; + InsertOptions options = 3; + TransactionContext transaction = 4; + + // Streaming envelope + string session_id = 5; // client-generated stable ID + int64 chunk_seq = 6; // 1,2,3,... + + // Payload + repeated Record rows = 7; + + // Optional client hint + bool last = 8; } -message InsertSummary { - uint64 received = 1; uint64 inserted = 2; uint64 updated = 3; - uint64 ignored = 4; uint64 failed = 5; - repeated RowError errors = 6; // truncate server-side if large - google.protobuf.Timestamp started_at = 7; - google.protobuf.Timestamp finished_at = 8; +// ----------------------------------------------------------------------------- +// Insert: Bidirectional +// ----------------------------------------------------------------------------- + +message InsertRequest { + oneof msg { + Start start = 1; + InsertChunk chunk = 2; + Commit commit = 3; + } +} + +message Start { + string database = 1; + DatabaseCredentials credentials = 2; + InsertOptions options = 3; + TransactionContext transaction = 4; + string session_id = 5; // may be empty; server can echo a generated one } -message BulkInsertRequest { - InsertOptions options = 1; repeated Record rows = 2; +message Commit { + string session_id = 1; + bool commit = 2; // if both false, server may default to commit + bool rollback = 3; } -// message BulkInsertRequest { -// string database = 1; -// string type = 2; -// Record record = 3; -// DatabaseCredentials credentials = 4; -// } +message InsertResponse { + oneof msg { + Started started = 1; + BatchAck batch_ack = 2; + Committed committed = 3; + InsertError error = 4; // immediate error if any + } +} + +message Started { + string session_id = 1; +} -// message BulkInsertResponse { -// int64 total_inserted = 1; -// int64 total_failed = 2; -// repeated string errors = 3; -// } +message BatchAck { + string session_id = 1; + int64 chunk_seq = 2; -message StartInsert { InsertOptions options = 1; } -message CommitInsert { string session_id = 1; } + // Tallies for this chunk only + int64 inserted = 3; + int64 updated = 4; + int64 ignored = 5; + int64 failed = 6; -message Started { string session_id = 1; } -message BatchAck { - string session_id = 1; uint64 chunk_seq = 2; - uint64 inserted = 3; uint64 updated = 4; uint64 ignored = 5; uint64 failed = 6; - repeated RowError errors = 7; + repeated InsertError errors = 7; // batch-scoped } -message Committed { InsertSummary summary = 1; } -message InsertRequest { oneof msg { StartInsert start = 1; InsertChunk chunk = 2; CommitInsert commit = 3; } } -message InsertResponse { oneof msg { Started started = 1; BatchAck batch_ack = 2; Committed committed = 3; } } +message Committed { + InsertSummary summary = 1; // full-stream totals +} -message PingRequest { - // Optional: include credentials if your server enforces auth on ping - DatabaseCredentials credentials = 1; +// ----------------------------------------------------------------------------- +// Admin service +// ----------------------------------------------------------------------------- + +service ArcadeDbAdminService { + rpc Ping (PingRequest) returns (PingResponse); + rpc GetServerInfo (GetServerInfoRequest) returns (GetServerInfoResponse); + + rpc ListDatabases (ListDatabasesRequest) returns (ListDatabasesResponse); + rpc ExistsDatabase (ExistsDatabaseRequest) returns (ExistsDatabaseResponse); + rpc CreateDatabase (CreateDatabaseRequest) returns (CreateDatabaseResponse); + rpc DropDatabase (DropDatabaseRequest) returns (DropDatabaseResponse); + + rpc GetDatabaseInfo (GetDatabaseInfoRequest) returns (GetDatabaseInfoResponse); + + rpc CreateUser (CreateUserRequest) returns (CreateUserResponse); + rpc DeleteUser (DeleteUserRequest) returns (DeleteUserResponse); } +message PingRequest { + DatabaseCredentials credentials = 1; // optional if you allow unauth ping +} message PingResponse { bool ok = 1; - int64 server_time_ms = 2; // epoch millis - int64 timestamp = 3; - string message = 4; + int64 server_time_ms = 2; } -// Server operations -message ServerStatusResponse { - string version = 1; - string status = 2; - int64 uptime_ms = 3; - int32 active_connections = 4; - map metrics = 5; +message GetServerInfoRequest { + DatabaseCredentials credentials = 1; +} +message GetServerInfoResponse { + string version = 1; // e.g., "25.8.1-SNAPSHOT" + string edition = 2; // "CE"/"EE" if applicable + int64 start_time_ms = 3; + int64 uptime_ms = 4; + int32 http_port = 5; // -1 if not available + int32 grpc_port = 6; // -1 if not available + int32 binary_port = 7; // -1 if not available + int32 databases_count = 8; + map features = 9; // optional flags } -// Lightweight server info; extend as needed -message GetServerInfoRequest { +message ListDatabasesRequest { DatabaseCredentials credentials = 1; } -message GetServerInfoResponse { - string version = 1; // e.g. "25.8.1-SNAPSHOT" - string edition = 2; // "CE"/"EE" (if applicable) - int64 start_time_ms = 3; // epoch millis - int64 uptime_ms = 4; - int32 http_port = 5; - int32 grpc_port = 6; - int32 binary_port = 7; - int32 databases_count = 8; - map features = 9; // arbitrary flags: {"ssl":"true",...} -} - -// If you don't already have this pair: +message ListDatabasesResponse { + repeated string databases = 1; +} + message ExistsDatabaseRequest { - string database = 1; - DatabaseCredentials credentials = 2; + DatabaseCredentials credentials = 1; + string name = 2; // database name } message ExistsDatabaseResponse { bool exists = 1; } -// Optional user management (server-scope) +message CreateDatabaseRequest { + DatabaseCredentials credentials = 1; + string name = 2; // database name + string type = 3; // "graph" or "document" (logical) +} +message CreateDatabaseResponse {} + +message DropDatabaseRequest { + DatabaseCredentials credentials = 1; + string name = 2; // database name +} +message DropDatabaseResponse {} + +message GetDatabaseInfoRequest { + DatabaseCredentials credentials = 1; + string name = 2; // database name +} +message GetDatabaseInfoResponse { + string database = 1; + string type = 2; // "graph"/"document" + int64 records = 3; + int32 classes = 4; + int32 indexes = 5; + int64 size_bytes = 6; // optional if you can provide + map properties = 7; // extra info +} + message CreateUserRequest { - DatabaseCredentials credentials = 1; // admin credentials - string user = 2; + DatabaseCredentials credentials = 1; // admin creds + string user = 2; string password = 3; - string role = 4; // or permissions string; adjust to your security model + string role = 4; // optional, depending on your model } - message CreateUserResponse { bool success = 1; string message = 2; } message DeleteUserRequest { - DatabaseCredentials credentials = 1; // admin credentials + DatabaseCredentials credentials = 1; // admin creds string user = 2; } - message DeleteUserResponse { bool success = 1; string message = 2; diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcAdminService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcAdminService.java index 3cf79d45d1..c8af198481 100644 --- a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcAdminService.java +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcAdminService.java @@ -98,18 +98,7 @@ public void listDatabases(ListDatabasesRequest req, StreamObserver names = new ArrayList<>(getDatabaseNames()); names.sort(String.CASE_INSENSITIVE_ORDER); - List allDatabaseInfos = new ArrayList<>(); - - for (String dbName : names) { - - //ServerDatabase db = arcadeServer.getDatabase(dbName); - - DatabaseInfo dbInfo = DatabaseInfo.newBuilder().setName(dbName).build(); - - allDatabaseInfos.add(dbInfo); - } - - resp.onNext(ListDatabasesResponse.newBuilder().addAllDatabases(allDatabaseInfos).build()); + resp.onNext(ListDatabasesResponse.newBuilder().addAllDatabases(names).build()); resp.onCompleted(); } catch (SecurityException se) { @@ -127,7 +116,7 @@ public void existsDatabase(ExistsDatabaseRequest req, StreamObserver t instanceof VertexType); if (vIter != null && vIter.iterator().hasNext()) - kind = "graph"; + type = "graph"; } catch (Throwable ignore) { } - DatabaseInfo dbInfo = DatabaseInfo.newBuilder().setName(name).setClasses(classes).setIndexes(indexes).setRecords(records).build(); - - GetDatabaseInfoResponse out = GetDatabaseInfoResponse.newBuilder().setInfo(dbInfo).build(); + GetDatabaseInfoResponse out = GetDatabaseInfoResponse.newBuilder() + .setDatabase(name) + .setClasses(classes).setIndexes(indexes).setRecords(records).setType(type) + .build(); resp.onNext(out); resp.onCompleted(); @@ -280,8 +270,11 @@ public void getDatabaseInfo(GetDatabaseInfoRequest req, StreamObserver resp) { + try { + authenticate(req.getCredentials()); + final String user = req.getUser(); final String pwd = req.getPassword(); final String role = req.getRole(); diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java index a01e4d5c83..e25b53dc71 100644 --- a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java @@ -1,6 +1,5 @@ package com.arcadedb.server.grpc; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -17,7 +16,6 @@ import com.arcadedb.database.MutableDocument; import com.arcadedb.database.RID; import com.arcadedb.engine.ComponentFile; -import com.arcadedb.engine.ComponentFile.MODE; import com.arcadedb.graph.Edge; import com.arcadedb.graph.MutableEdge; import com.arcadedb.graph.MutableVertex; @@ -29,7 +27,8 @@ import com.arcadedb.schema.Schema; import com.arcadedb.schema.VertexType; import com.arcadedb.server.ArcadeDBServer; -import com.arcadedb.server.ServerDatabase; +import com.arcadedb.server.grpc.InsertOptions.ConflictMode; +import com.arcadedb.server.grpc.InsertOptions.TransactionMode; import com.google.protobuf.Timestamp; import com.google.protobuf.Value; @@ -96,118 +95,6 @@ public void close() { activeTransactions.clear(); } - @Override - public void createDatabase(CreateDatabaseRequest request, StreamObserver responseObserver) { - - try { - // Validate credentials if needed - validateCredentials(request.getCredentials()); - - ServerDatabase db = arcadeServer.createDatabase(request.getDatabaseName(), MODE.READ_WRITE); - - CreateDatabaseResponse response = CreateDatabaseResponse.newBuilder().setSuccess(true).setMessage("Database created successfully") - .setDatabaseId(request.getDatabaseName()).build(); - - responseObserver.onNext(response); - responseObserver.onCompleted(); - - } - catch (Exception e) { - logger.error("Error creating database: {}", e.getMessage(), e); - responseObserver.onError(Status.INTERNAL.withDescription("Failed to create database: " + e.getMessage()).asException()); - } - } - - @Override - public void dropDatabase(DropDatabaseRequest req, StreamObserver resp) { - - try { - - final String name = req.getDatabaseName(); - - if (name == null || name.isEmpty()) - throw new IllegalArgumentException("database name is required"); - - try (// Create new database factory for this specific database - DatabaseFactory dbFactory = new DatabaseFactory(databasePath + "/" + req.getDatabaseName())) { - // Create the database - Database database = dbFactory.create(); - database.drop(); - } - - resp.onNext(DropDatabaseResponse.newBuilder().setSuccess(true).build()); - resp.onCompleted(); - } - catch (Exception e) { - resp.onError(Status.INTERNAL.withDescription("DropDatabase: " + e.getMessage()).asException()); - } - } - - @Override - public void listDatabases(ListDatabasesRequest req, StreamObserver resp) { - - try { - - validateCredentials(req.getCredentials()); - - java.util.Collection names = arcadeServer.getDatabaseNames(); - java.util.ArrayList out = new java.util.ArrayList<>(names); - - java.util.Collections.sort(out, String.CASE_INSENSITIVE_ORDER); - - List allDatabaseInfos = new ArrayList<>(); - - for (String dbName : out) { - - //ServerDatabase db = arcadeServer.getDatabase(dbName); - - DatabaseInfo dbInfo = DatabaseInfo.newBuilder().setName(dbName).build(); - - allDatabaseInfos.add(dbInfo); - } - - // 3) Build and send response - ListDatabasesResponse respMsg = ListDatabasesResponse.newBuilder() - .addAllDatabases(allDatabaseInfos) - .build(); - - resp.onNext(respMsg); - resp.onCompleted(); - } - catch (SecurityException se) { - resp.onError(io.grpc.Status.PERMISSION_DENIED - .withDescription("ListDatabases: " + se.getMessage()) - .asException()); - } - catch (Exception e) { - resp.onError(io.grpc.Status.INTERNAL - .withDescription("ListDatabases: " + e.getMessage()) - .asException()); - } - } - - @Override - public void getDatabaseInfo(GetDatabaseInfoRequest req, StreamObserver resp) { - - try { - final String name = req.getDatabaseName(); - Database db = getDatabase(name, req.getCredentials()); - - Schema schema = db.getSchema(); - int typeCount = schema.getTypes().size(); - - DatabaseInfo databaseInfo = DatabaseInfo.newBuilder().setName(name).setClasses(typeCount).build(); - - GetDatabaseInfoResponse.Builder b = GetDatabaseInfoResponse.newBuilder().setInfo(databaseInfo); - - resp.onNext(b.build()); - resp.onCompleted(); - } - catch (Exception e) { - resp.onError(Status.INTERNAL.withDescription("GetDatabaseInfo: " + e.getMessage()).asException()); - } - } - @Override public void executeCommand(ExecuteCommandRequest req, StreamObserver resp) { final long t0 = System.nanoTime(); @@ -332,9 +219,9 @@ else if (beganHere) { @Override public void createRecord(CreateRecordRequest req, StreamObserver resp) { - + try { - + Database db = getDatabase(req.getDatabase(), req.getCredentials()); final String cls = req.getType(); // or req.getTargetClass() if that’s your proto @@ -346,21 +233,21 @@ public void createRecord(CreateRecordRequest req, StreamObserver props = req.getPropertiesMap(); + // All properties from the request (proto map) — nested under "record" + final java.util.Map props = req.getRecord().getPropertiesMap(); // --- Vertex --- - if (dt instanceof VertexType) { + if (dt instanceof com.arcadedb.schema.VertexType) { com.arcadedb.graph.MutableVertex v = db.newVertex(cls); - // apply properties + + // apply properties with proper coercion props.forEach((k, val) -> v.set(k, toJavaForProperty(db, dt, k, val))); + v.save(); - CreateRecordResponse.Builder b = CreateRecordResponse.newBuilder(); - - // Adjust to your actual response field name (e.g., setIdentity / setRidStr) - b.setIdentity(v.getIdentity().toString()); - + // Build response: proto has setRid(String) + CreateRecordResponse.Builder b = CreateRecordResponse.newBuilder().setRid(v.getIdentity().toString()); + resp.onNext(b.build()); resp.onCompleted(); return; @@ -402,10 +289,12 @@ public void createRecord(CreateRecordRequest req, StreamObserver resp) { + public void lookupByRid(LookupByRidRequest req, StreamObserver resp) { try { Database db = getDatabase(req.getDatabase(), req.getCredentials()); - String ridStr = req.getRid(); - if (ridStr == null || ridStr.isEmpty()) + + final String ridStr = req.getRid(); + if (ridStr == null || ridStr.isBlank()) throw new IllegalArgumentException("rid is required"); var el = db.lookupByRID(new RID(ridStr), true); if (el == null) { - resp.onError(Status.NOT_FOUND.withDescription("Record not found: " + ridStr).asException()); + resp.onNext(LookupByRidResponse.newBuilder().setFound(false).build()); + resp.onCompleted(); return; } - resp.onNext(GetRecordResponse.newBuilder().setRecord(convertToGrpcRecord(el.getRecord())).build()); + + resp.onNext(LookupByRidResponse.newBuilder().setFound(true).setRecord(convertToGrpcRecord(el.getRecord())).build()); resp.onCompleted(); } catch (Exception e) { - resp.onError(Status.INTERNAL.withDescription("GetRecord: " + e.getMessage()).asException()); + resp.onError(Status.INTERNAL.withDescription("LookupByRid: " + e.getMessage()).asException()); } } @Override public void updateRecord(UpdateRecordRequest req, StreamObserver resp) { + Database db = null; boolean beganHere = false; @@ -483,7 +377,12 @@ public void updateRecord(UpdateRecordRequest req, StreamObserver mdoc.set(k, toJavaForProperty(dbRef, dtype, k, v))); + + final java.util.Map props = req.hasRecord() ? req.getRecord().getPropertiesMap() + : req.hasPartial() ? req.getPartial().getPropertiesMap() : java.util.Collections.emptyMap(); + + // apply updates + props.forEach((k, v) -> mdoc.set(k, toJavaForProperty(dbRef, dtype, k, v))); mdoc.save(); @@ -518,26 +417,66 @@ else if (beganHere) { @Override public void deleteRecord(DeleteRecordRequest req, StreamObserver resp) { + Database db = null; + boolean beganHere = false; + try { - Database db = getDatabase(req.getDatabase(), req.getCredentials()); - String ridStr = req.getRid(); - if (ridStr == null || ridStr.isEmpty()) + db = getDatabase(req.getDatabase(), req.getCredentials()); + + final String ridStr = req.getRid(); + if (ridStr == null || ridStr.isBlank()) throw new IllegalArgumentException("rid is required"); - var el = db.lookupByRID(new RID(ridStr), true); + // Optional tx begin + if (req.hasTransaction() && req.getTransaction().getBegin()) { + db.begin(); + beganHere = true; + } + var el = db.lookupByRID(new RID(ridStr), true); if (el == null) { - resp.onNext(DeleteRecordResponse.newBuilder().setSuccess(false).setDeleted(false).build()); + // Soft "not found" + resp.onNext( + DeleteRecordResponse.newBuilder().setSuccess(true).setDeleted(false).setMessage("Record not found: " + ridStr).build()); resp.onCompleted(); + // If we began here and no explicit rollback flag, default to commit (or + // rollback—your policy) + if (beganHere && req.hasTransaction() && !req.getTransaction().getRollback()) + db.commit(); return; } - el.delete(); + el.delete(); // deletes document/vertex/edge + + // Optional tx end controls + if (req.hasTransaction()) { + var tx = req.getTransaction(); + if (tx.getCommit()) + db.commit(); + else if (tx.getRollback()) + db.rollback(); + else if (beganHere) + db.commit(); // began here with no explicit end → commit by default + } + else if (beganHere) { + db.commit(); + } + resp.onNext(DeleteRecordResponse.newBuilder().setSuccess(true).setDeleted(true).build()); resp.onCompleted(); + } catch (Exception e) { - resp.onError(Status.INTERNAL.withDescription("DeleteRecord: " + e.getMessage()).asException()); + // Best-effort rollback if we started the tx + try { + if (beganHere && db != null) + db.rollback(); + } + catch (Exception ignore) { + } + + resp.onError( + Status.INTERNAL.withDescription("DeleteRecord: " + (e.getMessage() == null ? e.toString() : e.getMessage())).asException()); } } @@ -678,83 +617,93 @@ public void rollbackTransaction(RollbackTransactionRequest request, StreamObserv @Override public void streamQuery(StreamQueryRequest request, StreamObserver responseObserver) { - + final ServerCallStreamObserver scso = (ServerCallStreamObserver) responseObserver; - final java.util.concurrent.atomic.AtomicBoolean cancelled = new java.util.concurrent.atomic.AtomicBoolean(false); - scso.setOnCancelHandler(() -> cancelled.set(true)); - - Database db = null; - boolean beganHere = false; - - try { - db = getDatabase(request.getDatabase(), request.getCredentials()); - final int batchSize = Math.max(1, request.getBatchSize()); - - // --- TX begin if requested --- - final boolean hasTx = request.hasTransaction(); - final var tx = hasTx ? request.getTransaction() : null; - if (hasTx && tx.getBegin()) { - db.begin(); - beganHere = true; - } - - // --- Dispatch on mode (helpers do NOT manage transactions) --- - switch (request.getRetrievalMode()) { - case MATERIALIZE_ALL -> streamMaterialized(db, request, batchSize, scso, cancelled); - case PAGED -> streamPaged(db, request, batchSize, scso, cancelled); - case CURSOR -> streamCursor(db, request, batchSize, scso, cancelled); - default -> streamCursor(db, request, batchSize, scso, cancelled); - } - - // If the client cancelled mid-stream, choose rollback unless caller explicitly asked to commit/rollback. - if (cancelled.get()) { - if (hasTx) { - if (tx.getRollback()) { - db.rollback(); - } else if (tx.getCommit()) { - db.commit(); // caller explicitly wanted commit even if they cancelled - } else if (beganHere) { - db.rollback(); // safe default on cancellation - } - } - return; // don't call onCompleted() - } - - // --- TX end (normal path) — precedence: rollback > commit > begin-only ⇒ commit --- - if (hasTx) { - if (tx.getRollback()) { - db.rollback(); - } else if (tx.getCommit()) { - db.commit(); - } else if (beganHere) { - db.commit(); - } - } - - scso.onCompleted(); - - } catch (Exception e) { - // Best-effort rollback only if we began here and there wasn't an explicit commit/rollback - try { - if (beganHere && db != null) { - if (request.hasTransaction()) { - var tx = request.getTransaction(); - if (!tx.getCommit() && !tx.getRollback()) db.rollback(); - } else { - db.rollback(); - } - } - } catch (Exception ignore) { /* no-op */ } - - if (!cancelled.get()) { - responseObserver.onError(Status.INTERNAL - .withDescription("Stream query failed: " + e.getMessage()) - .asException()); - } - } + final java.util.concurrent.atomic.AtomicBoolean cancelled = new java.util.concurrent.atomic.AtomicBoolean(false); + scso.setOnCancelHandler(() -> cancelled.set(true)); + + Database db = null; + boolean beganHere = false; + + try { + db = getDatabase(request.getDatabase(), request.getCredentials()); + final int batchSize = Math.max(1, request.getBatchSize()); + + // --- TX begin if requested --- + final boolean hasTx = request.hasTransaction(); + final var tx = hasTx ? request.getTransaction() : null; + if (hasTx && tx.getBegin()) { + db.begin(); + beganHere = true; + } + + // --- Dispatch on mode (helpers do NOT manage transactions) --- + switch (request.getRetrievalMode()) { + case MATERIALIZE_ALL -> streamMaterialized(db, request, batchSize, scso, cancelled); + case PAGED -> streamPaged(db, request, batchSize, scso, cancelled); + case CURSOR -> streamCursor(db, request, batchSize, scso, cancelled); + default -> streamCursor(db, request, batchSize, scso, cancelled); + } + + // If the client cancelled mid-stream, choose rollback unless caller explicitly + // asked to commit/rollback. + if (cancelled.get()) { + if (hasTx) { + if (tx.getRollback()) { + db.rollback(); + } + else if (tx.getCommit()) { + db.commit(); // caller explicitly wanted commit even if they cancelled + } + else if (beganHere) { + db.rollback(); // safe default on cancellation + } + } + return; // don't call onCompleted() + } + + // --- TX end (normal path) — precedence: rollback > commit > begin-only ⇒ + // commit --- + if (hasTx) { + if (tx.getRollback()) { + db.rollback(); + } + else if (tx.getCommit()) { + db.commit(); + } + else if (beganHere) { + db.commit(); + } + } + + scso.onCompleted(); + + } + catch (Exception e) { + // Best-effort rollback only if we began here and there wasn't an explicit + // commit/rollback + try { + if (beganHere && db != null) { + if (request.hasTransaction()) { + var tx = request.getTransaction(); + if (!tx.getCommit() && !tx.getRollback()) + db.rollback(); + } + else { + db.rollback(); + } + } + } + catch (Exception ignore) { + /* no-op */ } + + if (!cancelled.get()) { + responseObserver.onError(Status.INTERNAL.withDescription("Stream query failed: " + e.getMessage()).asException()); + } + } } - + /** * Mode 1 (existing behavior-ish): run once and iterate results, batching as we * go. @@ -942,57 +891,88 @@ public void bulkInsert(BulkInsertRequest req, StreamObserver resp public StreamObserver insertStream(StreamObserver resp) { final ServerCallStreamObserver call = (ServerCallStreamObserver) resp; - // manual pull of inbound request messages (chunks) call.disableAutoInboundFlowControl(); final long startedAt = System.currentTimeMillis(); - // server defaults; if you want options via first chunk, parse sessionId and - // look them up - final InsertOptions opts = defaults(InsertOptions.newBuilder().build()); - - final InsertContext ctx; - try { - ctx = new InsertContext(opts); // begins tx for PER_STREAM/PER_BATCH if needed - } - catch (Exception e) { - resp.onError(Status.FAILED_PRECONDITION.withDescription(e.getMessage()).asException()); - // return a no-op observer - return new StreamObserver<>() { - public void onNext(InsertChunk c) { - } + final java.util.concurrent.atomic.AtomicBoolean cancelled = new java.util.concurrent.atomic.AtomicBoolean(false); + final java.util.concurrent.atomic.AtomicReference ctxRef = new java.util.concurrent.atomic.AtomicReference<>(); - public void onError(Throwable t) { - } + final Counts totals = new Counts(); - public void onCompleted() { - } - }; - } + // cache the first-chunk effective options to validate consistency + final java.util.concurrent.atomic.AtomicReference firstOptsRef = new java.util.concurrent.atomic.AtomicReference<>(); - // accumulate totals across chunks - final Counts totals = new Counts(); - final java.util.concurrent.atomic.AtomicBoolean cancelled = new java.util.concurrent.atomic.AtomicBoolean(false); call.setOnCancelHandler(() -> cancelled.set(true)); - // start pulling the first inbound message + // Pull the very first inbound message call.request(1); return new StreamObserver<>() { + @Override public void onNext(InsertChunk c) { - if (cancelled.get()) + + if (cancelled.get()) { return; + } + try { - Counts cts = insertRows(ctx, c.getRowsList().iterator()); // returns per-chunk counts + + InsertContext ctx = ctxRef.get(); + + if (ctx == null) { + + // -------- First chunk: derive effective options and build context -------- + + // Options may be provided on the first chunk; otherwise use server defaults + InsertOptions sent = c.hasOptions() ? c.getOptions() : InsertOptions.getDefaultInstance(); + + InsertOptions effective = defaults(sent); // your existing default-merging helper + + // (Optional) If your chunk carries db/creds (recommended), you can validate/set + // them here + + // Otherwise InsertContext will resolve them via 'effective' or server defaults + // Example if your proto carries these on the chunk: + // String dbName = c.getDatabase(); + // DatabaseCredentials creds = c.getCredentials(); + // and pass them to the InsertContext ctor if it expects them. + + // Create and cache context + ctx = new InsertContext(effective); // begins tx if PER_STREAM / PER_BATCH per your logic + + ctxRef.set(ctx); + + firstOptsRef.set(effective); + + if (logger.isDebugEnabled()) + logger.debug("insertStream: initialized context with options:\n{}", effective); + } + else { + + // -------- Subsequent chunks: optionally validate option consistency -------- + if (c.hasOptions()) { + InsertOptions prev = firstOptsRef.get(); + InsertOptions cur = defaults(c.getOptions()); + // Minimal consistency check (customize as you wish) + if (!sameInsertContract(prev, cur)) { + // contract changed mid-stream -> fail fast + throw new IllegalArgumentException("insertStream: options changed after first chunk"); + } + } + } + + // -------- Insert rows for this chunk -------- + Counts cts = insertRows(ctxRef.get(), c.getRowsList().iterator()); totals.add(cts); } catch (Exception e) { - // surface as a failed row group; you can also fail the whole call if you prefer - totals.err(Math.max(0, ctx.received - 1), "DB_ERROR", e.getMessage(), ""); + // Register as an error on totals; keep stream alive unless you prefer to fail + // the call + totals.err(Math.max(0, (ctxRef.get() != null ? ctxRef.get().received : 0) - 1), "DB_ERROR", e.getMessage(), ""); } finally { - // ask gRPC to deliver the next chunk if (!cancelled.get()) call.request(1); } @@ -1000,16 +980,27 @@ public void onNext(InsertChunk c) { @Override public void onError(Throwable t) { - // client cancelled or errored mid-stream - ctx.closeQuietly(); + InsertContext ctx = ctxRef.get(); + if (ctx != null) + ctx.closeQuietly(); } @Override public void onCompleted() { try { - ctx.flushCommit(true); // commit if not validate_only + + InsertContext ctx = ctxRef.get(); + if (ctx == null) { + // Client closed without sending a first chunk → nothing to do + resp.onNext(InsertSummary.newBuilder().setReceived(0).setInserted(0).setUpdated(0).setIgnored(0).setFailed(0) + .setExecutionTimeMs(System.currentTimeMillis() - startedAt).build()); + + resp.onCompleted(); + return; + } + + ctx.flushCommit(true); // commit if not validate-only if (!cancelled.get()) { - // build summary FROM totals + startedAt resp.onNext(ctx.summary(totals, startedAt)); resp.onCompleted(); } @@ -1018,113 +1009,163 @@ public void onCompleted() { resp.onError(Status.INTERNAL.withDescription("insertStream: " + e.getMessage()).asException()); } finally { - ctx.closeQuietly(); + InsertContext ctx = ctxRef.get(); + if (ctx != null) + ctx.closeQuietly(); } } }; } - // --- 3) Bi-di with per-batch ACKs --- + /** Minimal consistency check: same "contract" for the stream. */ + private boolean sameInsertContract(InsertOptions a, InsertOptions b) { + if (a == null || b == null) + return a == b; + // lock down the knobs that must not change mid-stream + if (a.getTargetClass() != null && !a.getTargetClass().equals(b.getTargetClass())) + return false; + if (a.getConflictMode() != b.getConflictMode()) + return false; + if (a.getTransactionMode() != b.getTransactionMode()) + return false; + // You can also compare key/update columns if you require strict equality: + if (!a.getKeyColumnsList().equals(b.getKeyColumnsList())) + return false; + if (!a.getUpdateColumnsOnConflictList().equals(b.getUpdateColumnsOnConflictList())) + return false; + return true; + } + @Override public StreamObserver insertBidirectional(StreamObserver resp) { final ServerCallStreamObserver call = (ServerCallStreamObserver) resp; call.disableAutoInboundFlowControl(); - // track one active session InsertContext final java.util.concurrent.atomic.AtomicReference ref = new java.util.concurrent.atomic.AtomicReference<>(); final java.util.concurrent.atomic.AtomicBoolean cancelled = new java.util.concurrent.atomic.AtomicBoolean(false); + final java.util.concurrent.atomic.AtomicBoolean started = new java.util.concurrent.atomic.AtomicBoolean(false); call.setOnCancelHandler(() -> { cancelled.set(true); - InsertContext ctx = ref.get(); + final InsertContext ctx = ref.getAndSet(null); if (ctx != null) { sessionWatermark.remove(ctx.sessionId); ctx.closeQuietly(); } }); - // start pulling inbound messages call.request(1); return new StreamObserver<>() { @Override - public void onNext(InsertRequest req) { + public void onNext(InsertRequest reqMsg) { if (cancelled.get()) return; - switch (req.getMsgCase()) { - case START -> { - // Build context with defaults+client opts; begin tx as needed - InsertOptions opts = defaults(req.getStart().getOptions()); - InsertContext ctx = new InsertContext(opts); - // init per-session metadata - ctx.startedAt = System.currentTimeMillis(); - ctx.totals = new Counts(); + try { + switch (reqMsg.getMsgCase()) { - ref.set(ctx); - sessionWatermark.put(ctx.sessionId, 0L); + case START -> { + if (!started.compareAndSet(false, true)) { + resp.onError( + Status.FAILED_PRECONDITION.withDescription("insertBidirectional: START already received").asException()); + return; + } - resp.onNext(InsertResponse.newBuilder().setStarted(Started.newBuilder().setSessionId(ctx.sessionId)).build()); + final InsertOptions opts = defaults(reqMsg.getStart().getOptions()); + final InsertContext ctx = new InsertContext(opts); + ctx.startedAt = System.currentTimeMillis(); + ctx.totals = new Counts(); - call.request(1); - } + ref.set(ctx); + sessionWatermark.put(ctx.sessionId, 0L); - case CHUNK -> { - InsertContext ctx = require(ref.get(), "session not started"); - InsertChunk c = req.getChunk(); + resp.onNext(InsertResponse.newBuilder().setStarted(Started.newBuilder().setSessionId(ctx.sessionId).build()).build()); - // Idempotent replay guard - long hi = sessionWatermark.getOrDefault(ctx.sessionId, 0L); - if (c.getChunkSeq() <= hi) { - resp.onNext(InsertResponse.newBuilder() - .setBatchAck(BatchAck.newBuilder().setSessionId(ctx.sessionId).setChunkSeq(c.getChunkSeq())).build()); + // pull next message call.request(1); - return; } - // Process rows and accumulate totals - Counts perChunk = insertRows(ctx, c.getRowsList().iterator()); - ctx.totals.add(perChunk); + case CHUNK -> { + final InsertContext ctx = require(ref.get(), "session not started"); + final InsertChunk c = reqMsg.getChunk(); + + // idempotent replay guard + final long hi = sessionWatermark.getOrDefault(ctx.sessionId, 0L); + if (c.getChunkSeq() <= hi) { + resp.onNext(InsertResponse.newBuilder().setBatchAck(BatchAck.newBuilder().setSessionId(ctx.sessionId) + .setChunkSeq(c.getChunkSeq()).setInserted(0).setUpdated(0).setIgnored(0).setFailed(0).build()).build()); + call.request(1); + return; + } - sessionWatermark.put(ctx.sessionId, c.getChunkSeq()); + Counts perChunk; + try { + perChunk = insertRows(ctx, c.getRowsList().iterator()); + ctx.totals.add(perChunk); + sessionWatermark.put(ctx.sessionId, c.getChunkSeq()); + } + catch (Exception e) { + // surface as failed chunk and continue (or switch to resp.onError(...) if you + // want to fail fast) + + perChunk = new Counts(); + + perChunk.failed = c.getRowsCount(); + + perChunk.errors.add(InsertError.newBuilder().setRowIndex(Math.max(0, ctx.received - 1)).setCode("DB_ERROR") + .setMessage(String.valueOf(e.getMessage())).build()); + ctx.totals.add(perChunk); + // intentionally do not advance watermark on failure; client may replay safely + } - // Ack this chunk with per-chunk counts - resp.onNext( - InsertResponse.newBuilder() - .setBatchAck(BatchAck.newBuilder().setSessionId(ctx.sessionId).setChunkSeq(c.getChunkSeq()) - .setInserted(perChunk.inserted).setUpdated(perChunk.updated).setIgnored(perChunk.ignored) - .setFailed(perChunk.failed)) - .build()); + resp.onNext(InsertResponse.newBuilder() + .setBatchAck(BatchAck.newBuilder().setSessionId(ctx.sessionId).setChunkSeq(c.getChunkSeq()) + .setInserted(perChunk.inserted).setUpdated(perChunk.updated).setIgnored(perChunk.ignored) + .setFailed(perChunk.failed).addAllErrors(perChunk.errors).build()) + .build()); - call.request(1); - } + call.request(1); + } - case COMMIT -> { - InsertContext ctx = require(ref.get(), "session not started"); - try { - ctx.flushCommit(true); - // final committed summary must use TOTALS + startedAt - InsertSummary sum = ctx.summary(ctx.totals, ctx.startedAt); - resp.onNext(InsertResponse.newBuilder().setCommitted(Committed.newBuilder().setSummary(sum)).build()); - resp.onCompleted(); + case COMMIT -> { + final InsertContext ctx = require(ref.get(), "session not started"); + try { + ctx.flushCommit(true); // commit unless validate_only in your InsertContext logic + final InsertSummary sum = ctx.summary(ctx.totals, ctx.startedAt); + resp.onNext(InsertResponse.newBuilder().setCommitted(Committed.newBuilder().setSummary(sum).build()).build()); + resp.onCompleted(); + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("commit: " + e.getMessage()).asException()); + } + finally { + sessionWatermark.remove(ctx.sessionId); + ctx.closeQuietly(); + ref.set(null); + } } - catch (Exception e) { - resp.onError(Status.INTERNAL.withDescription("commit: " + e.getMessage()).asException()); + + case MSG_NOT_SET -> { + // ignore + call.request(1); } - finally { + } + } + catch (Exception unexpected) { + // defensive: fail fast on unexpected exceptions + resp.onError(Status.INTERNAL.withDescription("insertBidirectional: " + unexpected.getMessage()).asException()); + final InsertContext ctx = ref.getAndSet(null); + if (ctx != null) { sessionWatermark.remove(ctx.sessionId); ctx.closeQuietly(); } } - - case MSG_NOT_SET -> { - /* ignore */ } - } } @Override public void onError(Throwable t) { - InsertContext ctx = ref.get(); + final InsertContext ctx = ref.getAndSet(null); if (ctx != null) { sessionWatermark.remove(ctx.sessionId); ctx.closeQuietly(); @@ -1133,11 +1174,13 @@ public void onError(Throwable t) { @Override public void onCompleted() { - // client half-closed without COMMIT -> choose policy (rollback/commit/ignore) - InsertContext ctx = ref.get(); + // Define your policy for half-close without COMMIT: + final InsertContext ctx = ref.getAndSet(null); if (ctx != null) { try { - ctx.flushCommit(true); + // Safer default is rollback; if you prefer auto-commit on close, call + // flushCommit(true) + ctx.flushCommit(false); // or ctx.rollbackQuietly() if you have a helper } catch (Exception ignore) { } @@ -1201,8 +1244,10 @@ private boolean tryUpsertDocument(InsertContext ctx, MutableDocument incoming) { // ---------- Core insert plumbing ---------- private static final class Counts { + long received, inserted, updated, ignored, failed; - final java.util.List errors = new java.util.ArrayList<>(); + + final java.util.List errors = new java.util.ArrayList<>(); void add(Counts o) { received += o.received; @@ -1215,12 +1260,14 @@ void add(Counts o) { void err(long rowIndex, String code, String msg, String field) { failed++; - errors.add(RowError.newBuilder().setRowIndex(rowIndex).setCode(code).setMessage(msg).setField(field).build()); + errors.add(InsertError.newBuilder().setRowIndex(rowIndex).setCode(code).setMessage(msg).setField(field).build()); } } private InsertSummary toSummary(Counts c, long startedAtMs) { + long now = System.currentTimeMillis(); + return InsertSummary.newBuilder().setReceived(c.received).setInserted(c.inserted).setUpdated(c.updated).setIgnored(c.ignored) .setFailed(c.failed).addAllErrors(c.errors).setStartedAt(ts(startedAtMs)).setFinishedAt(ts(now)).build(); } @@ -1235,6 +1282,7 @@ private Counts insertRows(InsertContext ctx, java.util.Iterator it) { boolean isEdge = dt instanceof EdgeType; while (it.hasNext()) { + Record r = it.next(); c.received++; ctx.received++; @@ -1260,8 +1308,10 @@ else if (isEdge) { String inRid = getStringProp(r, "in"); if (outRid == null || inRid == null) { + c.failed++; - c.errors.add(RowError.newBuilder().setRowIndex(ctx.received - 1).setCode("MISSING_ENDPOINTS") + + c.errors.add(InsertError.newBuilder().setRowIndex(ctx.received - 1).setCode("MISSING_ENDPOINTS") .setMessage("Edge requires 'out' and 'in'").build()); } else { @@ -1446,31 +1496,37 @@ private boolean tryUpsert(InsertContext ctx, MutableDocument doc) { } private static Object toJava(Value v) { - if (v == null) return null; - switch (v.getKindCase()) { - case STRING_VALUE: return v.getStringValue(); - case NUMBER_VALUE: return v.getNumberValue(); // Double - case BOOL_VALUE: return v.getBoolValue(); - case NULL_VALUE: return null; - - case STRUCT_VALUE: { - // Convert each nested Value -> Java recursively - java.util.LinkedHashMap m = new java.util.LinkedHashMap<>(); - v.getStructValue().getFieldsMap().forEach((k, vv) -> m.put(k, toJava(vv))); - return m; // plain Map - } - - case LIST_VALUE: { - java.util.ArrayList list = new java.util.ArrayList<>(v.getListValue().getValuesCount()); - for (com.google.protobuf.Value item : v.getListValue().getValuesList()) { - list.add(toJava(item)); - } - return list; - } - - case KIND_NOT_SET: - default: return null; - } + if (v == null) + return null; + switch (v.getKindCase()) { + case STRING_VALUE: + return v.getStringValue(); + case NUMBER_VALUE: + return v.getNumberValue(); // Double + case BOOL_VALUE: + return v.getBoolValue(); + case NULL_VALUE: + return null; + + case STRUCT_VALUE: { + // Convert each nested Value -> Java recursively + java.util.LinkedHashMap m = new java.util.LinkedHashMap<>(); + v.getStructValue().getFieldsMap().forEach((k, vv) -> m.put(k, toJava(vv))); + return m; // plain Map + } + + case LIST_VALUE: { + java.util.ArrayList list = new java.util.ArrayList<>(v.getListValue().getValuesCount()); + for (com.google.protobuf.Value item : v.getListValue().getValuesList()) { + list.add(toJava(item)); + } + return list; + } + + case KIND_NOT_SET: + default: + return null; + } } private InsertOptions defaults(InsertOptions in) { @@ -1493,20 +1549,29 @@ private static Timestamp ts(long ms) { } private final class InsertContext implements AutoCloseable { + protected Counts totals; + final InsertOptions opts; + final Database db; + final java.util.List keyCols; final java.util.List updateCols; + long startedAt; final String sessionId = java.util.UUID.randomUUID().toString(); long received = 0; InsertContext(InsertOptions opts) { + this.opts = opts; + this.db = getDatabase(opts.getDatabase(), opts.getCredentials()); + this.keyCols = opts.getKeyColumnsList(); + this.updateCols = opts.getUpdateColumnsOnConflictList(); if (!opts.getValidateOnly()) { @@ -1517,6 +1582,7 @@ private final class InsertContext implements AutoCloseable { } void flushCommit(boolean end) { + if (opts.getValidateOnly()) { if (end) db.rollback(); @@ -1524,6 +1590,7 @@ void flushCommit(boolean end) { } switch (opts.getTransactionMode()) { case PER_ROW -> db.commit(); + case PER_REQUEST -> db.commit(); case PER_BATCH -> { db.commit(); if (!end) @@ -1690,7 +1757,7 @@ private void validateCredentials(DatabaseCredentials credentials) { // Implement credential validation logic // This is a placeholder - integrate with ArcadeDB's security system - + if (credentials == null || credentials.getUsername().isEmpty()) { throw new IllegalArgumentException("Invalid credentials"); } @@ -1732,10 +1799,12 @@ private Map convertFromProtobufValues(Map values) } private Record convertToGrpcRecord(com.arcadedb.database.Record dbRecord) { + Record.Builder builder = Record.newBuilder().setRid(dbRecord.getIdentity().toString()); // Handle different record types if (dbRecord instanceof Document) { + Document doc = (Document) dbRecord; builder.setType(doc.getTypeName()); @@ -1841,9 +1910,9 @@ private void applyGrpcRecord(MutableEdge e, com.arcadedb.server.grpc.Record r, D // If property type is EMBEDDED and client sent a Struct/Map, create embedded // document private Object toJavaForProperty(Database db, DocumentType dt, String key, com.google.protobuf.Value v) { - + var p = (dt != null) ? dt.getPropertyIfExists(key) : null; - + if (p != null && p.getType() == com.arcadedb.schema.Type.EMBEDDED && v.hasStructValue()) { MutableDocument emb = db.newDocument(dt.getName()); v.getStructValue().getFieldsMap().forEach((k2, vv) -> emb.set(k2, toJava(vv))); diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcServerPlugin.java b/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcServerPlugin.java index b180acba34..41b1a25b48 100644 --- a/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcServerPlugin.java +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcServerPlugin.java @@ -127,7 +127,10 @@ private void startStandardServer(ContextConfiguration config) throws IOException // Configure the server configureServer(serverBuilder, config); - grpcServer = serverBuilder.build().start(); + grpcServer = serverBuilder + .maxInboundMessageSize(256 * 1024 * 1024) + .maxInboundMetadataSize(32 * 1024 * 1024) + .build().start(); // Build status message StringBuilder status = new StringBuilder(); @@ -161,7 +164,10 @@ private void startXdsServer(ContextConfiguration config) throws IOException { // Configure the XDS server as a ServerBuilder configureServer(xdsBuilder, config); - xdsServer = xdsBuilder.build().start(); + xdsServer = xdsBuilder + .maxInboundMessageSize(256 * 1024 * 1024) + .maxInboundMetadataSize(32 * 1024 * 1024) + .build().start(); logger.info("gRPC XDS server started on port {} (xDS management enabled)", port); } From ed6bd1182f1c622fca95b82f3f137ab1e984033a Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Mon, 1 Sep 2025 14:02:40 -0700 Subject: [PATCH 10/35] Checkpoint Commit --- .../remote/grpc/RemoteGrpcDatabase.java | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java index 5a91f5fe8b..0cd12935d5 100644 --- a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java @@ -1452,11 +1452,19 @@ public void onCompleted() { * @throws InterruptedException if waiting for completion is interrupted */ public com.arcadedb.server.grpc.InsertSummary ingestBidi(final java.util.List rows, - final com.arcadedb.server.grpc.InsertOptions opts, final int chunkSize, final int maxInflight) throws InterruptedException { - return ingestBidiCore(rows, opts, chunkSize, maxInflight, /* timeoutMs */ 5 * 60_000L, // or pass a parameter if you prefer + final com.arcadedb.server.grpc.InsertOptions opts, final int chunkSize, final int maxInflight, final long timeoutMs) throws InterruptedException { + + return ingestBidiCore(rows, opts, chunkSize, maxInflight, timeoutMs, (Object o) -> toProtoRecordFromDbRecord((com.arcadedb.database.Record) o)); } + public com.arcadedb.server.grpc.InsertSummary ingestBidi(final java.util.List rows, + final com.arcadedb.server.grpc.InsertOptions opts, final int chunkSize, final int maxInflight) throws InterruptedException { + + return ingestBidiCore(rows, opts, chunkSize, maxInflight, /* timeoutMs */ 5 * 60_000L, + (Object o) -> toProtoRecordFromDbRecord((com.arcadedb.database.Record) o)); + } + /** * Pushes map-shaped rows (property map per row) via InsertBidirectional with * per-batch ACKs. @@ -1475,13 +1483,22 @@ public com.arcadedb.server.grpc.InsertSummary ingestBidi(final java.util.List> rows, final int chunkSize, final int maxInflight) + throws InterruptedException { + + return ingestBidiCore(rows, options, chunkSize, maxInflight, /* timeoutMs */ 5 * 60_000L, + (Object o) -> toProtoRecordFromMap((java.util.Map) o)); + } + public com.arcadedb.server.grpc.InsertSummary ingestBidi(final com.arcadedb.server.grpc.InsertOptions options, final java.util.List> rows, final int chunkSize, final int maxInflight, final long timeoutMs) throws InterruptedException { + return ingestBidiCore(rows, options, chunkSize, maxInflight, timeoutMs, (Object o) -> toProtoRecordFromMap((java.util.Map) o)); } - + // Map -> PROTO Value private com.arcadedb.server.grpc.Record toProtoRecord(Map row) { com.arcadedb.server.grpc.Record.Builder b = com.arcadedb.server.grpc.Record.newBuilder(); From cbb8db8540d42aad3fd06be67d120051425e9a38 Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Mon, 1 Sep 2025 14:37:30 -0700 Subject: [PATCH 11/35] Checkpoint Commit --- .../remote/grpc/RemoteGrpcDatabase.java | 146 +++++++++++++----- .../server/grpc/ArcadeDbGrpcService.java | 36 +++++ 2 files changed, 146 insertions(+), 36 deletions(-) diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java index 0cd12935d5..77d7f32a88 100644 --- a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java @@ -8,7 +8,6 @@ import java.util.List; import java.util.Map; import java.util.NoSuchElementException; -import java.util.UUID; import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; @@ -36,10 +35,8 @@ import com.arcadedb.remote.RemoteSchema; import com.arcadedb.remote.RemoteTransactionExplicitLock; import com.arcadedb.server.grpc.ArcadeDbServiceGrpc; -import com.arcadedb.server.grpc.BatchAck; import com.arcadedb.server.grpc.BeginTransactionRequest; import com.arcadedb.server.grpc.BeginTransactionResponse; -import com.arcadedb.server.grpc.Commit; import com.arcadedb.server.grpc.CommitTransactionRequest; import com.arcadedb.server.grpc.CommitTransactionResponse; import com.arcadedb.server.grpc.DatabaseCredentials; @@ -49,17 +46,12 @@ import com.arcadedb.server.grpc.ExecuteCommandResponse; import com.arcadedb.server.grpc.ExecuteQueryRequest; import com.arcadedb.server.grpc.ExecuteQueryResponse; -import com.arcadedb.server.grpc.InsertChunk; import com.arcadedb.server.grpc.InsertOptions; -import com.arcadedb.server.grpc.InsertRequest; -import com.arcadedb.server.grpc.InsertResponse; -import com.arcadedb.server.grpc.InsertSummary; import com.arcadedb.server.grpc.LookupByRidRequest; import com.arcadedb.server.grpc.LookupByRidResponse; import com.arcadedb.server.grpc.QueryResult; import com.arcadedb.server.grpc.RollbackTransactionRequest; import com.arcadedb.server.grpc.RollbackTransactionResponse; -import com.arcadedb.server.grpc.Start; import com.arcadedb.server.grpc.StreamQueryRequest; import com.arcadedb.server.grpc.TransactionContext; import com.arcadedb.server.grpc.TransactionIsolation; @@ -1303,13 +1295,17 @@ private com.arcadedb.server.grpc.InsertSummary ingestBidiCore(final java.util.Li // 1) Ensure options carry DB + credentials the server expects final com.arcadedb.server.grpc.InsertOptions.Builder ob = options.toBuilder(); + if (options.getDatabase() == null || options.getDatabase().isEmpty()) { ob.setDatabase(getName()); } + final boolean missingCreds = !options.hasCredentials() || options.getCredentials().getUsername().isEmpty(); + if (missingCreds) { ob.setCredentials(buildCredentials()); } + final com.arcadedb.server.grpc.InsertOptions effectiveOpts = ob.build(); // 2) Pre-map rows → proto once (outside the observer) @@ -1326,6 +1322,58 @@ private com.arcadedb.server.grpc.InsertSummary ingestBidiCore(final java.util.Li final java.util.concurrent.atomic.AtomicReference committed = new java.util.concurrent.atomic.AtomicReference<>(); final java.util.List acks = java.util.Collections.synchronizedList(new java.util.ArrayList<>()); + // small holder to access req inside helpers + final java.util.concurrent.atomic.AtomicReference> observerRef = + new java.util.concurrent.atomic.AtomicReference<>(); + + final java.util.concurrent.atomic.AtomicBoolean commitSent = new java.util.concurrent.atomic.AtomicBoolean(false); + final java.util.concurrent.ScheduledExecutorService scheduler = java.util.concurrent.Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "grpc-ack-grace-timer"); + t.setDaemon(true); + return t; + }); + final long ackGraceMillis = Math.min(Math.max(timeoutMs / 10, 1_000L), 10_000L); // e.g., 1s..10s window + final Object timerLock = new Object(); + final java.util.concurrent.atomic.AtomicReference> ackGraceFuture = new java.util.concurrent.atomic.AtomicReference<>(); + + // helper: schedule COMMIT if still waiting + final Runnable sendCommitIfNeeded = () -> { + if (commitSent.compareAndSet(false, true)) { + try { + if (observerRef.get() != null) { + observerRef.get().onNext(com.arcadedb.server.grpc.InsertRequest.newBuilder() + .setCommit(com.arcadedb.server.grpc.Commit.newBuilder().setSessionId(sessionId)).build()); + observerRef.get().onCompleted(); + } + } + catch (Throwable ignore) { + /* best effort */ } + } + }; + + // helper: (re)arm the grace timer + final Runnable armAckGraceTimer = () -> { + synchronized (timerLock) { + var prev = ackGraceFuture.getAndSet(null); + if (prev != null) + prev.cancel(false); + // only arm if we've sent all chunks but acks are still pending + if (cursor.get() >= protoRows.size() && acked.get() < sent.get() && !commitSent.get()) { + var fut = scheduler.schedule(sendCommitIfNeeded, ackGraceMillis, java.util.concurrent.TimeUnit.MILLISECONDS); + ackGraceFuture.set(fut); + } + } + }; + + // helper: cancel timer (e.g., once all ACKed or after COMMIT observed) + final Runnable cancelAckGraceTimer = () -> { + synchronized (timerLock) { + var prev = ackGraceFuture.getAndSet(null); + if (prev != null) + prev.cancel(false); + } + }; + final io.grpc.stub.ClientResponseObserver observer = new io.grpc.stub.ClientResponseObserver<>() { io.grpc.stub.ClientCallStreamObserver req; @@ -1334,45 +1382,54 @@ private com.arcadedb.server.grpc.InsertSummary ingestBidiCore(final java.util.Li @Override public void beforeStart(io.grpc.stub.ClientCallStreamObserver r) { this.req = r; + observerRef.set(r); // <-- make req visible to helper r.disableAutoInboundFlowControl(); r.setOnReadyHandler(this::drain); } private void drain() { + if (!req.isReady()) return; - // Send START exactly once if (!started) { + req.onNext(com.arcadedb.server.grpc.InsertRequest.newBuilder() .setStart(com.arcadedb.server.grpc.Start.newBuilder().setOptions(effectiveOpts)).build()); + started = true; + + // Now that the call is started, it's safe to pull the first response + req.request(1); } - // While there is capacity (backpressure) and in-flight < maxInflight → send - // chunks while (req.isReady() && (sent.get() - acked.get()) < maxInflight) { final int start = cursor.get(); if (start >= protoRows.size()) break; final int end = Math.min(start + chunkSize, protoRows.size()); - final java.util.List slice = protoRows.subList(start, end); + final var slice = protoRows.subList(start, end); - final com.arcadedb.server.grpc.InsertChunk chunk = com.arcadedb.server.grpc.InsertChunk.newBuilder().setSessionId(sessionId) + final var chunk = com.arcadedb.server.grpc.InsertChunk.newBuilder().setSessionId(sessionId) .setChunkSeq(seq.getAndIncrement()).addAllRows(slice).build(); req.onNext(com.arcadedb.server.grpc.InsertRequest.newBuilder().setChunk(chunk).build()); - cursor.set(end); sent.incrementAndGet(); } - // If we sent everything and all ACKed → send COMMIT and half-close - if (cursor.get() >= protoRows.size() && acked.get() >= sent.get()) { - req.onNext(com.arcadedb.server.grpc.InsertRequest.newBuilder() - .setCommit(com.arcadedb.server.grpc.Commit.newBuilder().setSessionId(sessionId)).build()); - req.onCompleted(); + // If all chunks sent: + if (cursor.get() >= protoRows.size()) { + if (acked.get() >= sent.get()) { + // all acked → commit immediately + sendCommitIfNeeded.run(); + } + else { + // not all acked → (re)arm grace timer; if last ACK never arrives, timer will + // COMMIT + armAckGraceTimer.run(); + } } } @@ -1384,32 +1441,49 @@ public void onNext(com.arcadedb.server.grpc.InsertResponse v) { case BATCH_ACK -> { acks.add(v.getBatchAck()); acked.incrementAndGet(); - // Now that capacity freed up, try to send more + // Each ACK may free capacity: push more & manage timer drain(); + if (cursor.get() >= protoRows.size()) { + if (acked.get() >= sent.get()) { + cancelAckGraceTimer.run(); + sendCommitIfNeeded.run(); + } + else { + armAckGraceTimer.run(); + } + } + } + case COMMITTED -> { + committed.set(v.getCommitted().getSummary()); + cancelAckGraceTimer.run(); } - case COMMITTED -> committed.set(v.getCommitted().getSummary()); case MSG_NOT_SET -> { /* ignore */ } } - // NOTE: no req.request(1) here; responses are server-pushed + req.request(1); // continue pulling responses } @Override public void onError(Throwable t) { + cancelAckGraceTimer.run(); done.countDown(); } @Override public void onCompleted() { + cancelAckGraceTimer.run(); done.countDown(); } }; - // 4) Kick off the bidi call with deadline - this.asyncStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS).insertBidirectional(observer); - // 5) Wait for completion - done.await(timeoutMs + 5_000, java.util.concurrent.TimeUnit.MILLISECONDS); + try { + asyncStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS).insertBidirectional(observer); + boolean finished = done.await(timeoutMs + 5_000, java.util.concurrent.TimeUnit.MILLISECONDS); + } + finally { + scheduler.shutdownNow(); + } // 6) Prefer COMMITTED summary if present; otherwise aggregate ACKs final com.arcadedb.server.grpc.InsertSummary finalSummary = committed.get(); @@ -1452,19 +1526,20 @@ public void onCompleted() { * @throws InterruptedException if waiting for completion is interrupted */ public com.arcadedb.server.grpc.InsertSummary ingestBidi(final java.util.List rows, - final com.arcadedb.server.grpc.InsertOptions opts, final int chunkSize, final int maxInflight, final long timeoutMs) throws InterruptedException { - + final com.arcadedb.server.grpc.InsertOptions opts, final int chunkSize, final int maxInflight, final long timeoutMs) + throws InterruptedException { + return ingestBidiCore(rows, opts, chunkSize, maxInflight, timeoutMs, (Object o) -> toProtoRecordFromDbRecord((com.arcadedb.database.Record) o)); } public com.arcadedb.server.grpc.InsertSummary ingestBidi(final java.util.List rows, final com.arcadedb.server.grpc.InsertOptions opts, final int chunkSize, final int maxInflight) throws InterruptedException { - - return ingestBidiCore(rows, opts, chunkSize, maxInflight, /* timeoutMs */ 5 * 60_000L, + + return ingestBidiCore(rows, opts, chunkSize, maxInflight, /* timeoutMs */ 5 * 60_000L, (Object o) -> toProtoRecordFromDbRecord((com.arcadedb.database.Record) o)); } - + /** * Pushes map-shaped rows (property map per row) via InsertBidirectional with * per-batch ACKs. @@ -1484,9 +1559,8 @@ public com.arcadedb.server.grpc.InsertSummary ingestBidi(final java.util.List> rows, final int chunkSize, final int maxInflight) - throws InterruptedException { - + final java.util.List> rows, final int chunkSize, final int maxInflight) throws InterruptedException { + return ingestBidiCore(rows, options, chunkSize, maxInflight, /* timeoutMs */ 5 * 60_000L, (Object o) -> toProtoRecordFromMap((java.util.Map) o)); } @@ -1494,11 +1568,11 @@ public com.arcadedb.server.grpc.InsertSummary ingestBidi(final com.arcadedb.serv public com.arcadedb.server.grpc.InsertSummary ingestBidi(final com.arcadedb.server.grpc.InsertOptions options, final java.util.List> rows, final int chunkSize, final int maxInflight, final long timeoutMs) throws InterruptedException { - + return ingestBidiCore(rows, options, chunkSize, maxInflight, timeoutMs, (Object o) -> toProtoRecordFromMap((java.util.Map) o)); } - + // Map -> PROTO Value private com.arcadedb.server.grpc.Record toProtoRecord(Map row) { com.arcadedb.server.grpc.Record.Builder b = com.arcadedb.server.grpc.Record.newBuilder(); diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java index e25b53dc71..bca617a597 100644 --- a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java @@ -1044,7 +1044,27 @@ public StreamObserver insertBidirectional(StreamObserver ref = new java.util.concurrent.atomic.AtomicReference<>(); final java.util.concurrent.atomic.AtomicBoolean cancelled = new java.util.concurrent.atomic.AtomicBoolean(false); final java.util.concurrent.atomic.AtomicBoolean started = new java.util.concurrent.atomic.AtomicBoolean(false); + + final java.util.concurrent.ConcurrentLinkedQueue outQueue = new java.util.concurrent.ConcurrentLinkedQueue<>(); + // drain helper + final Runnable drain = () -> { + while (call.isReady()) { + InsertResponse next = outQueue.poll(); + if (next == null) + break; + resp.onNext(next); + } + }; + + // enqueue or send immediately + final java.util.function.Consumer sendOrQueue = (ir) -> { + outQueue.offer(ir); + drain.run(); + }; + + call.setOnReadyHandler(drain); + call.setOnCancelHandler(() -> { cancelled.set(true); final InsertContext ctx = ref.getAndSet(null); @@ -1080,6 +1100,10 @@ public void onNext(InsertRequest reqMsg) { ref.set(ctx); sessionWatermark.put(ctx.sessionId, 0L); + sendOrQueue.accept(InsertResponse.newBuilder() + .setStarted(Started.newBuilder().setSessionId(ctx.sessionId).build()) + .build()); + resp.onNext(InsertResponse.newBuilder().setStarted(Started.newBuilder().setSessionId(ctx.sessionId).build()).build()); // pull next message @@ -1092,6 +1116,7 @@ public void onNext(InsertRequest reqMsg) { // idempotent replay guard final long hi = sessionWatermark.getOrDefault(ctx.sessionId, 0L); + if (c.getChunkSeq() <= hi) { resp.onNext(InsertResponse.newBuilder().setBatchAck(BatchAck.newBuilder().setSessionId(ctx.sessionId) .setChunkSeq(c.getChunkSeq()).setInserted(0).setUpdated(0).setIgnored(0).setFailed(0).build()).build()); @@ -1104,6 +1129,17 @@ public void onNext(InsertRequest reqMsg) { perChunk = insertRows(ctx, c.getRowsList().iterator()); ctx.totals.add(perChunk); sessionWatermark.put(ctx.sessionId, c.getChunkSeq()); + + sendOrQueue.accept(InsertResponse.newBuilder() + .setBatchAck(BatchAck.newBuilder() + .setSessionId(ctx.sessionId).setChunkSeq(c.getChunkSeq()) + .setInserted(perChunk.inserted).setUpdated(perChunk.updated) + .setIgnored(perChunk.ignored).setFailed(perChunk.failed) + .addAllErrors(perChunk.errors) + .build()) + .build()); + + } catch (Exception e) { // surface as failed chunk and continue (or switch to resp.onError(...) if you From e11b28492ce43a894e07eba09604ea5d347206c1 Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Mon, 1 Sep 2025 14:52:30 -0700 Subject: [PATCH 12/35] Checkpoint Commit --- .../server/grpc/ArcadeDbGrpcService.java | 39 +++++++++++++------ 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java index bca617a597..0f9c1cbc23 100644 --- a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java @@ -1099,13 +1099,12 @@ public void onNext(InsertRequest reqMsg) { ref.set(ctx); sessionWatermark.put(ctx.sessionId, 0L); - - sendOrQueue.accept(InsertResponse.newBuilder() + + sendOrQueue.accept( + InsertResponse.newBuilder() .setStarted(Started.newBuilder().setSessionId(ctx.sessionId).build()) .build()); - resp.onNext(InsertResponse.newBuilder().setStarted(Started.newBuilder().setSessionId(ctx.sessionId).build()).build()); - // pull next message call.request(1); } @@ -1118,8 +1117,10 @@ public void onNext(InsertRequest reqMsg) { final long hi = sessionWatermark.getOrDefault(ctx.sessionId, 0L); if (c.getChunkSeq() <= hi) { + resp.onNext(InsertResponse.newBuilder().setBatchAck(BatchAck.newBuilder().setSessionId(ctx.sessionId) .setChunkSeq(c.getChunkSeq()).setInserted(0).setUpdated(0).setIgnored(0).setFailed(0).build()).build()); + call.request(1); return; } @@ -1155,12 +1156,25 @@ public void onNext(InsertRequest reqMsg) { // intentionally do not advance watermark on failure; client may replay safely } - resp.onNext(InsertResponse.newBuilder() - .setBatchAck(BatchAck.newBuilder().setSessionId(ctx.sessionId).setChunkSeq(c.getChunkSeq()) - .setInserted(perChunk.inserted).setUpdated(perChunk.updated).setIgnored(perChunk.ignored) - .setFailed(perChunk.failed).addAllErrors(perChunk.errors).build()) - .build()); - +// resp.onNext(InsertResponse.newBuilder() +// .setBatchAck(BatchAck.newBuilder().setSessionId(ctx.sessionId).setChunkSeq(c.getChunkSeq()) +// .setInserted(perChunk.inserted).setUpdated(perChunk.updated).setIgnored(perChunk.ignored) +// .setFailed(perChunk.failed).addAllErrors(perChunk.errors).build()) +// .build()); + + sendOrQueue.accept( + InsertResponse.newBuilder() + .setBatchAck(BatchAck.newBuilder() + .setSessionId(ctx.sessionId) + .setChunkSeq(c.getChunkSeq()) + .setInserted(perChunk.inserted) + .setUpdated(perChunk.updated) + .setIgnored(perChunk.ignored) + .setFailed(perChunk.failed) + .addAllErrors(perChunk.errors) + .build()) + .build()); + call.request(1); } @@ -1228,11 +1242,14 @@ public void onCompleted() { } private boolean tryUpsertVertex(InsertContext ctx, com.arcadedb.graph.MutableVertex incoming) { + var keys = ctx.keyCols; + if (keys.isEmpty()) return false; String where = String.join(" AND ", keys.stream().map(k -> k + " = ?").toList()); + Object[] params = keys.stream() // read key values from the incoming vertex as a document .map(k -> ((MutableDocument) incoming).get(k)).toArray(); @@ -1245,7 +1262,7 @@ private boolean tryUpsertVertex(InsertContext ctx, com.arcadedb.graph.MutableVer return false; // mutate via MutableDocument view (valid for vertex records) - MutableDocument existingDoc = (MutableDocument) res.getElement().get().asDocument(true); + MutableVertex existingDoc = (MutableVertex) res.getElement().get().asDocument(true); for (String col : ctx.updateCols) { existingDoc.set(col, ((MutableDocument) incoming).get(col)); } From 923e3bd6c0fb4f09d9c4bc3c22d75add8a4069ca Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Mon, 1 Sep 2025 15:58:33 -0700 Subject: [PATCH 13/35] Checkpoint Commit --- .../server/grpc/ArcadeDbGrpcService.java | 110 +++++++++--------- 1 file changed, 58 insertions(+), 52 deletions(-) diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java index 0f9c1cbc23..b631fcb444 100644 --- a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java @@ -1044,7 +1044,7 @@ public StreamObserver insertBidirectional(StreamObserver ref = new java.util.concurrent.atomic.AtomicReference<>(); final java.util.concurrent.atomic.AtomicBoolean cancelled = new java.util.concurrent.atomic.AtomicBoolean(false); final java.util.concurrent.atomic.AtomicBoolean started = new java.util.concurrent.atomic.AtomicBoolean(false); - + final java.util.concurrent.ConcurrentLinkedQueue outQueue = new java.util.concurrent.ConcurrentLinkedQueue<>(); // drain helper @@ -1061,10 +1061,10 @@ public StreamObserver insertBidirectional(StreamObserver sendOrQueue = (ir) -> { outQueue.offer(ir); drain.run(); - }; + }; call.setOnReadyHandler(drain); - + call.setOnCancelHandler(() -> { cancelled.set(true); final InsertContext ctx = ref.getAndSet(null); @@ -1099,11 +1099,9 @@ public void onNext(InsertRequest reqMsg) { ref.set(ctx); sessionWatermark.put(ctx.sessionId, 0L); - + sendOrQueue.accept( - InsertResponse.newBuilder() - .setStarted(Started.newBuilder().setSessionId(ctx.sessionId).build()) - .build()); + InsertResponse.newBuilder().setStarted(Started.newBuilder().setSessionId(ctx.sessionId).build()).build()); // pull next message call.request(1); @@ -1115,12 +1113,12 @@ public void onNext(InsertRequest reqMsg) { // idempotent replay guard final long hi = sessionWatermark.getOrDefault(ctx.sessionId, 0L); - + if (c.getChunkSeq() <= hi) { - + resp.onNext(InsertResponse.newBuilder().setBatchAck(BatchAck.newBuilder().setSessionId(ctx.sessionId) .setChunkSeq(c.getChunkSeq()).setInserted(0).setUpdated(0).setIgnored(0).setFailed(0).build()).build()); - + call.request(1); return; } @@ -1130,26 +1128,22 @@ public void onNext(InsertRequest reqMsg) { perChunk = insertRows(ctx, c.getRowsList().iterator()); ctx.totals.add(perChunk); sessionWatermark.put(ctx.sessionId, c.getChunkSeq()); - + sendOrQueue.accept(InsertResponse.newBuilder() - .setBatchAck(BatchAck.newBuilder() - .setSessionId(ctx.sessionId).setChunkSeq(c.getChunkSeq()) - .setInserted(perChunk.inserted).setUpdated(perChunk.updated) - .setIgnored(perChunk.ignored).setFailed(perChunk.failed) - .addAllErrors(perChunk.errors) - .build()) - .build()); - - + .setBatchAck(BatchAck.newBuilder().setSessionId(ctx.sessionId).setChunkSeq(c.getChunkSeq()) + .setInserted(perChunk.inserted).setUpdated(perChunk.updated).setIgnored(perChunk.ignored) + .setFailed(perChunk.failed).addAllErrors(perChunk.errors).build()) + .build()); + } catch (Exception e) { // surface as failed chunk and continue (or switch to resp.onError(...) if you // want to fail fast) - + perChunk = new Counts(); - + perChunk.failed = c.getRowsCount(); - + perChunk.errors.add(InsertError.newBuilder().setRowIndex(Math.max(0, ctx.received - 1)).setCode("DB_ERROR") .setMessage(String.valueOf(e.getMessage())).build()); ctx.totals.add(perChunk); @@ -1162,19 +1156,12 @@ public void onNext(InsertRequest reqMsg) { // .setFailed(perChunk.failed).addAllErrors(perChunk.errors).build()) // .build()); - sendOrQueue.accept( - InsertResponse.newBuilder() - .setBatchAck(BatchAck.newBuilder() - .setSessionId(ctx.sessionId) - .setChunkSeq(c.getChunkSeq()) - .setInserted(perChunk.inserted) - .setUpdated(perChunk.updated) - .setIgnored(perChunk.ignored) - .setFailed(perChunk.failed) - .addAllErrors(perChunk.errors) - .build()) - .build()); - + sendOrQueue.accept(InsertResponse.newBuilder() + .setBatchAck(BatchAck.newBuilder().setSessionId(ctx.sessionId).setChunkSeq(c.getChunkSeq()) + .setInserted(perChunk.inserted).setUpdated(perChunk.updated).setIgnored(perChunk.ignored) + .setFailed(perChunk.failed).addAllErrors(perChunk.errors).build()) + .build()); + call.request(1); } @@ -1241,57 +1228,76 @@ public void onCompleted() { }; } - private boolean tryUpsertVertex(InsertContext ctx, com.arcadedb.graph.MutableVertex incoming) { + private boolean tryUpsertVertex(final InsertContext ctx, final com.arcadedb.graph.MutableVertex incoming) { var keys = ctx.keyCols; if (keys.isEmpty()) return false; + // Read incoming values via the (read-only) document view + var inDoc = incoming.asDocument(); + String where = String.join(" AND ", keys.stream().map(k -> k + " = ?").toList()); - - Object[] params = keys.stream() - // read key values from the incoming vertex as a document - .map(k -> ((MutableDocument) incoming).get(k)).toArray(); + Object[] params = keys.stream().map(inDoc::get).toArray(); + + // Prefer selecting the element so we can get a mutable vertex directly + final String sql = "SELECT FROM " + ctx.opts.getTargetClass() + " WHERE " + where; - try (var rs = ctx.db.query("sql", "SELECT FROM " + ctx.opts.getTargetClass() + " WHERE " + where, params)) { + try (var rs = ctx.db.query("sql", sql, params)) { if (!rs.hasNext()) return false; + var res = rs.next(); if (!res.isElement()) return false; - // mutate via MutableDocument view (valid for vertex records) - MutableVertex existingDoc = (MutableVertex) res.getElement().get().asDocument(true); + Vertex v = res.getElement().get().asVertex(); + + MutableVertex existingV = v.modify(); + + // Apply the updates for (String col : ctx.updateCols) { - existingDoc.set(col, ((MutableDocument) incoming).get(col)); + existingV.set(col, inDoc.get(col)); } - existingDoc.save(); + + existingV.save(); + return true; } } private boolean tryUpsertDocument(InsertContext ctx, MutableDocument incoming) { + var keys = ctx.keyCols; + if (keys.isEmpty()) return false; String where = String.join(" AND ", keys.stream().map(k -> k + " = ?").toList()); Object[] params = keys.stream().map(incoming::get).toArray(); - - try (var rs = ctx.db.query("sql", "SELECT FROM " + ctx.opts.getTargetClass() + " WHERE " + where, params)) { - if (!rs.hasNext()) - return false; + + try (ResultSet rs = ctx.db.query("sql", "SELECT FROM " + ctx.opts.getTargetClass() + " WHERE " + where, params)) { + + if (!rs.hasNext()) { + return false; + } + var res = rs.next(); if (!res.isElement()) return false; - var existing = (MutableDocument) res.getElement().get().asDocument(true); + Document d = res.getElement().get().asDocument(); + + var existing = d.modify(); + for (String col : ctx.updateCols) existing.set(col, incoming.get(col)); + existing.save(); + return true; - } + } } // ---------- Core insert plumbing ---------- From 69183d512d0cc0b0f1940fbbaf5bb8af3b9cb3a2 Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Mon, 1 Sep 2025 16:22:19 -0700 Subject: [PATCH 14/35] Checkpoint Commit --- .../main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java index b631fcb444..cca9674ea9 100644 --- a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java @@ -370,7 +370,7 @@ public void updateRecord(UpdateRecordRequest req, StreamObserver Date: Mon, 1 Sep 2025 16:22:51 -0700 Subject: [PATCH 15/35] Checkpoint Commit --- server/TMP/grpc/ArcadeDbGrpcService.java | 1082 ----------------- server/TMP/grpc/CompressionAwareService.java | 86 -- server/TMP/grpc/GrpcAuthInterceptor.java | 142 --- .../TMP/grpc/GrpcCompressionInterceptor.java | 83 -- server/TMP/grpc/GrpcLoggingInterceptor.java | 81 -- server/TMP/grpc/GrpcMetricsInterceptor.java | 78 -- server/TMP/grpc/GrpcServerPlugin.java | 386 ------ 7 files changed, 1938 deletions(-) delete mode 100644 server/TMP/grpc/ArcadeDbGrpcService.java delete mode 100644 server/TMP/grpc/CompressionAwareService.java delete mode 100644 server/TMP/grpc/GrpcAuthInterceptor.java delete mode 100644 server/TMP/grpc/GrpcCompressionInterceptor.java delete mode 100644 server/TMP/grpc/GrpcLoggingInterceptor.java delete mode 100644 server/TMP/grpc/GrpcMetricsInterceptor.java delete mode 100644 server/TMP/grpc/GrpcServerPlugin.java diff --git a/server/TMP/grpc/ArcadeDbGrpcService.java b/server/TMP/grpc/ArcadeDbGrpcService.java deleted file mode 100644 index 1a48868c03..0000000000 --- a/server/TMP/grpc/ArcadeDbGrpcService.java +++ /dev/null @@ -1,1082 +0,0 @@ -package com.arcadedb.server.grpc; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.arcadedb.database.Database; -import com.arcadedb.database.DatabaseFactory; -import com.arcadedb.database.Document; -import com.arcadedb.database.MutableDocument; -import com.arcadedb.engine.ComponentFile; -import com.arcadedb.exception.DuplicatedKeyException; -import com.arcadedb.exception.ValidationException; -import com.arcadedb.graph.Edge; -import com.arcadedb.graph.Vertex; -import com.arcadedb.query.sql.executor.Result; -import com.arcadedb.query.sql.executor.ResultSet; -import com.arcadedb.server.ArcadeDBServer; -import com.google.protobuf.Empty; -import com.google.protobuf.Timestamp; -import com.google.protobuf.Value; - -import io.grpc.Status; -import io.grpc.StatusRuntimeException; -import io.grpc.stub.ServerCallStreamObserver; -import io.grpc.stub.StreamObserver; - -/** - * gRPC Service implementation for ArcadeDB - */ -public class ArcadeDbGrpcService extends ArcadeDbServiceGrpc.ArcadeDbServiceImplBase { - - private static final Logger logger = LoggerFactory.getLogger(ArcadeDbGrpcService.class); - - // Transaction management - private final Map activeTransactions = new ConcurrentHashMap<>(); - - // Database connection pool - private final Map databasePool = new ConcurrentHashMap<>(); - - // Track highest acknowledged chunk per session for idempotency - private final Map sessionWatermark = new ConcurrentHashMap<>(); - - - // ArcadeDB server reference (optional, for accessing existing databases) - private final ArcadeDBServer arcadeServer; - - // Database directory path - private final String databasePath; - - public ArcadeDbGrpcService(String databasePath, ArcadeDBServer server) { - this.databasePath = databasePath; - this.arcadeServer = server; - } - - public void close() { - // Close all open databases - for (Database db : databasePool.values()) { - try { - if (db != null && db.isOpen()) { - db.close(); - } - } catch (Exception e) { - logger.error("Error closing database", e); - } - } - databasePool.clear(); - - // Clean up transactions - for (Database db : activeTransactions.values()) { - try { - if (db != null && db.isOpen()) { - db.rollback(); - db.close(); - } - } catch (Exception e) { - logger.error("Error closing transaction database", e); - } - } - activeTransactions.clear(); - } - - @Override - public void createDatabase(CreateDatabaseRequest request, - StreamObserver responseObserver) { - try { - // Validate credentials if needed - validateCredentials(request.getCredentials()); - - // Create new database factory for this specific database - DatabaseFactory dbFactory = new DatabaseFactory(databasePath + "/" + request.getDatabaseName()); - - // Create the database - Database database = dbFactory.create(); - database.close(); - - CreateDatabaseResponse response = CreateDatabaseResponse.newBuilder() - .setSuccess(true) - .setMessage("Database created successfully") - .setDatabaseId(request.getDatabaseName()) - .build(); - - responseObserver.onNext(response); - responseObserver.onCompleted(); - - } catch (Exception e) { - logger.error("Error creating database: {}", e.getMessage(), e); - responseObserver.onError(Status.INTERNAL - .withDescription("Failed to create database: " + e.getMessage()) - .asException()); - } - } - - @Override - public void executeQuery(ExecuteQueryRequest request, - StreamObserver responseObserver) { - try { - - // Force compression for streaming (usually beneficial) - CompressionAwareService.setResponseCompression(responseObserver, "gzip"); - - Database database = getDatabase(request.getDatabase(), request.getCredentials()); - - // Check if this is part of a transaction - if (request.hasTransaction()) { - database = activeTransactions.get(request.getTransaction().getTransactionId()); - if (database == null) { - throw new IllegalArgumentException("Invalid transaction ID"); - } - } - - // Execute the query - long startTime = System.currentTimeMillis(); - ResultSet resultSet = database.query("sql", request.getQuery(), - convertParameters(request.getParametersMap())); - - logger.debug("executeQuery(): to get resultSet = {}", (System.currentTimeMillis() - startTime)); - - // Build response - QueryResult.Builder resultBuilder = QueryResult.newBuilder(); - - // Process results - int count = 0; - while (resultSet.hasNext()) { - Result result = resultSet.next(); - if (result.isElement()) { - com.arcadedb.database.Record dbRecord = result.getElement().get(); - Record grpcRecord = convertToGrpcRecord(dbRecord); - resultBuilder.addRecords(grpcRecord); - count++; - - // Apply limit if specified - if (request.getLimit() > 0 && count >= request.getLimit()) { - break; - } - } - } - - resultBuilder.setTotalRecordsInBatch(count); - - long executionTime = System.currentTimeMillis() - startTime; - - ExecuteQueryResponse response = ExecuteQueryResponse.newBuilder() - .addResults(resultBuilder.build()) - .setExecutionTimeMs(executionTime) - .build(); - - logger.debug("executeQuery(): executionTime + response generation = {}", executionTime); - - responseObserver.onNext(response); - responseObserver.onCompleted(); - - } catch (Exception e) { - logger.error("Error executing query: {}", e.getMessage(), e); - responseObserver.onError(Status.INTERNAL - .withDescription("Query execution failed: " + e.getMessage()) - .asException()); - } - } - - @Override - public void beginTransaction(BeginTransactionRequest request, - StreamObserver responseObserver) { - try { - Database database = getDatabase(request.getDatabase(), request.getCredentials()); - - // Begin transaction - database.begin(); - - // Generate transaction ID - String transactionId = generateTransactionId(); - activeTransactions.put(transactionId, database); - - BeginTransactionResponse response = BeginTransactionResponse.newBuilder() - .setTransactionId(transactionId) - .setTimestamp(System.currentTimeMillis()) - .build(); - - responseObserver.onNext(response); - responseObserver.onCompleted(); - - } catch (Exception e) { - logger.error("Error beginning transaction: {}", e.getMessage(), e); - responseObserver.onError(Status.INTERNAL - .withDescription("Failed to begin transaction: " + e.getMessage()) - .asException()); - } - } - - @Override - public void commitTransaction(CommitTransactionRequest request, - StreamObserver responseObserver) { - try { - Database database = activeTransactions.remove(request.getTransaction().getTransactionId()); - if (database == null) { - throw new IllegalArgumentException("Invalid transaction ID"); - } - - // Commit transaction - database.commit(); - - CommitTransactionResponse response = CommitTransactionResponse.newBuilder() - .setSuccess(true) - .setMessage("Transaction committed successfully") - .setTimestamp(System.currentTimeMillis()) - .build(); - - responseObserver.onNext(response); - responseObserver.onCompleted(); - - } catch (Exception e) { - logger.error("Error committing transaction: {}", e.getMessage(), e); - responseObserver.onError(Status.INTERNAL - .withDescription("Failed to commit transaction: " + e.getMessage()) - .asException()); - } - } - - @Override - public void rollbackTransaction(RollbackTransactionRequest request, - StreamObserver responseObserver) { - try { - Database database = activeTransactions.remove(request.getTransaction().getTransactionId()); - if (database == null) { - throw new IllegalArgumentException("Invalid transaction ID"); - } - - // Rollback transaction - database.rollback(); - - RollbackTransactionResponse response = RollbackTransactionResponse.newBuilder() - .setSuccess(true) - .setMessage("Transaction rolled back successfully") - .build(); - - responseObserver.onNext(response); - responseObserver.onCompleted(); - - } catch (Exception e) { - logger.error("Error rolling back transaction: {}", e.getMessage(), e); - responseObserver.onError(Status.INTERNAL - .withDescription("Failed to rollback transaction: " + e.getMessage()) - .asException()); - } - } - - @Override - public void streamQuery(StreamQueryRequest request, StreamObserver responseObserver) { - final ServerCallStreamObserver scso = - (ServerCallStreamObserver) responseObserver; - - final AtomicBoolean cancelled = new AtomicBoolean(false); - scso.setOnCancelHandler(() -> cancelled.set(true)); - - try { - Database database = getDatabase(request.getDatabase(), request.getCredentials()); - final int batchSize = Math.max(1, request.getBatchSize()); - - // Dispatch on mode - switch (request.getRetrievalMode()) { - case MATERIALIZE_ALL -> streamMaterialized(database, request, batchSize, scso, cancelled); - case PAGED -> streamPaged(database, request, batchSize, scso, cancelled); - case CURSOR -> streamCursor(database, request, batchSize, scso, cancelled); - default -> streamCursor(database, request, batchSize, scso, cancelled); - } - - if (!cancelled.get()) scso.onCompleted(); - } catch (Exception e) { - if (!cancelled.get()) { - responseObserver.onError( - Status.INTERNAL.withDescription("Stream query failed: " + e.getMessage()).asException()); - } - } - } - - /** Mode 1 (existing behavior-ish): run once and iterate results, batching as we go. */ - private void streamCursor(Database db, StreamQueryRequest req, int batchSize, ServerCallStreamObserver scso, - AtomicBoolean cancelled) { - long running = 0L; - QueryResult.Builder batch = QueryResult.newBuilder(); - int inBatch = 0; - - try (ResultSet rs = db.query("sql", req.getQuery(), convertParameters(req.getParametersMap()))) { - while (rs.hasNext()) { - if (cancelled.get()) - return; - waitUntilReady(scso, cancelled); - - Result r = rs.next(); - if (!r.isElement()) - continue; - - com.arcadedb.database.Record rec = r.getElement().get(); - batch.addRecords(convertToGrpcRecord(rec)); - inBatch++; - running++; - - if (inBatch >= batchSize) { - safeOnNext(scso, cancelled, - batch.setTotalRecordsInBatch(inBatch).setRunningTotalEmitted(running).setIsLastBatch(false).build()); - batch = QueryResult.newBuilder(); - inBatch = 0; - } - } - - if (!cancelled.get() && inBatch > 0) { - safeOnNext(scso, cancelled, batch.setTotalRecordsInBatch(inBatch).setRunningTotalEmitted(running).setIsLastBatch(true).build()); - } - } - } - - /** Mode 2: materialize everything first (simple, but can be memory-heavy). */ - private void streamMaterialized(Database db, StreamQueryRequest req, int batchSize, ServerCallStreamObserver scso, - AtomicBoolean cancelled) { - final java.util.ArrayList all = new java.util.ArrayList<>(); - try (ResultSet rs = db.query("sql", req.getQuery(), convertParameters(req.getParametersMap()))) { - while (rs.hasNext()) { - if (cancelled.get()) - return; - Result r = rs.next(); - if (!r.isElement()) - continue; - all.add(convertToGrpcRecord(r.getElement().get())); - } - } - - long running = 0L; - for (int i = 0; i < all.size(); i += batchSize) { - if (cancelled.get()) - return; - waitUntilReady(scso, cancelled); - - int end = Math.min(i + batchSize, all.size()); - QueryResult.Builder b = QueryResult.newBuilder(); - for (int j = i; j < end; j++) - b.addRecords(all.get(j)); - - running += (end - i); - safeOnNext(scso, cancelled, - b.setTotalRecordsInBatch(end - i).setRunningTotalEmitted(running).setIsLastBatch(end == all.size()).build()); - } - } - - /** Mode 3: only fetch one page’s worth of rows per emission via LIMIT/SKIP. */ - private void streamPaged(Database db, StreamQueryRequest req, int batchSize, ServerCallStreamObserver scso, - AtomicBoolean cancelled) { - - final String pagedSql = wrapWithSkipLimit(req.getQuery()); // see helper below - int offset = 0; - long running = 0L; - - while (true) { - if (cancelled.get()) - return; - waitUntilReady(scso, cancelled); - - java.util.Map params = new java.util.HashMap<>(convertParameters(req.getParametersMap())); - params.put("_skip", offset); - params.put("_limit", batchSize); - - int count = 0; - QueryResult.Builder b = QueryResult.newBuilder(); - - try (ResultSet rs = db.query("sql", pagedSql, params)) { - while (rs.hasNext()) { - if (cancelled.get()) - return; - Result r = rs.next(); - if (!r.isElement()) - continue; - b.addRecords(convertToGrpcRecord(r.getElement().get())); - count++; - } - } - - if (count == 0) - return; // no more rows - - running += count; - boolean last = count < batchSize; - - safeOnNext(scso, cancelled, b.setTotalRecordsInBatch(count).setRunningTotalEmitted(running).setIsLastBatch(last).build()); - - if (last) - return; - offset += batchSize; - } - } - - /** Wrap arbitrary SQL so we can safely inject LIMIT/SKIP outside. */ - private String wrapWithSkipLimit(String originalSql) { - - // Minimal defensive approach; you can do a real parser if needed. - - // ArcadeDB: SELECT FROM (...) [ORDER BY ...] SKIP :_skip LIMIT :_limit - - return "SELECT FROM (" + originalSql + ") ORDER BY @rid SKIP :_skip LIMIT :_limit"; - } - - private void safeOnNext(ServerCallStreamObserver scso, AtomicBoolean cancelled, QueryResult payload) { - if (cancelled.get()) - return; - try { - scso.onNext(payload); - } - catch (StatusRuntimeException e) { - if (e.getStatus().getCode() == Status.Code.CANCELLED) { - cancelled.set(true); - return; - } - throw e; - } - } - - private void waitUntilReady(ServerCallStreamObserver scso, AtomicBoolean cancelled) { - // Skip if you’re okay with best-effort pushes; otherwise honor transport readiness - if (scso.isReady()) - return; - while (!scso.isReady()) { - if (cancelled.get()) - return; - // avoid burning CPU: - try { - Thread.sleep(1); - } - catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - return; - } - } - } - - // --- 1) Unary bulk --- - @Override - public void bulkInsert(BulkInsertRequest req, StreamObserver resp) { - final InsertOptions opts = defaults(req.getOptions()); - try (InsertContext ctx = new InsertContext(opts)) { - insertRows(ctx, req.getRowsList().iterator()); - ctx.flushCommit(true); - resp.onNext(ctx.summary()); - resp.onCompleted(); - } - catch (Exception e) { - resp.onError(Status.INTERNAL.withDescription("bulkInsert: " + e.getMessage()).asException()); - } - } - - // --- 2) Client-streaming; single summary at end --- - @Override - public StreamObserver insertStream(StreamObserver resp) { - final ServerCallStreamObserver call = (ServerCallStreamObserver) resp; - call.disableAutoInboundFlowControl(); - - // You can pass options via the first chunk's session_id, or set server - // defaults; here: server defaults - final InsertOptions opts = defaults(InsertOptions.newBuilder().build()); - final InsertContext ctx; - try { - ctx = new InsertContext(opts); - } - catch (Exception e) { - resp.onError(Status.FAILED_PRECONDITION.withDescription(e.getMessage()).asException()); - return new StreamObserver<>() { - public void onNext(InsertChunk c) { - } - - public void onError(Throwable t) { - } - - public void onCompleted() { - } - }; - } - - return new StreamObserver<>() { - @Override - public void onNext(InsertChunk c) { - insertRows(ctx, c.getRowsList().iterator()); - call.request(1); - } - - @Override - public void onError(Throwable t) { - ctx.closeQuietly(); - } - - @Override - public void onCompleted() { - try { - ctx.flushCommit(true); - resp.onNext(ctx.summary()); - resp.onCompleted(); - } - catch (Exception e) { - resp.onError(Status.INTERNAL.withDescription("insertStream: " + e.getMessage()).asException()); - } - finally { - ctx.closeQuietly(); - } - } - }; - } - - // --- 3) Bi-di with per-batch ACKs --- - @Override - public StreamObserver insertBidirectional(StreamObserver resp) { - final ServerCallStreamObserver call = (ServerCallStreamObserver) resp; - call.disableAutoInboundFlowControl(); - - final AtomicReference ref = new AtomicReference<>(); - return new StreamObserver<>() { - @Override - public void onNext(InsertRequest req) { - switch (req.getMsgCase()) { - case START -> { - InsertOptions opts = defaults(req.getStart().getOptions()); - InsertContext ctx = new InsertContext(opts); - ref.set(ctx); - sessionWatermark.put(ctx.sessionId, 0L); - resp.onNext(InsertResponse.newBuilder().setStarted(Started.newBuilder().setSessionId(ctx.sessionId).build()).build()); - call.request(1); - } - case CHUNK -> { - InsertContext ctx = require(ref.get(), "session not started"); - InsertChunk c = req.getChunk(); - - // Idempotent replay guard - long hi = sessionWatermark.getOrDefault(ctx.sessionId, 0L); - if (c.getChunkSeq() <= hi) { - resp.onNext(InsertResponse.newBuilder() - .setBatchAck(BatchAck.newBuilder().setSessionId(ctx.sessionId).setChunkSeq(c.getChunkSeq()).build()).build()); - call.request(1); - return; - } - - BatchCounts counts = insertRows(ctx, c.getRowsList().iterator()); - sessionWatermark.put(ctx.sessionId, c.getChunkSeq()); - - resp.onNext(InsertResponse.newBuilder() - .setBatchAck(BatchAck.newBuilder().setSessionId(ctx.sessionId).setChunkSeq(c.getChunkSeq()) - .setInserted(counts.inserted).setUpdated(counts.updated).setIgnored(counts.ignored).setFailed(counts.failed) - .addAllErrors(counts.errors)) - .build()); - call.request(1); - } - case COMMIT -> { - InsertContext ctx = require(ref.get(), "session not started"); - try { - ctx.flushCommit(true); - resp.onNext(InsertResponse.newBuilder().setCommitted(Committed.newBuilder().setSummary(ctx.summary()).build()).build()); - resp.onCompleted(); - } - catch (Exception e) { - resp.onError(Status.INTERNAL.withDescription("commit: " + e.getMessage()).asException()); - } - finally { - sessionWatermark.remove(ctx.sessionId); - ctx.closeQuietly(); - } - } - case MSG_NOT_SET -> { - /* ignore */ } - } - } - - @Override - public void onError(Throwable t) { - InsertContext ctx = ref.get(); - if (ctx != null) - ctx.closeQuietly(); - } - - @Override - public void onCompleted() { - /* if client half-closes without COMMIT, choose policy */ } - }; - } - - // ---------- Core insert plumbing ---------- - - private static final class BatchCounts { - long inserted, updated, ignored, failed; - final List errors = new ArrayList<>(); - - void addError(long rowIndex, String code, String msg, String field) { - failed++; - errors.add(RowError.newBuilder().setRowIndex(rowIndex).setCode(code).setMessage(msg).setField(field).build()); - } - } - - private BatchCounts insertRows(InsertContext ctx, Iterator it) { - - BatchCounts c = new BatchCounts(); - - int inBatch = 0; - - while (it.hasNext()) { - Record r = it.next(); - ctx.received++; - try { - - MutableDocument doc = ctx.db.newDocument(ctx.opts.getTargetClass()); - - // map fields: - applyGrpcRecordToDocument(r, doc); - - switch (ctx.opts.getConflictMode()) { - case CONFLICT_ABORT -> { - doc.save(); - c.inserted++; - } - case CONFLICT_IGNORE -> { - try { - doc.save(); - c.inserted++; - } - catch (Exception dup) { - c.ignored++; - } - } - case CONFLICT_UPDATE -> { - if (tryUpsert(ctx, doc)) - c.updated++; - else { - doc.save(); - c.inserted++; - } - } - } - - } - catch (ValidationException ve) { - c.addError(ctx.received - 1, "VALIDATION", ve.getMessage(), ""); - } - catch (DuplicatedKeyException dke) { - if (ctx.opts.getConflictMode() == ConflictMode.CONFLICT_ABORT) { - c.addError(ctx.received - 1, "CONFLICT", dke.getMessage(), ""); - } - else { - c.ignored++; - } - } - catch (Exception e) { - c.addError(ctx.received - 1, "DB_ERROR", e.getMessage(), ""); - } - - inBatch++; - if (ctx.opts.getTransactionMode() == TransactionMode.PER_BATCH && inBatch >= ctx.serverBatchSize()) { - ctx.flushCommit(false); - inBatch = 0; - } - else if (ctx.opts.getTransactionMode() == TransactionMode.PER_ROW) { - ctx.flushCommit(false); - inBatch = 0; - } - } - - return c; - } - - private void applyGrpcRecordToDocument(Record r, MutableDocument doc) { - // Example if proto = message Record { map - // properties = 1; } - if (hasMethod(r, "getPropertiesMap")) { - @SuppressWarnings("unchecked") - Map props = (Map) invokeNoArg(r, "getPropertiesMap"); - props.forEach((k, v) -> doc.set(k, toJava(v))); - return; - } - - // Example if proto = message Record { repeated Property properties = 1; message - // Property { string key=1; google.protobuf.Value value=2; } } - if (hasMethod(r, "getPropertiesList")) { - @SuppressWarnings("unchecked") - List list = (List) invokeNoArg(r, "getPropertiesList"); - for (Object p : list) { - String key = (String) invokeNoArg(p, "getKey"); - com.google.protobuf.Value val = (com.google.protobuf.Value) invokeNoArg(p, "getValue"); - doc.set(key, toJava(val)); - } - return; - } - - // Fallback: if your proto DOES have fields map after all - if (hasMethod(r, "getFieldsMap")) { - @SuppressWarnings("unchecked") - Map fields = (Map) invokeNoArg(r, "getFieldsMap"); - fields.forEach((k, v) -> doc.set(k, toJava(v))); - } - } - - // tiny reflection helpers (compile-time safe across proto variants) - private static boolean hasMethod(Object o, String name) { - try { - o.getClass().getMethod(name); - return true; - } - catch (NoSuchMethodException e) { - return false; - } - } - - private static Object invokeNoArg(Object o, String name) { - try { - return o.getClass().getMethod(name).invoke(o); - } - catch (Exception e) { - throw new RuntimeException(e); - } - } - - private boolean tryUpsert(InsertContext ctx, MutableDocument doc) { - - if (ctx.keyCols.isEmpty()) - return false; - - String where = String.join(" AND ", ctx.keyCols.stream().map(k -> k + " = ?").toList()); - - Object[] params = ctx.keyCols.stream().map(doc::get).toArray(); - - try (ResultSet rs = ctx.db.query("sql", "SELECT FROM " + ctx.opts.getTargetClass() + " WHERE " + where, params)) { - - if (!rs.hasNext()) - return false; - - var res = rs.next(); - - if (!res.isElement()) - return false; - - var existing = res.getElement().get(); - - MutableDocument m = (MutableDocument) existing.asDocument(true); - - for (String col : ctx.updateCols) { - m.set(col, doc.get(col)); - } - - m.save(); - - return true; - } - } - - private static Object toJava(Value v) { - return switch (v.getKindCase()) { - case STRING_VALUE -> v.getStringValue(); - case NUMBER_VALUE -> v.getNumberValue(); - case BOOL_VALUE -> v.getBoolValue(); - case NULL_VALUE -> null; - case STRUCT_VALUE -> new HashMap<>(v.getStructValue().getFieldsMap()); // or nested doc - case LIST_VALUE -> v.getListValue().getValuesList().stream().map(ArcadeDbGrpcService::toJava).toList(); - case KIND_NOT_SET -> null; - }; - } - - private InsertOptions defaults(InsertOptions in) { - InsertOptions.Builder b = in.toBuilder(); - if (in.getServerBatchSize() == 0) - b.setServerBatchSize(1000); - if (in.getTransactionMode() == TransactionMode.UNRECOGNIZED) - b.setTransactionMode(TransactionMode.PER_BATCH); - return b.build(); - } - - private static T require(T v, String msg) { - if (v == null) - throw Status.FAILED_PRECONDITION.withDescription(msg).asRuntimeException(); - return v; - } - - private static Timestamp ts(long ms) { - return Timestamp.newBuilder().setSeconds(ms / 1000).setNanos((int) ((ms % 1000) * 1_000_000)).build(); - } - - private final class InsertContext implements AutoCloseable { - - final InsertOptions opts; - final Database db; - final String sessionId = UUID.randomUUID().toString(); - final List keyCols; - final List updateCols; - long received = 0; - final long started = System.currentTimeMillis(); - - InsertContext(InsertOptions opts) { - this.opts = opts; - this.db = getDatabase(opts.getDatabase(), opts.getCredentials()); // your existing helper - this.keyCols = opts.getKeyColumnsList(); - this.updateCols = opts.getUpdateColumnsOnConflictList(); - if (opts.getTransactionMode() == TransactionMode.PER_STREAM) - db.begin(); - } - - int serverBatchSize() { - return opts.getServerBatchSize(); - } - - void flushCommit(boolean end) { - if (opts.getValidateOnly()) { - if (end && opts.getTransactionMode() == TransactionMode.PER_STREAM) - db.rollback(); - return; - } - if (opts.getTransactionMode() == TransactionMode.PER_BATCH) { - db.commit(); - db.begin(); - } - else if (end && opts.getTransactionMode() == TransactionMode.PER_STREAM) - db.commit(); - } - - InsertSummary summary() { - return InsertSummary.newBuilder().setReceived(received) // set by caller - // inserted/updated/ignored/failed are tracked in BatchCounts -> you can total - // them here if you prefer - .setStartedAt(ts(started)).setFinishedAt(ts(System.currentTimeMillis())).build(); - } - - @Override - public void close() { - } - - void closeQuietly() { - try { - close(); - } - catch (Exception ignore) { - } - } - } - - @Override - public void ping(Empty request, StreamObserver responseObserver) { - PingResponse response = PingResponse.newBuilder() - .setTimestamp(System.currentTimeMillis()) - .setMessage("pong") - .build(); - - responseObserver.onNext(response); - responseObserver.onCompleted(); - } - - @Override - public void getServerStatus(Empty request, StreamObserver responseObserver) { - try { - ServerStatusResponse response = ServerStatusResponse.newBuilder() - .setVersion("25.8.1-SNAPSHOT") - .setStatus("RUNNING") - .setUptimeMs(getUptime()) - .setActiveConnections(getActiveConnections()) - .putMetrics("active_transactions", String.valueOf(activeTransactions.size())) - .build(); - - responseObserver.onNext(response); - responseObserver.onCompleted(); - - } catch (Exception e) { - logger.error("Error getting server status: {}", e.getMessage(), e); - responseObserver.onError(Status.INTERNAL - .withDescription("Failed to get server status: " + e.getMessage()) - .asException()); - } - } - - // Helper methods - - private Database getDatabase(String databaseName, DatabaseCredentials credentials) { - - // Validate credentials - validateCredentials(credentials); - - // Use the same approach as Postgres/Redis plugins - if (arcadeServer != null) { - - // This is how other plugins do it - get the already-open database - Database db = arcadeServer.getDatabase(databaseName); - - logger.debug("getDatabase(): db = {} isOpen = {}", db, db.isOpen()); - - if (db != null) { - return db; - } - } - - // Check if database is already in the pool - String poolKey = databaseName; - Database database = databasePool.get(poolKey); - - if (database != null && database.isOpen()) { - // Return existing open database - return database; - } - - // Create new database connection - synchronized (databasePool) { - // Double-check after acquiring lock - database = databasePool.get(poolKey); - if (database != null && database.isOpen()) { - return database; - } - - // Create database factory for the specific database - DatabaseFactory dbFactory = new DatabaseFactory(databasePath + "/" + databaseName); - - try { - // Open database - ArcadeDB requires MODE parameter - if (dbFactory.exists()) { - // Try READ_ONLY first to avoid conflicts - try { - database = dbFactory.open(ComponentFile.MODE.READ_ONLY); - } catch (Exception e) { - // If READ_ONLY fails, try READ_WRITE - logger.debug("Opening database in READ_WRITE mode: {}", databaseName); - database = dbFactory.open(ComponentFile.MODE.READ_WRITE); - } - } else { - // Create if it doesn't exist - database = dbFactory.create(); - } - - // Add to pool - databasePool.put(poolKey, database); - return database; - - } catch (Exception e) { - logger.error("Failed to open database: {}", databaseName, e); - throw new RuntimeException("Cannot open database: " + databaseName + " - " + e.getMessage(), e); - } - } - } - - private void validateCredentials(DatabaseCredentials credentials) { - // Implement credential validation logic - // This is a placeholder - integrate with ArcadeDB's security system - if (credentials == null || credentials.getUsername().isEmpty()) { - throw new IllegalArgumentException("Invalid credentials"); - } - } - - private Map convertParameters(Map protoParams) { - Map params = new HashMap<>(); - for (Map.Entry entry : protoParams.entrySet()) { - params.put(entry.getKey(), convertFromProtobufValue(entry.getValue())); - } - return params; - } - - private Object convertFromProtobufValue(Value value) { - switch (value.getKindCase()) { - case NULL_VALUE: - return null; - case NUMBER_VALUE: - return value.getNumberValue(); - case STRING_VALUE: - return value.getStringValue(); - case BOOL_VALUE: - return value.getBoolValue(); - case STRUCT_VALUE: - return convertFromProtobufValues(value.getStructValue().getFieldsMap()); - case LIST_VALUE: - return value.getListValue().getValuesList().stream() - .map(this::convertFromProtobufValue) - .toArray(); - default: - return null; - } - } - - private Map convertFromProtobufValues(Map values) { - Map result = new HashMap<>(); - for (Map.Entry entry : values.entrySet()) { - result.put(entry.getKey(), convertFromProtobufValue(entry.getValue())); - } - return result; - } - - private Record convertToGrpcRecord(com.arcadedb.database.Record dbRecord) { - Record.Builder builder = Record.newBuilder() - .setRid(dbRecord.getIdentity().toString()); - - // Handle different record types - if (dbRecord instanceof Document) { - Document doc = (Document) dbRecord; - builder.setType(doc.getTypeName()); - - // Convert properties - Set properties = doc.getPropertyNames(); - for (String property : properties) { - Object value = doc.get(property); - if (value != null) { - builder.putProperties(property, convertToProtobufValue(value)); - } - } - } else if (dbRecord instanceof Vertex) { - Vertex vertex = (Vertex) dbRecord; - builder.setType(vertex.getTypeName()); - - // Convert properties - Set properties = vertex.getPropertyNames(); - for (String property : properties) { - Object value = vertex.get(property); - if (value != null) { - builder.putProperties(property, convertToProtobufValue(value)); - } - } - } else if (dbRecord instanceof Edge) { - Edge edge = (Edge) dbRecord; - builder.setType(edge.getTypeName()); - - // Convert properties - Set properties = edge.getPropertyNames(); - for (String property : properties) { - Object value = edge.get(property); - if (value != null) { - builder.putProperties(property, convertToProtobufValue(value)); - } - } - } - - return builder.build(); - } - - private Value convertToProtobufValue(Object value) { - if (value == null) { - return Value.newBuilder().setNullValue(com.google.protobuf.NullValue.NULL_VALUE).build(); - } else if (value instanceof Boolean) { - return Value.newBuilder().setBoolValue((Boolean) value).build(); - } else if (value instanceof Number) { - return Value.newBuilder().setNumberValue(((Number) value).doubleValue()).build(); - } else if (value instanceof String) { - return Value.newBuilder().setStringValue((String) value).build(); - } else { - // For complex objects, convert to string - return Value.newBuilder().setStringValue(value.toString()).build(); - } - } - - private String generateTransactionId() { - return "tx_" + System.nanoTime(); - } - - private long getUptime() { - // Implement uptime calculation - return System.currentTimeMillis(); - } - - private int getActiveConnections() { - // Implement active connections count - return 0; - } -} \ No newline at end of file diff --git a/server/TMP/grpc/CompressionAwareService.java b/server/TMP/grpc/CompressionAwareService.java deleted file mode 100644 index 860501d231..0000000000 --- a/server/TMP/grpc/CompressionAwareService.java +++ /dev/null @@ -1,86 +0,0 @@ -package com.arcadedb.server.grpc; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.grpc.stub.ServerCallStreamObserver; -import io.grpc.stub.StreamObserver; - - -/** - * Enhanced compression-aware service wrapper - */ -class CompressionAwareService { - private static final Logger logger = LoggerFactory.getLogger(CompressionAwareService.class); - - /** - * Force compression for a specific response - */ - public static void setResponseCompression(StreamObserver responseObserver, String compression) { - if (responseObserver instanceof ServerCallStreamObserver) { - ServerCallStreamObserver serverObserver = (ServerCallStreamObserver) responseObserver; - serverObserver.setCompression(compression); - logger.debug("Set response compression to: {}", compression); - } - } - - /** - * Check if current request was compressed (call from service method) - */ - public static boolean isCurrentRequestCompressed() { - GrpcCompressionInterceptor.CompressionInfo info = - GrpcCompressionInterceptor.COMPRESSION_KEY.get(); - return info != null && info.requestCompressed; - } - - /** - * Get current request compression encoding - */ - public static String getCurrentRequestEncoding() { - GrpcCompressionInterceptor.CompressionInfo info = - GrpcCompressionInterceptor.COMPRESSION_KEY.get(); - return info != null ? info.requestEncoding : "identity"; - } - - /** - * Get compression statistics for monitoring - */ - public static class CompressionStats { - private long compressedRequests = 0; - private long uncompressedRequests = 0; - private long compressedResponses = 0; - private long uncompressedResponses = 0; - - public synchronized void recordRequest(boolean compressed) { - if (compressed) { - compressedRequests++; - } else { - uncompressedRequests++; - } - } - - public synchronized void recordResponse(boolean compressed) { - if (compressed) { - compressedResponses++; - } else { - uncompressedResponses++; - } - } - - public synchronized String getStats() { - long totalRequests = compressedRequests + uncompressedRequests; - long totalResponses = compressedResponses + uncompressedResponses; - - double reqCompressionRate = totalRequests > 0 ? - (compressedRequests * 100.0 / totalRequests) : 0; - double respCompressionRate = totalResponses > 0 ? - (compressedResponses * 100.0 / totalResponses) : 0; - - return String.format( - "Compression Stats - Requests: %.1f%% (%d/%d), Responses: %.1f%% (%d/%d)", - reqCompressionRate, compressedRequests, totalRequests, - respCompressionRate, compressedResponses, totalResponses - ); - } - } -} \ No newline at end of file diff --git a/server/TMP/grpc/GrpcAuthInterceptor.java b/server/TMP/grpc/GrpcAuthInterceptor.java deleted file mode 100644 index 06274dcab1..0000000000 --- a/server/TMP/grpc/GrpcAuthInterceptor.java +++ /dev/null @@ -1,142 +0,0 @@ -package com.arcadedb.server.grpc; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.arcadedb.server.security.ServerSecurity; - -import io.grpc.Context; -import io.grpc.Contexts; -import io.grpc.Metadata; -import io.grpc.ServerCall; -import io.grpc.ServerCallHandler; -import io.grpc.ServerInterceptor; -import io.grpc.Status; - -/** - * Authentication interceptor for gRPC requests - */ -class GrpcAuthInterceptor implements ServerInterceptor { - private static final Logger logger = LoggerFactory.getLogger(GrpcAuthInterceptor.class); - - private static final String BEARER_TYPE = "Bearer"; - private static final Metadata.Key AUTHORIZATION_HEADER = - Metadata.Key.of("authorization", Metadata.ASCII_STRING_MARSHALLER); - private static final Metadata.Key USER_HEADER = - Metadata.Key.of("x-arcade-user", Metadata.ASCII_STRING_MARSHALLER); - private static final Metadata.Key PASSWORD_HEADER = - Metadata.Key.of("x-arcade-password", Metadata.ASCII_STRING_MARSHALLER); - private static final Metadata.Key DATABASE_HEADER = - Metadata.Key.of("x-arcade-database", Metadata.ASCII_STRING_MARSHALLER); - - private final ServerSecurity security; - private final boolean securityEnabled; - - public GrpcAuthInterceptor(ServerSecurity security) { - this.security = security; - // Check if security is enabled by checking if it's not null and has users configured - this.securityEnabled = (security != null && security.getUsers() != null && !security.getUsers().isEmpty()); - } - - @Override - public ServerCall.Listener interceptCall( - ServerCall call, - Metadata headers, - ServerCallHandler next) { - - String methodName = call.getMethodDescriptor().getFullMethodName(); - - // Skip auth for health check and reflection - if (methodName.startsWith("grpc.health.") || - methodName.startsWith("grpc.reflection.")) { - return next.startCall(call, headers); - } - - // If security is not enabled, allow all requests - if (!securityEnabled) { - return next.startCall(call, headers); - } - - try { - // Get database name from header (required for authentication) - String database = headers.get(DATABASE_HEADER); - if (database == null || database.isEmpty()) { - database = "default"; // Use default database if not specified - } - - // Try Bearer token authentication first - String authorization = headers.get(AUTHORIZATION_HEADER); - if (authorization != null && authorization.startsWith(BEARER_TYPE)) { - String token = authorization.substring(BEARER_TYPE.length()).trim(); - if (!validateToken(token, database)) { - call.close(Status.UNAUTHENTICATED.withDescription("Invalid token"), new Metadata()); - return new ServerCall.Listener() {}; - } - } else { - // Try basic authentication - String username = headers.get(USER_HEADER); - String password = headers.get(PASSWORD_HEADER); - - if (username == null || password == null) { - // No authentication provided for secured server - call.close(Status.UNAUTHENTICATED.withDescription("Authentication required"), new Metadata()); - return new ServerCall.Listener() {}; - } else { - // Validate credentials - if (!validateCredentials(username, password, database)) { - call.close(Status.UNAUTHENTICATED.withDescription("Invalid credentials"), new Metadata()); - return new ServerCall.Listener() {}; - } - // Add user to context - Context context = Context.current().withValue(USER_CONTEXT_KEY, username); - return Contexts.interceptCall(context, call, headers, next); - } - } - - // Add user context for downstream processing - Context context = Context.current(); - if (authorization != null) { - context = context.withValue(USER_CONTEXT_KEY, extractUserFromToken(authorization)); - } - - return Contexts.interceptCall(context, call, headers, next); - - } catch (Exception e) { - logger.error("Authentication error", e); - call.close(Status.INTERNAL.withDescription("Authentication error"), new Metadata()); - return new ServerCall.Listener() {}; - } - } - - private boolean validateToken(String token, String database) { - // Implement token validation logic - // This could integrate with JWT, OAuth2, or custom token validation - // For now, this is a placeholder - return true; - } - - private boolean validateCredentials(String username, String password, String database) { - if (security == null) { - return true; // No security configured - } - - try { - // ArcadeDB's authenticate method requires database name as well - // Returns a SecurityUser object if authentication succeeds, null otherwise - Object authenticatedUser = security.authenticate(username, password, database); - return authenticatedUser != null; - } catch (Exception e) { - logger.error("Failed to authenticate user: {} for database: {}", username, database, e); - return false; - } - } - - private String extractUserFromToken(String authorization) { - // Extract user information from token - // This is a placeholder - implement based on your token format - return "authenticated-user"; - } - - // Context key for storing authenticated user - public static final Context.Key USER_CONTEXT_KEY = Context.key("user"); -} \ No newline at end of file diff --git a/server/TMP/grpc/GrpcCompressionInterceptor.java b/server/TMP/grpc/GrpcCompressionInterceptor.java deleted file mode 100644 index c1221db2f9..0000000000 --- a/server/TMP/grpc/GrpcCompressionInterceptor.java +++ /dev/null @@ -1,83 +0,0 @@ -package com.arcadedb.server.grpc; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.grpc.Context; -import io.grpc.Contexts; -import io.grpc.ForwardingServerCall; -import io.grpc.Metadata; -import io.grpc.ServerCall; -import io.grpc.ServerCallHandler; -import io.grpc.ServerInterceptor; - -/** - * Compression interceptor that can force compression based on configuration - */ -class GrpcCompressionInterceptor implements ServerInterceptor { - private static final Logger logger = LoggerFactory.getLogger(GrpcCompressionInterceptor.class); - - // Context key to store compression info - public static final Context.Key COMPRESSION_KEY = Context.key("compression-info"); - - private final boolean forceCompression; - private final String compressionType; - private final int minMessageSizeForCompression; - - public GrpcCompressionInterceptor(boolean forceCompression, String compressionType, int minMessageSizeBytes) { - this.forceCompression = forceCompression; - this.compressionType = compressionType != null ? compressionType : "gzip"; - this.minMessageSizeForCompression = minMessageSizeBytes; - } - - @Override - public ServerCall.Listener interceptCall( - ServerCall call, - Metadata headers, - ServerCallHandler next) { - - String methodName = call.getMethodDescriptor().getFullMethodName(); - - // Check if request is compressed - String requestEncoding = headers.get(Metadata.Key.of("grpc-encoding", Metadata.ASCII_STRING_MARSHALLER)); - boolean requestCompressed = requestEncoding != null && !requestEncoding.equals("identity"); - - // Store compression info in context - CompressionInfo compressionInfo = new CompressionInfo(requestCompressed, requestEncoding); - Context context = Context.current().withValue(COMPRESSION_KEY, compressionInfo); - - // Wrap the call to control compression - ServerCall compressedCall = new ForwardingServerCall.SimpleForwardingServerCall(call) { - private boolean compressionSet = false; - - @Override - public void sendMessage(RespT message) { - if (!compressionSet && forceCompression) { - // Force compression for this response - if (this.getMethodDescriptor().getType().serverSendsOneMessage()) { - // For unary calls, we can set compression - setMessageCompression(true); - compressionSet = true; - logger.debug("Forced {} compression for method: {}", compressionType, methodName); - } - } - super.sendMessage(message); - } - }; - - return Contexts.interceptCall(context, compressedCall, headers, next); - } - - /** - * Helper class to store compression information - */ - public static class CompressionInfo { - public final boolean requestCompressed; - public final String requestEncoding; - - public CompressionInfo(boolean requestCompressed, String requestEncoding) { - this.requestCompressed = requestCompressed; - this.requestEncoding = requestEncoding; - } - } -} \ No newline at end of file diff --git a/server/TMP/grpc/GrpcLoggingInterceptor.java b/server/TMP/grpc/GrpcLoggingInterceptor.java deleted file mode 100644 index 1dcdc0145f..0000000000 --- a/server/TMP/grpc/GrpcLoggingInterceptor.java +++ /dev/null @@ -1,81 +0,0 @@ -package com.arcadedb.server.grpc; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import io.grpc.ForwardingServerCall; -import io.grpc.Metadata; -import io.grpc.ServerCall; -import io.grpc.ServerCallHandler; -import io.grpc.ServerInterceptor; -import io.grpc.Status; - -/** - * Logging interceptor for gRPC requests with compression detection - */ -class GrpcLoggingInterceptor implements ServerInterceptor { - private static final Logger logger = LoggerFactory.getLogger(GrpcLoggingInterceptor.class); - - private static final Metadata.Key GRPC_ENCODING_KEY = - Metadata.Key.of("grpc-encoding", Metadata.ASCII_STRING_MARSHALLER); - private static final Metadata.Key GRPC_ACCEPT_ENCODING_KEY = - Metadata.Key.of("grpc-accept-encoding", Metadata.ASCII_STRING_MARSHALLER); - - @Override - public ServerCall.Listener interceptCall( - ServerCall call, - Metadata headers, - ServerCallHandler next) { - - String methodName = call.getMethodDescriptor().getFullMethodName(); - long startTime = System.currentTimeMillis(); - - // Check if client sent compressed request - String requestEncoding = headers.get(GRPC_ENCODING_KEY); - String acceptEncoding = headers.get(GRPC_ACCEPT_ENCODING_KEY); - boolean requestCompressed = requestEncoding != null && !requestEncoding.equals("identity"); - boolean clientAcceptsCompression = acceptEncoding != null && acceptEncoding.contains("gzip"); - - logger.debug("gRPC call started: {} (request compression: {}, client accepts: {})", - methodName, - requestCompressed ? requestEncoding : "none", - clientAcceptsCompression ? acceptEncoding : "none"); - - ServerCall wrappedCall = new ForwardingServerCall.SimpleForwardingServerCall(call) { - private String responseCompression = "none"; - - @Override - public void sendHeaders(Metadata headers) { - // Check if we're sending compressed response - String encoding = headers.get(GRPC_ENCODING_KEY); - if (encoding != null && !encoding.equals("identity")) { - responseCompression = encoding; - } - super.sendHeaders(headers); - } - - @Override - public void close(Status status, Metadata trailers) { - long duration = System.currentTimeMillis() - startTime; - - // Add compression info to trailers for client visibility - trailers.put(Metadata.Key.of("x-grpc-compression-used", Metadata.ASCII_STRING_MARSHALLER), - responseCompression); - trailers.put(Metadata.Key.of("x-grpc-request-compressed", Metadata.ASCII_STRING_MARSHALLER), - String.valueOf(requestCompressed)); - - if (status.isOk()) { - logger.debug("gRPC call completed: {} ({}ms, req-compression: {}, resp-compression: {})", - methodName, duration, - requestCompressed ? requestEncoding : "none", - responseCompression); - } else { - logger.warn("gRPC call failed: {} - {} ({}ms)", methodName, status, duration); - } - super.close(status, trailers); - } - }; - - return next.startCall(wrappedCall, headers); - } -} \ No newline at end of file diff --git a/server/TMP/grpc/GrpcMetricsInterceptor.java b/server/TMP/grpc/GrpcMetricsInterceptor.java deleted file mode 100644 index 100a1ee665..0000000000 --- a/server/TMP/grpc/GrpcMetricsInterceptor.java +++ /dev/null @@ -1,78 +0,0 @@ -package com.arcadedb.server.grpc; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.arcadedb.server.ArcadeDBServer; - -import io.grpc.ForwardingServerCall; -import io.grpc.Metadata; -import io.grpc.ServerCall; -import io.grpc.ServerCallHandler; -import io.grpc.ServerInterceptor; -import io.grpc.Status; -import io.micrometer.core.instrument.Counter; -import io.micrometer.core.instrument.MeterRegistry; -import io.micrometer.core.instrument.Timer; -import io.micrometer.core.instrument.simple.SimpleMeterRegistry; - -/** - * Metrics interceptor for gRPC requests using Micrometer - */ -class GrpcMetricsInterceptor implements ServerInterceptor { - - private static final Logger logger = LoggerFactory.getLogger(GrpcMetricsInterceptor.class); - - private final MeterRegistry meterRegistry; - private final Counter requestCounter; - private final Counter errorCounter; - private final Timer requestTimer; - - public GrpcMetricsInterceptor(ArcadeDBServer server) { - // Try to get existing meter registry from server, or create a simple one - this.meterRegistry = new SimpleMeterRegistry(); // In production, integrate with server's meter registry - - this.requestCounter = Counter.builder("grpc.requests.total") - .description("Total number of gRPC requests") - .register(meterRegistry); - - this.errorCounter = Counter.builder("grpc.errors.total") - .description("Total number of gRPC errors") - .register(meterRegistry); - - this.requestTimer = Timer.builder("grpc.request.duration") - .description("gRPC request duration") - .register(meterRegistry); - } - - @Override - public ServerCall.Listener interceptCall( - ServerCall call, - Metadata headers, - ServerCallHandler next) { - - String methodName = call.getMethodDescriptor().getFullMethodName(); - Timer.Sample sample = Timer.start(meterRegistry); - - requestCounter.increment(); - - ServerCall wrappedCall = new ForwardingServerCall.SimpleForwardingServerCall(call) { - @Override - public void close(Status status, Metadata trailers) { - sample.stop(requestTimer); - - if (!status.isOk()) { - errorCounter.increment(); - } - - // Add metrics as trailers for observability - trailers.put(Metadata.Key.of("grpc-metrics-method", Metadata.ASCII_STRING_MARSHALLER), methodName); - trailers.put(Metadata.Key.of("grpc-metrics-status", Metadata.ASCII_STRING_MARSHALLER), status.getCode().toString()); - - super.close(status, trailers); - } - }; - - return next.startCall(wrappedCall, headers); - } -} \ No newline at end of file diff --git a/server/TMP/grpc/GrpcServerPlugin.java b/server/TMP/grpc/GrpcServerPlugin.java deleted file mode 100644 index b180acba34..0000000000 --- a/server/TMP/grpc/GrpcServerPlugin.java +++ /dev/null @@ -1,386 +0,0 @@ -package com.arcadedb.server.grpc; - -import java.io.File; -import java.io.IOException; -import java.util.concurrent.TimeUnit; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.arcadedb.ContextConfiguration; -import com.arcadedb.server.ArcadeDBServer; -import com.arcadedb.server.ServerPlugin; -import com.arcadedb.server.security.ServerSecurity; - -import io.grpc.CompressorRegistry; -import io.grpc.DecompressorRegistry; -import io.grpc.Grpc; -import io.grpc.InsecureServerCredentials; -import io.grpc.Server; -import io.grpc.ServerBuilder; -import io.grpc.ServerCredentials; -import io.grpc.TlsServerCredentials; -import io.grpc.health.v1.HealthCheckResponse; -import io.grpc.protobuf.services.HealthStatusManager; -import io.grpc.protobuf.services.ProtoReflectionService; -import io.grpc.xds.XdsServerBuilder; - -/** - * ArcadeDB gRPC Server Plugin - * - * Configuration options: - * - grpc.enabled: Enable/disable gRPC server (default: true) - * - grpc.port: Port for standard gRPC server (default: 50051) - * - grpc.host: Host to bind (default: 0.0.0.0) - * - grpc.mode: Server mode - "standard", "xds", or "both" (default: standard) - * - grpc.xds.port: Port for XDS server (default: 50052) - * - grpc.tls.enabled: Enable TLS (default: false) - * - grpc.tls.cert: Path to TLS certificate - * - grpc.tls.key: Path to TLS private key - * - grpc.maxMessageSize: Max message size in MB (default: 100) - * - grpc.reflection.enabled: Enable gRPC reflection (default: true) - * - grpc.health.enabled: Enable health checking (default: true) - */ -public class GrpcServerPlugin implements ServerPlugin { - - private static final Logger logger = LoggerFactory.getLogger(GrpcServerPlugin.class); - - private ArcadeDBServer arcadeServer; - private Server grpcServer; - private Server xdsServer; - private HealthStatusManager healthManager; - private ArcadeDbGrpcService grpcService; // Keep reference for cleanup - private Thread shutdownHook; - - // Configuration keys as simple strings - private static final String CONFIG_PREFIX = "arcadedb.grpc."; - private static final String CONFIG_ENABLED = CONFIG_PREFIX + "enabled"; - private static final String CONFIG_PORT = CONFIG_PREFIX + "port"; - private static final String CONFIG_HOST = CONFIG_PREFIX + "host"; - private static final String CONFIG_MODE = CONFIG_PREFIX + "mode"; - private static final String CONFIG_XDS_PORT = CONFIG_PREFIX + "xds.port"; - private static final String CONFIG_TLS_ENABLED = CONFIG_PREFIX + "tls.enabled"; - private static final String CONFIG_TLS_CERT = CONFIG_PREFIX + "tls.cert"; - private static final String CONFIG_TLS_KEY = CONFIG_PREFIX + "tls.key"; - private static final String CONFIG_MAX_MESSAGE_SIZE = CONFIG_PREFIX + "maxMessageSize"; - private static final String CONFIG_REFLECTION_ENABLED = CONFIG_PREFIX + "reflection.enabled"; - private static final String CONFIG_HEALTH_ENABLED = CONFIG_PREFIX + "health.enabled"; - private static final String CONFIG_COMPRESSION_ENABLED = CONFIG_PREFIX + "compression.enabled"; - private static final String CONFIG_COMPRESSION_FORCE = CONFIG_PREFIX + "compression.force"; - private static final String CONFIG_COMPRESSION_TYPE = CONFIG_PREFIX + "compression.type"; - - @Override - public void configure(ArcadeDBServer server, ContextConfiguration configuration) { - this.arcadeServer = server; - } - - @Override - public void startService() { - ContextConfiguration config = arcadeServer.getConfiguration(); - - // Get configuration values with defaults - boolean enabled = getConfigBoolean(config, CONFIG_ENABLED, true); - if (!enabled) { - logger.info("gRPC server is disabled"); - return; - } - - String mode = getConfigString(config, CONFIG_MODE, "standard").toLowerCase(); - - try { - switch (mode) { - case "standard": - startStandardServer(config); - break; - case "xds": - startXdsServer(config); - break; - case "both": - startStandardServer(config); - startXdsServer(config); - break; - default: - logger.error("Invalid gRPC mode: {}. Use 'standard', 'xds', or 'both'", mode); - } - - registerShutdownHook(); - - } catch (IOException e) { - logger.error("Failed to start gRPC server", e); - throw new RuntimeException("Failed to start gRPC server", e); - } - } - - private void startStandardServer(ContextConfiguration config) throws IOException { - int port = getConfigInt(config, CONFIG_PORT, 50051); - String host = getConfigString(config, CONFIG_HOST, "0.0.0.0"); - - ServerBuilder serverBuilder; - - // Configure TLS if enabled - if (getConfigBoolean(config, CONFIG_TLS_ENABLED, false)) { - serverBuilder = configureStandardTls(port, config); - } else { - serverBuilder = ServerBuilder.forPort(port); - } - - // Configure the server - configureServer(serverBuilder, config); - - grpcServer = serverBuilder.build().start(); - - // Build status message - StringBuilder status = new StringBuilder(); - status.append("gRPC server started on ").append(host).append(":").append(port); - status.append(" (mode: standard"); - - if (getConfigBoolean(config, CONFIG_TLS_ENABLED, false)) { - status.append(", TLS enabled"); - } - - if (getConfigBoolean(config, CONFIG_COMPRESSION_ENABLED, true)) { - status.append(", compression: "); - if (getConfigBoolean(config, CONFIG_COMPRESSION_FORCE, false)) { - status.append("forced-").append(getConfigString(config, CONFIG_COMPRESSION_TYPE, "gzip")); - } else { - status.append("available"); - } - } - - status.append(")"); - logger.info(status.toString()); - } - - private void startXdsServer(ContextConfiguration config) throws IOException { - int port = getConfigInt(config, CONFIG_XDS_PORT, 50052); - - // XDS server for service mesh integration - // XdsServerBuilder requires ServerCredentials - XdsServerBuilder xdsBuilder = XdsServerBuilder.forPort(port, InsecureServerCredentials.create()); - - // Configure the XDS server as a ServerBuilder - configureServer(xdsBuilder, config); - - xdsServer = xdsBuilder.build().start(); - - logger.info("gRPC XDS server started on port {} (xDS management enabled)", port); - } - - private void configureServer(ServerBuilder serverBuilder, ContextConfiguration config) { - // Get database directory path - String databasePath = arcadeServer.getRootPath() + File.separator + "databases"; - - // Create the main service - ArcadeDbGrpcService mainService = new ArcadeDbGrpcService(databasePath, arcadeServer); - - // Add the main service - serverBuilder.addService(mainService); - - // Add health service if enabled - if (getConfigBoolean(config, CONFIG_HEALTH_ENABLED, true)) { - healthManager = new HealthStatusManager(); - serverBuilder.addService(healthManager.getHealthService()); - - // Set initial health status - healthManager.setStatus( - ArcadeDbGrpcService.class.getName(), - HealthCheckResponse.ServingStatus.SERVING - ); - } - - // Add reflection service if enabled - if (getConfigBoolean(config, CONFIG_REFLECTION_ENABLED, true)) { - serverBuilder.addService(ProtoReflectionService.newInstance()); - } - - serverBuilder.compressorRegistry(CompressorRegistry.getDefaultInstance()).decompressorRegistry(DecompressorRegistry.getDefaultInstance()); - - // Configure max message size - int maxMessageSizeMB = getConfigInt(config, CONFIG_MAX_MESSAGE_SIZE, 100); - serverBuilder.maxInboundMessageSize(maxMessageSizeMB * 1024 * 1024); - - // Add interceptors for logging, metrics, auth, etc. - serverBuilder.intercept(new GrpcLoggingInterceptor()); - serverBuilder.intercept(new GrpcMetricsInterceptor(arcadeServer)); - - // Add compression interceptor if force compression is enabled - if (getConfigBoolean(config, CONFIG_COMPRESSION_FORCE, false)) { - String compressionType = getConfigString(config, CONFIG_COMPRESSION_TYPE, "gzip"); - serverBuilder.intercept(new GrpcCompressionInterceptor(true, compressionType, 1024)); - } - - // Add authentication interceptor if security is configured - ServerSecurity serverSecurity = arcadeServer.getSecurity(); - if (serverSecurity != null) { - serverBuilder.intercept(new GrpcAuthInterceptor(serverSecurity)); - } - } - - private ServerBuilder configureStandardTls(int port, ContextConfiguration config) { - String certPath = getConfigString(config, CONFIG_TLS_CERT, null); - String keyPath = getConfigString(config, CONFIG_TLS_KEY, null); - - if (certPath == null || keyPath == null) { - logger.warn("TLS enabled but certificate or key path not provided. Falling back to insecure."); - return ServerBuilder.forPort(port); - } - - File certFile = new File(certPath); - File keyFile = new File(keyPath); - - if (!certFile.exists() || !keyFile.exists()) { - logger.warn("TLS certificate or key file not found. Falling back to insecure."); - return ServerBuilder.forPort(port); - } - - try { - ServerCredentials credentials = TlsServerCredentials.create(certFile, keyFile); - // Use Grpc.newServerBuilderForPort for TLS - return Grpc.newServerBuilderForPort(port, credentials); - } catch (Exception e) { - logger.error("Failed to configure TLS", e); - return ServerBuilder.forPort(port); - } - } - - private ServerCredentials configureTlsCredentials(ContextConfiguration config) { - String certPath = getConfigString(config, CONFIG_TLS_CERT, null); - String keyPath = getConfigString(config, CONFIG_TLS_KEY, null); - - if (certPath == null || keyPath == null) { - logger.warn("TLS enabled but certificate or key path not provided. Using insecure credentials."); - return InsecureServerCredentials.create(); - } - - File certFile = new File(certPath); - File keyFile = new File(keyPath); - - if (!certFile.exists() || !keyFile.exists()) { - logger.warn("TLS certificate or key file not found. Using insecure credentials."); - return InsecureServerCredentials.create(); - } - - try { - return TlsServerCredentials.create(certFile, keyFile); - } catch (Exception e) { - logger.error("Failed to configure TLS credentials", e); - return InsecureServerCredentials.create(); - } - } - - private void registerShutdownHook() { - shutdownHook = new Thread(() -> { - logger.info("Shutting down gRPC server..."); - stopService(); - }); - Runtime.getRuntime().addShutdownHook(shutdownHook); - } - - @Override - public void stopService() { - try { - // Update health status to NOT_SERVING - if (healthManager != null) { - healthManager.setStatus( - ArcadeDbGrpcService.class.getName(), - HealthCheckResponse.ServingStatus.NOT_SERVING - ); - } - - // Close the gRPC service to release database connections - if (grpcService != null) { - try { - grpcService.close(); - logger.info("gRPC service closed and database connections released"); - } catch (Exception e) { - logger.error("Error closing gRPC service", e); - } - } - - // Shutdown servers gracefully - if (grpcServer != null) { - grpcServer.shutdown(); - if (!grpcServer.awaitTermination(30, TimeUnit.SECONDS)) { - grpcServer.shutdownNow(); - grpcServer.awaitTermination(5, TimeUnit.SECONDS); - } - logger.info("Standard gRPC server stopped"); - } - - if (xdsServer != null) { - xdsServer.shutdown(); - if (!xdsServer.awaitTermination(30, TimeUnit.SECONDS)) { - xdsServer.shutdownNow(); - xdsServer.awaitTermination(5, TimeUnit.SECONDS); - } - logger.info("XDS gRPC server stopped"); - } - - // Remove shutdown hook if it exists - if (shutdownHook != null) { - try { - Runtime.getRuntime().removeShutdownHook(shutdownHook); - } catch (IllegalStateException e) { - // Already shutting down - } - } - - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - logger.error("Interrupted while shutting down gRPC server", e); - } - } - - /** - * Get the status of the gRPC servers - */ - public ServerStatus getStatus() { - return new ServerStatus( - grpcServer != null && !grpcServer.isShutdown(), - xdsServer != null && !xdsServer.isShutdown(), - grpcServer != null ? grpcServer.getPort() : -1, - xdsServer != null ? xdsServer.getPort() : -1 - ); - } - - // Configuration helper methods - private String getConfigString(ContextConfiguration config, String key, String defaultValue) { - - return config.getValueAsString(key, defaultValue); - } - - private int getConfigInt(ContextConfiguration config, String key, int defaultValue) { - String value = getConfigString(config, key, null); - if (value != null) { - try { - return Integer.parseInt(value); - } catch (NumberFormatException e) { - logger.warn("Invalid integer value for {}: {}", key, value); - } - } - return defaultValue; - } - - private boolean getConfigBoolean(ContextConfiguration config, String key, boolean defaultValue) { - String value = getConfigString(config, key, null); - if (value != null) { - return Boolean.parseBoolean(value); - } - return defaultValue; - } - - public static class ServerStatus { - public final boolean standardServerRunning; - public final boolean xdsServerRunning; - public final int standardPort; - public final int xdsPort; - - public ServerStatus(boolean standardServerRunning, boolean xdsServerRunning, - int standardPort, int xdsPort) { - this.standardServerRunning = standardServerRunning; - this.xdsServerRunning = xdsServerRunning; - this.standardPort = standardPort; - this.xdsPort = xdsPort; - } - } -} \ No newline at end of file From e267b035832aba36737861a44bae3edb16fc1e5c Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Mon, 1 Sep 2025 21:11:35 -0700 Subject: [PATCH 16/35] Checkpoint Commit --- .../remote/grpc/RemoteGrpcDatabase.java | 21 +++- .../remote/grpc/utils/ProtoUtils.java | 110 ++++++++++++++++++ 2 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 grpc-client/src/main/java/com/arcadedb/remote/grpc/utils/ProtoUtils.java diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java index 77d7f32a88..d1729a2926 100644 --- a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java @@ -34,6 +34,7 @@ import com.arcadedb.remote.RemoteImmutableVertex; import com.arcadedb.remote.RemoteSchema; import com.arcadedb.remote.RemoteTransactionExplicitLock; +import com.arcadedb.remote.grpc.utils.ProtoUtils; import com.arcadedb.server.grpc.ArcadeDbServiceGrpc; import com.arcadedb.server.grpc.BeginTransactionRequest; import com.arcadedb.server.grpc.BeginTransactionResponse; @@ -1062,10 +1063,21 @@ public String createVertex(String cls, Map props, long timeoutMs public boolean updateRecord(String rid, Map props, long timeoutMs) { - // Build the nested proto Record payload for a PATCH com.arcadedb.server.grpc.Record record = com.arcadedb.server.grpc.Record.newBuilder().putAllProperties(convertParamsToProto(props)) .build(); + return updateRecord(rid, record, timeoutMs); + } + + public boolean updateRecord(String rid, Record dbRecord, long timeoutMs) { + + com.arcadedb.server.grpc.Record record = ProtoUtils.toProtoRecord(dbRecord); + + return updateRecord(rid, record, timeoutMs); + } + + private boolean updateRecord(String rid, com.arcadedb.server.grpc.Record record, long timeoutMs) { + // Build the Update request (use setRecord(...) for full replace instead) com.arcadedb.server.grpc.UpdateRecordRequest req = com.arcadedb.server.grpc.UpdateRecordRequest.newBuilder().setDatabase(getName()) .setRid(rid).setRecord(record) // <<<< oneof {record|partial} @@ -1087,7 +1099,7 @@ public boolean updateRecord(String rid, Map props, long timeoutM throw new RuntimeException("Failed to update record", e); } } - + public boolean deleteRecord(String rid, long timeoutMs) { var req = com.arcadedb.server.grpc.DeleteRecordRequest.newBuilder().setDatabase(getName()).setRid(rid).setCredentials(buildCredentials()) .build(); @@ -1861,15 +1873,20 @@ else if (typeName.contains("Edge") || typeName.startsWith("E_")) { } private Map convertParamsToProto(Map params) { + Map protoParams = new HashMap<>(); + for (Map.Entry entry : params.entrySet()) { protoParams.put(entry.getKey(), objectToValue(entry.getValue())); } + return protoParams; } private Value objectToValue(Object obj) { + Value.Builder builder = Value.newBuilder(); + if (obj == null) { builder.setNullValue(com.google.protobuf.NullValue.NULL_VALUE); } diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/utils/ProtoUtils.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/utils/ProtoUtils.java new file mode 100644 index 0000000000..d3bf6127b1 --- /dev/null +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/utils/ProtoUtils.java @@ -0,0 +1,110 @@ +package com.arcadedb.remote.grpc.utils; +import java.util.Map; + +import com.arcadedb.database.Document; +import com.arcadedb.database.Record; +import com.arcadedb.graph.Vertex; +import com.arcadedb.server.grpc.Record.Builder; +import com.google.protobuf.Value; +import com.google.protobuf.util.Values; + +public class ProtoUtils { + + /** + * Converts an ArcadeDB Record into a gRPC Record message. + * + * @param rec ArcadeDB record (Document, Vertex, or Edge) + * @return Proto Record message + */ + public static com.arcadedb.server.grpc.Record toProtoRecord(Record rec) { + + if (rec == null) { + throw new IllegalArgumentException("Record cannot be null"); + } + + Builder builder = com.arcadedb.server.grpc.Record.newBuilder() + .setRid(rec.getIdentity().toString()); + + // Determine logical type/class name + + String typeName = null; + + if (rec instanceof Vertex) { + + typeName = rec.asVertex().getType().getName(); + } + + if (rec instanceof Document) { + + typeName = rec.asDocument().getType().getName(); + } + + if (typeName != null && !typeName.isEmpty()) { + + builder.setType(typeName); + } + + // Convert properties + if (rec instanceof Document doc) { + for (String propName : doc.getPropertyNames()) { + Object value = doc.get(propName); + builder.putProperties(propName, toProtoValue(value)); + } + } + + return builder.build(); + } + + /** + * Converts a Java Object into a protobuf Value for Record.properties map. + * + * Supports primitives, strings, numbers, booleans, lists, and maps. + * + * @param value Object value + * @return Protobuf Value + */ + private static Value toProtoValue(Object value) { + + if (value == null) { + return Values.ofNull(); + } + if (value instanceof String s) { + return Values.of(s); + } + if (value instanceof Integer i) { + return Values.of(i); + } + if (value instanceof Long l) { + return Values.of(l); + } + if (value instanceof Float f) { + return Values.of(f); + } + if (value instanceof Double d) { + return Values.of(d); + } + if (value instanceof Boolean b) { + return Values.of(b); + } + if (value instanceof java.util.Date dt) { + // Serialize date as ISO8601 string + return Values.of(dt.toInstant().toString()); + } + if (value instanceof Iterable list) { + var lv = com.google.protobuf.ListValue.newBuilder(); + for (Object item : list) { + lv.addValues(toProtoValue(item)); + } + return Values.of(lv.build()); + } + if (value instanceof Map map) { + var structBuilder = com.google.protobuf.Struct.newBuilder(); + for (Map.Entry entry : map.entrySet()) { + structBuilder.putFields(String.valueOf(entry.getKey()), toProtoValue(entry.getValue())); + } + return Values.of(structBuilder.build()); + } + // fallback to string + return Values.of(String.valueOf(value)); + } +} \ No newline at end of file From a7ba6bef8ed29d2ed65eff0e681c573bc96de80a Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Mon, 1 Sep 2025 22:22:54 -0700 Subject: [PATCH 17/35] Checkpoint Commit --- .../remote/grpc/RemoteGrpcDatabase.java | 90 ++++++++++--------- .../remote/grpc/utils/ProtoUtils.java | 26 ++++-- 2 files changed, 68 insertions(+), 48 deletions(-) diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java index d1729a2926..9d17d71600 100644 --- a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java @@ -300,37 +300,6 @@ public void rollback() { } } - @Override - public Record lookupByRID(final RID rid, final boolean loadContent) { - - checkDatabaseIsOpen(); - - stats.readRecord.incrementAndGet(); - - if (rid == null) - throw new IllegalArgumentException("Record is null"); - - LookupByRidRequest request = LookupByRidRequest.newBuilder().setDatabase(getName()).setRid(rid.toString()) - .setCredentials(buildCredentials()).build(); - - try { - - LookupByRidResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).lookupByRid(request); - - if (!response.getFound()) { - throw new RecordNotFoundException("Record " + rid + " not found", rid); - } - return grpcRecordToRecord(response.getRecord()); - } - catch (StatusRuntimeException e) { - handleGrpcException(e); - return null; - } - catch (StatusException e) { - handleGrpcException(e); - return null; - } - } @Override public void deleteRecord(final Record record) { @@ -754,7 +723,7 @@ public boolean hasNext() { public Record next() { if (!hasNext()) throw new NoSuchElementException(); - return grpcRecordToRecord(currentBatch.next()); + return grpcRecordToDBRecord(currentBatch.next()); } }; } @@ -813,7 +782,7 @@ public boolean hasNext() { public Record next() { if (!hasNext()) throw new NoSuchElementException(); - return grpcRecordToRecord(currentBatch.next()); + return grpcRecordToDBRecord(currentBatch.next()); } }; } @@ -894,7 +863,7 @@ public boolean hasNext() { List converted = new ArrayList<>(qr.getRecordsCount()); for (com.arcadedb.server.grpc.Record gr : qr.getRecordsList()) { - converted.add(grpcRecordToRecord(gr)); + converted.add(grpcRecordToDBRecord(gr)); } nextBatch = new QueryBatch(converted, n, qr.getRunningTotalEmitted(), qr.getIsLastBatch()); @@ -1015,7 +984,7 @@ public QueryBatch next() { qr = respIter.read(); List converted = new ArrayList<>(qr.getRecordsCount()); for (com.arcadedb.server.grpc.Record gr : qr.getRecordsList()) { - converted.add(grpcRecordToRecord(gr)); + converted.add(grpcRecordToDBRecord(gr)); } return new QueryBatch(converted, qr.getTotalRecordsInBatch(), // int totalInBatch @@ -1080,7 +1049,8 @@ private boolean updateRecord(String rid, com.arcadedb.server.grpc.Record record, // Build the Update request (use setRecord(...) for full replace instead) com.arcadedb.server.grpc.UpdateRecordRequest req = com.arcadedb.server.grpc.UpdateRecordRequest.newBuilder().setDatabase(getName()) - .setRid(rid).setRecord(record) // <<<< oneof {record|partial} + .setRid(rid).setRecord(record) + .setTransaction(TransactionContext.newBuilder().setBegin(true).setCommit(true)) .setCredentials(buildCredentials()).build(); // Call RPC @@ -1154,7 +1124,39 @@ public long countType(final String typeName, final boolean polymorphic) { public Record lookupByRID(final RID rid) { return lookupByRID(rid, true); } + + @Override + public Record lookupByRID(final RID rid, final boolean loadContent) { + + checkDatabaseIsOpen(); + + stats.readRecord.incrementAndGet(); + + if (rid == null) + throw new IllegalArgumentException("Record is null"); + + LookupByRidRequest request = LookupByRidRequest.newBuilder().setDatabase(getName()).setRid(rid.toString()) + .setCredentials(buildCredentials()).build(); + + try { + + LookupByRidResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).lookupByRid(request); + if (!response.getFound()) { + throw new RecordNotFoundException("Record " + rid + " not found", rid); + } + return grpcRecordToDBRecord(response.getRecord()); + } + catch (StatusRuntimeException e) { + handleGrpcException(e); + return null; + } + catch (StatusException e) { + handleGrpcException(e); + return null; + } + } + @Override public boolean existsRecord(RID rid) { stats.existsRecord.incrementAndGet(); @@ -1791,7 +1793,7 @@ public Record next() { if (!hasNext()) { throw new NoSuchElementException(); } - return grpcRecordToRecord(currentBatch.next()); + return grpcRecordToDBRecord(currentBatch.next()); } }; } @@ -1807,7 +1809,7 @@ private ResultSet createGrpcResultSet(ExecuteQueryResponse response) { } private Result grpcRecordToResult(com.arcadedb.server.grpc.Record grpcRecord) { - Record record = grpcRecordToRecord(grpcRecord); + Record record = grpcRecordToDBRecord(grpcRecord); if (record == null) { Map properties = new HashMap<>(); grpcRecord.getPropertiesMap().forEach((k, v) -> properties.put(k, valueToObject(v))); @@ -1816,7 +1818,7 @@ private Result grpcRecordToResult(com.arcadedb.server.grpc.Record grpcRecord) { return new ResultInternal(record); } - private Record grpcRecordToRecord(com.arcadedb.server.grpc.Record grpcRecord) { + private Record grpcRecordToDBRecord(com.arcadedb.server.grpc.Record grpcRecord) { Map map = new HashMap<>(); // Convert properties @@ -1877,7 +1879,15 @@ private Map convertParamsToProto(Map params) { Map protoParams = new HashMap<>(); for (Map.Entry entry : params.entrySet()) { - protoParams.put(entry.getKey(), objectToValue(entry.getValue())); + + Value value = objectToValue(entry.getValue()); + + if (value != null) { + + System.out.println("Converting object to Value: " + entry.getKey() + " " + entry.getValue().getClass() + " -> " + value.hasStructValue()); + } + + protoParams.put(entry.getKey(), value); } return protoParams; diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/utils/ProtoUtils.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/utils/ProtoUtils.java index d3bf6127b1..369c4ad7a4 100644 --- a/grpc-client/src/main/java/com/arcadedb/remote/grpc/utils/ProtoUtils.java +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/utils/ProtoUtils.java @@ -68,35 +68,36 @@ private static Value toProtoValue(Object value) { if (value == null) { return Values.ofNull(); } - if (value instanceof String s) { + + if (value instanceof String s) { return Values.of(s); } + if (value instanceof Integer i) { return Values.of(i); } + if (value instanceof Long l) { return Values.of(l); } + if (value instanceof Float f) { return Values.of(f); } + if (value instanceof Double d) { return Values.of(d); } + if (value instanceof Boolean b) { return Values.of(b); } + if (value instanceof java.util.Date dt) { // Serialize date as ISO8601 string return Values.of(dt.toInstant().toString()); } - if (value instanceof Iterable list) { - var lv = com.google.protobuf.ListValue.newBuilder(); - for (Object item : list) { - lv.addValues(toProtoValue(item)); - } - return Values.of(lv.build()); - } + if (value instanceof Map map) { var structBuilder = com.google.protobuf.Struct.newBuilder(); for (Map.Entry entry : map.entrySet()) { @@ -104,6 +105,15 @@ private static Value toProtoValue(Object value) { } return Values.of(structBuilder.build()); } + + if (value instanceof Iterable list) { + var lv = com.google.protobuf.ListValue.newBuilder(); + for (Object item : list) { + lv.addValues(toProtoValue(item)); + } + return Values.of(lv.build()); + } + // fallback to string return Values.of(String.valueOf(value)); } From 7d9dd9161555277ce98b5b765f82d3e196cc8a18 Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Tue, 2 Sep 2025 08:13:49 -0700 Subject: [PATCH 18/35] Revised updateRecord logic to support EMBEDDED types properly --- .../remote/grpc/RemoteGrpcDatabase.java | 8 +- .../server/grpc/ArcadeDbGrpcService.java | 426 ++++++++++++++---- 2 files changed, 351 insertions(+), 83 deletions(-) diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java index 9d17d71600..80c6db4777 100644 --- a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java @@ -1882,10 +1882,10 @@ private Map convertParamsToProto(Map params) { Value value = objectToValue(entry.getValue()); - if (value != null) { - - System.out.println("Converting object to Value: " + entry.getKey() + " " + entry.getValue().getClass() + " -> " + value.hasStructValue()); - } +// if (value != null) { +// +// System.out.println("Converting object to Value: " + entry.getKey() + " " + entry.getValue().getClass() + " -> " + value.hasStructValue()); +// } protoParams.put(entry.getKey(), value); } diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java index cca9674ea9..ed96c59298 100644 --- a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java @@ -241,7 +241,7 @@ public void createRecord(CreateRecordRequest req, StreamObserver v.set(k, toJavaForProperty(db, dt, k, val))); + props.forEach((k, val) -> v.set(k, toJavaForProperty(db, v, dt, k, val))); v.save(); @@ -282,7 +282,7 @@ public void createRecord(CreateRecordRequest req, StreamObserver { if (!"out".equals(k) && !"in".equals(k)) { - e.set(k, toJavaForProperty(db, dt, k, val)); + e.set(k, toJavaForProperty(db, e, dt, k, val)); } }); @@ -300,7 +300,7 @@ public void createRecord(CreateRecordRequest req, StreamObserver d.set(k, toJavaForProperty(db, dt, k, val))); + props.forEach((k, val) -> d.set(k, toJavaForProperty(db, d, dt, k, val))); d.save(); CreateRecordResponse.Builder b = CreateRecordResponse.newBuilder(); @@ -356,7 +356,9 @@ public void updateRecord(UpdateRecordRequest req, StreamObserver props = req.hasRecord() ? req.getRecord().getPropertiesMap() + : req.hasPartial() ? req.getPartial().getPropertiesMap() : java.util.Collections.emptyMap(); + + // Exclude ArcadeDB system fields during update + + props.forEach((k, v) -> { + String key = k.trim().toLowerCase(); + if (key.equals("@rid") || key.equals("@type") || key.equals("@cat")) { + // Skip internal fields to prevent accidental overwrites + logger.debug("Skipping internal field during update: {}", k); + return; + } + // Perform the update for user-defined fields + mvertex.set(k, toJavaForProperty(dbRef, mvertex, dtype, k, v)); + }); + + mvertex.save(); + } + else if (el instanceof Document) { - // Apply updates + logger.debug("updateRecord(): Processing Document ..."); - final java.util.Map props = req.hasRecord() ? req.getRecord().getPropertiesMap() - : req.hasPartial() ? req.getPartial().getPropertiesMap() : java.util.Collections.emptyMap(); + // Get mutable view for updates (works for docs, vertices, edges) + MutableDocument mdoc = (MutableDocument) el.asDocument(true).modify(); - // apply updates - props.forEach((k, v) -> mdoc.set(k, toJavaForProperty(dbRef, dtype, k, v))); + var dtype = db.getSchema().getType(mdoc.getTypeName()); - mdoc.save(); + // Apply updates + + final java.util.Map props = req.hasRecord() ? req.getRecord().getPropertiesMap() + : req.hasPartial() ? req.getPartial().getPropertiesMap() : java.util.Collections.emptyMap(); + + // Exclude ArcadeDB system fields during update + + props.forEach((k, v) -> { + String key = k.trim().toLowerCase(); + if (key.equals("@rid") || key.equals("@type") || key.equals("@cat")) { + // Skip internal fields to prevent accidental overwrites + logger.debug("Skipping internal field during update: {}", k); + return; + } + // Perform the update for user-defined fields + mdoc.set(k, toJavaForProperty(dbRef, mdoc, dtype, k, v)); + }); + + mdoc.save(); + } // Commit/rollback with proper precedence if (hasTx) { + if (tx.getRollback()) { db.rollback(); } @@ -400,10 +450,13 @@ else if (beganHere) { } resp.onNext(UpdateRecordResponse.newBuilder().setUpdated(true).setSuccess(true).build()); - resp.onCompleted(); + resp.onCompleted(); } catch (Exception e) { + + logger.error("ERROR in updateRecord", e); + try { if (beganHere && db != null) db.rollback(); @@ -411,6 +464,7 @@ else if (beganHere) { catch (Exception ignore) { // ignore rollback errors } + resp.onError(Status.INTERNAL.withDescription("UpdateRecord: " + e.getMessage()).asException()); } } @@ -1229,9 +1283,9 @@ public void onCompleted() { } private boolean tryUpsertVertex(final InsertContext ctx, final com.arcadedb.graph.MutableVertex incoming) { - + var keys = ctx.keyCols; - + if (keys.isEmpty()) return false; @@ -1253,16 +1307,17 @@ private boolean tryUpsertVertex(final InsertContext ctx, final com.arcadedb.grap return false; Vertex v = res.getElement().get().asVertex(); - + MutableVertex existingV = v.modify(); // Apply the updates + for (String col : ctx.updateCols) { existingV.set(col, inDoc.get(col)); } - + existingV.save(); - + return true; } } @@ -1276,13 +1331,13 @@ private boolean tryUpsertDocument(InsertContext ctx, MutableDocument incoming) { String where = String.join(" AND ", keys.stream().map(k -> k + " = ?").toList()); Object[] params = keys.stream().map(incoming::get).toArray(); - + try (ResultSet rs = ctx.db.query("sql", "SELECT FROM " + ctx.opts.getTargetClass() + " WHERE " + where, params)) { - + if (!rs.hasNext()) { - return false; - } - + return false; + } + var res = rs.next(); if (!res.isElement()) return false; @@ -1291,13 +1346,16 @@ private boolean tryUpsertDocument(InsertContext ctx, MutableDocument incoming) { var existing = d.modify(); - for (String col : ctx.updateCols) + // Apply the updates + + for (String col : ctx.updateCols) { existing.set(col, incoming.get(col)); + } existing.save(); return true; - } + } } // ---------- Core insert plumbing ---------- @@ -1868,11 +1926,11 @@ private Record convertToGrpcRecord(com.arcadedb.database.Record dbRecord) { builder.setType(doc.getTypeName()); // Convert properties - Set properties = doc.getPropertyNames(); - for (String property : properties) { - Object value = doc.get(property); + Set propertyNames = doc.getPropertyNames(); + for (String propertyName : propertyNames) { + Object value = doc.get(propertyName); if (value != null) { - builder.putProperties(property, convertToProtobufValue(value)); + builder.putProperties(propertyName, convertToProtobufValue(propertyName, value)); } } } @@ -1881,11 +1939,11 @@ else if (dbRecord instanceof Vertex) { builder.setType(vertex.getTypeName()); // Convert properties - Set properties = vertex.getPropertyNames(); - for (String property : properties) { - Object value = vertex.get(property); + Set propertyNames = vertex.getPropertyNames(); + for (String propertyName : propertyNames) { + Object value = vertex.get(propertyName); if (value != null) { - builder.putProperties(property, convertToProtobufValue(value)); + builder.putProperties(propertyName, convertToProtobufValue(propertyName, value)); } } } @@ -1894,11 +1952,11 @@ else if (dbRecord instanceof Edge) { builder.setType(edge.getTypeName()); // Convert properties - Set properties = edge.getPropertyNames(); - for (String property : properties) { - Object value = edge.get(property); + Set propertyNames = edge.getPropertyNames(); + for (String propertyName : propertyNames) { + Object value = edge.get(propertyName); if (value != null) { - builder.putProperties(property, convertToProtobufValue(value)); + builder.putProperties(propertyName, convertToProtobufValue(propertyName, value)); } } } @@ -1906,84 +1964,294 @@ else if (dbRecord instanceof Edge) { return builder.build(); } - private Value convertToProtobufValue(Object value) { - if (value == null) { - return Value.newBuilder().setNullValue(com.google.protobuf.NullValue.NULL_VALUE).build(); - } - else if (value instanceof Boolean) { - return Value.newBuilder().setBoolValue((Boolean) value).build(); - } - else if (value instanceof Number) { - return Value.newBuilder().setNumberValue(((Number) value).doubleValue()).build(); - } - else if (value instanceof String) { - return Value.newBuilder().setStringValue((String) value).build(); - } - else { - // For complex objects, convert to string - return Value.newBuilder().setStringValue(value.toString()).build(); - } + private Value convertToProtobufValue(String propName, Object value) { + + // logger.debug("convertToProtobufValue(): propName = {}, value.class = {}", + // propName, value.getClass()); + + return toProtoValue(value); } - // Proto Value builder from Java (mirror of toJava) private com.google.protobuf.Value toProtoValue(Object o) { - com.google.protobuf.Value.Builder b = com.google.protobuf.Value.newBuilder(); if (o == null) - return b.setNullValue(com.google.protobuf.NullValue.NULL_VALUE).build(); + return com.google.protobuf.Value.newBuilder().setNullValue(com.google.protobuf.NullValue.NULL_VALUE).build(); + if (o instanceof String s) - return b.setStringValue(s).build(); + return com.google.protobuf.Value.newBuilder().setStringValue(s).build(); + if (o instanceof Boolean b) + return com.google.protobuf.Value.newBuilder().setBoolValue(b).build(); if (o instanceof Number n) - return b.setNumberValue(n.doubleValue()).build(); - if (o instanceof Boolean bo) - return b.setBoolValue(bo).build(); + return com.google.protobuf.Value.newBuilder().setNumberValue(n.doubleValue()).build(); + + if (o instanceof java.math.BigDecimal bd) + return com.google.protobuf.Value.newBuilder().setStringValue(bd.toPlainString()).build(); + + if (o instanceof byte[] bytes) + return com.google.protobuf.Value.newBuilder().setStringValue(java.util.Base64.getEncoder().encodeToString(bytes)).build(); + + // Date/Time → ISO-8601 + if (o instanceof java.util.Date d) + return com.google.protobuf.Value.newBuilder().setStringValue(d.toInstant().toString()).build(); + if (o instanceof java.time.Instant inst) + return com.google.protobuf.Value.newBuilder().setStringValue(inst.toString()).build(); + if (o instanceof java.time.LocalDate ld) + return com.google.protobuf.Value.newBuilder().setStringValue(ld.toString()).build(); + if (o instanceof java.time.LocalDateTime ldt) + return com.google.protobuf.Value.newBuilder().setStringValue(ldt.toString()).build(); + + // Embedded docs (mutable/immutable) + if (o instanceof com.arcadedb.database.Document doc) { + var sb = com.google.protobuf.Struct.newBuilder(); + for (String k : doc.getPropertyNames()) { + sb.putFields(k, toProtoValue(doc.get(k))); + } + return com.google.protobuf.Value.newBuilder().setStructValue(sb).build(); + } + if (o instanceof java.util.Map m) { var sb = com.google.protobuf.Struct.newBuilder(); m.forEach((k, v) -> sb.putFields(String.valueOf(k), toProtoValue(v))); return com.google.protobuf.Value.newBuilder().setStructValue(sb).build(); } - if (o instanceof java.util.List list) { + + if (o instanceof java.util.Collection c) { + var lb = com.google.protobuf.ListValue.newBuilder(); + c.forEach(v -> lb.addValues(toProtoValue(v))); + return com.google.protobuf.Value.newBuilder().setListValue(lb).build(); + } + + if (o.getClass().isArray()) { + int len = java.lang.reflect.Array.getLength(o); var lb = com.google.protobuf.ListValue.newBuilder(); - list.forEach(v -> lb.addValues(toProtoValue(v))); + for (int i = 0; i < len; i++) { + lb.addValues(toProtoValue(java.lang.reflect.Array.get(o, i))); + } return com.google.protobuf.Value.newBuilder().setListValue(lb).build(); } - return b.setStringValue(String.valueOf(o)).build(); + + // RID or Identifiable → string rid + if (o instanceof com.arcadedb.database.Identifiable id) + return com.google.protobuf.Value.newBuilder().setStringValue(id.getIdentity().toString()).build(); + if (o instanceof com.arcadedb.database.RID rid) + return com.google.protobuf.Value.newBuilder().setStringValue(rid.toString()).build(); + + // Fallback + return com.google.protobuf.Value.newBuilder().setStringValue(String.valueOf(o)).build(); + } + + private com.arcadedb.server.grpc.Record toProtoRecordFromDbRecord(com.arcadedb.database.Record rec) { + var doc = rec.asDocument(); // read-only view if not a document + var b = com.arcadedb.server.grpc.Record.newBuilder().setRid(rec.getIdentity() != null ? rec.getIdentity().toString() : "") + .setType(doc != null ? doc.getTypeName() : ""); + if (doc != null) { + for (String name : doc.getPropertyNames()) { + b.putProperties(name, toProtoValue(doc.get(name))); + } + } + return b.build(); } // Apply properties from proto Record to a document/vertex/edge, schema-aware // for EMBEDDED private void applyGrpcRecord(MutableDocument d, com.arcadedb.server.grpc.Record r, Database db, String targetClass) { DocumentType dt = db.getSchema().getType(targetClass); - r.getPropertiesMap().forEach((k, val) -> d.set(k, toJavaForProperty(db, dt, k, val))); + r.getPropertiesMap().forEach((k, val) -> d.set(k, toJavaForProperty(db, d, dt, k, val))); } private void applyGrpcRecord(MutableVertex v, com.arcadedb.server.grpc.Record r, Database db) { DocumentType dt = db.getSchema().getType(v.getTypeName()); - r.getPropertiesMap().forEach((k, val) -> v.set(k, toJavaForProperty(db, dt, k, val))); + r.getPropertiesMap().forEach((k, val) -> v.set(k, toJavaForProperty(db, v, dt, k, val))); } private void applyGrpcRecord(MutableEdge e, com.arcadedb.server.grpc.Record r, Database db, String edgeClass) { DocumentType dt = db.getSchema().getType(edgeClass); - r.getPropertiesMap().forEach((k, val) -> e.set(k, toJavaForProperty(db, dt, k, val))); + r.getPropertiesMap().forEach((k, val) -> e.set(k, toJavaForProperty(db, e, dt, k, val))); + } + + // Generic fallback (schema-agnostic) for nested values + private Object toJavaLoose(com.google.protobuf.Value v) { + return switch (v.getKindCase()) { + case NULL_VALUE -> null; + case STRING_VALUE -> v.getStringValue(); + case NUMBER_VALUE -> v.getNumberValue(); + case BOOL_VALUE -> v.getBoolValue(); + case LIST_VALUE -> v.getListValue().getValuesList().stream().map(this::toJavaLoose).toList(); + case STRUCT_VALUE -> { + var m = new java.util.LinkedHashMap(); + v.getStructValue().getFieldsMap().forEach((k, val) -> m.put(k, toJavaLoose(val))); + yield m; + } + case KIND_NOT_SET -> null; + }; } - // If property type is EMBEDDED and client sent a Struct/Map, create embedded - // document - private Object toJavaForProperty(Database db, DocumentType dt, String key, com.google.protobuf.Value v) { + private Object toJavaForProperty(final Database db, final com.arcadedb.database.MutableDocument parent, final com.arcadedb.schema.DocumentType dtype, + final String propName, final com.google.protobuf.Value v) { + + logger.debug("toJavaForProperty(): propName = {}, dtype = {}, v = {}", propName, dtype, v); + + com.arcadedb.schema.Property p = null; + + try { + + p = (dtype != null) ? dtype.getProperty(propName) : null; + } + catch (com.arcadedb.exception.SchemaException schemaException) { + + //logger.warn("SchemaException: " + schemaException.getMessage()); + } + + if (p == null) + return toJavaLoose(v); + + final var prop = dtype != null ? dtype.getProperty(propName) : null; + final var t = prop != null ? prop.getType() : null; + + switch (t) { + + case STRING: + return (v.getKindCase() == com.google.protobuf.Value.KindCase.STRING_VALUE) ? v.getStringValue() : String.valueOf(toJavaLoose(v)); + + case BOOLEAN: + return (v.getKindCase() == com.google.protobuf.Value.KindCase.BOOL_VALUE) ? v.getBoolValue() + : Boolean.valueOf(String.valueOf(toJavaLoose(v))); + + case BYTE: + // treat as number 0..255 (or string), store as Byte + if (v.getKindCase() == com.google.protobuf.Value.KindCase.NUMBER_VALUE) + return (byte) ((int) v.getNumberValue()); + if (v.getKindCase() == com.google.protobuf.Value.KindCase.STRING_VALUE) + return Byte.valueOf(v.getStringValue()); + return (byte) 0; + + case SHORT: + if (v.getKindCase() == com.google.protobuf.Value.KindCase.NUMBER_VALUE) + return (short) ((int) v.getNumberValue()); + return Short.valueOf(String.valueOf(toJavaLoose(v))); + + case INTEGER: + if (v.getKindCase() == com.google.protobuf.Value.KindCase.NUMBER_VALUE) + return (int) v.getNumberValue(); + return Integer.valueOf(String.valueOf(toJavaLoose(v))); + + case LONG: + if (v.getKindCase() == com.google.protobuf.Value.KindCase.NUMBER_VALUE) + return (long) v.getNumberValue(); + return Long.valueOf(String.valueOf(toJavaLoose(v))); + + case FLOAT: + if (v.getKindCase() == com.google.protobuf.Value.KindCase.NUMBER_VALUE) + return (float) v.getNumberValue(); + return Float.valueOf(String.valueOf(toJavaLoose(v))); + + case DOUBLE: + if (v.getKindCase() == com.google.protobuf.Value.KindCase.NUMBER_VALUE) + return v.getNumberValue(); + return Double.valueOf(String.valueOf(toJavaLoose(v))); + + case DECIMAL: + // use string to preserve precision + if (v.getKindCase() == com.google.protobuf.Value.KindCase.STRING_VALUE) + return new java.math.BigDecimal(v.getStringValue()); + if (v.getKindCase() == com.google.protobuf.Value.KindCase.NUMBER_VALUE) + return java.math.BigDecimal.valueOf(v.getNumberValue()); + return null; + + case DATE: + // prefer ISO (yyyy-MM-dd). Accept epoch millis as number. + if (v.getKindCase() == com.google.protobuf.Value.KindCase.STRING_VALUE) + return java.sql.Date.valueOf(java.time.LocalDate.parse(v.getStringValue())); + if (v.getKindCase() == com.google.protobuf.Value.KindCase.NUMBER_VALUE) + return new java.util.Date((long) v.getNumberValue()); + return null; + + case DATETIME: + case DATETIME_MICROS: + case DATETIME_NANOS: + case DATETIME_SECOND: + if (v.getKindCase() == com.google.protobuf.Value.KindCase.STRING_VALUE) { + // ISO-8601 → Instant + java.time.Instant inst = java.time.Instant.parse(v.getStringValue()); + return java.util.Date.from(inst); + } + if (v.getKindCase() == com.google.protobuf.Value.KindCase.NUMBER_VALUE) { + // epoch millis + return new java.util.Date((long) v.getNumberValue()); + } + return null; + + case BINARY: + // base64 string on the wire + if (v.getKindCase() == com.google.protobuf.Value.KindCase.STRING_VALUE) + return java.util.Base64.getDecoder().decode(v.getStringValue()); + return null; + + case MAP: + // Struct → Map (NOT a document) + if (v.getKindCase() != com.google.protobuf.Value.KindCase.STRUCT_VALUE) + return java.util.Map.of(); + var mm = new java.util.LinkedHashMap(); + v.getStructValue().getFieldsMap().forEach((k, val) -> mm.put(k, toJavaLoose(val))); + return mm; - var p = (dt != null) ? dt.getPropertyIfExists(key) : null; + case LIST: + if (v.getKindCase() != com.google.protobuf.Value.KindCase.LIST_VALUE) + return java.util.List.of(); + return v.getListValue().getValuesList().stream().map(this::toJavaLoose).toList(); - if (p != null && p.getType() == com.arcadedb.schema.Type.EMBEDDED && v.hasStructValue()) { - MutableDocument emb = db.newDocument(dt.getName()); - v.getStructValue().getFieldsMap().forEach((k2, vv) -> emb.set(k2, toJava(vv))); - return emb; + case EMBEDDED: { + + // Must be a STRUCT in proto + if (v.getKindCase() != com.google.protobuf.Value.KindCase.STRUCT_VALUE) { + // choose policy (reject or coerce); rejecting is safer: + throw new IllegalArgumentException("EMBEDDED property '" + propName + "' expects STRUCT"); + } + + String propTypeName = prop.getOfType(); + + logger.debug("EMBEDDED: property '" + propName + "' expects STRUCT of type '" + propTypeName + "'. Received: " + v.getStructValue().getFieldsMap().keySet()); + + DocumentType propType = db.getSchema().getType(propTypeName); + + // Guard: if someone mislinked a vertex/edge type here, fail loudly + if (propType instanceof com.arcadedb.schema.VertexType || propType instanceof com.arcadedb.schema.EdgeType) { + throw new IllegalArgumentException( + "EMBEDDED property '" + propName + "' cannot link to non-document type: " + propType.getName()); + } + + // Create embedded from the parent. Passing null is allowed for “untyped” embedded docs. + final com.arcadedb.database.MutableEmbeddedDocument ed = parent.newEmbeddedDocument(propTypeName, propName); + + // populate fields + v.getStructValue().getFieldsMap().forEach((k, vv) -> ed.set(k, toJavaLoose(vv))); + return ed; + } + + case LINK: { + // Expect "#:" in string; adjust if you send a struct instead + if (v.getKindCase() == com.google.protobuf.Value.KindCase.STRING_VALUE) { + return new com.arcadedb.database.RID(v.getStringValue()); + } + return null; } - return toJava(v); - } + // Collections of links/embedded/arrays → treat as list/map with toJavaLoose + case ARRAY_OF_SHORTS: + case ARRAY_OF_INTEGERS: + case ARRAY_OF_LONGS: + case ARRAY_OF_FLOATS: + case ARRAY_OF_DOUBLES: + return toJavaLoose(v); + + default: + return toJavaLoose(v); + } + } + private String generateTransactionId() { return "tx_" + System.nanoTime(); } - + private long getUptime() { // Implement uptime calculation return System.currentTimeMillis(); From e242927050fb055e0270db3bc4088e4968d6b849 Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Wed, 3 Sep 2025 06:29:10 -0700 Subject: [PATCH 19/35] Added Admin Service to GRPC Plugin --- .../server/grpc/ArcadeDbGrpcAdminService.java | 20 ++++++++----------- .../server/grpc/GrpcServerPlugin.java | 10 +++++++++- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcAdminService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcAdminService.java index c8af198481..eee4d73cca 100644 --- a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcAdminService.java +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcAdminService.java @@ -13,6 +13,8 @@ import com.arcadedb.schema.DocumentType; import com.arcadedb.schema.Schema; import com.arcadedb.schema.VertexType; +import com.arcadedb.server.ArcadeDBServer; +import com.arcadedb.server.security.credential.CredentialsValidator; import io.grpc.Status; import io.grpc.stub.StreamObserver; @@ -26,10 +28,11 @@ public class ArcadeDbGrpcAdminService extends ArcadeDbAdminServiceGrpc.ArcadeDbAdminServiceImplBase { // Replace with your concrete server class if needed - private final Object server; + private final ArcadeDBServer server; private final CredentialsValidator credentialsValidator; - public ArcadeDbGrpcAdminService(final Object server, final CredentialsValidator credentialsValidator) { + public ArcadeDbGrpcAdminService(final ArcadeDBServer server, CredentialsValidator credentialsValidator) { + this.server = Objects.requireNonNull(server, "server"); this.credentialsValidator = Objects.requireNonNull(credentialsValidator, "credentialsValidator"); } @@ -325,16 +328,16 @@ public void deleteUser(DeleteUserRequest req, StreamObserver // ------------------------------------------------------------------------------------ private void authenticate(DatabaseCredentials creds) { + if (creds == null) throw new SecurityException("Authentication required"); final String user = creds.getUsername(); // matches your proto (not getUser()) final String pass = creds.getPassword(); + if (user == null || user.isBlank()) throw new SecurityException("Authentication required"); - if (!credentialsValidator.authenticateServer(user, pass)) { - throw new SecurityException("Invalid credentials"); - } + credentialsValidator.validateCredentials(user, pass); } /** @@ -518,11 +521,4 @@ private int safeBinaryPort() { } return -1; } - - // ------------------------------------------------------------------------------------ - // Auth hook - // ------------------------------------------------------------------------------------ - public interface CredentialsValidator { - boolean authenticateServer(String user, String password); - } } \ No newline at end of file diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcServerPlugin.java b/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcServerPlugin.java index 41b1a25b48..d3d5855aa9 100644 --- a/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcServerPlugin.java +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcServerPlugin.java @@ -11,6 +11,7 @@ import com.arcadedb.server.ArcadeDBServer; import com.arcadedb.server.ServerPlugin; import com.arcadedb.server.security.ServerSecurity; +import com.arcadedb.server.security.credential.DefaultCredentialsValidator; import io.grpc.CompressorRegistry; import io.grpc.DecompressorRegistry; @@ -173,7 +174,8 @@ private void startXdsServer(ContextConfiguration config) throws IOException { } private void configureServer(ServerBuilder serverBuilder, ContextConfiguration config) { - // Get database directory path + + // Get database directory path String databasePath = arcadeServer.getRootPath() + File.separator + "databases"; // Create the main service @@ -182,6 +184,12 @@ private void configureServer(ServerBuilder serverBuilder, ContextConfiguratio // Add the main service serverBuilder.addService(mainService); + // Create the Admin service + ArcadeDbGrpcAdminService adminService = new ArcadeDbGrpcAdminService(arcadeServer, new DefaultCredentialsValidator()); + + // Add the Admin service + serverBuilder.addService(adminService); + // Add health service if enabled if (getConfigBoolean(config, CONFIG_HEALTH_ENABLED, true)) { healthManager = new HealthStatusManager(); From ff26b2398b49acaedf8cd617b65fff2b141a36e9 Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Thu, 4 Sep 2025 17:38:06 -0700 Subject: [PATCH 20/35] Introduce GrpcValue and GrpcRecord constructs --- .../remote/grpc/RemoteGrpcDatabase.java | 965 ++++-------- .../remote/grpc/utils/ProtoUtils.java | 359 ++++- .../remote/grpc/ArcadeDbHTTPTvsGRPCBench.java | 4 +- grpc/pom.xml | 27 +- grpc/src/main/proto/arcadedb-server.proto | 80 +- .../server/grpc/ArcadeDbGrpcService.java | 1303 +++++++++++++---- 6 files changed, 1715 insertions(+), 1023 deletions(-) diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java index 80c6db4777..8716a51975 100644 --- a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java @@ -1,7 +1,6 @@ package com.arcadedb.remote.grpc; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; @@ -36,10 +35,15 @@ import com.arcadedb.remote.RemoteTransactionExplicitLock; import com.arcadedb.remote.grpc.utils.ProtoUtils; import com.arcadedb.server.grpc.ArcadeDbServiceGrpc; +import com.arcadedb.server.grpc.BatchAck; import com.arcadedb.server.grpc.BeginTransactionRequest; import com.arcadedb.server.grpc.BeginTransactionResponse; +import com.arcadedb.server.grpc.BulkInsertRequest; +import com.arcadedb.server.grpc.Commit; import com.arcadedb.server.grpc.CommitTransactionRequest; import com.arcadedb.server.grpc.CommitTransactionResponse; +import com.arcadedb.server.grpc.CreateRecordRequest; +import com.arcadedb.server.grpc.CreateRecordResponse; import com.arcadedb.server.grpc.DatabaseCredentials; import com.arcadedb.server.grpc.DeleteRecordRequest; import com.arcadedb.server.grpc.DeleteRecordResponse; @@ -47,16 +51,25 @@ import com.arcadedb.server.grpc.ExecuteCommandResponse; import com.arcadedb.server.grpc.ExecuteQueryRequest; import com.arcadedb.server.grpc.ExecuteQueryResponse; +import com.arcadedb.server.grpc.GrpcRecord; +import com.arcadedb.server.grpc.GrpcValue; +import com.arcadedb.server.grpc.InsertChunk; import com.arcadedb.server.grpc.InsertOptions; +import com.arcadedb.server.grpc.InsertRequest; +import com.arcadedb.server.grpc.InsertResponse; +import com.arcadedb.server.grpc.InsertSummary; import com.arcadedb.server.grpc.LookupByRidRequest; import com.arcadedb.server.grpc.LookupByRidResponse; +import com.arcadedb.server.grpc.PropertiesUpdate; import com.arcadedb.server.grpc.QueryResult; import com.arcadedb.server.grpc.RollbackTransactionRequest; import com.arcadedb.server.grpc.RollbackTransactionResponse; +import com.arcadedb.server.grpc.Start; import com.arcadedb.server.grpc.StreamQueryRequest; import com.arcadedb.server.grpc.TransactionContext; import com.arcadedb.server.grpc.TransactionIsolation; -import com.google.protobuf.Struct; +import com.arcadedb.server.grpc.UpdateRecordRequest; +import com.arcadedb.server.grpc.UpdateRecordResponse; import com.google.protobuf.Value; import io.grpc.CallCredentials; @@ -69,6 +82,9 @@ import io.grpc.StatusException; import io.grpc.StatusRuntimeException; import io.grpc.stub.BlockingClientCall; +import io.grpc.stub.ClientCallStreamObserver; +import io.grpc.stub.ClientResponseObserver; +import io.grpc.stub.StreamObserver; /** * Remote Database implementation using gRPC protocol instead of HTTP. Extends @@ -155,25 +171,7 @@ public void thisUsesUnstableApi() { } protected CallCredentials createCredentials() { - return new CallCredentials() { - @Override - public void applyRequestMetadata(RequestInfo requestInfo, Executor appExecutor, MetadataApplier applier) { - Metadata headers = new Metadata(); - headers.put(Metadata.Key.of("username", Metadata.ASCII_STRING_MARSHALLER), userName); - headers.put(Metadata.Key.of("password", Metadata.ASCII_STRING_MARSHALLER), userPassword); - headers.put(Metadata.Key.of("x-arcade-user", Metadata.ASCII_STRING_MARSHALLER), userName); - headers.put(Metadata.Key.of("x-arcade-password", Metadata.ASCII_STRING_MARSHALLER), userPassword); - headers.put(Metadata.Key.of("x-arcade-database", Metadata.ASCII_STRING_MARSHALLER), databaseName); - applier.apply(headers); - } - - // x-arcade-user: root" -H "x-arcade-password: oY9uU2uJ8nD8iY7t" -H - // "x-arcade-database: local_shakeiq_curonix_poc-app" - @Override - public void thisUsesUnstableApi() { - // Required by the interface - } - }; + return createCallCredentials(userName, userPassword); } /** @@ -210,8 +208,7 @@ public void close() { if (!channel.awaitTermination(5, TimeUnit.SECONDS)) { channel.shutdownNow(); } - } - catch (InterruptedException e) { + } catch (InterruptedException e) { channel.shutdownNow(); } } @@ -230,11 +227,9 @@ public void begin(final Database.TRANSACTION_ISOLATION_LEVEL isolationLevel) { transactionId = response.getTransactionId(); // Store transaction ID in parent class session management setSessionId(transactionId); - } - catch (StatusRuntimeException e) { + } catch (StatusRuntimeException e) { throw new TransactionException("Error on transaction begin", e); - } - catch (StatusException e) { + } catch (StatusException e) { throw new TransactionException("Error on transaction begin", e); } } @@ -256,14 +251,11 @@ public void commit() { if (!response.getSuccess()) { throw new TransactionException("Failed to commit transaction: " + response.getMessage()); } - } - catch (StatusRuntimeException e) { + } catch (StatusRuntimeException e) { handleGrpcException(e); - } - catch (StatusException e) { + } catch (StatusException e) { handleGrpcException(e); - } - finally { + } finally { transactionId = null; setSessionId(null); } @@ -287,23 +279,18 @@ public void rollback() { if (!response.getSuccess()) { throw new TransactionException("Failed to rollback transaction: " + response.getMessage()); } - } - catch (StatusRuntimeException e) { + } catch (StatusRuntimeException e) { throw new TransactionException("Error on transaction rollback", e); - } - catch (StatusException e) { - throw new TransactionException("Error on transaction begin", e); - } - finally { + } catch (StatusException e) { + throw new TransactionException("Error on transaction rollback", e); + } finally { transactionId = null; setSessionId(null); } } - @Override public void deleteRecord(final Record record) { - checkDatabaseIsOpen(); stats.deleteRecord.incrementAndGet(); @@ -318,11 +305,9 @@ public void deleteRecord(final Record record) { if (!response.getSuccess()) { throw new DatabaseOperationException("Failed to delete record: " + response.getMessage()); } - } - catch (StatusRuntimeException e) { + } catch (StatusRuntimeException e) { handleGrpcException(e); - } - catch (StatusException e) { + } catch (StatusException e) { handleGrpcException(e); } } @@ -353,19 +338,17 @@ public ResultSet query(final String language, final String query, final Map params, long timeoutMs) { + public ExecuteCommandResponse execSql(String db, String sql, Map params, long timeoutMs) { return executeCommand(db, "sql", sql, params, /* returnRows */ false, /* maxRows */ 0, txBeginCommit(), timeoutMs); } - public com.arcadedb.server.grpc.ExecuteCommandResponse execSql(String sql, Map params, long timeoutMs) { + public ExecuteCommandResponse execSql(String sql, Map params, long timeoutMs) { return executeCommand(databaseName, "sql", sql, params, /* returnRows */ false, /* maxRows */ 0, txBeginCommit(), timeoutMs); } - public com.arcadedb.server.grpc.ExecuteCommandResponse executeCommand(String language, String command, Map params, - boolean returnRows, int maxRows, com.arcadedb.server.grpc.TransactionContext tx, long timeoutMs) { + public ExecuteCommandResponse executeCommand(String language, String command, Map params, + boolean returnRows, int maxRows, TransactionContext tx, long timeoutMs) { - var reqB = com.arcadedb.server.grpc.ExecuteCommandRequest.newBuilder().setDatabase(databaseName).setCommand(command) - .putAllParameters(convertParamsToProto(params)).setLanguage(langOrDefault(language)).setReturnRows(returnRows) + var reqB = ExecuteCommandRequest.newBuilder().setDatabase(databaseName).setCommand(command) + .putAllParameters(convertParamsToGrpcValue(params)).setLanguage(langOrDefault(language)).setReturnRows(returnRows) .setMaxRows(maxRows > 0 ? maxRows : 0); if (tx != null) @@ -442,20 +423,17 @@ public com.arcadedb.server.grpc.ExecuteCommandResponse executeCommand(String lan reqB.setCredentials(buildCredentials()); try { - - return blockingStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS).executeCommand(reqB.build()); - } - catch (StatusException e) { - + return blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).executeCommand(reqB.build()); + } catch (StatusException e) { throw new RuntimeException("Failed to execute command: " + e.getMessage(), e); } } - public com.arcadedb.server.grpc.ExecuteCommandResponse executeCommand(String database, String language, String command, - Map params, boolean returnRows, int maxRows, com.arcadedb.server.grpc.TransactionContext tx, long timeoutMs) { + public ExecuteCommandResponse executeCommand(String database, String language, String command, + Map params, boolean returnRows, int maxRows, TransactionContext tx, long timeoutMs) { - var reqB = com.arcadedb.server.grpc.ExecuteCommandRequest.newBuilder().setDatabase(database).setCommand(command) - .putAllParameters(convertParamsToProto(params)).setLanguage(langOrDefault(language)).setReturnRows(returnRows) + var reqB = ExecuteCommandRequest.newBuilder().setDatabase(database).setCommand(command) + .putAllParameters(convertParamsToGrpcValue(params)).setLanguage(langOrDefault(language)).setReturnRows(returnRows) .setMaxRows(maxRows > 0 ? maxRows : 0); if (tx != null) @@ -464,10 +442,8 @@ public com.arcadedb.server.grpc.ExecuteCommandResponse executeCommand(String dat reqB.setCredentials(buildCredentials()); try { - return blockingStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS).executeCommand(reqB.build()); - } - catch (StatusException e) { - + return blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).executeCommand(reqB.build()); + } catch (StatusException e) { throw new RuntimeException("Failed to execute command: " + e.getMessage(), e); } } @@ -479,186 +455,72 @@ protected RID saveRecord(final MutableDocument record) { final RID rid = record.getIdentity(); if (rid != null) { - // -------- UPDATE (partial) -------- - // Build the nested proto Record for the "partial" oneof - com.arcadedb.server.grpc.Record partial = com.arcadedb.server.grpc.Record.newBuilder() - .putAllProperties(convertParamsToProto(record.toMap(false))).build(); - - com.arcadedb.server.grpc.UpdateRecordRequest request = com.arcadedb.server.grpc.UpdateRecordRequest.newBuilder() - .setDatabase(getName()).setRid(rid.toString()).setRecord(partial).setDatabase(databaseName) + PropertiesUpdate partial = PropertiesUpdate.newBuilder() + .putAllProperties(convertParamsToGrpcValue(record.toMap(false))) + .build(); + + UpdateRecordRequest request = UpdateRecordRequest.newBuilder() + .setDatabase(getName()) + .setRid(rid.toString()) + .setPartial(partial) + .setDatabase(databaseName) .setCredentials(buildCredentials()).build(); try { - com.arcadedb.server.grpc.UpdateRecordResponse response = blockingStub - .withDeadlineAfter(getTimeout(), java.util.concurrent.TimeUnit.MILLISECONDS).updateRecord(request); + UpdateRecordResponse response = blockingStub + .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .updateRecord(request); // If your proto has flags, you can check response.getSuccess()/getUpdated() // Otherwise, treat non-exception as success. return rid; - } - catch (io.grpc.StatusRuntimeException e) { + } catch (StatusRuntimeException e) { handleGrpcException(e); return null; - } - catch (io.grpc.StatusException e) { + } catch (StatusException e) { handleGrpcException(e); return null; } - } - else { + } else { // -------- CREATE -------- - com.arcadedb.server.grpc.Record recMsg = com.arcadedb.server.grpc.Record.newBuilder() - .putAllProperties(convertParamsToProto(record.toMap(false))).build(); - - com.arcadedb.server.grpc.CreateRecordRequest request = com.arcadedb.server.grpc.CreateRecordRequest.newBuilder() - .setDatabase(getName()).setType(record.getTypeName()).setRecord(recMsg) // nested Record payload + GrpcRecord recMsg = GrpcRecord.newBuilder() + .putAllProperties(convertParamsToGrpcValue(record.toMap(false))) + .build(); + + CreateRecordRequest request = CreateRecordRequest.newBuilder() + .setDatabase(getName()) + .setType(record.getTypeName()) + .setRecord(recMsg) // nested GrpcRecord payload .setCredentials(buildCredentials()).build(); try { - com.arcadedb.server.grpc.CreateRecordResponse response = blockingStub - .withDeadlineAfter(getTimeout(), java.util.concurrent.TimeUnit.MILLISECONDS).createRecord(request); + CreateRecordResponse response = blockingStub + .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .createRecord(request); // Proto returns the newly created RID as a string final String ridStr = response.getRid(); if (ridStr == null || ridStr.isEmpty()) { - throw new com.arcadedb.exception.DatabaseOperationException("Failed to create record (empty RID)"); + throw new DatabaseOperationException("Failed to create record (empty RID)"); } // Construct a RID from the returned string - // Prefer the single-arg ctor if available; otherwise use the (Database, String) - // ctor. try { - return new com.arcadedb.database.RID(ridStr); - } - catch (NoSuchMethodError | IllegalArgumentException ex) { + return new RID(ridStr); + } catch (NoSuchMethodError | IllegalArgumentException ex) { // Fallback for older APIs expecting (Database, String) - return new com.arcadedb.database.RID(this, ridStr); + return new RID(this, ridStr); } - } - catch (io.grpc.StatusRuntimeException e) { + } catch (StatusRuntimeException e) { handleGrpcException(e); return null; - } - catch (io.grpc.StatusException e) { + } catch (StatusException e) { handleGrpcException(e); return null; } } } -// /** -// * Executes a streaming query that returns results in batches. -// * Useful for processing large result sets without loading everything into memory. -// * -// * @param language the query language (typically "sql") -// * @param query the query string -// * @param batchSize the number of records per batch (default 100 if <= 0) -// * @return Iterator of records that fetches results in batches -// */ -// public Iterator queryStream(final String language, final String query, final int batchSize) { -// checkDatabaseIsOpen(); -// stats.queries.incrementAndGet(); -// -// StreamQueryRequest request = StreamQueryRequest.newBuilder() -// .setDatabase(getName()) -// .setQuery(query) -// .setCredentials(buildCredentials()) -// .setBatchSize(batchSize > 0 ? batchSize : 100) -// .build(); -// -// Iterator responseIterator = blockingStub -// .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) -// .streamQuery(request); -// -// return new Iterator() { -// private Iterator currentBatch = Collections.emptyIterator(); -// -// @Override -// public boolean hasNext() { -// if (currentBatch.hasNext()) { -// return true; -// } -// if (responseIterator.hasNext()) { -// QueryResult result = responseIterator.next(); -// currentBatch = result.getRecordsList().iterator(); -// return currentBatch.hasNext(); -// } -// return false; -// } -// -// @Override -// public Record next() { -// if (!hasNext()) { -// throw new NoSuchElementException(); -// } -// return grpcRecordToRecord(currentBatch.next()); -// } -// }; -// } -// -// /** -// * Executes a streaming query with parameters. -// */ -// public Iterator queryStream(final String language, final String query, -// final Map params, final int batchSize) { -// checkDatabaseIsOpen(); -// stats.queries.incrementAndGet(); -// -// StreamQueryRequest.Builder requestBuilder = StreamQueryRequest.newBuilder() -// .setDatabase(getName()) -// .setQuery(query) -// .setCredentials(buildCredentials()) -// .setBatchSize(batchSize > 0 ? batchSize : 100); -// -// if (params != null && !params.isEmpty()) { -// requestBuilder.putAllParameters(convertParamsToProto(params)); -// } -// -// Iterator responseIterator = blockingStub -// .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) -// .streamQuery(requestBuilder.build()); -// -// // Same iterator implementation as above -// return new Iterator() { -// private Iterator currentBatch = Collections.emptyIterator(); -// -// @Override -// public boolean hasNext() { -// if (currentBatch.hasNext()) { -// return true; -// } -// if (responseIterator.hasNext()) { -// QueryResult result = responseIterator.next(); -// currentBatch = result.getRecordsList().iterator(); -// return currentBatch.hasNext(); -// } -// return false; -// } -// -// @Override -// public Record next() { -// if (!hasNext()) { -// throw new NoSuchElementException(); -// } -// return grpcRecordToRecord(currentBatch.next()); -// } -// }; -// } -// -// /** -// * Convenience method with default batch size -// */ -// public Iterator queryStream(final String language, final String query) { -// return queryStream(language, query, 100); -// } - - /* - * StreamQueryRequest req = StreamQueryRequest.newBuilder() .setDatabase(db) - * .setQuery(sql) .setBatchSize(500) - * .setRetrievalMode(StreamQueryRequest.RetrievalMode.PAGED) - * .setTransaction(TransactionContext.newBuilder() .setBegin(true) - * .setCommit(true) // or setRollback(true) .build()) .build(); - */ // Convenience: default batch size stays 100, default mode = CURSOR public Iterator queryStream(final String language, final String query) { @@ -679,12 +541,11 @@ public Iterator queryStream(final String language, final String query, f .setBatchSize(batchSize > 0 ? batchSize : 100).setRetrievalMode(mode) // <--- NEW .build(); - BlockingClientCall responseIterator = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + final BlockingClientCall responseIterator = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) .streamQuery(request); return new Iterator() { - - private Iterator currentBatch = Collections.emptyIterator(); + private Iterator currentBatch = Collections.emptyIterator(); @Override public boolean hasNext() { @@ -693,7 +554,6 @@ public boolean hasNext() { try { while (responseIterator.hasNext()) { - QueryResult result = responseIterator.read(); // Defensive: skip any empty batches the server might send @@ -708,12 +568,10 @@ public boolean hasNext() { } } catch (InterruptedException e) { - - throw new RuntimeException("queryStream: interrupted", e); + throw new RuntimeException("Interrupted while waiting for stream", e); } catch (StatusException e) { - - throw new RuntimeException("queryStream: error", e); + handleGrpcException(e); } return false; @@ -738,14 +596,14 @@ public Iterator queryStream(final String language, final String query, f .setBatchSize(batchSize > 0 ? batchSize : 100).setRetrievalMode(mode); // <--- NEW if (params != null && !params.isEmpty()) { - b.putAllParameters(convertParamsToProto(params)); + b.putAllParameters(convertParamsToGrpcValue(params)); } - BlockingClientCall responseIterator = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + final BlockingClientCall responseIterator = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) .streamQuery(b.build()); return new Iterator() { - private Iterator currentBatch = Collections.emptyIterator(); + private Iterator currentBatch = Collections.emptyIterator(); @Override public boolean hasNext() { @@ -753,7 +611,9 @@ public boolean hasNext() { return true; try { + while (responseIterator.hasNext()) { + QueryResult result = responseIterator.read(); if (result.getRecordsCount() == 0) { @@ -765,17 +625,13 @@ public boolean hasNext() { currentBatch = result.getRecordsList().iterator(); return true; } - } - catch (InterruptedException e) { - throw new RuntimeException("queryStream: interrupted", e); - } - catch (StatusException e) { - - throw new RuntimeException("queryStream: error", e); + return false; } - - return false; + catch (Exception e) { + + throw new RuntimeException("Interrupted while waiting for stream", e); + } } @Override @@ -831,10 +687,10 @@ public Iterator queryStreamBatches(final String language, final Stri .setBatchSize(batchSize > 0 ? batchSize : 100).setRetrievalMode(mode); if (params != null && !params.isEmpty()) { - b.putAllParameters(convertParamsToProto(params)); + b.putAllParameters(convertParamsToGrpcValue(params)); } - BlockingClientCall responseIterator = blockingStub.withWaitForReady() // optional, improves robustness + final BlockingClientCall responseIterator = blockingStub.withWaitForReady() // optional, improves robustness .withDeadlineAfter(/* e.g. */ 10, TimeUnit.MINUTES).streamQuery(b.build()); return new Iterator() { @@ -847,10 +703,13 @@ public boolean hasNext() { return true; if (drained) return false; - + try { + while (responseIterator.hasNext()) { + QueryResult qr = responseIterator.read(); + int n = qr.getTotalRecordsInBatch(); // server-populated // Guard: some servers could omit this; fallback to list size. if (n == 0) @@ -862,7 +721,7 @@ public boolean hasNext() { } List converted = new ArrayList<>(qr.getRecordsCount()); - for (com.arcadedb.server.grpc.Record gr : qr.getRecordsList()) { + for (GrpcRecord gr : qr.getRecordsList()) { converted.add(grpcRecordToDBRecord(gr)); } @@ -873,18 +732,14 @@ public boolean hasNext() { } return true; } - } - catch (InterruptedException e) { - throw new RuntimeException("queryStreamBatches: interrupted", e); - } - catch (StatusException e) { - - throw new RuntimeException("queryStreamBatches: error", e); + drained = true; + return false; } - - drained = true; - return false; + catch (Exception e) { + + throw new RuntimeException("Interrupted while waiting for stream", e); + } } @Override @@ -898,92 +753,91 @@ public QueryBatch next() { }; } - public Iterator queryStream(String database, String sql, Map params, int batchSize, - com.arcadedb.server.grpc.StreamQueryRequest.RetrievalMode mode, com.arcadedb.server.grpc.TransactionContext tx, long timeoutMs) { + public Iterator queryStream(String database, String sql, Map params, int batchSize, + StreamQueryRequest.RetrievalMode mode, TransactionContext tx, long timeoutMs) { - var reqB = com.arcadedb.server.grpc.StreamQueryRequest.newBuilder().setDatabase(database).setQuery(sql) - .putAllParameters(convertParamsToProto(params)).setCredentials(buildCredentials()).setBatchSize(batchSize > 0 ? batchSize : 100) + var reqB = StreamQueryRequest.newBuilder().setDatabase(database).setQuery(sql) + .putAllParameters(convertParamsToGrpcValue(params)).setCredentials(buildCredentials()).setBatchSize(batchSize > 0 ? batchSize : 100) .setRetrievalMode(mode); if (tx != null) reqB.setTransaction(tx); - BlockingClientCall it = blockingStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS) + final BlockingClientCall it = blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS) .streamQuery(reqB.build()); return new Iterator<>() { - - private Iterator curr = java.util.Collections.emptyIterator(); + + private Iterator curr = Collections.emptyIterator(); public boolean hasNext() { - + if (curr.hasNext()) return true; try { + if (it.hasNext()) { + curr = it.read().getRecordsList().iterator(); return hasNext(); } } catch (InterruptedException e) { - - throw new RuntimeException("queryStream: interrupted", e); + throw new RuntimeException("Interrupted while waiting for stream", e); } catch (StatusException e) { - - throw new RuntimeException("queryStream: error", e); + throw new RuntimeException("Stream query failed: " + e.getStatus().getDescription()); } return false; } - public com.arcadedb.server.grpc.Record next() { + public GrpcRecord next() { if (!hasNext()) - throw new java.util.NoSuchElementException(); + throw new NoSuchElementException(); return curr.next(); } }; } public Iterator queryStreamBatches(String passLabel, String sql, Map params, int batchSize, - com.arcadedb.server.grpc.StreamQueryRequest.RetrievalMode mode, com.arcadedb.server.grpc.TransactionContext tx, long timeoutMs) { + StreamQueryRequest.RetrievalMode mode, TransactionContext tx, long timeoutMs) { - var reqB = com.arcadedb.server.grpc.StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(sql) - .putAllParameters(convertParamsToProto(params)).setCredentials(buildCredentials()).setBatchSize(batchSize > 0 ? batchSize : 100) + var reqB = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(sql) + .putAllParameters(convertParamsToGrpcValue(params)).setCredentials(buildCredentials()).setBatchSize(batchSize > 0 ? batchSize : 100) .setRetrievalMode(mode); if (tx != null) reqB.setTransaction(tx); - BlockingClientCall respIter = blockingStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS) + final BlockingClientCall respIter = blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS) .streamQuery(reqB.build()); return new Iterator<>() { - + public boolean hasNext() { - + try { return respIter.hasNext(); } catch (InterruptedException e) { - - throw new RuntimeException("queryStreamBatches: interrupted", e); + throw new RuntimeException(e); } catch (StatusException e) { - - throw new RuntimeException("queryStreamBatches: error", e); + throw new RuntimeException(e); } } public QueryBatch next() { QueryResult qr; - + try { qr = respIter.read(); List converted = new ArrayList<>(qr.getRecordsCount()); - for (com.arcadedb.server.grpc.Record gr : qr.getRecordsList()) { + + for (GrpcRecord gr : qr.getRecordsList()) { converted.add(grpcRecordToDBRecord(gr)); } @@ -993,95 +847,96 @@ public QueryBatch next() { ); } catch (InterruptedException e) { - - throw new RuntimeException("queryStreamBatches: interrupted", e); + throw new RuntimeException(e); } catch (StatusException e) { - - throw new RuntimeException("queryStreamBatches: error", e); - } + throw new RuntimeException(e); + } } }; } public String createVertex(String cls, Map props, long timeoutMs) { - // Build the nested proto Record payload - com.arcadedb.server.grpc.Record recMsg = com.arcadedb.server.grpc.Record.newBuilder().putAllProperties(convertParamsToProto(props)) + // Build the nested proto GrpcRecord payload + GrpcRecord recMsg = GrpcRecord.newBuilder().putAllProperties(convertParamsToGrpcValue(props)) .build(); // Build the Create request - com.arcadedb.server.grpc.CreateRecordRequest req = com.arcadedb.server.grpc.CreateRecordRequest.newBuilder().setDatabase(getName()) + CreateRecordRequest req = CreateRecordRequest.newBuilder().setDatabase(getName()) .setType(cls).setRecord(recMsg) // <<<< NESTED RECORD (not top-level properties) .setCredentials(buildCredentials()).build(); // Call RPC - com.arcadedb.server.grpc.CreateRecordResponse res; + CreateRecordResponse res; try { - - res = blockingStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS).createRecord(req); - + res = blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).createRecord(req); // Response carries new RID string return res.getRid(); // e.g., "#12:0" - } - catch (StatusException e) { - + } catch (StatusException e) { throw new RuntimeException("Failed to create vertex", e); } } public boolean updateRecord(String rid, Map props, long timeoutMs) { - - com.arcadedb.server.grpc.Record record = com.arcadedb.server.grpc.Record.newBuilder().putAllProperties(convertParamsToProto(props)) + PropertiesUpdate partial = PropertiesUpdate.newBuilder() + .putAllProperties(convertParamsToGrpcValue(props)) .build(); - return updateRecord(rid, record, timeoutMs); + return updateRecord(rid, partial, timeoutMs); } public boolean updateRecord(String rid, Record dbRecord, long timeoutMs) { - - com.arcadedb.server.grpc.Record record = ProtoUtils.toProtoRecord(dbRecord); - - return updateRecord(rid, record, timeoutMs); + GrpcRecord record = ProtoUtils.toProtoRecord(dbRecord); + return updateRecordFull(rid, record, timeoutMs); } - private boolean updateRecord(String rid, com.arcadedb.server.grpc.Record record, long timeoutMs) { - - // Build the Update request (use setRecord(...) for full replace instead) - com.arcadedb.server.grpc.UpdateRecordRequest req = com.arcadedb.server.grpc.UpdateRecordRequest.newBuilder().setDatabase(getName()) - .setRid(rid).setRecord(record) + private boolean updateRecord(String rid, PropertiesUpdate partial, long timeoutMs) { + // Build the Update request using setPartial for partial update + UpdateRecordRequest req = UpdateRecordRequest.newBuilder().setDatabase(getName()) + .setRid(rid).setPartial(partial) .setTransaction(TransactionContext.newBuilder().setBegin(true).setCommit(true)) .setCredentials(buildCredentials()).build(); // Call RPC - com.arcadedb.server.grpc.UpdateRecordResponse res; + UpdateRecordResponse res; try { - res = blockingStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS).updateRecord(req); - + res = blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).updateRecord(req); // Choose the flag your proto defines. Most builds expose getSuccess(). return res.getSuccess(); - // If your generated class exposes getUpdated() instead, use: - // return res.getUpdated(); + } catch (StatusException e) { + throw new RuntimeException("Failed to update record", e); } - catch (StatusException e) { + } + + private boolean updateRecordFull(String rid, GrpcRecord record, long timeoutMs) { + // Build the Update request using setRecord for full replacement + UpdateRecordRequest req = UpdateRecordRequest.newBuilder().setDatabase(getName()) + .setRid(rid).setRecord(record) + .setTransaction(TransactionContext.newBuilder().setBegin(true).setCommit(true)) + .setCredentials(buildCredentials()).build(); + + // Call RPC + UpdateRecordResponse res; + try { + res = blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).updateRecord(req); + return res.getSuccess(); + } catch (StatusException e) { throw new RuntimeException("Failed to update record", e); } } public boolean deleteRecord(String rid, long timeoutMs) { - var req = com.arcadedb.server.grpc.DeleteRecordRequest.newBuilder().setDatabase(getName()).setRid(rid).setCredentials(buildCredentials()) + var req = DeleteRecordRequest.newBuilder().setDatabase(getName()).setRid(rid).setCredentials(buildCredentials()) .build(); DeleteRecordResponse res; try { - - res = blockingStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS).deleteRecord(req); + res = blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).deleteRecord(req); return res.getDeleted(); - } - catch (StatusException e) { - + } catch (StatusException e) { throw new RuntimeException("Failed to delete record", e); } } @@ -1127,9 +982,7 @@ public Record lookupByRID(final RID rid) { @Override public Record lookupByRID(final RID rid, final boolean loadContent) { - checkDatabaseIsOpen(); - stats.readRecord.incrementAndGet(); if (rid == null) @@ -1139,18 +992,17 @@ public Record lookupByRID(final RID rid, final boolean loadContent) { .setCredentials(buildCredentials()).build(); try { - LookupByRidResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).lookupByRid(request); if (!response.getFound()) { throw new RecordNotFoundException("Record " + rid + " not found", rid); } return grpcRecordToDBRecord(response.getRecord()); - } + } catch (StatusRuntimeException e) { handleGrpcException(e); return null; - } + } catch (StatusException e) { handleGrpcException(e); return null; @@ -1164,17 +1016,16 @@ public boolean existsRecord(RID rid) { throw new IllegalArgumentException("Record is null"); try { return lookupByRID(rid, false) != null; - } - catch (RecordNotFoundException e) { + } catch (RecordNotFoundException e) { return false; } } - public com.arcadedb.server.grpc.InsertSummary insertBulk(final com.arcadedb.server.grpc.InsertOptions options, - final java.util.List protoRows, final long timeoutMs) { + public InsertSummary insertBulk(final InsertOptions options, + final List protoRows, final long timeoutMs) { // Ensure options carry DB + credentials as the server expects - com.arcadedb.server.grpc.InsertOptions.Builder ob = options.toBuilder(); + InsertOptions.Builder ob = options.toBuilder(); if (options.getDatabase() == null || options.getDatabase().isEmpty()) { ob.setDatabase(getName()); // your wrapper's DB name @@ -1187,37 +1038,35 @@ public com.arcadedb.server.grpc.InsertSummary insertBulk(final com.arcadedb.serv InsertOptions newOptions = ob.build(); - com.arcadedb.server.grpc.BulkInsertRequest req = com.arcadedb.server.grpc.BulkInsertRequest.newBuilder().setOptions(newOptions) + BulkInsertRequest req = BulkInsertRequest.newBuilder().setOptions(newOptions) .addAllRows(protoRows).build(); try { - return blockingStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS).bulkInsert(req); - } - catch (StatusException e) { - + return blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).bulkInsert(req); + } catch (StatusException e) { throw new RuntimeException("insertBulk() -> failed: " + e.getStatus()); } } // Convenience overload that accepts domain rows (convert first) - public com.arcadedb.server.grpc.InsertSummary insertBulkAsListOfMaps(final com.arcadedb.server.grpc.InsertOptions options, - final java.util.List> rows, final long timeoutMs) { + public InsertSummary insertBulkAsListOfMaps(final InsertOptions options, + final List> rows, final long timeoutMs) { - java.util.List protoRows = rows.stream().map(this::toProtoRecordFromMap) // your converter + List protoRows = rows.stream().map(this::toProtoRecordFromMap) // your converter .collect(java.util.stream.Collectors.toList()); return insertBulk(options, protoRows, timeoutMs); } - public com.arcadedb.server.grpc.InsertSummary ingestStream(final com.arcadedb.server.grpc.InsertOptions options, - final java.util.List protoRows, final int chunkSize, final long timeoutMs) + public InsertSummary ingestStream(final InsertOptions options, + final List protoRows, final int chunkSize, final long timeoutMs) throws InterruptedException { final java.util.concurrent.CountDownLatch done = new java.util.concurrent.CountDownLatch(1); - final java.util.concurrent.atomic.AtomicReference summaryRef = new java.util.concurrent.atomic.AtomicReference<>(); + final java.util.concurrent.atomic.AtomicReference summaryRef = new java.util.concurrent.atomic.AtomicReference<>(); // Ensure options carry DB + credentials as the server expects - com.arcadedb.server.grpc.InsertOptions.Builder ob = options.toBuilder(); + InsertOptions.Builder ob = options.toBuilder(); if (options.getDatabase() == null || options.getDatabase().isEmpty()) { ob.setDatabase(getName()); // your wrapper's DB name @@ -1229,9 +1078,9 @@ public com.arcadedb.server.grpc.InsertSummary ingestStream(final com.arcadedb.se ob.setCredentials(buildCredentials()); // your existing helper used by queries } - io.grpc.stub.StreamObserver resp = new io.grpc.stub.StreamObserver<>() { + StreamObserver resp = new StreamObserver<>() { @Override - public void onNext(com.arcadedb.server.grpc.InsertSummary value) { + public void onNext(InsertSummary value) { summaryRef.set(value); } @@ -1247,9 +1096,9 @@ public void onCompleted() { }; // Use the write service async stub, per-call deadline - var stub = this.asyncStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS); + var stub = this.asyncStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS); - final io.grpc.stub.StreamObserver req = stub.insertStream(resp); + final StreamObserver req = stub.insertStream(resp); // Stream chunks final String sessionId = "sess-" + System.nanoTime(); @@ -1257,10 +1106,9 @@ public void onCompleted() { long seq = 1; for (int i = 0; i < protoRows.size(); i += chunkSize) { - final int end = Math.min(i + chunkSize, protoRows.size()); - final com.arcadedb.server.grpc.InsertChunk chunk = com.arcadedb.server.grpc.InsertChunk.newBuilder().setSessionId(sessionId) + final InsertChunk chunk = InsertChunk.newBuilder().setSessionId(sessionId) .setOptions(ob.build()).setChunkSeq(seq++).addAllRows(protoRows.subList(i, end)).build(); req.onNext(chunk); @@ -1268,47 +1116,31 @@ public void onCompleted() { req.onCompleted(); - done.await(timeoutMs + 5_000, java.util.concurrent.TimeUnit.MILLISECONDS); + done.await(timeoutMs + 5_000, TimeUnit.MILLISECONDS); - final com.arcadedb.server.grpc.InsertSummary s = summaryRef.get(); - return (s != null) ? s : com.arcadedb.server.grpc.InsertSummary.newBuilder().setReceived(protoRows.size()).build(); + final InsertSummary s = summaryRef.get(); + return (s != null) ? s : InsertSummary.newBuilder().setReceived(protoRows.size()).build(); } // Convenience overload - public com.arcadedb.server.grpc.InsertSummary ingestStreamAsListOfMaps(final com.arcadedb.server.grpc.InsertOptions options, - final java.util.List> rows, final int chunkSize, final long timeoutMs) throws InterruptedException { + public InsertSummary ingestStreamAsListOfMaps(final InsertOptions options, + final List> rows, final int chunkSize, final long timeoutMs) throws InterruptedException { - java.util.List protoRows = rows.stream().map(this::toProtoRecordFromMap) + List protoRows = rows.stream().map(this::toProtoRecordFromMap) .collect(java.util.stream.Collectors.toList()); return ingestStream(options, protoRows, chunkSize, timeoutMs); } /** - * Core implementation of InsertBidirectional ingest: - Sends a START once with - * effective InsertOptions (db/creds enforced) - Streams chunks of mapped rows - * (using chunkSize) while respecting backpressure (maxInflight) - Collects - * per-batch ACKs and returns COMMITTED summary if present, else aggregates ACKs - * - * @param row type (e.g., com.arcadedb.database.Record or - * Map) - * @param rows source rows - * @param options base InsertOptions; db/credentials will be injected if - * missing - * @param chunkSize max rows per chunk - * @param maxInflight max number of unacknowledged chunks allowed - * @param timeoutMs overall client-side timeout - * @param mapper maps T → grpc Record (must be pure & fast) - * @return InsertSummary as returned by COMMITTED, or aggregated from ACKs if - * COMMITTED missing - * @throws InterruptedException if the await is interrupted + * Core implementation of InsertBidirectional ingest */ - private com.arcadedb.server.grpc.InsertSummary ingestBidiCore(final java.util.List rows, - final com.arcadedb.server.grpc.InsertOptions options, final int chunkSize, final int maxInflight, final long timeoutMs, - final java.util.function.Function mapper) throws InterruptedException { + private InsertSummary ingestBidiCore(final List rows, + final InsertOptions options, final int chunkSize, final int maxInflight, final long timeoutMs, + final java.util.function.Function mapper) throws InterruptedException { // 1) Ensure options carry DB + credentials the server expects - final com.arcadedb.server.grpc.InsertOptions.Builder ob = options.toBuilder(); + final InsertOptions.Builder ob = options.toBuilder(); if (options.getDatabase() == null || options.getDatabase().isEmpty()) { ob.setDatabase(getName()); @@ -1320,10 +1152,10 @@ private com.arcadedb.server.grpc.InsertSummary ingestBidiCore(final java.util.Li ob.setCredentials(buildCredentials()); } - final com.arcadedb.server.grpc.InsertOptions effectiveOpts = ob.build(); + final InsertOptions effectiveOpts = ob.build(); // 2) Pre-map rows → proto once (outside the observer) - final java.util.List protoRows = rows.stream().map(mapper) + final List protoRows = rows.stream().map(mapper) .collect(java.util.stream.Collectors.toList()); // 3) Streaming state @@ -1333,11 +1165,11 @@ private com.arcadedb.server.grpc.InsertSummary ingestBidiCore(final java.util.Li final java.util.concurrent.atomic.AtomicInteger cursor = new java.util.concurrent.atomic.AtomicInteger(0); final java.util.concurrent.atomic.AtomicInteger sent = new java.util.concurrent.atomic.AtomicInteger(0); final java.util.concurrent.atomic.AtomicInteger acked = new java.util.concurrent.atomic.AtomicInteger(0); - final java.util.concurrent.atomic.AtomicReference committed = new java.util.concurrent.atomic.AtomicReference<>(); - final java.util.List acks = java.util.Collections.synchronizedList(new java.util.ArrayList<>()); + final java.util.concurrent.atomic.AtomicReference committed = new java.util.concurrent.atomic.AtomicReference<>(); + final List acks = java.util.Collections.synchronizedList(new ArrayList<>()); // small holder to access req inside helpers - final java.util.concurrent.atomic.AtomicReference> observerRef = + final java.util.concurrent.atomic.AtomicReference> observerRef = new java.util.concurrent.atomic.AtomicReference<>(); final java.util.concurrent.atomic.AtomicBoolean commitSent = new java.util.concurrent.atomic.AtomicBoolean(false); @@ -1355,12 +1187,11 @@ private com.arcadedb.server.grpc.InsertSummary ingestBidiCore(final java.util.Li if (commitSent.compareAndSet(false, true)) { try { if (observerRef.get() != null) { - observerRef.get().onNext(com.arcadedb.server.grpc.InsertRequest.newBuilder() - .setCommit(com.arcadedb.server.grpc.Commit.newBuilder().setSessionId(sessionId)).build()); + observerRef.get().onNext(InsertRequest.newBuilder() + .setCommit(Commit.newBuilder().setSessionId(sessionId)).build()); observerRef.get().onCompleted(); } - } - catch (Throwable ignore) { + } catch (Throwable ignore) { /* best effort */ } } }; @@ -1373,7 +1204,7 @@ private com.arcadedb.server.grpc.InsertSummary ingestBidiCore(final java.util.Li prev.cancel(false); // only arm if we've sent all chunks but acks are still pending if (cursor.get() >= protoRows.size() && acked.get() < sent.get() && !commitSent.get()) { - var fut = scheduler.schedule(sendCommitIfNeeded, ackGraceMillis, java.util.concurrent.TimeUnit.MILLISECONDS); + var fut = scheduler.schedule(sendCommitIfNeeded, ackGraceMillis, TimeUnit.MILLISECONDS); ackGraceFuture.set(fut); } } @@ -1388,13 +1219,13 @@ private com.arcadedb.server.grpc.InsertSummary ingestBidiCore(final java.util.Li } }; - final io.grpc.stub.ClientResponseObserver observer = new io.grpc.stub.ClientResponseObserver<>() { + final ClientResponseObserver observer = new ClientResponseObserver<>() { - io.grpc.stub.ClientCallStreamObserver req; + ClientCallStreamObserver req; volatile boolean started = false; @Override - public void beforeStart(io.grpc.stub.ClientCallStreamObserver r) { + public void beforeStart(ClientCallStreamObserver r) { this.req = r; observerRef.set(r); // <-- make req visible to helper r.disableAutoInboundFlowControl(); @@ -1402,14 +1233,12 @@ public void beforeStart(io.grpc.stub.ClientCallStreamObserver= sent.get()) { // all acked → commit immediately sendCommitIfNeeded.run(); - } - else { - // not all acked → (re)arm grace timer; if last ACK never arrives, timer will - // COMMIT + } else { + // not all acked → (re)arm grace timer; if last ACK never arrives, timer will COMMIT armAckGraceTimer.run(); } } } @Override - public void onNext(com.arcadedb.server.grpc.InsertResponse v) { + public void onNext(InsertResponse v) { switch (v.getMsgCase()) { case STARTED -> { /* ok */ } @@ -1461,8 +1288,7 @@ public void onNext(com.arcadedb.server.grpc.InsertResponse v) { if (acked.get() >= sent.get()) { cancelAckGraceTimer.run(); sendCommitIfNeeded.run(); - } - else { + } else { armAckGraceTimer.run(); } } @@ -1492,141 +1318,92 @@ public void onCompleted() { // 4) Kick off the bidi call with deadline try { - asyncStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS).insertBidirectional(observer); - boolean finished = done.await(timeoutMs + 5_000, java.util.concurrent.TimeUnit.MILLISECONDS); - } - finally { + asyncStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).insertBidirectional(observer); + boolean finished = done.await(timeoutMs + 5_000, TimeUnit.MILLISECONDS); + } finally { scheduler.shutdownNow(); } // 6) Prefer COMMITTED summary if present; otherwise aggregate ACKs - final com.arcadedb.server.grpc.InsertSummary finalSummary = committed.get(); + final InsertSummary finalSummary = committed.get(); if (finalSummary != null) return finalSummary; - final long ins = acks.stream().mapToLong(com.arcadedb.server.grpc.BatchAck::getInserted).sum(); - final long upd = acks.stream().mapToLong(com.arcadedb.server.grpc.BatchAck::getUpdated).sum(); - final long ign = acks.stream().mapToLong(com.arcadedb.server.grpc.BatchAck::getIgnored).sum(); - final long fail = acks.stream().mapToLong(com.arcadedb.server.grpc.BatchAck::getFailed).sum(); + final long ins = acks.stream().mapToLong(BatchAck::getInserted).sum(); + final long upd = acks.stream().mapToLong(BatchAck::getUpdated).sum(); + final long ign = acks.stream().mapToLong(BatchAck::getIgnored).sum(); + final long fail = acks.stream().mapToLong(BatchAck::getFailed).sum(); - return com.arcadedb.server.grpc.InsertSummary.newBuilder().setReceived(protoRows.size()).setInserted(ins).setUpdated(upd).setIgnored(ign) + return InsertSummary.newBuilder().setReceived(protoRows.size()).setInserted(ins).setUpdated(upd).setIgnored(ign) .setFailed(fail).build(); } /** - * Pushes domain {@code com.arcadedb.database.Record} rows via - * InsertBidirectional with per-batch ACKs. - * - *

- * Behavior: - *

    - *
  • Sends a {@code START} with effective {@link InsertOptions} - * (DB/credentials auto-injected if missing).
    • - *
  • Streams chunks of rows (size = {@code chunkSize}) while respecting - * backpressure (at most {@code maxInflight} unacknowledged chunks).
    • - *
  • Each server {@code BATCH_ACK} reduces in-flight count and triggers - * sending more.
    • - *
  • When all chunks are acked, sends {@code COMMIT} and returns the final - * {@code InsertSummary} from the server if available; otherwise aggregates ACKs - * to form a summary.
    • - *
- * - * @param rows domain records to send - * @param opts base InsertOptions (target class, conflict mode, tx mode). - * DB/credentials will be filled if missing - * @param chunkSize number of rows per chunk (must be > 0) - * @param maxInflight max unacknowledged chunks allowed (backpressure window) - * @return InsertSummary from COMMITTED or aggregated from ACKs - * @throws InterruptedException if waiting for completion is interrupted + * Pushes domain {@code com.arcadedb.database.Record} rows via InsertBidirectional with per-batch ACKs. */ - public com.arcadedb.server.grpc.InsertSummary ingestBidi(final java.util.List rows, - final com.arcadedb.server.grpc.InsertOptions opts, final int chunkSize, final int maxInflight, final long timeoutMs) + public InsertSummary ingestBidi(final List rows, + final InsertOptions opts, final int chunkSize, final int maxInflight, final long timeoutMs) throws InterruptedException { return ingestBidiCore(rows, opts, chunkSize, maxInflight, timeoutMs, (Object o) -> toProtoRecordFromDbRecord((com.arcadedb.database.Record) o)); } - public com.arcadedb.server.grpc.InsertSummary ingestBidi(final java.util.List rows, - final com.arcadedb.server.grpc.InsertOptions opts, final int chunkSize, final int maxInflight) throws InterruptedException { + public InsertSummary ingestBidi(final List rows, + final InsertOptions opts, final int chunkSize, final int maxInflight) throws InterruptedException { return ingestBidiCore(rows, opts, chunkSize, maxInflight, /* timeoutMs */ 5 * 60_000L, (Object o) -> toProtoRecordFromDbRecord((com.arcadedb.database.Record) o)); } /** - * Pushes map-shaped rows (property map per row) via InsertBidirectional with - * per-batch ACKs. - * - *

- * Behavior is identical to {@link #ingestBidi(List, InsertOptions, int, int)} - * but accepts pre-materialized property maps. This is useful when ingesting - * from JSON/CSV or an HTTP pipeline. - * - * @param options base InsertOptions (target class, conflict mode, tx mode). - * DB/credentials will be filled if missing - * @param rows list of maps (one per record) to insert/update - * @param chunkSize rows per chunk - * @param maxInflight max unacknowledged chunks allowed - * @param timeoutMs overall client-side timeout - * @return InsertSummary from COMMITTED or aggregated from ACKs - * @throws InterruptedException if the await is interrupted + * Pushes map-shaped rows (property map per row) via InsertBidirectional with per-batch ACKs. */ - public com.arcadedb.server.grpc.InsertSummary ingestBidi(final com.arcadedb.server.grpc.InsertOptions options, - final java.util.List> rows, final int chunkSize, final int maxInflight) throws InterruptedException { + public InsertSummary ingestBidi(final InsertOptions options, + final List> rows, final int chunkSize, final int maxInflight) throws InterruptedException { return ingestBidiCore(rows, options, chunkSize, maxInflight, /* timeoutMs */ 5 * 60_000L, - (Object o) -> toProtoRecordFromMap((java.util.Map) o)); + (Object o) -> toProtoRecordFromMap((Map) o)); } - public com.arcadedb.server.grpc.InsertSummary ingestBidi(final com.arcadedb.server.grpc.InsertOptions options, - final java.util.List> rows, final int chunkSize, final int maxInflight, final long timeoutMs) + public InsertSummary ingestBidi(final InsertOptions options, + final List> rows, final int chunkSize, final int maxInflight, final long timeoutMs) throws InterruptedException { return ingestBidiCore(rows, options, chunkSize, maxInflight, timeoutMs, - (Object o) -> toProtoRecordFromMap((java.util.Map) o)); + (Object o) -> toProtoRecordFromMap((Map) o)); } - // Map -> PROTO Value - private com.arcadedb.server.grpc.Record toProtoRecord(Map row) { - com.arcadedb.server.grpc.Record.Builder b = com.arcadedb.server.grpc.Record.newBuilder(); - // Example if your proto uses a map - // fields/properties - row.forEach((k, v) -> b.putProperties(k, toProtoValue(v))); + // Map -> GrpcRecord + private GrpcRecord toProtoRecordFromMap(Map row) { + GrpcRecord.Builder b = GrpcRecord.newBuilder(); + row.forEach((k, v) -> b.putProperties(k, objectToGrpcValue(v))); return b.build(); } - // Map -> PROTO Record - private com.arcadedb.server.grpc.Record toProtoRecordFromMap(java.util.Map row) { - com.arcadedb.server.grpc.Record.Builder b = com.arcadedb.server.grpc.Record.newBuilder(); - row.forEach((k, v) -> b.putProperties(k, toProtoValue(v))); - return b.build(); + // Domain Record (storage) -> GrpcRecord + private GrpcRecord toProtoRecordFromDbRecord(com.arcadedb.database.Record rec) { + // Use ProtoUtils for proper conversion + return ProtoUtils.toProtoRecord(rec); } - // Domain Record (storage) -> PROTO Record - private com.arcadedb.server.grpc.Record toProtoRecordFromDbRecord(com.arcadedb.database.Record rec) { - - // Promote to a read-only Document; null if this record isn't a document - com.arcadedb.database.Document doc = rec.asDocument(); - if (doc == null) { - return com.arcadedb.server.grpc.Record.getDefaultInstance(); - } - - com.arcadedb.server.grpc.Record.Builder b = com.arcadedb.server.grpc.Record.newBuilder(); - - // Iterate document properties - for (String name : doc.getPropertyNames()) { - // Depending on your ArcadeDB version, use ONE of these: - Object v = doc.get(name); // <-- common in ArcadeDB - // Object v = doc.getProperty(name); // <-- use this if doc.get(...) doesn't - // exist in your API - b.putProperties(name, toProtoValue(v)); // or putFields(...) if your proto uses "fields" + // Query Result -> GrpcRecord + private GrpcRecord toProtoRecordFromResult(Result res) { + GrpcRecord.Builder b = GrpcRecord.newBuilder(); + for (String name : res.getPropertyNames()) { + Object v = res.getProperty(name); + b.putProperties(name, objectToGrpcValue(v)); } return b.build(); } + // Convert Java object -> GrpcValue (extend as needed) + private GrpcValue objectToGrpcValue(Object v) { + return ProtoUtils.toGrpcValue(v); + } + // google.protobuf.Value -> Java (mirror of your toProtoValue) - private static Object fromProtoValue(com.google.protobuf.Value v) { + private static Object fromProtoValue(Value v) { if (v == null) return null; switch (v.getKindCase()) { @@ -1639,16 +1416,16 @@ private static Object fromProtoValue(com.google.protobuf.Value v) { case STRING_VALUE: return v.getStringValue(); case LIST_VALUE: { - java.util.List out = new java.util.ArrayList<>(); - for (com.google.protobuf.Value item : v.getListValue().getValuesList()) { + List out = new ArrayList<>(); + for (Value item : v.getListValue().getValuesList()) { out.add(fromProtoValue(item)); } return out; } case STRUCT_VALUE: { - java.util.Map m = new java.util.LinkedHashMap<>(); + Map m = new java.util.LinkedHashMap<>(); v.getStructValue().getFieldsMap().forEach((k, vv) -> m.put(k, fromProtoValue(vv))); - return m; // server-side schema can later coerce EMBEDDED if needed + return m; } case KIND_NOT_SET: default: @@ -1656,67 +1433,6 @@ private static Object fromProtoValue(com.google.protobuf.Value v) { } } - // Query Result -> PROTO Record - private com.arcadedb.server.grpc.Record toProtoRecordFromResult(Result res) { - com.arcadedb.server.grpc.Record.Builder b = com.arcadedb.server.grpc.Record.newBuilder(); - for (String name : res.getPropertyNames()) { - Object v = res.getProperty(name); - b.putProperties(name, toProtoValue(v)); // or putFields(...) - } - return b.build(); - } - - // Convert Java object -> protobuf Value (extend as needed) - private com.google.protobuf.Value toProtoValue(Object v) { - com.google.protobuf.Value.Builder vb = com.google.protobuf.Value.newBuilder(); - if (v == null) - return vb.setNullValue(com.google.protobuf.NullValue.NULL_VALUE).build(); - if (v instanceof String s) - return vb.setStringValue(s).build(); - if (v instanceof Boolean b) - return vb.setBoolValue(b).build(); - if (v instanceof Number n) - return vb.setNumberValue(n.doubleValue()).build(); - if (v instanceof java.time.Instant t) - return vb.setStringValue(t.toString()).build(); // or a Struct for Timestamp if you prefer - if (v instanceof java.util.Map m) { - var sb = com.google.protobuf.Struct.newBuilder(); - for (var e : m.entrySet()) - sb.putFields(String.valueOf(e.getKey()), toProtoValue(e.getValue())); - return vb.setStructValue(sb.build()).build(); - } - if (v instanceof java.util.List list) { - var lb = com.google.protobuf.ListValue.newBuilder(); - for (Object e : list) - lb.addValues(toProtoValue(e)); - return vb.setListValue(lb.build()).build(); - } - return vb.setStringValue(String.valueOf(v)).build(); - } - - // Override HTTP-specific methods to prevent usage -// @Override -// HttpRequest.Builder createRequestBuilder(String httpMethod, String url) { -// throw new UnsupportedOperationException("HTTP operations not supported in gRPC implementation"); -// } -// -// @Override -// protected ResultSet createResultSet(JSONObject response) { -// throw new UnsupportedOperationException("JSON operations not supported in gRPC implementation"); -// } -// -// @Override -// protected Result json2Result(JSONObject result) { -// throw new UnsupportedOperationException("JSON operations not supported in gRPC implementation"); -// } -// -// @Override -// protected Record json2Record(JSONObject result) { -// throw new UnsupportedOperationException("JSON operations not supported in gRPC implementation"); -// } - - // gRPC-specific helper methods - public DatabaseCredentials buildCredentials() { return DatabaseCredentials.newBuilder().setUsername(getUserName()).setPassword(getUserPassword()).build(); } @@ -1737,12 +1453,12 @@ private TransactionIsolation mapIsolationLevel(Database.TRANSACTION_ISOLATION_LE } // ---- TX helpers ------------------------------------------------------------- - private static com.arcadedb.server.grpc.TransactionContext txBeginCommit() { - return com.arcadedb.server.grpc.TransactionContext.newBuilder().setBegin(true).setCommit(true).build(); + private static TransactionContext txBeginCommit() { + return TransactionContext.newBuilder().setBegin(true).setCommit(true).build(); } - private static com.arcadedb.server.grpc.TransactionContext txNone() { - return com.arcadedb.server.grpc.TransactionContext.getDefaultInstance(); + private static TransactionContext txNone() { + return TransactionContext.getDefaultInstance(); } // Optional: language defaulting (server defaults to "sql" too) @@ -1751,38 +1467,36 @@ private static String langOrDefault(String language) { } private Iterator streamQuery(final String query) { - StreamQueryRequest request = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(query).setCredentials(buildCredentials()) .setBatchSize(100).build(); - final BlockingClientCall blockingClientCall = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + final BlockingClientCall responseIterator = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) .streamQuery(request); return new Iterator() { - private Iterator currentBatch = Collections.emptyIterator(); + private Iterator currentBatch = Collections.emptyIterator(); @Override public boolean hasNext() { - + if (currentBatch.hasNext()) { return true; } try { + if (responseIterator.hasNext()) { - if (blockingClientCall.hasNext()) { - QueryResult result = blockingClientCall.read(); - currentBatch = result.getRecordsList().iterator(); - return currentBatch.hasNext(); + QueryResult result; + result = responseIterator.read(); + currentBatch = result.getRecordsList().iterator(); + return currentBatch.hasNext(); } } catch (InterruptedException e) { - - throw new RuntimeException("streamQuery: Interrupted", e); + throw new RuntimeException(e); } catch (StatusException e) { - - throw new RuntimeException("streamQuery: error occured", e); + throw new RuntimeException(e); } return false; @@ -1801,28 +1515,28 @@ public Record next() { private ResultSet createGrpcResultSet(ExecuteQueryResponse response) { InternalResultSet resultSet = new InternalResultSet(); for (QueryResult queryResult : response.getResultsList()) { - for (com.arcadedb.server.grpc.Record record : queryResult.getRecordsList()) { + for (GrpcRecord record : queryResult.getRecordsList()) { resultSet.add(grpcRecordToResult(record)); } } return resultSet; } - private Result grpcRecordToResult(com.arcadedb.server.grpc.Record grpcRecord) { + private Result grpcRecordToResult(GrpcRecord grpcRecord) { Record record = grpcRecordToDBRecord(grpcRecord); if (record == null) { Map properties = new HashMap<>(); - grpcRecord.getPropertiesMap().forEach((k, v) -> properties.put(k, valueToObject(v))); + grpcRecord.getPropertiesMap().forEach((k, v) -> properties.put(k, grpcValueToObject(v))); return new ResultInternal(properties); } return new ResultInternal(record); } - private Record grpcRecordToDBRecord(com.arcadedb.server.grpc.Record grpcRecord) { + private Record grpcRecordToDBRecord(GrpcRecord grpcRecord) { Map map = new HashMap<>(); // Convert properties - grpcRecord.getPropertiesMap().forEach((k, v) -> map.put(k, valueToObject(v))); + grpcRecord.getPropertiesMap().forEach((k, v) -> map.put(k, grpcValueToObject(v))); // Add metadata map.put("@rid", grpcRecord.getRid()); @@ -1842,7 +1556,7 @@ private Record grpcRecordToDBRecord(com.arcadedb.server.grpc.Record grpcRecord) } } - private String mapRecordType(com.arcadedb.server.grpc.Record grpcRecord) { + private String mapRecordType(GrpcRecord grpcRecord) { // Determine record category from type name String typeName = grpcRecord.getType(); @@ -1852,103 +1566,37 @@ private String mapRecordType(com.arcadedb.server.grpc.Record grpcRecord) { Object type = getSchema().getType(typeName); if (type instanceof com.arcadedb.schema.VertexType) { return "v"; - } - else if (type instanceof com.arcadedb.schema.EdgeType) { + } else if (type instanceof com.arcadedb.schema.EdgeType) { return "e"; } } - } - catch (Exception e) { + } catch (Exception e) { // Fall back to name-based detection } // Fall back to name-based detection if (typeName.contains("Vertex") || typeName.startsWith("V_")) { return "v"; - } - else if (typeName.contains("Edge") || typeName.startsWith("E_")) { + } else if (typeName.contains("Edge") || typeName.startsWith("E_")) { return "e"; - } - else { + } else { return "d"; } } - private Map convertParamsToProto(Map params) { - - Map protoParams = new HashMap<>(); + private Map convertParamsToGrpcValue(Map params) { + Map grpcParams = new HashMap<>(); for (Map.Entry entry : params.entrySet()) { - - Value value = objectToValue(entry.getValue()); - -// if (value != null) { -// -// System.out.println("Converting object to Value: " + entry.getKey() + " " + entry.getValue().getClass() + " -> " + value.hasStructValue()); -// } - - protoParams.put(entry.getKey(), value); + GrpcValue value = objectToGrpcValue(entry.getValue()); + grpcParams.put(entry.getKey(), value); } - return protoParams; - } - - private Value objectToValue(Object obj) { - - Value.Builder builder = Value.newBuilder(); - - if (obj == null) { - builder.setNullValue(com.google.protobuf.NullValue.NULL_VALUE); - } - else if (obj instanceof Boolean) { - builder.setBoolValue((Boolean) obj); - } - else if (obj instanceof Number) { - builder.setNumberValue(((Number) obj).doubleValue()); - } - else if (obj instanceof String) { - builder.setStringValue((String) obj); - } - else if (obj instanceof RID) { - builder.setStringValue(obj.toString()); - } - else if (obj instanceof Map) { - Struct.Builder structBuilder = Struct.newBuilder(); - ((Map) obj).forEach((k, v) -> structBuilder.putFields(k.toString(), objectToValue(v))); - builder.setStructValue(structBuilder.build()); - } - else if (obj instanceof Collection) { - com.google.protobuf.ListValue.Builder listBuilder = com.google.protobuf.ListValue.newBuilder(); - ((Collection) obj).forEach(item -> listBuilder.addValues(objectToValue(item))); - builder.setListValue(listBuilder.build()); - } - else { - builder.setStringValue(obj.toString()); - } - return builder.build(); + return grpcParams; } - private Object valueToObject(Value value) { - switch (value.getKindCase()) { - case NULL_VALUE: - return null; - case NUMBER_VALUE: - return value.getNumberValue(); - case STRING_VALUE: - return value.getStringValue(); - case BOOL_VALUE: - return value.getBoolValue(); - case STRUCT_VALUE: - Map map = new HashMap<>(); - value.getStructValue().getFieldsMap().forEach((k, v) -> map.put(k, valueToObject(v))); - return map; - case LIST_VALUE: - List list = new ArrayList<>(); - value.getListValue().getValuesList().forEach(v -> list.add(valueToObject(v))); - return list; - default: - return null; - } + private Object grpcValueToObject(GrpcValue grpcValue) { + return ProtoUtils.fromGrpcValue(grpcValue); } private void handleGrpcException(StatusRuntimeException e) { @@ -2011,4 +1659,11 @@ protected ManagedChannel createChannel(String server, int port) { .keepAliveTimeout(10, TimeUnit.SECONDS).keepAliveWithoutCalls(true).build(); } } + + // Add the missing RemoteGrpcTransactionExplicitLock class reference + private static class RemoteGrpcTransactionExplicitLock extends RemoteTransactionExplicitLock { + public RemoteGrpcTransactionExplicitLock(RemoteGrpcDatabase database) { + super(database); + } + } } \ No newline at end of file diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/utils/ProtoUtils.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/utils/ProtoUtils.java index 369c4ad7a4..f49641efc0 100644 --- a/grpc-client/src/main/java/com/arcadedb/remote/grpc/utils/ProtoUtils.java +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/utils/ProtoUtils.java @@ -1,54 +1,74 @@ package com.arcadedb.remote.grpc.utils; + import java.util.Map; +import java.util.HashMap; +import java.util.List; +import java.util.ArrayList; +import java.util.Collection; import com.arcadedb.database.Document; import com.arcadedb.database.Record; +import com.arcadedb.database.RID; import com.arcadedb.graph.Vertex; -import com.arcadedb.server.grpc.Record.Builder; +import com.arcadedb.graph.Edge; +import com.arcadedb.server.grpc.GrpcRecord; +import com.arcadedb.server.grpc.GrpcValue; +import com.arcadedb.server.grpc.GrpcValueType; import com.google.protobuf.Value; -import com.google.protobuf.util.Values; +import com.google.protobuf.Struct; +import com.google.protobuf.ListValue; +import com.google.protobuf.NullValue; public class ProtoUtils { /** - * Converts an ArcadeDB Record into a gRPC Record message. + * Converts an ArcadeDB Record into a gRPC GrpcRecord message. * * @param rec ArcadeDB record (Document, Vertex, or Edge) - * @return Proto Record message + * @return Proto GrpcRecord message */ - public static com.arcadedb.server.grpc.Record toProtoRecord(Record rec) { - - if (rec == null) { + public static GrpcRecord toProtoRecord(Record rec) { + if (rec == null) { throw new IllegalArgumentException("Record cannot be null"); } - Builder builder = com.arcadedb.server.grpc.Record.newBuilder() - .setRid(rec.getIdentity().toString()); + GrpcRecord.Builder builder = GrpcRecord.newBuilder(); + + // Set RID if available + if (rec.getIdentity() != null) { + builder.setRid(rec.getIdentity().toString()); + } // Determine logical type/class name - String typeName = null; if (rec instanceof Vertex) { - - typeName = rec.asVertex().getType().getName(); - } - - if (rec instanceof Document) { - - typeName = rec.asDocument().getType().getName(); + Vertex vertex = rec.asVertex(); + if (vertex != null && vertex.getType() != null) { + typeName = vertex.getType().getName(); + } + } else if (rec instanceof Edge) { + Edge edge = rec.asEdge(); + if (edge != null && edge.getType() != null) { + typeName = edge.getType().getName(); + } + } else if (rec instanceof Document) { + Document doc = rec.asDocument(); + if (doc != null && doc.getType() != null) { + typeName = doc.getType().getName(); + } } if (typeName != null && !typeName.isEmpty()) { - - builder.setType(typeName); + builder.setType(typeName); } // Convert properties - if (rec instanceof Document doc) { + if (rec instanceof Document) { + Document doc = (Document) rec; for (String propName : doc.getPropertyNames()) { Object value = doc.get(propName); - builder.putProperties(propName, toProtoValue(value)); + builder.putProperties(propName, toGrpcValue(value)); } } @@ -56,65 +76,278 @@ public static com.arcadedb.server.grpc.Record toProtoRecord(Record rec) { } /** - * Converts a Java Object into a protobuf Value for Record.properties map. - * - * Supports primitives, strings, numbers, booleans, lists, and maps. + * Converts a Map into a gRPC GrpcRecord message. * - * @param value Object value - * @return Protobuf Value + * @param map Map with properties + * @param rid Optional RID + * @param type Optional type name + * @return Proto GrpcRecord message */ - private static Value toProtoValue(Object value) { - - if (value == null) { - return Values.ofNull(); - } - - if (value instanceof String s) { - return Values.of(s); - } + public static GrpcRecord mapToProtoRecord(Map map, String rid, String type) { + GrpcRecord.Builder builder = GrpcRecord.newBuilder(); - if (value instanceof Integer i) { - return Values.of(i); + if (rid != null && !rid.isEmpty()) { + builder.setRid(rid); } - if (value instanceof Long l) { - return Values.of(l); + if (type != null && !type.isEmpty()) { + builder.setType(type); } - if (value instanceof Float f) { - return Values.of(f); + for (Map.Entry entry : map.entrySet()) { + builder.putProperties(entry.getKey(), toGrpcValue(entry.getValue())); } - if (value instanceof Double d) { - return Values.of(d); - } + return builder.build(); + } + + /** + * Converts a Java Object into a GrpcValue for GrpcRecord.properties map. + * + * Supports primitives, strings, numbers, booleans, lists, maps, dates, and RIDs. + * + * @param value Object value + * @return GrpcValue + */ + public static GrpcValue toGrpcValue(Object value) { + GrpcValue.Builder builder = GrpcValue.newBuilder(); + Value.Builder valueBuilder = Value.newBuilder(); - if (value instanceof Boolean b) { - return Values.of(b); + if (value == null) { + valueBuilder.setNullValue(NullValue.NULL_VALUE); + builder.setValueType(GrpcValueType.RECORD_VALUE_TYPE_UNSPECIFIED); + } else if (value instanceof Boolean) { + valueBuilder.setBoolValue((Boolean) value); + builder.setValueType(GrpcValueType.BOOLEAN); + } else if (value instanceof Byte) { + valueBuilder.setNumberValue(((Byte) value).doubleValue()); + builder.setValueType(GrpcValueType.BYTE); + } else if (value instanceof Short) { + valueBuilder.setNumberValue(((Short) value).doubleValue()); + builder.setValueType(GrpcValueType.SHORT); + } else if (value instanceof Integer) { + valueBuilder.setNumberValue(((Integer) value).doubleValue()); + builder.setValueType(GrpcValueType.INTEGER); + } else if (value instanceof Long) { + valueBuilder.setNumberValue(((Long) value).doubleValue()); + builder.setValueType(GrpcValueType.LONG); + } else if (value instanceof Float) { + valueBuilder.setNumberValue(((Float) value).doubleValue()); + builder.setValueType(GrpcValueType.FLOAT); + } else if (value instanceof Double) { + valueBuilder.setNumberValue((Double) value); + builder.setValueType(GrpcValueType.DOUBLE); + } else if (value instanceof String) { + valueBuilder.setStringValue((String) value); + builder.setValueType(GrpcValueType.STRING); + } else if (value instanceof RID) { + valueBuilder.setStringValue(value.toString()); + builder.setValueType(GrpcValueType.LINK); + } else if (value instanceof java.util.Date) { + // Serialize date as ISO8601 string + valueBuilder.setStringValue(((java.util.Date) value).toInstant().toString()); + builder.setValueType(GrpcValueType.DATETIME); + } else if (value instanceof java.time.LocalDate) { + valueBuilder.setStringValue(value.toString()); + builder.setValueType(GrpcValueType.DATE); + } else if (value instanceof java.time.LocalDateTime) { + valueBuilder.setStringValue(value.toString()); + builder.setValueType(GrpcValueType.DATETIME); + } else if (value instanceof java.time.Instant) { + valueBuilder.setStringValue(value.toString()); + builder.setValueType(GrpcValueType.DATETIME); + } else if (value instanceof byte[]) { + // For binary data, encode as base64 string + valueBuilder.setStringValue(java.util.Base64.getEncoder().encodeToString((byte[]) value)); + builder.setValueType(GrpcValueType.BINARY); + } else if (value instanceof Map) { + Struct.Builder structBuilder = Struct.newBuilder(); + Map map = (Map) value; + + // Determine if this is an embedded document or a regular map + boolean isEmbedded = map.containsKey("@type") || map.containsKey("@class"); + + for (Map.Entry entry : map.entrySet()) { + GrpcValue nestedValue = toGrpcValue(entry.getValue()); + structBuilder.putFields(String.valueOf(entry.getKey()), nestedValue.getValue()); + } + + valueBuilder.setStructValue(structBuilder.build()); + builder.setValueType(isEmbedded ? GrpcValueType.EMBEDDED : GrpcValueType.MAP); + + // Set ofType if this is an embedded document with a known type + if (isEmbedded && map.containsKey("@type")) { + builder.setOfType(String.valueOf(map.get("@type"))); + } + } else if (value instanceof Collection) { + ListValue.Builder listBuilder = ListValue.newBuilder(); + Collection collection = (Collection) value; + + String elementType = null; + for (Object item : collection) { + GrpcValue nestedValue = toGrpcValue(item); + listBuilder.addValues(nestedValue.getValue()); + + // Try to infer element type from first non-null element + if (elementType == null && item != null) { + elementType = inferTypeString(item); + } + } + + valueBuilder.setListValue(listBuilder.build()); + builder.setValueType(GrpcValueType.LIST); + + if (elementType != null) { + builder.setOfType(elementType); + } + } else if (value instanceof Document) { + // Handle embedded documents + Document doc = (Document) value; + Map docMap = new HashMap<>(); + + for (String propName : doc.getPropertyNames()) { + docMap.put(propName, doc.get(propName)); + } + + if (doc.getType() != null) { + docMap.put("@type", doc.getType().getName()); + } + + // Recursive call to handle as map + return toGrpcValue(docMap); + } else { + // Fallback to string representation + valueBuilder.setStringValue(value.toString()); + builder.setValueType(GrpcValueType.STRING); } - if (value instanceof java.util.Date dt) { - // Serialize date as ISO8601 string - return Values.of(dt.toInstant().toString()); + builder.setValue(valueBuilder.build()); + return builder.build(); + } + + /** + * Converts a GrpcValue back to a Java Object. + * + * @param grpcValue GrpcValue to convert + * @return Java Object + */ + public static Object fromGrpcValue(GrpcValue grpcValue) { + if (grpcValue == null) { + return null; } - if (value instanceof Map map) { - var structBuilder = com.google.protobuf.Struct.newBuilder(); - for (Map.Entry entry : map.entrySet()) { - structBuilder.putFields(String.valueOf(entry.getKey()), toProtoValue(entry.getValue())); + Value value = grpcValue.getValue(); + GrpcValueType type = grpcValue.getValueType(); + + switch (value.getKindCase()) { + case NULL_VALUE: + return null; + + case BOOL_VALUE: + return value.getBoolValue(); + + case NUMBER_VALUE: + double numValue = value.getNumberValue(); + // Use the type hint to determine the specific numeric type + switch (type) { + case BYTE: + return (byte) numValue; + case SHORT: + return (short) numValue; + case INTEGER: + return (int) numValue; + case LONG: + return (long) numValue; + case FLOAT: + return (float) numValue; + case DOUBLE: + case DECIMAL: + default: + return numValue; + } + + case STRING_VALUE: + String strValue = value.getStringValue(); + // Check if this is a special string type + switch (type) { + case LINK: + // Parse as RID if it looks like one + if (strValue.startsWith("#")) { + try { + return new RID(strValue); + } catch (Exception e) { + return strValue; + } + } + return strValue; + case BINARY: + // Decode base64 string back to byte array + try { + return java.util.Base64.getDecoder().decode(strValue); + } catch (Exception e) { + return strValue; + } + case DATE: + case DATETIME: + // Could parse as date/time if needed + // For now, return as string and let the caller handle parsing + return strValue; + default: + return strValue; } - return Values.of(structBuilder.build()); + + case STRUCT_VALUE: + Map map = new HashMap<>(); + value.getStructValue().getFieldsMap().forEach((k, v) -> { + // Create a GrpcValue wrapper for recursive conversion + // We don't have type info for nested values, so use UNSPECIFIED + GrpcValue nestedValue = GrpcValue.newBuilder() + .setValue(v) + .setValueType(GrpcValueType.RECORD_VALUE_TYPE_UNSPECIFIED) + .build(); + map.put(k, fromGrpcValue(nestedValue)); + }); + return map; + + case LIST_VALUE: + List list = new ArrayList<>(); + value.getListValue().getValuesList().forEach(v -> { + // Create a GrpcValue wrapper for recursive conversion + GrpcValue nestedValue = GrpcValue.newBuilder() + .setValue(v) + .setValueType(GrpcValueType.RECORD_VALUE_TYPE_UNSPECIFIED) + .build(); + list.add(fromGrpcValue(nestedValue)); + }); + return list; + + default: + return null; } - - if (value instanceof Iterable list) { - var lv = com.google.protobuf.ListValue.newBuilder(); - for (Object item : list) { - lv.addValues(toProtoValue(item)); + } + + /** + * Helper method to infer type string from an object. + * Used for setting the ofType field in collections. + */ + private static String inferTypeString(Object obj) { + if (obj == null) return null; + if (obj instanceof String) return "String"; + if (obj instanceof Integer) return "Integer"; + if (obj instanceof Long) return "Long"; + if (obj instanceof Double) return "Double"; + if (obj instanceof Float) return "Float"; + if (obj instanceof Boolean) return "Boolean"; + if (obj instanceof java.util.Date) return "Date"; + if (obj instanceof RID) return "Link"; + if (obj instanceof Map) { + Map map = (Map) obj; + if (map.containsKey("@type")) { + return String.valueOf(map.get("@type")); } - return Values.of(lv.build()); + return "Map"; } - - // fallback to string - return Values.of(String.valueOf(value)); + if (obj instanceof Collection) return "List"; + return obj.getClass().getSimpleName(); } } \ No newline at end of file diff --git a/grpc-client/src/test/java/com/arcadedb/remote/grpc/ArcadeDbHTTPTvsGRPCBench.java b/grpc-client/src/test/java/com/arcadedb/remote/grpc/ArcadeDbHTTPTvsGRPCBench.java index f0f767b6e7..a1f669dfd6 100644 --- a/grpc-client/src/test/java/com/arcadedb/remote/grpc/ArcadeDbHTTPTvsGRPCBench.java +++ b/grpc-client/src/test/java/com/arcadedb/remote/grpc/ArcadeDbHTTPTvsGRPCBench.java @@ -131,7 +131,7 @@ else if (a.startsWith("--pass=")) }); time("gRPC update by id", () -> { grpc.begin(); - grpc.executeCommand("sql", "UPDATE UserFeedback SET feedback = 'UPDATED_GRPC' WHERE id = 'UF-0001'", Map.of(), false, 0, + grpc.command("sql", "UPDATE UserFeedback SET feedback = 'UPDATED_GRPC' WHERE id = 'UF-0001'", Map.of(), false, 0, /* tx */ null, 60_000); grpc.commit(); }); @@ -232,12 +232,14 @@ private static void cleanupGRPC(RemoteGrpcDatabase grpc) { private static void runStreamBench(RemoteGrpcDatabase grpc, String sql, int batchSize, StreamQueryRequest.RetrievalMode mode) { time("gRPC stream " + mode + " (batchSize=" + batchSize + ")", () -> { + var it = grpc.queryStream(DB_NAME, sql, Map.of(), batchSize, mode, /* tx */ null, 120_000); var total = new AtomicInteger(0); while (it.hasNext()) { it.next(); total.incrementAndGet(); } + System.out.println(" rows=" + total.get()); }); } diff --git a/grpc/pom.xml b/grpc/pom.xml index 537e8717f1..6e75f4c65e 100644 --- a/grpc/pom.xml +++ b/grpc/pom.xml @@ -13,6 +13,13 @@ + + com.arcadedb + arcadedb-engine + ${project.parent.version} + provided + + io.grpc @@ -110,11 +117,13 @@ - + + io.github.ascopes protobuf-maven-plugin - + + org.codehaus.mojo @@ -135,6 +144,20 @@ + + + org.apache.maven.plugins + maven-clean-plugin + 3.3.1 + + + + src/generated + + + + + \ No newline at end of file diff --git a/grpc/src/main/proto/arcadedb-server.proto b/grpc/src/main/proto/arcadedb-server.proto index 0a013a85b2..9d97fb3ce2 100644 --- a/grpc/src/main/proto/arcadedb-server.proto +++ b/grpc/src/main/proto/arcadedb-server.proto @@ -33,11 +33,49 @@ message RowError { uint64 row_index = 1; string code = 2; string message = 3; string field = 4; } -// Generic record payload (document/vertex/edge) -message Record { +message GrpcRecord { string rid = 1; string type = 2; // optional logical type/class name - map properties = 3; + map properties = 3; +} + +message GrpcValue { + + GrpcValueType valueType = 1; + google.protobuf.Value value = 2; + + // Optional type parameter for complex types + // For LIST: element type (e.g., "String", "OrderItem") + // For MAP: value type (e.g., "Integer", "Address") + // For EMBEDDED: document type (e.g., "Address") + // For LINK: target type (e.g., "Customer", "Product") + optional string ofType = 3; +} + +enum GrpcValueType { + + // Default value (0), typically used to indicate an unset/unspecified type + RECORD_VALUE_TYPE_UNSPECIFIED = 0; + + // Primitive types + BOOLEAN = 1; + BYTE = 2; + SHORT = 3; + INTEGER = 4; + LONG = 5; + STRING = 6; + LINK = 7; + BINARY = 8; + DATE = 9; + DATETIME = 10; + FLOAT = 11; + DOUBLE = 12; + DECIMAL = 13; + + // Complex types (use with ofType) + LIST = 14; // ofType specifies element type + MAP = 15; // ofType specifies value type + EMBEDDED = 16; // ofType specifies document type } // ----------------------------------------------------------------------------- @@ -45,10 +83,14 @@ message Record { // ----------------------------------------------------------------------------- message StreamQueryRequest { + string database = 1; string query = 2; - map parameters = 3; + + map parameters = 3; + DatabaseCredentials credentials = 4; + int32 batch_size = 5; enum RetrievalMode { @@ -63,7 +105,7 @@ message StreamQueryRequest { } message QueryResult { - repeated Record records = 1; + repeated GrpcRecord records = 1; int32 total_records_in_batch = 2; int64 running_total_emitted = 3; bool is_last_batch = 4; @@ -160,9 +202,13 @@ message DeleteRecordResponse { // ----------------------------------------------------------------------------- message ExecuteCommandRequest { + string database = 1; + string command = 2; - map parameters = 3; + + map parameters = 3; + DatabaseCredentials credentials = 4; TransactionContext transaction = 5; @@ -179,7 +225,7 @@ message ExecuteCommandResponse { int64 affected_records = 3; // server-computed (elements + numeric scalars) int64 execution_time_ms = 4; - repeated Record records = 5; // optional row payload if return_rows=true + repeated GrpcRecord records = 5; // optional row payload if return_rows=true } // ----------------------------------------------------------------------------- @@ -187,9 +233,12 @@ message ExecuteCommandResponse { // ----------------------------------------------------------------------------- message ExecuteQueryRequest { + string database = 1; string query = 2; - map parameters = 3; + + map parameters = 3; + DatabaseCredentials credentials = 4; TransactionContext transaction = 5; int32 limit = 6; @@ -203,23 +252,24 @@ message ExecuteQueryResponse { } // ----------------------------------------------------------------------------- -// Record CRUD helpers +// GrpcRecord CRUD helpers // ----------------------------------------------------------------------------- message CreateRecordRequest { string database = 1; DatabaseCredentials credentials = 2; string type = 3; // target class/type - Record record = 4; // payload + GrpcRecord record = 4; // payload TransactionContext transaction = 5; } + message CreateRecordResponse { string rid = 1; } // Helper wrapper so we can use a map with oneof message PropertiesUpdate { - map properties = 1; + map properties = 1; } message UpdateRecordRequest { @@ -228,7 +278,7 @@ message UpdateRecordRequest { string rid = 3; oneof payload { - Record record = 4; // full replacement + GrpcRecord record = 4; // full replacement PropertiesUpdate partial = 5; // partial update (map wrapped) } @@ -247,7 +297,7 @@ message LookupByRidRequest { } message LookupByRidResponse { bool found = 1; - Record record = 2; // present if found + GrpcRecord record = 2; // present if found } // ----------------------------------------------------------------------------- @@ -316,7 +366,7 @@ message BulkInsertRequest { DatabaseCredentials credentials = 2; InsertOptions options = 3; - repeated Record rows = 4; + repeated GrpcRecord rows = 4; TransactionContext transaction = 5; // optional } @@ -337,7 +387,7 @@ message InsertChunk { int64 chunk_seq = 6; // 1,2,3,... // Payload - repeated Record rows = 7; + repeated GrpcRecord rows = 7; // Optional client hint bool last = 8; diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java index ed96c59298..6cff3aec51 100644 --- a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java @@ -3,6 +3,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; @@ -13,19 +14,25 @@ import com.arcadedb.database.Database; import com.arcadedb.database.DatabaseFactory; import com.arcadedb.database.Document; +import com.arcadedb.database.Identifiable; import com.arcadedb.database.MutableDocument; +import com.arcadedb.database.MutableEmbeddedDocument; import com.arcadedb.database.RID; import com.arcadedb.engine.ComponentFile; import com.arcadedb.graph.Edge; import com.arcadedb.graph.MutableEdge; import com.arcadedb.graph.MutableVertex; import com.arcadedb.graph.Vertex; +import com.arcadedb.index.Index; import com.arcadedb.query.sql.executor.Result; import com.arcadedb.query.sql.executor.ResultSet; import com.arcadedb.schema.DocumentType; import com.arcadedb.schema.EdgeType; +import com.arcadedb.schema.Property; import com.arcadedb.schema.Schema; +import com.arcadedb.schema.Type; import com.arcadedb.schema.VertexType; +import com.arcadedb.serializer.json.JSONObject; import com.arcadedb.server.ArcadeDBServer; import com.arcadedb.server.grpc.InsertOptions.ConflictMode; import com.arcadedb.server.grpc.InsertOptions.TransactionMode; @@ -125,29 +132,40 @@ public void executeCommand(ExecuteCommandRequest req, StreamObserver props = req.getRecord().getPropertiesMap(); + final java.util.Map props = req.getRecord().getPropertiesMap(); // --- Vertex --- if (dt instanceof com.arcadedb.schema.VertexType) { @@ -255,21 +273,28 @@ public void createRecord(CreateRecordRequest req, StreamObserver { if (!"out".equals(k) && !"in".equals(k)) { e.set(k, toJavaForProperty(db, e, dt, k, val)); @@ -312,6 +338,7 @@ public void createRecord(CreateRecordRequest req, StreamObserver props = req.hasRecord() ? req.getRecord().getPropertiesMap() + final java.util.Map props = req.hasRecord() ? req.getRecord().getPropertiesMap() : req.hasPartial() ? req.getPartial().getPropertiesMap() : java.util.Collections.emptyMap(); // Exclude ArcadeDB system fields during update - + props.forEach((k, v) -> { - String key = k.trim().toLowerCase(); - if (key.equals("@rid") || key.equals("@type") || key.equals("@cat")) { - // Skip internal fields to prevent accidental overwrites - logger.debug("Skipping internal field during update: {}", k); - return; - } - // Perform the update for user-defined fields - mvertex.set(k, toJavaForProperty(dbRef, mvertex, dtype, k, v)); + String key = k.trim().toLowerCase(); + if (key.equals("@rid") || key.equals("@type") || key.equals("@cat")) { + // Skip internal fields to prevent accidental overwrites + logger.debug("Skipping internal field during update: {}", k); + return; + } + // Perform the update for user-defined fields + mvertex.set(k, toJavaForProperty(dbRef, mvertex, dtype, k, v)); }); - + mvertex.save(); } else if (el instanceof Document) { @@ -416,20 +443,20 @@ else if (el instanceof Document) { // Apply updates - final java.util.Map props = req.hasRecord() ? req.getRecord().getPropertiesMap() + final java.util.Map props = req.hasRecord() ? req.getRecord().getPropertiesMap() : req.hasPartial() ? req.getPartial().getPropertiesMap() : java.util.Collections.emptyMap(); // Exclude ArcadeDB system fields during update - + props.forEach((k, v) -> { - String key = k.trim().toLowerCase(); - if (key.equals("@rid") || key.equals("@type") || key.equals("@cat")) { - // Skip internal fields to prevent accidental overwrites - logger.debug("Skipping internal field during update: {}", k); - return; - } - // Perform the update for user-defined fields - mdoc.set(k, toJavaForProperty(dbRef, mdoc, dtype, k, v)); + String key = k.trim().toLowerCase(); + if (key.equals("@rid") || key.equals("@type") || key.equals("@cat")) { + // Skip internal fields to prevent accidental overwrites + logger.debug("Skipping internal field during update: {}", k); + return; + } + // Perform the update for user-defined fields + mdoc.set(k, toJavaForProperty(dbRef, mdoc, dtype, k, v)); }); mdoc.save(); @@ -536,6 +563,7 @@ else if (beganHere) { @Override public void executeQuery(ExecuteQueryRequest request, StreamObserver responseObserver) { + try { // Force compression for streaming (usually beneficial) @@ -563,10 +591,14 @@ public void executeQuery(ExecuteQueryRequest request, StreamObserver scso, AtomicBoolean cancelled) { - final java.util.ArrayList all = new java.util.ArrayList<>(); + + final java.util.List all = new java.util.ArrayList<>(); + try (ResultSet rs = db.query("sql", req.getQuery(), convertParameters(req.getParametersMap()))) { + while (rs.hasNext()) { if (cancelled.get()) return; Result r = rs.next(); if (!r.isElement()) continue; - all.add(convertToGrpcRecord(r.getElement().get())); + all.add(convertToGrpcRecord(r.getElement().get(), db)); } } @@ -856,7 +891,7 @@ private void streamPaged(Database db, StreamQueryRequest req, int batchSize, Ser Result r = rs.next(); if (!r.isElement()) continue; - b.addRecords(convertToGrpcRecord(r.getElement().get())); + b.addRecords(convertToGrpcRecord(r.getElement().get(), db)); count++; } } @@ -1389,7 +1424,7 @@ private InsertSummary toSummary(Counts c, long startedAtMs) { .setFailed(c.failed).addAllErrors(c.errors).setStartedAt(ts(startedAtMs)).setFinishedAt(ts(now)).build(); } - private Counts insertRows(InsertContext ctx, java.util.Iterator it) { + private Counts insertRows(InsertContext ctx, java.util.Iterator it) { Counts c = new Counts(); int inBatch = 0; @@ -1400,7 +1435,8 @@ private Counts insertRows(InsertContext ctx, java.util.Iterator it) { while (it.hasNext()) { - Record r = it.next(); + GrpcRecord r = it.next(); + c.received++; ctx.received++; @@ -1479,35 +1515,24 @@ else if (ctx.opts.getTransactionMode() == TransactionMode.PER_ROW) { return c; } - /** - * Safely extract a String property from a gRPC Record. - * - * @param r the gRPC record - * @param key the property key to lookup - * @return the string value, or null if not present - */ - private static String getStringProp(com.arcadedb.server.grpc.Record r, String key) { + private String getStringProp(GrpcRecord r, String key) { + if (r == null || key == null || key.isEmpty()) { return null; } - // Adjust to your proto structure: - // If you use getPropertiesMap(): + if (r.getPropertiesMap().containsKey(key)) { - var v = r.getPropertiesMap().get(key); - if (v.hasStringValue()) { - return v.getStringValue(); + + GrpcValue grpcVal = r.getPropertiesMap().get(key); + + if (grpcVal.getValueType() == GrpcValueType.STRING || grpcVal.getValue().hasStringValue()) { + + return grpcVal.getValue().getStringValue(); } - return v.toString(); // fallback to raw string - } - // If your proto uses getFieldsMap() instead: - // if (r.getFieldsMap().containsKey(key)) { - // var v = r.getFieldsMap().get(key); - // if (v.hasStringValue()) { - // return v.getStringValue(); - // } - // return v.toString(); - // } + // Try to convert to string + return String.valueOf(fromGrpcValue(grpcVal)); + } return null; } @@ -1516,19 +1541,327 @@ private int serverBatchSize(InsertContext ctx) { return ctx.opts.getServerBatchSize() == 0 ? 1000 : ctx.opts.getServerBatchSize(); } - private void applyGrpcRecord(MutableDocument doc, com.arcadedb.server.grpc.Record r) { - r.getPropertiesMap().forEach((k, val) -> doc.set(k, toJava(val))); + private void applyGrpcRecord(MutableDocument doc, GrpcRecord r) { + r.getPropertiesMap().forEach((k, grpcVal) -> { + // Skip system fields + if (k.startsWith("@")) + return; + doc.set(k, fromGrpcValue(grpcVal)); + }); + } + + private void applyGrpcRecord(MutableVertex vertex, GrpcRecord r) { + r.getPropertiesMap().forEach((k, grpcVal) -> { + // Skip system fields + if (k.startsWith("@")) + return; + vertex.set(k, fromGrpcValue(grpcVal)); + }); } - private void applyGrpcRecord(MutableVertex vertex, com.arcadedb.server.grpc.Record r) { - r.getPropertiesMap().forEach((k, val) -> vertex.set(k, toJava(val))); + private void applyGrpcRecord(MutableEdge edge, GrpcRecord r) { + r.getPropertiesMap().forEach((k, grpcVal) -> { + // Skip system fields + if (k.startsWith("@")) + return; + edge.set(k, fromGrpcValue(grpcVal)); + }); + } + + private Object fromGrpcValue(GrpcValue grpcValue) { + + if (grpcValue == null) + return null; + + Value value = grpcValue.getValue(); + GrpcValueType type = grpcValue.getValueType(); + + // Use the type hint to properly convert + switch (type) { + + case BOOLEAN: + return value.getBoolValue(); + + case BYTE: + if (value.hasNumberValue()) + return (byte) value.getNumberValue(); + if (value.hasStringValue()) + return Byte.parseByte(value.getStringValue()); + return null; + + case SHORT: + if (value.hasNumberValue()) + return (short) value.getNumberValue(); + if (value.hasStringValue()) + return Short.parseShort(value.getStringValue()); + return null; + + case INTEGER: + if (value.hasNumberValue()) + return (int) value.getNumberValue(); + if (value.hasStringValue()) + return Integer.parseInt(value.getStringValue()); + return null; + + case LONG: + if (value.hasNumberValue()) + return (long) value.getNumberValue(); + if (value.hasStringValue()) + return Long.parseLong(value.getStringValue()); + return null; + + case FLOAT: + if (value.hasNumberValue()) + return (float) value.getNumberValue(); + if (value.hasStringValue()) + return Float.parseFloat(value.getStringValue()); + return null; + + case DOUBLE: + return value.getNumberValue(); + + case STRING: + return value.getStringValue(); + + case DECIMAL: + if (value.hasStringValue()) + return new java.math.BigDecimal(value.getStringValue()); + if (value.hasNumberValue()) + return java.math.BigDecimal.valueOf(value.getNumberValue()); + return null; + + case DATE: + if (value.hasStringValue()) { + return java.sql.Date.valueOf(java.time.LocalDate.parse(value.getStringValue())); + } + if (value.hasNumberValue()) { + return new java.util.Date((long) value.getNumberValue()); + } + return null; + + case DATETIME: + if (value.hasStringValue()) { + return java.util.Date.from(java.time.Instant.parse(value.getStringValue())); + } + if (value.hasNumberValue()) { + return new java.util.Date((long) value.getNumberValue()); + } + return null; + + case BINARY: + if (value.hasStringValue()) { + return java.util.Base64.getDecoder().decode(value.getStringValue()); + } + return null; + + case LINK: + if (value.hasStringValue()) { + return new com.arcadedb.database.RID(value.getStringValue()); + } + return null; + + case EMBEDDED: + // Handle embedded document - create from struct + if (value.hasStructValue()) { + // This would need database context to create proper embedded document + return convertStructToMap(value.getStructValue()); + } + return null; + + case MAP: + if (value.hasStructValue()) { + return convertStructToMap(value.getStructValue()); + } + return null; + + case LIST: + if (value.hasListValue()) { + return value.getListValue().getValuesList().stream().map(this::convertProtobufValueToJava) + .collect(java.util.stream.Collectors.toList()); + } + return null; + + default: + return toJava(value); + } } - private void applyGrpcRecord(MutableEdge edge, com.arcadedb.server.grpc.Record r) { - r.getPropertiesMap().forEach((k, val) -> edge.set(k, toJava(val))); + private Map convertStructToMap(com.google.protobuf.Struct struct) { + if (struct == null) { + return new java.util.LinkedHashMap<>(); + } + + Map map = new java.util.LinkedHashMap<>(); + struct.getFieldsMap().forEach((key, value) -> { + map.put(key, convertProtobufValueToJava(value)); + }); + + return map; } - private void applyGrpcRecordToDocument(Record r, MutableDocument doc) { + private Object convertProtobufValueToJava(com.google.protobuf.Value value) { + if (value == null) { + return null; + } + + switch (value.getKindCase()) { + case NULL_VALUE: + return null; + + case NUMBER_VALUE: + double num = value.getNumberValue(); + // Try to preserve integer types when possible + if (num == Math.floor(num) && !Double.isInfinite(num)) { + long longVal = (long) num; + if (longVal >= Integer.MIN_VALUE && longVal <= Integer.MAX_VALUE) { + return (int) longVal; + } + return longVal; + } + return num; + + case STRING_VALUE: + return value.getStringValue(); + + case BOOL_VALUE: + return value.getBoolValue(); + + case STRUCT_VALUE: + // Recursively convert nested struct + return convertStructToMap(value.getStructValue()); + + case LIST_VALUE: + // Convert list values + java.util.List list = new java.util.ArrayList<>(); + value.getListValue().getValuesList().forEach(item -> { + list.add(convertProtobufValueToJava(item)); + }); + return list; + + case KIND_NOT_SET: + default: + return null; + } + } + + // Alternative version that preserves more type information if needed + private Map convertStructToMapWithTypeHints(com.google.protobuf.Struct struct, Map typeHints) { + + if (struct == null) { + return new java.util.LinkedHashMap<>(); + } + + Map map = new java.util.LinkedHashMap<>(); + struct.getFieldsMap().forEach((key, value) -> { + GrpcValueType hint = typeHints != null ? typeHints.get(key) : null; + map.put(key, convertProtobufValueWithTypeHint(value, hint)); + }); + + return map; + } + + private Object convertProtobufValueWithTypeHint(com.google.protobuf.Value value, GrpcValueType typeHint) { + + if (value == null || value.hasNullValue()) { + return null; + } + + // If we have a type hint, use it for more precise conversion + if (typeHint != null) { + switch (typeHint) { + case BYTE: + if (value.hasNumberValue()) + return (byte) value.getNumberValue(); + if (value.hasStringValue()) + return Byte.parseByte(value.getStringValue()); + break; + + case SHORT: + if (value.hasNumberValue()) + return (short) value.getNumberValue(); + if (value.hasStringValue()) + return Short.parseShort(value.getStringValue()); + break; + + case INTEGER: + if (value.hasNumberValue()) + return (int) value.getNumberValue(); + if (value.hasStringValue()) + return Integer.parseInt(value.getStringValue()); + break; + + case LONG: + if (value.hasNumberValue()) + return (long) value.getNumberValue(); + if (value.hasStringValue()) + return Long.parseLong(value.getStringValue()); + break; + + case FLOAT: + if (value.hasNumberValue()) + return (float) value.getNumberValue(); + if (value.hasStringValue()) + return Float.parseFloat(value.getStringValue()); + break; + + case DECIMAL: + if (value.hasStringValue()) + return new java.math.BigDecimal(value.getStringValue()); + if (value.hasNumberValue()) + return java.math.BigDecimal.valueOf(value.getNumberValue()); + break; + + case DATE: + if (value.hasStringValue()) { + try { + return java.sql.Date.valueOf(java.time.LocalDate.parse(value.getStringValue())); + } + catch (Exception e) { + logger.debug("Failed to parse date: {}", value.getStringValue()); + } + } + if (value.hasNumberValue()) { + return new java.sql.Date((long) value.getNumberValue()); + } + break; + + case DATETIME: + if (value.hasStringValue()) { + try { + return java.util.Date.from(java.time.Instant.parse(value.getStringValue())); + } + catch (Exception e) { + logger.debug("Failed to parse datetime: {}", value.getStringValue()); + } + } + if (value.hasNumberValue()) { + return new java.util.Date((long) value.getNumberValue()); + } + break; + + case BINARY: + if (value.hasStringValue()) { + return java.util.Base64.getDecoder().decode(value.getStringValue()); + } + break; + + case LINK: + if (value.hasStringValue()) { + return new com.arcadedb.database.RID(value.getStringValue()); + } + break; + + default: + // Fall through to generic conversion + break; + } + } + + // Generic conversion without type hint + return convertProtobufValueToJava(value); + } + + private void applyGrpcRecordToDocument(GrpcRecord r, MutableDocument doc) { // Example if proto = message Record { map // properties = 1; } if (hasMethod(r, "getPropertiesMap")) { @@ -1579,39 +1912,6 @@ private static Object invokeNoArg(Object o, String name) { } } - private boolean tryUpsert(InsertContext ctx, MutableDocument doc) { - - if (ctx.keyCols.isEmpty()) - return false; - - String where = String.join(" AND ", ctx.keyCols.stream().map(k -> k + " = ?").toList()); - - Object[] params = ctx.keyCols.stream().map(doc::get).toArray(); - - try (ResultSet rs = ctx.db.query("sql", "SELECT FROM " + ctx.opts.getTargetClass() + " WHERE " + where, params)) { - - if (!rs.hasNext()) - return false; - - var res = rs.next(); - - if (!res.isElement()) - return false; - - var existing = res.getElement().get(); - - MutableDocument m = (MutableDocument) existing.asDocument(true); - - for (String col : ctx.updateCols) { - m.set(col, doc.get(col)); - } - - m.save(); - - return true; - } - } - private static Object toJava(Value v) { if (v == null) return null; @@ -1880,96 +2180,242 @@ private void validateCredentials(DatabaseCredentials credentials) { } } - private Map convertParameters(Map protoParams) { + private Map convertParameters(Map protoParams) { + Map params = new HashMap<>(); - for (Map.Entry entry : protoParams.entrySet()) { - params.put(entry.getKey(), convertFromProtobufValue(entry.getValue())); - } - return params; - } - private Object convertFromProtobufValue(Value value) { - switch (value.getKindCase()) { - case NULL_VALUE: - return null; - case NUMBER_VALUE: - return value.getNumberValue(); - case STRING_VALUE: - return value.getStringValue(); - case BOOL_VALUE: - return value.getBoolValue(); - case STRUCT_VALUE: - return convertFromProtobufValues(value.getStructValue().getFieldsMap()); - case LIST_VALUE: - return value.getListValue().getValuesList().stream().map(this::convertFromProtobufValue).toArray(); - default: - return null; - } - } + for (Map.Entry entry : protoParams.entrySet()) { - private Map convertFromProtobufValues(Map values) { - Map result = new HashMap<>(); - for (Map.Entry entry : values.entrySet()) { - result.put(entry.getKey(), convertFromProtobufValue(entry.getValue())); + params.put(entry.getKey(), fromGrpcValue(entry.getValue())); } - return result; + + return params; } - private Record convertToGrpcRecord(com.arcadedb.database.Record dbRecord) { + private GrpcRecord convertToGrpcRecord(com.arcadedb.database.Record dbRecord, Database db) { - Record.Builder builder = Record.newBuilder().setRid(dbRecord.getIdentity().toString()); + GrpcRecord.Builder builder = GrpcRecord.newBuilder().setRid(dbRecord.getIdentity().toString()); - // Handle different record types if (dbRecord instanceof Document) { Document doc = (Document) dbRecord; builder.setType(doc.getTypeName()); - // Convert properties + // Get schema information for type-aware conversion + DocumentType docType = null; + try { + docType = db.getSchema().getType(doc.getTypeName()); + } + catch (Exception e) { + // Type might not exist in schema + } + Set propertyNames = doc.getPropertyNames(); for (String propertyName : propertyNames) { Object value = doc.get(propertyName); if (value != null) { - builder.putProperties(propertyName, convertToProtobufValue(propertyName, value)); + builder.putProperties(propertyName, convertToGrpcValue(propertyName, value, doc, docType)); } } - } - else if (dbRecord instanceof Vertex) { - Vertex vertex = (Vertex) dbRecord; - builder.setType(vertex.getTypeName()); - // Convert properties - Set propertyNames = vertex.getPropertyNames(); - for (String propertyName : propertyNames) { - Object value = vertex.get(propertyName); - if (value != null) { - builder.putProperties(propertyName, convertToProtobufValue(propertyName, value)); + // Add special handling for edges to include endpoints + if (dbRecord instanceof Edge) { + Edge edge = (Edge) dbRecord; + // Add out/in as properties if not already present + if (!propertyNames.contains("@out")) { + builder.putProperties("@out", convertToGrpcValue("@out", edge.getOut(), doc, docType)); + } + if (!propertyNames.contains("@in")) { + builder.putProperties("@in", convertToGrpcValue("@in", edge.getIn(), doc, docType)); } } } - else if (dbRecord instanceof Edge) { - Edge edge = (Edge) dbRecord; - builder.setType(edge.getTypeName()); - // Convert properties - Set propertyNames = edge.getPropertyNames(); - for (String propertyName : propertyNames) { - Object value = edge.get(propertyName); - if (value != null) { - builder.putProperties(propertyName, convertToProtobufValue(propertyName, value)); - } + return builder.build(); + } + + private GrpcValue convertPropToGrpcValue(String propName, Result result) { + + Object propValue = result.getProperty(propName); + + GrpcValue.Builder builder = GrpcValue.newBuilder(); + + GrpcValueType valueType = determineValueType(propValue); + String ofType = inferOfType(propValue); + + builder.setValueType(valueType); + builder.setValue(toProtoValue(propValue)); + + if (ofType != null) { + builder.setOfType(ofType); + } + + return builder.build(); + } + + private GrpcValue convertToGrpcValue(String propName, Object value, Document parentDoc, DocumentType docType) { + + GrpcValue.Builder builder = GrpcValue.newBuilder(); + + // Try to get schema information for this property + com.arcadedb.schema.Property schemaProp = null; + + if (docType != null) { + try { + schemaProp = docType.getProperty(propName); + } + catch (Exception e) { + // Property not defined in schema + } + } + + // Determine the value type + GrpcValueType valueType; + String ofType = null; + + if (schemaProp != null) { + // Use schema type if available + valueType = schemaTypeToGrpcType(schemaProp.getType()); + if (schemaProp.getOfType() != null) { + ofType = schemaProp.getOfType(); } } + else { + // Infer type from value + valueType = determineValueType(value); + ofType = inferOfType(value); + } + + builder.setValueType(valueType); + builder.setValue(toProtoValue(value)); + + if (ofType != null) { + builder.setOfType(ofType); + } return builder.build(); } - private Value convertToProtobufValue(String propName, Object value) { + private GrpcValueType schemaTypeToGrpcType(com.arcadedb.schema.Type schemaType) { - // logger.debug("convertToProtobufValue(): propName = {}, value.class = {}", - // propName, value.getClass()); + switch (schemaType) { + case BOOLEAN: + return GrpcValueType.BOOLEAN; + case BYTE: + return GrpcValueType.BYTE; + case SHORT: + return GrpcValueType.SHORT; + case INTEGER: + return GrpcValueType.INTEGER; + case LONG: + return GrpcValueType.LONG; + case FLOAT: + return GrpcValueType.FLOAT; + case DOUBLE: + return GrpcValueType.DOUBLE; + case DECIMAL: + return GrpcValueType.DECIMAL; + case STRING: + return GrpcValueType.STRING; + case BINARY: + return GrpcValueType.BINARY; + case DATE: + return GrpcValueType.DATE; + case DATETIME: + case DATETIME_MICROS: + case DATETIME_NANOS: + case DATETIME_SECOND: + return GrpcValueType.DATETIME; + case LINK: + return GrpcValueType.LINK; + case EMBEDDED: + return GrpcValueType.EMBEDDED; + case LIST: + return GrpcValueType.LIST; + case MAP: + return GrpcValueType.MAP; + default: + return GrpcValueType.STRING; + } + } - return toProtoValue(value); + private String inferOfType(Object value) { + if (value instanceof Document) { + return ((Document) value).getTypeName(); + } + if (value instanceof Identifiable) { + try { + Document linked = ((Identifiable) value).getRecord().asDocument(); + return linked.getTypeName(); + } + catch (Exception e) { + return null; + } + } + if (value instanceof List && !((List) value).isEmpty()) { + Object first = ((List) value).get(0); + if (first instanceof Document) { + return ((Document) first).getTypeName(); + } + } + return null; + } + + private GrpcValueType determineValueType(Object value) { + if (value == null) + return GrpcValueType.RECORD_VALUE_TYPE_UNSPECIFIED; + if (value instanceof Boolean) + return GrpcValueType.BOOLEAN; + if (value instanceof Byte) + return GrpcValueType.BYTE; + if (value instanceof Short) + return GrpcValueType.SHORT; + if (value instanceof Integer) + return GrpcValueType.INTEGER; + if (value instanceof Long) + return GrpcValueType.LONG; + if (value instanceof Float) + return GrpcValueType.FLOAT; + if (value instanceof Double) + return GrpcValueType.DOUBLE; + if (value instanceof String) + return GrpcValueType.STRING; + if (value instanceof java.math.BigDecimal) + return GrpcValueType.DECIMAL; + if (value instanceof byte[]) + return GrpcValueType.BINARY; + if (value instanceof java.util.Date || value instanceof java.time.LocalDate) + return GrpcValueType.DATE; + if (value instanceof java.time.Instant || value instanceof java.time.LocalDateTime) + return GrpcValueType.DATETIME; + if (value instanceof com.arcadedb.database.RID || value instanceof com.arcadedb.database.Identifiable) + return GrpcValueType.LINK; + if (value instanceof com.arcadedb.database.Document) + return GrpcValueType.EMBEDDED; + if (value instanceof java.util.Map) + return GrpcValueType.MAP; + if (value instanceof java.util.Collection || value.getClass().isArray()) + return GrpcValueType.LIST; + + return GrpcValueType.STRING; // fallback + } + + private String determineOfType(Object value, String propName, String parentTypeName) { + if (value instanceof com.arcadedb.database.Document) { + return ((Document) value).getTypeName(); + } + if (value instanceof com.arcadedb.database.Identifiable) { + // For links, return the target type if known + try { + Document linked = ((Identifiable) value).getRecord().asDocument(); + return linked.getTypeName(); + } + catch (Exception e) { + return null; + } + } + // For collections, you might need schema info to determine element type + return null; } private com.google.protobuf.Value toProtoValue(Object o) { @@ -2039,31 +2485,19 @@ private com.google.protobuf.Value toProtoValue(Object o) { return com.google.protobuf.Value.newBuilder().setStringValue(String.valueOf(o)).build(); } - private com.arcadedb.server.grpc.Record toProtoRecordFromDbRecord(com.arcadedb.database.Record rec) { - var doc = rec.asDocument(); // read-only view if not a document - var b = com.arcadedb.server.grpc.Record.newBuilder().setRid(rec.getIdentity() != null ? rec.getIdentity().toString() : "") - .setType(doc != null ? doc.getTypeName() : ""); - if (doc != null) { - for (String name : doc.getPropertyNames()) { - b.putProperties(name, toProtoValue(doc.get(name))); - } - } - return b.build(); - } - // Apply properties from proto Record to a document/vertex/edge, schema-aware // for EMBEDDED - private void applyGrpcRecord(MutableDocument d, com.arcadedb.server.grpc.Record r, Database db, String targetClass) { + private void applyGrpcRecord(MutableDocument d, GrpcRecord r, Database db, String targetClass) { DocumentType dt = db.getSchema().getType(targetClass); r.getPropertiesMap().forEach((k, val) -> d.set(k, toJavaForProperty(db, d, dt, k, val))); } - private void applyGrpcRecord(MutableVertex v, com.arcadedb.server.grpc.Record r, Database db) { + private void applyGrpcRecord(MutableVertex v, GrpcRecord r, Database db) { DocumentType dt = db.getSchema().getType(v.getTypeName()); r.getPropertiesMap().forEach((k, val) -> v.set(k, toJavaForProperty(db, v, dt, k, val))); } - private void applyGrpcRecord(MutableEdge e, com.arcadedb.server.grpc.Record r, Database db, String edgeClass) { + private void applyGrpcRecord(MutableEdge e, GrpcRecord r, Database db, String edgeClass) { DocumentType dt = db.getSchema().getType(edgeClass); r.getPropertiesMap().forEach((k, val) -> e.set(k, toJavaForProperty(db, e, dt, k, val))); } @@ -2085,181 +2519,476 @@ private Object toJavaLoose(com.google.protobuf.Value v) { }; } - private Object toJavaForProperty(final Database db, final com.arcadedb.database.MutableDocument parent, final com.arcadedb.schema.DocumentType dtype, - final String propName, final com.google.protobuf.Value v) { - - logger.debug("toJavaForProperty(): propName = {}, dtype = {}, v = {}", propName, dtype, v); - - com.arcadedb.schema.Property p = null; - + private Object toJavaForProperty(final Database db, final MutableDocument parent, final DocumentType dtype, final String propName, + final GrpcValue grpcValue) { + + if (grpcValue == null) + return null; + + // Try to get schema information + com.arcadedb.schema.Property prop = null; try { - - p = (dtype != null) ? dtype.getProperty(propName) : null; + prop = (dtype != null) ? dtype.getProperty(propName) : null; } - catch (com.arcadedb.exception.SchemaException schemaException) { - - //logger.warn("SchemaException: " + schemaException.getMessage()); + catch (com.arcadedb.exception.SchemaException e) { + // Property not in schema, use the type hint from GrpcValue } - - if (p == null) - return toJavaLoose(v); - final var prop = dtype != null ? dtype.getProperty(propName) : null; - final var t = prop != null ? prop.getType() : null; + // If we have schema info, use it; otherwise use the type from GrpcValue + if (prop != null) { + return convertWithSchemaType(db, parent, prop, propName, grpcValue); + } + else { + // No schema, use the type hint from the GrpcValue + return fromGrpcValue(grpcValue); + } + } - switch (t) { + private Object convertWithSchemaType(Database db, MutableDocument parent, com.arcadedb.schema.Property prop, String propName, + GrpcValue grpcValue) { + Value v = grpcValue.getValue(); + com.arcadedb.schema.Type schemaType = prop.getType(); - case STRING: - return (v.getKindCase() == com.google.protobuf.Value.KindCase.STRING_VALUE) ? v.getStringValue() : String.valueOf(toJavaLoose(v)); + // Handle null values + if (v.hasNullValue()) { + return null; + } + switch (schemaType) { case BOOLEAN: - return (v.getKindCase() == com.google.protobuf.Value.KindCase.BOOL_VALUE) ? v.getBoolValue() - : Boolean.valueOf(String.valueOf(toJavaLoose(v))); + if (v.hasBoolValue()) + return v.getBoolValue(); + if (v.hasStringValue()) + return Boolean.parseBoolean(v.getStringValue()); + return null; case BYTE: - // treat as number 0..255 (or string), store as Byte - if (v.getKindCase() == com.google.protobuf.Value.KindCase.NUMBER_VALUE) - return (byte) ((int) v.getNumberValue()); - if (v.getKindCase() == com.google.protobuf.Value.KindCase.STRING_VALUE) - return Byte.valueOf(v.getStringValue()); - return (byte) 0; + if (v.hasNumberValue()) + return (byte) v.getNumberValue(); + if (v.hasStringValue()) + return Byte.parseByte(v.getStringValue()); + return null; case SHORT: - if (v.getKindCase() == com.google.protobuf.Value.KindCase.NUMBER_VALUE) - return (short) ((int) v.getNumberValue()); - return Short.valueOf(String.valueOf(toJavaLoose(v))); + if (v.hasNumberValue()) + return (short) v.getNumberValue(); + if (v.hasStringValue()) + return Short.parseShort(v.getStringValue()); + return null; case INTEGER: - if (v.getKindCase() == com.google.protobuf.Value.KindCase.NUMBER_VALUE) + if (v.hasNumberValue()) return (int) v.getNumberValue(); - return Integer.valueOf(String.valueOf(toJavaLoose(v))); + if (v.hasStringValue()) + return Integer.parseInt(v.getStringValue()); + return null; case LONG: - if (v.getKindCase() == com.google.protobuf.Value.KindCase.NUMBER_VALUE) + if (v.hasNumberValue()) return (long) v.getNumberValue(); - return Long.valueOf(String.valueOf(toJavaLoose(v))); + if (v.hasStringValue()) + return Long.parseLong(v.getStringValue()); + return null; case FLOAT: - if (v.getKindCase() == com.google.protobuf.Value.KindCase.NUMBER_VALUE) + if (v.hasNumberValue()) return (float) v.getNumberValue(); - return Float.valueOf(String.valueOf(toJavaLoose(v))); + if (v.hasStringValue()) + return Float.parseFloat(v.getStringValue()); + return null; case DOUBLE: - if (v.getKindCase() == com.google.protobuf.Value.KindCase.NUMBER_VALUE) + if (v.hasNumberValue()) return v.getNumberValue(); - return Double.valueOf(String.valueOf(toJavaLoose(v))); + if (v.hasStringValue()) + return Double.parseDouble(v.getStringValue()); + return null; case DECIMAL: - // use string to preserve precision - if (v.getKindCase() == com.google.protobuf.Value.KindCase.STRING_VALUE) + if (v.hasStringValue()) return new java.math.BigDecimal(v.getStringValue()); - if (v.getKindCase() == com.google.protobuf.Value.KindCase.NUMBER_VALUE) + if (v.hasNumberValue()) return java.math.BigDecimal.valueOf(v.getNumberValue()); return null; + case STRING: + if (v.hasStringValue()) + return v.getStringValue(); + // Convert other types to string representation + if (v.hasNumberValue()) + return String.valueOf(v.getNumberValue()); + if (v.hasBoolValue()) + return String.valueOf(v.getBoolValue()); + return null; + + case BINARY: + if (v.hasStringValue()) { + // Expect base64 encoded string + return java.util.Base64.getDecoder().decode(v.getStringValue()); + } + return null; + case DATE: - // prefer ISO (yyyy-MM-dd). Accept epoch millis as number. - if (v.getKindCase() == com.google.protobuf.Value.KindCase.STRING_VALUE) + if (v.hasStringValue()) { + // Parse ISO date format (yyyy-MM-dd) return java.sql.Date.valueOf(java.time.LocalDate.parse(v.getStringValue())); - if (v.getKindCase() == com.google.protobuf.Value.KindCase.NUMBER_VALUE) - return new java.util.Date((long) v.getNumberValue()); + } + if (v.hasNumberValue()) { + // Epoch millis + return new java.sql.Date((long) v.getNumberValue()); + } return null; case DATETIME: case DATETIME_MICROS: case DATETIME_NANOS: case DATETIME_SECOND: - if (v.getKindCase() == com.google.protobuf.Value.KindCase.STRING_VALUE) { - // ISO-8601 → Instant - java.time.Instant inst = java.time.Instant.parse(v.getStringValue()); - return java.util.Date.from(inst); + if (v.hasStringValue()) { + // Parse ISO-8601 datetime + java.time.Instant instant = java.time.Instant.parse(v.getStringValue()); + return java.util.Date.from(instant); } - if (v.getKindCase() == com.google.protobuf.Value.KindCase.NUMBER_VALUE) { - // epoch millis + if (v.hasNumberValue()) { + // Epoch millis return new java.util.Date((long) v.getNumberValue()); } return null; - case BINARY: - // base64 string on the wire - if (v.getKindCase() == com.google.protobuf.Value.KindCase.STRING_VALUE) - return java.util.Base64.getDecoder().decode(v.getStringValue()); + case LINK: + if (v.hasStringValue()) { + // Parse RID string format (#cluster:position) + return new com.arcadedb.database.RID(v.getStringValue()); + } return null; - case MAP: - // Struct → Map (NOT a document) - if (v.getKindCase() != com.google.protobuf.Value.KindCase.STRUCT_VALUE) - return java.util.Map.of(); - var mm = new java.util.LinkedHashMap(); - v.getStructValue().getFieldsMap().forEach((k, val) -> mm.put(k, toJavaLoose(val))); - return mm; + case EMBEDDED: + if (v.hasStructValue()) { + String embeddedTypeName = grpcValue.hasOfType() ? grpcValue.getOfType() : prop.getOfType(); - case LIST: - if (v.getKindCase() != com.google.protobuf.Value.KindCase.LIST_VALUE) - return java.util.List.of(); - return v.getListValue().getValuesList().stream().map(this::toJavaLoose).toList(); - - case EMBEDDED: { - - // Must be a STRUCT in proto - if (v.getKindCase() != com.google.protobuf.Value.KindCase.STRUCT_VALUE) { - // choose policy (reject or coerce); rejecting is safer: - throw new IllegalArgumentException("EMBEDDED property '" + propName + "' expects STRUCT"); - } - - String propTypeName = prop.getOfType(); - - logger.debug("EMBEDDED: property '" + propName + "' expects STRUCT of type '" + propTypeName + "'. Received: " + v.getStructValue().getFieldsMap().keySet()); - - DocumentType propType = db.getSchema().getType(propTypeName); - - // Guard: if someone mislinked a vertex/edge type here, fail loudly - if (propType instanceof com.arcadedb.schema.VertexType || propType instanceof com.arcadedb.schema.EdgeType) { - throw new IllegalArgumentException( - "EMBEDDED property '" + propName + "' cannot link to non-document type: " + propType.getName()); - } - - // Create embedded from the parent. Passing null is allowed for “untyped” embedded docs. - final com.arcadedb.database.MutableEmbeddedDocument ed = parent.newEmbeddedDocument(propTypeName, propName); - - // populate fields - v.getStructValue().getFieldsMap().forEach((k, vv) -> ed.set(k, toJavaLoose(vv))); - return ed; - } - - case LINK: { - // Expect "#:" in string; adjust if you send a struct instead - if (v.getKindCase() == com.google.protobuf.Value.KindCase.STRING_VALUE) { - return new com.arcadedb.database.RID(v.getStringValue()); + MutableEmbeddedDocument ed = parent.newEmbeddedDocument(embeddedTypeName, propName); + + // Recursively convert struct fields + v.getStructValue().getFieldsMap().forEach((k, vv) -> { + // Try to get schema for embedded type + DocumentType embeddedType = null; + com.arcadedb.schema.Property embeddedProp = null; + + if (embeddedTypeName != null) { + try { + embeddedType = db.getSchema().getType(embeddedTypeName); + embeddedProp = embeddedType.getProperty(k); + } + catch (Exception e) { + // Property not in schema + } + } + + if (embeddedProp != null && embeddedType != null) { + // Create a simple GrpcValue wrapper for recursive conversion + GrpcValue.Builder rvBuilder = GrpcValue.newBuilder().setValue(vv) + .setValueType(schemaTypeToGrpcType(embeddedProp.getType())); + ed.set(k, convertWithSchemaType(db, ed, embeddedProp, k, rvBuilder.build())); + } + else { + // No schema, use generic conversion + ed.set(k, toJava(vv)); + } + }); + return ed; } return null; - } - // Collections of links/embedded/arrays → treat as list/map with toJavaLoose + case LIST: case ARRAY_OF_SHORTS: case ARRAY_OF_INTEGERS: case ARRAY_OF_LONGS: case ARRAY_OF_FLOATS: case ARRAY_OF_DOUBLES: - return toJavaLoose(v); + if (v.hasListValue()) { + java.util.List list = new java.util.ArrayList<>(); + + // Determine element type + com.arcadedb.schema.Type elementType = getListElementType(schemaType); + + for (com.google.protobuf.Value item : v.getListValue().getValuesList()) { + if (elementType != null) { + // Create wrapper for typed conversion + GrpcValue.Builder rvBuilder = GrpcValue.newBuilder().setValue(item).setValueType(schemaTypeToGrpcType(elementType)); + + // Create a dummy property for element conversion + com.arcadedb.schema.Property elementProp = createVirtualProperty(elementType, prop.getOfType()); + list.add(convertWithSchemaType(db, parent, elementProp, propName + "[element]", rvBuilder.build())); + } + else { + // Generic conversion + list.add(toJava(item)); + } + } + + // Convert to appropriate array type if needed + if (schemaType == com.arcadedb.schema.Type.ARRAY_OF_SHORTS) { + return list.stream().mapToInt(o -> ((Number) o).shortValue()).toArray(); + } + else if (schemaType == com.arcadedb.schema.Type.ARRAY_OF_INTEGERS) { + return list.stream().mapToInt(o -> ((Number) o).intValue()).toArray(); + } + else if (schemaType == com.arcadedb.schema.Type.ARRAY_OF_LONGS) { + return list.stream().mapToLong(o -> ((Number) o).longValue()).toArray(); + } + else if (schemaType == com.arcadedb.schema.Type.ARRAY_OF_FLOATS) { + float[] array = new float[list.size()]; + for (int i = 0; i < list.size(); i++) { + array[i] = ((Number) list.get(i)).floatValue(); + } + return array; + } + else if (schemaType == com.arcadedb.schema.Type.ARRAY_OF_DOUBLES) { + return list.stream().mapToDouble(o -> ((Number) o).doubleValue()).toArray(); + } + + return list; + } + return null; + + case MAP: + if (v.hasStructValue()) { + java.util.Map map = new java.util.LinkedHashMap<>(); + v.getStructValue().getFieldsMap().forEach((k, vv) -> { + map.put(k, toJava(vv)); + }); + return map; + } + return null; default: - return toJavaLoose(v); + // Fallback to generic conversion + return toJava(v); } } - - private String generateTransactionId() { - return "tx_" + System.nanoTime(); - } - - private long getUptime() { - // Implement uptime calculation - return System.currentTimeMillis(); + + private com.arcadedb.schema.Property createVirtualProperty(final com.arcadedb.schema.Type type, final String ofType) { + + return new com.arcadedb.schema.Property() { + private final Map customValues = new HashMap<>(); + private Object defaultValue = null; + private boolean readonly = false; + private boolean mandatory = false; + private boolean notNull = false; + private boolean hidden = false; + private String max = null; + private String min = null; + private String regexp = null; + + @Override + public Index createIndex(Schema.INDEX_TYPE indexType, boolean unique) { + // Virtual properties cannot create indexes + throw new UnsupportedOperationException("Cannot create index on virtual property"); + } + + @Override + public Index getOrCreateIndex(Schema.INDEX_TYPE indexType, boolean unique) { + // Virtual properties don't have indexes + return null; + } + + @Override + public String getName() { + return "__virtual_" + type.name().toLowerCase(); + } + + @Override + public Type getType() { + return type; + } + + @Override + public int getId() { + // Virtual properties don't have database IDs + return -1; + } + + @Override + public Object getDefaultValue() { + return defaultValue; + } + + @Override + public Property setDefaultValue(Object defaultValue) { + this.defaultValue = defaultValue; + return this; + } + + @Override + public String getOfType() { + return ofType; + } + + @Override + public Property setOfType(String ofType) { + // Cannot modify virtual property's ofType after creation + throw new UnsupportedOperationException("Cannot modify virtual property's ofType"); + } + + @Override + public Property setReadonly(boolean readonly) { + this.readonly = readonly; + return this; + } + + @Override + public boolean isReadonly() { + return readonly; + } + + @Override + public Property setMandatory(boolean mandatory) { + this.mandatory = mandatory; + return this; + } + + @Override + public boolean isMandatory() { + return mandatory; + } + + @Override + public Property setNotNull(boolean notNull) { + this.notNull = notNull; + return this; + } + + @Override + public boolean isNotNull() { + return notNull; + } + + @Override + public Property setHidden(boolean hidden) { + this.hidden = hidden; + return this; + } + + @Override + public boolean isHidden() { + return hidden; + } + + @Override + public Property setMax(String max) { + this.max = max; + return this; + } + + @Override + public String getMax() { + return max; + } + + @Override + public Property setMin(String min) { + this.min = min; + return this; + } + + @Override + public String getMin() { + return min; + } + + @Override + public Property setRegexp(String regexp) { + this.regexp = regexp; + return this; + } + + @Override + public String getRegexp() { + return regexp; + } + + @Override + public Set getCustomKeys() { + return customValues.keySet(); + } + + @Override + public Object getCustomValue(String key) { + return customValues.get(key); + } + + @Override + public Object setCustomValue(String key, Object value) { + return customValues.put(key, value); + } + + @Override + public JSONObject toJSON() { + JSONObject json = new JSONObject(); + json.put("name", getName()); + json.put("type", type.name()); + if (ofType != null) { + json.put("ofType", ofType); + } + if (defaultValue != null) { + json.put("default", defaultValue); + } + json.put("readonly", readonly); + json.put("mandatory", mandatory); + json.put("notNull", notNull); + json.put("hidden", hidden); + if (max != null) + json.put("max", max); + if (min != null) + json.put("min", min); + if (regexp != null) + json.put("regexp", regexp); + if (!customValues.isEmpty()) { + json.put("custom", new JSONObject(customValues)); + } + return json; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (!(obj instanceof com.arcadedb.schema.Property)) + return false; + com.arcadedb.schema.Property other = (com.arcadedb.schema.Property) obj; + return type == other.getType() && Objects.equals(ofType, other.getOfType()); + } + + @Override + public int hashCode() { + return Objects.hash(type, ofType); + } + + @Override + public String toString() { + return "VirtualProperty{type=" + type + (ofType != null ? ", ofType=" + ofType : "") + "}"; + } + }; } - private int getActiveConnections() { - // Implement active connections count - return 0; + // Helper method to determine element type for arrays/lists + private com.arcadedb.schema.Type getListElementType(com.arcadedb.schema.Type listType) { + switch (listType) { + case ARRAY_OF_SHORTS: + return com.arcadedb.schema.Type.SHORT; + case ARRAY_OF_INTEGERS: + return com.arcadedb.schema.Type.INTEGER; + case ARRAY_OF_LONGS: + return com.arcadedb.schema.Type.LONG; + case ARRAY_OF_FLOATS: + return com.arcadedb.schema.Type.FLOAT; + case ARRAY_OF_DOUBLES: + return com.arcadedb.schema.Type.DOUBLE; + default: + return null; + } } + private String generateTransactionId() { + return "tx_" + System.nanoTime(); + } } \ No newline at end of file From f661950497bb3a1781983220d7eac1ee9595cb1b Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Thu, 4 Sep 2025 21:26:37 -0700 Subject: [PATCH 21/35] Checkpoint Commit --- .../remote/grpc/RemoteGrpcDatabase.java | 61 ++++- .../RemoteGrpcDatabaseRegressionTest.java | 223 ++++++++++++++++++ .../server/grpc/ArcadeDbGrpcService.java | 149 ++++++++---- 3 files changed, 377 insertions(+), 56 deletions(-) create mode 100644 grpc-client/src/test/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseRegressionTest.java diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java index 8716a51975..4264be198a 100644 --- a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java @@ -342,13 +342,17 @@ public ResultSet query(final String language, final String query, final Map 0) { Map result = new HashMap<>(); result.put("affected", response.getAffectedRecords()); @@ -1533,6 +1543,7 @@ private Result grpcRecordToResult(GrpcRecord grpcRecord) { } private Record grpcRecordToDBRecord(GrpcRecord grpcRecord) { + Map map = new HashMap<>(); // Convert properties @@ -1544,6 +1555,12 @@ private Record grpcRecordToDBRecord(GrpcRecord grpcRecord) { map.put("@cat", mapRecordType(grpcRecord)); String cat = (String) map.get("@cat"); + + if (cat == null) { + + return null; + } + switch (cat) { case "d": return new RemoteImmutableDocument(this, map); @@ -1557,31 +1574,49 @@ private Record grpcRecordToDBRecord(GrpcRecord grpcRecord) { } private String mapRecordType(GrpcRecord grpcRecord) { + // Determine record category from type name String typeName = grpcRecord.getType(); // Check schema to determine actual type try { + if (getSchema().existsType(typeName)) { + Object type = getSchema().getType(typeName); - if (type instanceof com.arcadedb.schema.VertexType) { + + if (type instanceof com.arcadedb.schema.VertexType) { return "v"; - } else if (type instanceof com.arcadedb.schema.EdgeType) { + } + else if (type instanceof com.arcadedb.schema.EdgeType) { return "e"; } + else if (type instanceof com.arcadedb.schema.DocumentType) { + return "d"; + } + else { + return null; + } } - } catch (Exception e) { + else { + + return null; + } + } + catch (Exception e) { // Fall back to name-based detection } - // Fall back to name-based detection - if (typeName.contains("Vertex") || typeName.startsWith("V_")) { - return "v"; - } else if (typeName.contains("Edge") || typeName.startsWith("E_")) { - return "e"; - } else { - return "d"; - } + return null; + +// // Fall back to name-based detection +// if (typeName.contains("Vertex") || typeName.startsWith("V_")) { +// return "v"; +// } else if (typeName.contains("Edge") || typeName.startsWith("E_")) { +// return "e"; +// } else { +// return "d"; +// } } private Map convertParamsToGrpcValue(Map params) { diff --git a/grpc-client/src/test/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseRegressionTest.java b/grpc-client/src/test/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseRegressionTest.java new file mode 100644 index 0000000000..23615c23aa --- /dev/null +++ b/grpc-client/src/test/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseRegressionTest.java @@ -0,0 +1,223 @@ +package com.arcadedb.remote.grpc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +import com.arcadedb.query.sql.executor.Result; +import com.arcadedb.query.sql.executor.ResultSet; +import com.arcadedb.server.grpc.InsertOptions; +import com.arcadedb.server.grpc.InsertOptions.ConflictMode; +import com.arcadedb.server.grpc.InsertOptions.TransactionMode; +import com.arcadedb.server.grpc.InsertSummary; + +/** + * Regression tests that exercise ONLY the gRPC Remote database client. + * + * Requirements: - ArcadeDB server must be running with gRPC enabled. - + * Connection parameters may be provided via env vars: ARCADE_DB, + * ARCADE_GRPC_HOST, ARCADE_GRPC_PORT, ARCADE_HTTP_PORT, ARCADE_USER, + * ARCADE_PASS + * + * Defaults (match the sample bench): DB=ArcadeDB, gRPC=127.0.0.1:50059, + * HTTP=127.0.0.1:2489, user=root, pass=root1234 + * + * These tests create an isolated vertex type "RG_Feedback" and clean up their + * own data. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class RemoteGrpcDatabaseRegressionTest { + + // -------- Config (env overrides supported) -------- + + static final String DB_NAME = System.getenv().getOrDefault("ARCADE_DB", "test"); + static final String GRPC_HOST = System.getenv().getOrDefault("ARCADE_GRPC_HOST", "127.0.0.1"); + static final int GRPC_PORT = Integer.parseInt(System.getenv().getOrDefault("ARCADE_GRPC_PORT", "50051")); + static final String HTTP_HOST = System.getenv().getOrDefault("ARCADE_HTTP_HOST", "127.0.0.1"); + static final int HTTP_PORT = Integer.parseInt(System.getenv().getOrDefault("ARCADE_HTTP_PORT", "2480")); + static final String USER = System.getenv().getOrDefault("ARCADE_USER", "root"); + static final String PASS = System.getenv().getOrDefault("ARCADE_PASS", "oY9uU2uJ8nD8iY7t"); + + // Test type & props + static final String TYPE = "RG_Feedback"; + + private RemoteGrpcDatabase grpc; + + @BeforeAll + void ensureDatabaseExists() { + // Prefer using the gRPC admin helper if available (same package). + try (RemoteGrpcServer admin = new RemoteGrpcServer(GRPC_HOST, GRPC_PORT, USER, PASS)) { + if (!admin.existsDatabase(DB_NAME)) { + admin.createDatabase(DB_NAME); + } + } + catch (Throwable t) { + // If admin tooling isn't on the classpath, tests may still pass provided DB + // exists. + System.err.println("[WARN] Could not verify/create DB via RemoteGrpcServer: " + t.getMessage()); + } + } + + @BeforeEach + void open() { + grpc = new RemoteGrpcDatabase(GRPC_HOST, GRPC_PORT, HTTP_PORT, DB_NAME, USER, PASS); + // Create isolated schema for these tests (id unique, name string, n integer) + grpc.command("sql", "CREATE VERTEX TYPE `" + TYPE + "` IF NOT EXISTS", Map.of()); + grpc.command("sql", "CREATE PROPERTY `" + TYPE + "`.id IF NOT EXISTS STRING", Map.of()); + grpc.command("sql", "CREATE PROPERTY `" + TYPE + "`.name IF NOT EXISTS STRING", Map.of()); + grpc.command("sql", "CREATE PROPERTY `" + TYPE + "`.n IF NOT EXISTS INTEGER", Map.of()); + grpc.command("sql", "CREATE INDEX IF NOT EXISTS ON " + TYPE + " (id) UNIQUE", Map.of()); + // Ensure clean slate for each test method + grpc.command("sql", "DELETE FROM `" + TYPE + "`", Map.of()); + } + + @AfterEach + void close() { + if (grpc != null) { + try { + grpc.rollback(); + } + catch (Throwable ignore) { + } + grpc.close(); + } + } + + // ---------- Helpers ---------- + + private InsertOptions defaultInsertOptions(final String targetClass, final List keyCols, final List updateCols) { + return InsertOptions.newBuilder().setDatabase(DB_NAME).setTargetClass(targetClass).addAllKeyColumns(keyCols) + .setConflictMode(ConflictMode.CONFLICT_UPDATE) // idempotent: upsert by keys + .addAllUpdateColumnsOnConflict(updateCols) // LWW on these fields + .setTransactionMode(TransactionMode.PER_BATCH).setServerBatchSize(256).setCredentials(grpc.buildCredentials()) // package-private + // in same + // package + .build(); + } + + private long countAll(String type) { + + String sql = "SELECT count(*) AS c FROM " + type; + + try (ResultSet rs = grpc.query("sql", sql, Map.of())) { + long c = 0; + while (rs.hasNext()) { + Result r = rs.next(); + //System.out.println("Count all: r = " + r); + Number n = r.getProperty("c"); + c = (n == null) ? 0 : n.longValue(); + } + return c; + } + } + + private Map row(String id, String name, int n) { + Map m = new HashMap<>(); + m.put("id", id); + m.put("name", name); + m.put("n", n); + return m; + } + + // ---------- Tests ---------- + + @Test + @DisplayName("Bulk insert via gRPC is idempotent by key and supports updates on conflict") + void bulkInsertIdempotentAndUpdate() { + // Prepare rows + List> rows = new ArrayList<>(); + rows.add(row("r1", "alpha", 1)); + rows.add(row("r2", "beta", 2)); + rows.add(row("r3", "gamma", 3)); + + grpc.begin(); + InsertOptions opts = defaultInsertOptions(TYPE, List.of("id"), Arrays.asList("name", "n")); + InsertSummary s1 = grpc.insertBulkAsListOfMaps(opts, rows, 60_000); + grpc.commit(); + + assertEquals(3, s1.getInserted(), "first insert should insert 3"); + assertEquals(0, s1.getUpdated(), "first insert should not update"); + assertEquals(3, countAll(TYPE), "row count after first insert"); + + // Re-insert with one changed record (r2) to force an update-on-conflict + rows.set(1, row("r2", "beta-UPDATED", 22)); + + grpc.begin(); + InsertSummary s2 = grpc.insertBulkAsListOfMaps(opts, rows, 60_000); + grpc.commit(); + + assertEquals(0, s2.getInserted(), "second insert should not insert new rows"); + assertTrue(s2.getUpdated() >= 1, "should update at least 1 row on conflict"); + assertEquals(3, countAll(TYPE), "row count unchanged after upsert"); + + // Verify the updated record + try (ResultSet rs = grpc.query("sql", "SELECT from `" + TYPE + "` WHERE id = :id", Map.of("id", "r2"))) { + assertTrue(rs.hasNext(), "record r2 must exist"); + Result r = rs.next(); + assertEquals("beta-UPDATED", r.getProperty("name")); + assertEquals(22, r.getProperty("n").intValue()); + } + } + + @Test + @DisplayName("Basic CRUD via SQL commands over gRPC") + void basicCrudViaCommand() { + // Create + grpc.command("sql", "INSERT INTO `" + TYPE + "` set id = :id, name = :name, n = :n", Map.of("id", "x1", "name", "one", "n", 1)); + grpc.command("sql", "INSERT INTO `" + TYPE + "` set id = :id, name = :name, n = :n", Map.of("id", "x2", "name", "two", "n", 2)); + + assertEquals(2, countAll(TYPE), "two rows inserted"); + + // Read + try (ResultSet rs = grpc.query("sql", "SELECT from `" + TYPE + "` WHERE id = :id", Map.of("id", "x1"))) { + assertTrue(rs.hasNext()); + Result r = rs.next(); + assertEquals("one", r.getProperty("name")); + } + + // Update + grpc.begin(); + grpc.command("sql", "UPDATE `" + TYPE + "` SET name = :name, n = :n WHERE id = :id", Map.of("name", "ONE!", "n", 11, "id", "x1")); + grpc.commit(); + + try (ResultSet rs = grpc.query("sql", "SELECT from `" + TYPE + "` WHERE id = :id", Map.of("id", "x1"))) { + Result r = rs.next(); + assertEquals("ONE!", r.getProperty("name")); + assertEquals(11, r.getProperty("n").intValue()); + } + + // Delete + grpc.command("sql", "DELETE FROM `" + TYPE + "` WHERE id = :id", Map.of("id", "x2")); + assertEquals(1, countAll(TYPE), "one row remains after delete"); + } + + @Test + @DisplayName("Transaction: rollback undoes changes; commit persists") + void transactionsRollbackAndCommit() { + long before = countAll(TYPE); + + // Rollback path + grpc.begin(); + grpc.command("sql", "INSERT INTO `" + TYPE + "` set id = :id, name = :name, n = :n", Map.of("id", "tx1", "name", "temp", "n", 99)); + grpc.rollback(); + assertEquals(before, countAll(TYPE), "rollback must revert insert"); + + // Commit path + grpc.begin(); + grpc.command("sql", "INSERT INTO `" + TYPE + "` set id = :id, name = :name, n = :n", Map.of("id", "tx2", "name", "persisted", "n", 100)); + grpc.commit(); + assertEquals(before + 1, countAll(TYPE), "commit must persist insert"); + } +} diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java index 6cff3aec51..a59d3a7ba7 100644 --- a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java @@ -104,13 +104,15 @@ public void close() { @Override public void executeCommand(ExecuteCommandRequest req, StreamObserver resp) { + final long t0 = System.nanoTime(); Database db = null; boolean beganHere = false; try { - // Resolve DB + params + + // Resolve DB + params db = getDatabase(req.getDatabase(), req.getCredentials()); final java.util.Map params = convertParameters(req.getParametersMap()); @@ -119,79 +121,100 @@ public void executeCommand(ExecuteCommandRequest req, StreamObserver 0 ? req.getMaxRows() : DEFAULT_MAX_COMMAND_ROWS; ExecuteCommandResponse.Builder out = ExecuteCommandResponse.newBuilder().setSuccess(true).setMessage("OK"); // Execute the command - + + logger.info("executeCommand(): command = {}", req.getCommand()); + try (ResultSet rs = db.command(language, req.getCommand(), params)) { - if (returnRows) { - int emitted = 0; - while (rs.hasNext()) { - - Result r = rs.next(); - - if (r.isElement()) { - affected++; // count modified/returned records - if (emitted < maxRows) { - out.addRecords(convertToGrpcRecord(r.getElement().get(), db)); - emitted++; + if (rs != null) { + + logger.info("executeCommand(): rs = {}", rs); + + if (returnRows) { + + logger.info("executeCommand(): returning rows ..."); + + int emitted = 0; + + while (rs.hasNext()) { + + Result r = rs.next(); + + if (r.isElement()) { + affected++; // count modified/returned records + if (emitted < maxRows) { + out.addRecords(convertToGrpcRecord(r.getElement().get(), db)); + emitted++; + } } - } - else { + else { - // Scalar / projection row (e.g., RETURN COUNT) + // Scalar / projection row (e.g., RETURN COUNT) - if (emitted < maxRows) { + if (emitted < maxRows) { - GrpcRecord.Builder recB = GrpcRecord.newBuilder(); + GrpcRecord.Builder recB = GrpcRecord.newBuilder(); - for (String p : r.getPropertyNames()) { + for (String p : r.getPropertyNames()) { - recB.putProperties(p, convertPropToGrpcValue(p, r)); - } + recB.putProperties(p, convertPropToGrpcValue(p, r)); + } - out.addRecords(recB.build()); + out.addRecords(recB.build()); - emitted++; - } + emitted++; + } - for (String p : r.getPropertyNames()) { - Object v = r.getProperty(p); - if (v instanceof Number n) - affected += n.longValue(); + for (String p : r.getPropertyNames()) { + Object v = r.getProperty(p); + if (v instanceof Number n) + affected += n.longValue(); + } } } } - } - else { - // Not returning rows: still consume to compute 'affected' - while (rs.hasNext()) { - Result r = rs.next(); - if (r.isElement()) { - affected++; - } - else { - for (String p : r.getPropertyNames()) { - Object v = r.getProperty(p); - if (v instanceof Number n) - affected += n.longValue(); + else { + + logger.info("executeCommand(): not returning rows ... rs = {}", rs); + + // Not returning rows: still consume to compute 'affected' + while (rs.hasNext()) { + Result r = rs.next(); + if (r.isElement()) { + affected++; + } + else { + for (String p : r.getPropertyNames()) { + Object v = r.getProperty(p); + if (v instanceof Number n) + affected += n.longValue(); + } } } } } } + logger.info("executeCommand(): after - hasTx = {} tx ={}", hasTx, tx); + // Transaction end — precedence: rollback > commit > begin-only⇒commit if (hasTx) { if (tx.getRollback()) { @@ -214,6 +237,9 @@ else if (beganHere) { } catch (Exception e) { + + logger.error("ERROR", e); + // Best-effort rollback if we began here and failed try { if (beganHere && db != null) @@ -573,6 +599,9 @@ public void executeQuery(ExecuteQueryRequest request, StreamObserver Date: Thu, 4 Sep 2025 21:27:34 -0700 Subject: [PATCH 22/35] Checkpoint Commit --- .../server/grpc/ArcadeDbGrpcService.java | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java index a59d3a7ba7..ba1243c53a 100644 --- a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java @@ -124,7 +124,7 @@ public void executeCommand(ExecuteCommandRequest req, StreamObserver commit > begin-only⇒commit if (hasTx) { @@ -600,7 +600,7 @@ public void executeQuery(ExecuteQueryRequest request, StreamObserver Date: Fri, 5 Sep 2025 06:16:01 -0700 Subject: [PATCH 23/35] Revised GrpcValue structure --- grpc/src/main/proto/arcadedb-server.proto | 82 +- .../server/grpc/ArcadeDbGrpcService.java | 1628 +++++------------ 2 files changed, 452 insertions(+), 1258 deletions(-) diff --git a/grpc/src/main/proto/arcadedb-server.proto b/grpc/src/main/proto/arcadedb-server.proto index 9d97fb3ce2..d459520699 100644 --- a/grpc/src/main/proto/arcadedb-server.proto +++ b/grpc/src/main/proto/arcadedb-server.proto @@ -33,49 +33,57 @@ message RowError { uint64 row_index = 1; string code = 2; string message = 3; string field = 4; } +// ----------------------------------------------------------------------------- +// GRPC Record +// ----------------------------------------------------------------------------- + +// A record (vertex/document) message GrpcRecord { - string rid = 1; - string type = 2; // optional logical type/class name + string rid = 1; // empty for embedded docs + string type = 2; // ArcadeDB class/type name (optional for embedded) map properties = 3; } +// Typed values, recursively composable message GrpcValue { + oneof kind { + bool bool_value = 1; + int32 int32_value = 2; + int64 int64_value = 3; // epoch ms, counters, etc. + float float_value = 4; + double double_value = 5; + string string_value = 6; + bytes bytes_value = 7; // binary + google.protobuf.Timestamp timestamp_value = 8; // date/datetime + GrpcList list_value = 9; // LIST + GrpcMap map_value = 10; // schemaless MAP + GrpcEmbedded embedded_value = 11; // embedded doc + GrpcLink link_value = 12; // RID link + GrpcDecimal decimal_value = 13; // exact decimal + } + + // Optional semantic hint (non-breaking): "date", "datetime", "rid", "decimal(38,9)", etc. + string logical_type = 14; +} + +// Containers +message GrpcList { repeated GrpcValue values = 1; } +message GrpcMap { map entries = 1; } +message GrpcEmbedded{ + string type = 1; // ArcadeDB class name if present; may be empty + map fields = 2; +} + +// RID link +message GrpcLink { + string rid = 1; // e.g., "#12:0" + string type = 2; // optional target class +} - GrpcValueType valueType = 1; - google.protobuf.Value value = 2; - - // Optional type parameter for complex types - // For LIST: element type (e.g., "String", "OrderItem") - // For MAP: value type (e.g., "Integer", "Address") - // For EMBEDDED: document type (e.g., "Address") - // For LINK: target type (e.g., "Customer", "Product") - optional string ofType = 3; -} - -enum GrpcValueType { - - // Default value (0), typically used to indicate an unset/unspecified type - RECORD_VALUE_TYPE_UNSPECIFIED = 0; - - // Primitive types - BOOLEAN = 1; - BYTE = 2; - SHORT = 3; - INTEGER = 4; - LONG = 5; - STRING = 6; - LINK = 7; - BINARY = 8; - DATE = 9; - DATETIME = 10; - FLOAT = 11; - DOUBLE = 12; - DECIMAL = 13; - - // Complex types (use with ofType) - LIST = 14; // ofType specifies element type - MAP = 15; // ofType specifies value type - EMBEDDED = 16; // ofType specifies document type +// Exact decimal without loss +message GrpcDecimal { + sint64 unscaled = 1; // unscaled value + int32 scale = 2; // number of decimal places } // ----------------------------------------------------------------------------- diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java index ba1243c53a..6805daf2f6 100644 --- a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java @@ -1,10 +1,7 @@ package com.arcadedb.server.grpc; import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.Objects; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; @@ -14,7 +11,6 @@ import com.arcadedb.database.Database; import com.arcadedb.database.DatabaseFactory; import com.arcadedb.database.Document; -import com.arcadedb.database.Identifiable; import com.arcadedb.database.MutableDocument; import com.arcadedb.database.MutableEmbeddedDocument; import com.arcadedb.database.RID; @@ -23,21 +19,16 @@ import com.arcadedb.graph.MutableEdge; import com.arcadedb.graph.MutableVertex; import com.arcadedb.graph.Vertex; -import com.arcadedb.index.Index; import com.arcadedb.query.sql.executor.Result; import com.arcadedb.query.sql.executor.ResultSet; import com.arcadedb.schema.DocumentType; import com.arcadedb.schema.EdgeType; -import com.arcadedb.schema.Property; import com.arcadedb.schema.Schema; -import com.arcadedb.schema.Type; import com.arcadedb.schema.VertexType; -import com.arcadedb.serializer.json.JSONObject; import com.arcadedb.server.ArcadeDBServer; import com.arcadedb.server.grpc.InsertOptions.ConflictMode; import com.arcadedb.server.grpc.InsertOptions.TransactionMode; import com.google.protobuf.Timestamp; -import com.google.protobuf.Value; import io.grpc.Status; import io.grpc.StatusRuntimeException; @@ -104,15 +95,15 @@ public void close() { @Override public void executeCommand(ExecuteCommandRequest req, StreamObserver resp) { - + final long t0 = System.nanoTime(); Database db = null; boolean beganHere = false; try { - - // Resolve DB + params + + // Resolve DB + params db = getDatabase(req.getDatabase(), req.getCredentials()); final java.util.Map params = convertParameters(req.getParametersMap()); @@ -121,7 +112,7 @@ public void executeCommand(ExecuteCommandRequest req, StreamObserver 0 ? req.getMaxRows() : DEFAULT_MAX_COMMAND_ROWS; ExecuteCommandResponse.Builder out = ExecuteCommandResponse.newBuilder().setSuccess(true).setMessage("OK"); // Execute the command - + logger.debug("executeCommand(): command = {}", req.getCommand()); - + try (ResultSet rs = db.command(language, req.getCommand(), params)) { if (rs != null) { - + logger.debug("executeCommand(): rs = {}", rs); if (returnRows) { - + logger.debug("executeCommand(): returning rows ..."); - + int emitted = 0; - + while (rs.hasNext()) { Result r = rs.next(); @@ -192,7 +183,7 @@ public void executeCommand(ExecuteCommandRequest req, StreamObserver commit > begin-only⇒commit if (hasTx) { if (tx.getRollback()) { @@ -237,9 +228,9 @@ else if (beganHere) { } catch (Exception e) { - + logger.error("ERROR", e); - + // Best-effort rollback if we began here and failed try { if (beganHere && db != null) @@ -305,14 +296,14 @@ public void createRecord(CreateRecordRequest req, StreamObserver scso, AtomicBoolean cancelled) { + long running = 0L; + QueryResult.Builder batch = QueryResult.newBuilder(); int inBatch = 0; @@ -1132,9 +1125,11 @@ public void onError(Throwable t) { @Override public void onCompleted() { + try { InsertContext ctx = ctxRef.get(); + if (ctx == null) { // Client closed without sending a first chunk → nothing to do resp.onNext(InsertSummary.newBuilder().setReceived(0).setInserted(0).setUpdated(0).setIgnored(0).setFailed(0) @@ -1145,6 +1140,7 @@ public void onCompleted() { } ctx.flushCommit(true); // commit if not validate-only + if (!cancelled.get()) { resp.onNext(ctx.summary(totals, startedAt)); resp.onCompleted(); @@ -1295,12 +1291,6 @@ public void onNext(InsertRequest reqMsg) { // intentionally do not advance watermark on failure; client may replay safely } -// resp.onNext(InsertResponse.newBuilder() -// .setBatchAck(BatchAck.newBuilder().setSessionId(ctx.sessionId).setChunkSeq(c.getChunkSeq()) -// .setInserted(perChunk.inserted).setUpdated(perChunk.updated).setIgnored(perChunk.ignored) -// .setFailed(perChunk.failed).addAllErrors(perChunk.errors).build()) -// .build()); - sendOrQueue.accept(InsertResponse.newBuilder() .setBatchAck(BatchAck.newBuilder().setSessionId(ctx.sessionId).setChunkSeq(c.getChunkSeq()) .setInserted(perChunk.inserted).setUpdated(perChunk.updated).setIgnored(perChunk.ignored) @@ -1472,16 +1462,10 @@ void err(long rowIndex, String code, String msg, String field) { } } - private InsertSummary toSummary(Counts c, long startedAtMs) { - - long now = System.currentTimeMillis(); - - return InsertSummary.newBuilder().setReceived(c.received).setInserted(c.inserted).setUpdated(c.updated).setIgnored(c.ignored) - .setFailed(c.failed).addAllErrors(c.errors).setStartedAt(ts(startedAtMs)).setFinishedAt(ts(now)).build(); - } - private Counts insertRows(InsertContext ctx, java.util.Iterator it) { + Counts c = new Counts(); + int inBatch = 0; Schema schema = ctx.db.getSchema(); @@ -1552,6 +1536,7 @@ else if (isEdge) { case CONFLICT_IGNORE -> c.ignored++; case CONFLICT_ABORT, UNRECOGNIZED -> c.err(ctx.received - 1, "CONFLICT", dup.getMessage(), ""); case CONFLICT_UPDATE -> c.err(ctx.received - 1, "CONFLICT", dup.getMessage(), ""); + case CONFLICT_ERROR -> c.err(ctx.received - 1, "CONFLICT", dup.getMessage(), ""); } } catch (Exception e) { @@ -1572,25 +1557,14 @@ else if (ctx.opts.getTransactionMode() == TransactionMode.PER_ROW) { } private String getStringProp(GrpcRecord r, String key) { - - if (r == null || key == null || key.isEmpty()) { + if (r == null || key == null || key.isEmpty()) return null; - } - - if (r.getPropertiesMap().containsKey(key)) { - - GrpcValue grpcVal = r.getPropertiesMap().get(key); - - if (grpcVal.getValueType() == GrpcValueType.STRING || grpcVal.getValue().hasStringValue()) { - - return grpcVal.getValue().getStringValue(); - } - - // Try to convert to string - return String.valueOf(fromGrpcValue(grpcVal)); - } - - return null; + GrpcValue v = r.getPropertiesMap().get(key); + if (v == null) + return null; + if (v.getKindCase() == GrpcValue.KindCase.STRING_VALUE) + return v.getStringValue(); + return String.valueOf(fromGrpcValue(v)); } private int serverBatchSize(InsertContext ctx) { @@ -1624,382 +1598,151 @@ private void applyGrpcRecord(MutableEdge edge, GrpcRecord r) { }); } - private Object fromGrpcValue(GrpcValue grpcValue) { - - if (grpcValue == null) - return null; - - Value value = grpcValue.getValue(); - GrpcValueType type = grpcValue.getValueType(); - - // Use the type hint to properly convert - switch (type) { - - case BOOLEAN: - return value.getBoolValue(); - - case BYTE: - if (value.hasNumberValue()) - return (byte) value.getNumberValue(); - if (value.hasStringValue()) - return Byte.parseByte(value.getStringValue()); - return null; - - case SHORT: - if (value.hasNumberValue()) - return (short) value.getNumberValue(); - if (value.hasStringValue()) - return Short.parseShort(value.getStringValue()); - return null; - - case INTEGER: - if (value.hasNumberValue()) - return (int) value.getNumberValue(); - if (value.hasStringValue()) - return Integer.parseInt(value.getStringValue()); - return null; - - case LONG: - if (value.hasNumberValue()) - return (long) value.getNumberValue(); - if (value.hasStringValue()) - return Long.parseLong(value.getStringValue()); - return null; - - case FLOAT: - if (value.hasNumberValue()) - return (float) value.getNumberValue(); - if (value.hasStringValue()) - return Float.parseFloat(value.getStringValue()); - return null; - - case DOUBLE: - return value.getNumberValue(); - - case STRING: - return value.getStringValue(); - - case DECIMAL: - if (value.hasStringValue()) - return new java.math.BigDecimal(value.getStringValue()); - if (value.hasNumberValue()) - return java.math.BigDecimal.valueOf(value.getNumberValue()); - return null; - - case DATE: - if (value.hasStringValue()) { - return java.sql.Date.valueOf(java.time.LocalDate.parse(value.getStringValue())); - } - if (value.hasNumberValue()) { - return new java.util.Date((long) value.getNumberValue()); - } - return null; - - case DATETIME: - if (value.hasStringValue()) { - return java.util.Date.from(java.time.Instant.parse(value.getStringValue())); - } - if (value.hasNumberValue()) { - return new java.util.Date((long) value.getNumberValue()); - } - return null; - - case BINARY: - if (value.hasStringValue()) { - return java.util.Base64.getDecoder().decode(value.getStringValue()); - } - return null; - - case LINK: - if (value.hasStringValue()) { - return new com.arcadedb.database.RID(value.getStringValue()); - } - return null; - - case EMBEDDED: - // Handle embedded document - create from struct - if (value.hasStructValue()) { - // This would need database context to create proper embedded document - return convertStructToMap(value.getStructValue()); - } - return null; - - case MAP: - if (value.hasStructValue()) { - return convertStructToMap(value.getStructValue()); - } - return null; + private static long tsToMillis(com.google.protobuf.Timestamp ts) { + return ts.getSeconds() * 1000L + ts.getNanos() / 1_000_000L; + } - case LIST: - if (value.hasListValue()) { - return value.getListValue().getValuesList().stream().map(this::convertProtobufValueToJava) - .collect(java.util.stream.Collectors.toList()); - } + private Object fromGrpcValue(GrpcValue v) { + if (v == null) return null; + switch (v.getKindCase()) { + case BOOL_VALUE: + return v.getBoolValue(); + case INT32_VALUE: + return v.getInt32Value(); + case INT64_VALUE: + return v.getInt64Value(); + case FLOAT_VALUE: + return v.getFloatValue(); + case DOUBLE_VALUE: + return v.getDoubleValue(); + case STRING_VALUE: + return v.getStringValue(); + case BYTES_VALUE: + return v.getBytesValue().toByteArray(); + case TIMESTAMP_VALUE: + return new java.util.Date(tsToMillis(v.getTimestampValue())); + case LINK_VALUE: + return new com.arcadedb.database.RID(v.getLinkValue().getRid()); - default: - return toJava(value); + case DECIMAL_VALUE: { + var d = v.getDecimalValue(); + return new java.math.BigDecimal(java.math.BigInteger.valueOf(d.getUnscaled()), d.getScale()); } - } - private Map convertStructToMap(com.google.protobuf.Struct struct) { - if (struct == null) { - return new java.util.LinkedHashMap<>(); + case LIST_VALUE: { + var out = new java.util.ArrayList<>(); + for (GrpcValue e : v.getListValue().getValuesList()) + out.add(fromGrpcValue(e)); + return out; } - Map map = new java.util.LinkedHashMap<>(); - struct.getFieldsMap().forEach((key, value) -> { - map.put(key, convertProtobufValueToJava(value)); - }); - - return map; - } - - private Object convertProtobufValueToJava(com.google.protobuf.Value value) { - if (value == null) { - return null; + case MAP_VALUE: { + var out = new java.util.LinkedHashMap(); + v.getMapValue().getEntriesMap().forEach((k, vv) -> out.put(k, fromGrpcValue(vv))); + return out; } - switch (value.getKindCase()) { - case NULL_VALUE: - return null; - - case NUMBER_VALUE: - double num = value.getNumberValue(); - // Try to preserve integer types when possible - if (num == Math.floor(num) && !Double.isInfinite(num)) { - long longVal = (long) num; - if (longVal >= Integer.MIN_VALUE && longVal <= Integer.MAX_VALUE) { - return (int) longVal; - } - return longVal; - } - return num; - - case STRING_VALUE: - return value.getStringValue(); - - case BOOL_VALUE: - return value.getBoolValue(); - - case STRUCT_VALUE: - // Recursively convert nested struct - return convertStructToMap(value.getStructValue()); - - case LIST_VALUE: - // Convert list values - java.util.List list = new java.util.ArrayList<>(); - value.getListValue().getValuesList().forEach(item -> { - list.add(convertProtobufValueToJava(item)); - }); - return list; + case EMBEDDED_VALUE: { + var out = new java.util.LinkedHashMap(); + v.getEmbeddedValue().getFieldsMap().forEach((k, vv) -> out.put(k, fromGrpcValue(vv))); + return out; + } case KIND_NOT_SET: - default: return null; } + return null; } - // Alternative version that preserves more type information if needed - private Map convertStructToMapWithTypeHints(com.google.protobuf.Struct struct, Map typeHints) { - - if (struct == null) { - return new java.util.LinkedHashMap<>(); - } - - Map map = new java.util.LinkedHashMap<>(); - struct.getFieldsMap().forEach((key, value) -> { - GrpcValueType hint = typeHints != null ? typeHints.get(key) : null; - map.put(key, convertProtobufValueWithTypeHint(value, hint)); - }); - - return map; + private static com.google.protobuf.Timestamp msToTimestamp(long ms) { + long seconds = Math.floorDiv(ms, 1000L); + int nanos = (int) Math.floorMod(ms, 1000L) * 1_000_000; + return com.google.protobuf.Timestamp.newBuilder().setSeconds(seconds).setNanos(nanos).build(); } - private Object convertProtobufValueWithTypeHint(com.google.protobuf.Value value, GrpcValueType typeHint) { - - if (value == null || value.hasNullValue()) { - return null; - } - - // If we have a type hint, use it for more precise conversion - if (typeHint != null) { - switch (typeHint) { - case BYTE: - if (value.hasNumberValue()) - return (byte) value.getNumberValue(); - if (value.hasStringValue()) - return Byte.parseByte(value.getStringValue()); - break; - - case SHORT: - if (value.hasNumberValue()) - return (short) value.getNumberValue(); - if (value.hasStringValue()) - return Short.parseShort(value.getStringValue()); - break; - - case INTEGER: - if (value.hasNumberValue()) - return (int) value.getNumberValue(); - if (value.hasStringValue()) - return Integer.parseInt(value.getStringValue()); - break; - - case LONG: - if (value.hasNumberValue()) - return (long) value.getNumberValue(); - if (value.hasStringValue()) - return Long.parseLong(value.getStringValue()); - break; - - case FLOAT: - if (value.hasNumberValue()) - return (float) value.getNumberValue(); - if (value.hasStringValue()) - return Float.parseFloat(value.getStringValue()); - break; - - case DECIMAL: - if (value.hasStringValue()) - return new java.math.BigDecimal(value.getStringValue()); - if (value.hasNumberValue()) - return java.math.BigDecimal.valueOf(value.getNumberValue()); - break; - - case DATE: - if (value.hasStringValue()) { - try { - return java.sql.Date.valueOf(java.time.LocalDate.parse(value.getStringValue())); - } - catch (Exception e) { - logger.debug("Failed to parse date: {}", value.getStringValue()); - } - } - if (value.hasNumberValue()) { - return new java.sql.Date((long) value.getNumberValue()); - } - break; - - case DATETIME: - if (value.hasStringValue()) { - try { - return java.util.Date.from(java.time.Instant.parse(value.getStringValue())); - } - catch (Exception e) { - logger.debug("Failed to parse datetime: {}", value.getStringValue()); - } - } - if (value.hasNumberValue()) { - return new java.util.Date((long) value.getNumberValue()); - } - break; - - case BINARY: - if (value.hasStringValue()) { - return java.util.Base64.getDecoder().decode(value.getStringValue()); - } - break; + private GrpcValue toGrpcValue(Object o) { + GrpcValue.Builder b = GrpcValue.newBuilder(); + if (o == null) + return b.build(); - case LINK: - if (value.hasStringValue()) { - return new com.arcadedb.database.RID(value.getStringValue()); - } - break; + if (o instanceof Boolean v) + return b.setBoolValue(v).build(); + if (o instanceof Integer v) + return b.setInt32Value(v).build(); + if (o instanceof Long v) + return b.setInt64Value(v).build(); + if (o instanceof Float v) + return b.setFloatValue(v).build(); + if (o instanceof Double v) + return b.setDoubleValue(v).build(); + if (o instanceof CharSequence v) + return b.setStringValue(v.toString()).build(); + if (o instanceof byte[] v) + return b.setBytesValue(com.google.protobuf.ByteString.copyFrom(v)).build(); - default: - // Fall through to generic conversion - break; - } + if (o instanceof java.util.Date v) { + return GrpcValue.newBuilder().setTimestampValue(msToTimestamp(v.getTime())).setLogicalType("datetime").build(); } - // Generic conversion without type hint - return convertProtobufValueToJava(value); - } - - private void applyGrpcRecordToDocument(GrpcRecord r, MutableDocument doc) { - // Example if proto = message Record { map - // properties = 1; } - if (hasMethod(r, "getPropertiesMap")) { - @SuppressWarnings("unchecked") - Map props = (Map) invokeNoArg(r, "getPropertiesMap"); - props.forEach((k, v) -> doc.set(k, toJava(v))); - return; + if (o instanceof com.arcadedb.database.RID rid) { + return GrpcValue.newBuilder().setLinkValue(GrpcLink.newBuilder().setRid(rid.toString()).build()).setLogicalType("rid").build(); } - // Example if proto = message Record { repeated Property properties = 1; message - // Property { string key=1; google.protobuf.Value value=2; } } - if (hasMethod(r, "getPropertiesList")) { - @SuppressWarnings("unchecked") - List list = (List) invokeNoArg(r, "getPropertiesList"); - for (Object p : list) { - String key = (String) invokeNoArg(p, "getKey"); - com.google.protobuf.Value val = (com.google.protobuf.Value) invokeNoArg(p, "getValue"); - doc.set(key, toJava(val)); + if (o instanceof java.math.BigDecimal v) { + var unscaled = v.unscaledValue(); + if (unscaled.bitLength() <= 63) { + return GrpcValue.newBuilder().setDecimalValue(GrpcDecimal.newBuilder().setUnscaled(unscaled.longValue()).setScale(v.scale())) + .setLogicalType("decimal").build(); + } + else { + // if you need >64-bit unscaled, switch GrpcDecimal.unscaled to bytes in the + // proto + return GrpcValue.newBuilder().setStringValue(v.toPlainString()).setLogicalType("decimal").build(); } - return; - } - - // Fallback: if your proto DOES have fields map after all - if (hasMethod(r, "getFieldsMap")) { - @SuppressWarnings("unchecked") - Map fields = (Map) invokeNoArg(r, "getFieldsMap"); - fields.forEach((k, v) -> doc.set(k, toJava(v))); } - } - // tiny reflection helpers (compile-time safe across proto variants) - private static boolean hasMethod(Object o, String name) { - try { - o.getClass().getMethod(name); - return true; - } - catch (NoSuchMethodException e) { - return false; + if (o instanceof com.arcadedb.database.Document edoc && edoc.getIdentity() == null) { + GrpcEmbedded.Builder eb = GrpcEmbedded.newBuilder(); + if (edoc.getType() != null) + eb.setType(edoc.getTypeName()); + for (String k : edoc.getPropertyNames()) { + eb.putFields(k, toGrpcValue(edoc.get(k))); + } + return GrpcValue.newBuilder().setEmbeddedValue(eb.build()).build(); } - } - private static Object invokeNoArg(Object o, String name) { - try { - return o.getClass().getMethod(name).invoke(o); - } - catch (Exception e) { - throw new RuntimeException(e); + if (o instanceof java.util.Map m) { + GrpcMap.Builder mb = GrpcMap.newBuilder(); + for (var e : m.entrySet()) { + mb.putEntries(String.valueOf(e.getKey()), toGrpcValue(e.getValue())); + } + return GrpcValue.newBuilder().setMapValue(mb.build()).build(); } - } - - private static Object toJava(Value v) { - if (v == null) - return null; - switch (v.getKindCase()) { - case STRING_VALUE: - return v.getStringValue(); - case NUMBER_VALUE: - return v.getNumberValue(); // Double - case BOOL_VALUE: - return v.getBoolValue(); - case NULL_VALUE: - return null; - case STRUCT_VALUE: { - // Convert each nested Value -> Java recursively - java.util.LinkedHashMap m = new java.util.LinkedHashMap<>(); - v.getStructValue().getFieldsMap().forEach((k, vv) -> m.put(k, toJava(vv))); - return m; // plain Map + if (o instanceof java.util.Collection c) { + GrpcList.Builder lb = GrpcList.newBuilder(); + for (Object e : c) + lb.addValues(toGrpcValue(e)); + return GrpcValue.newBuilder().setListValue(lb.build()).build(); } - case LIST_VALUE: { - java.util.ArrayList list = new java.util.ArrayList<>(v.getListValue().getValuesCount()); - for (com.google.protobuf.Value item : v.getListValue().getValuesList()) { - list.add(toJava(item)); + if (o.getClass().isArray()) { + int len = java.lang.reflect.Array.getLength(o); + GrpcList.Builder lb = GrpcList.newBuilder(); + for (int i = 0; i < len; i++) { + lb.addValues(toGrpcValue(java.lang.reflect.Array.get(o, i))); } - return list; + return GrpcValue.newBuilder().setListValue(lb.build()).build(); } - case KIND_NOT_SET: - default: - return null; - } + // RID/Identifiable string fallback + if (o instanceof com.arcadedb.database.Identifiable id) + return GrpcValue.newBuilder().setLinkValue(GrpcLink.newBuilder().setRid(id.getIdentity().toString()).build()).setLogicalType("rid") + .build(); + + // Fallback + return GrpcValue.newBuilder().setStringValue(String.valueOf(o)).build(); } private InsertOptions defaults(InsertOptions in) { @@ -2079,7 +1822,9 @@ void flushCommit(boolean end) { } InsertSummary summary(Counts c, long startedAtMs) { + long now = System.currentTimeMillis(); + return InsertSummary.newBuilder().setReceived(c.received).setInserted(c.inserted).setUpdated(c.updated).setIgnored(c.ignored) .setFailed(c.failed).addAllErrors(c.errors).setStartedAt(ts(startedAtMs)).setFinishedAt(ts(now)).build(); } @@ -2097,65 +1842,6 @@ void closeQuietly() { } } - // --- TX helpers -------------------------------------------------------------- - - private static final class TxScope { - final boolean beganHere; - final String txId; - - TxScope(boolean beganHere, String txId) { - this.beganHere = beganHere; - this.txId = txId; - } - } - - /** - * Begin a transaction if requested by TransactionContext. Policy: - If tx.begin - * == true: begin() and mark beganHere=true (we’re not doing per-tx registry in - * this patch). - If tx.timeout_ms > 0 or tx.read_only set: optionally apply - * hints (no-ops here unless your DB supports). Returns a TxScope to be passed - * to endTx(). - */ - private TxScope beginTx(Database db, com.arcadedb.server.grpc.TransactionContext tx) { - if (tx == null) - return new TxScope(false, null); - boolean begin = tx.getBegin(); - String txId = tx.getTransactionId().isEmpty() ? null : tx.getTransactionId(); - - if (begin) { - db.begin(); // You can add isolation/RO if your ArcadeDB build supports it - return new TxScope(true, txId); - } - return new TxScope(false, txId); - } - - /** - * End the transaction according to commit/rollback flags. Precedence: rollback - * > commit > (if we beganHere and neither flag set) commit() - */ - private void endTx(Database db, com.arcadedb.server.grpc.TransactionContext tx, TxScope scope) { - if (tx == null) - return; - try { - if (tx.getRollback()) { - db.rollback(); - return; - } - if (tx.getCommit()) { - db.commit(); - return; - } - if (scope.beganHere) { - // Policy for “begin only”: commit by default (you can change to leave-open if - // you add a tx registry) - db.commit(); - } - } - catch (Exception ignore) { - // swallow – upstream handler will report the original RPC result - } - } - // Helper methods private Database getDatabase(String databaseName, DatabaseCredentials credentials) { @@ -2249,806 +1935,306 @@ private Map convertParameters(Map protoParams } private GrpcRecord convertToGrpcRecord(com.arcadedb.database.Record dbRecord, Database db) { - GrpcRecord.Builder builder = GrpcRecord.newBuilder().setRid(dbRecord.getIdentity().toString()); - logger.debug("convertToGrpcRecord(): dbRecord = {}, db = {}", dbRecord, db); - - if (dbRecord instanceof Document) { - - Document doc = (Document) dbRecord; - builder.setType(doc.getTypeName()); - - // Get schema information for type-aware conversion - DocumentType docType = null; - try { - docType = db.getSchema().getType(doc.getTypeName()); - } - catch (Exception e) { - // Type might not exist in schema - } + if (dbRecord instanceof Document doc) { + if (doc.getType() != null) + builder.setType(doc.getTypeName()); - Set propertyNames = doc.getPropertyNames(); - for (String propertyName : propertyNames) { + // set all properties + for (String propertyName : doc.getPropertyNames()) { Object value = doc.get(propertyName); - if (value != null) { - builder.putProperties(propertyName, convertToGrpcValue(propertyName, value, doc, docType)); - } + if (value != null) + builder.putProperties(propertyName, toGrpcValue(value)); } - // Add special handling for edges to include endpoints - if (dbRecord instanceof Edge) { - Edge edge = (Edge) dbRecord; - // Add out/in as properties if not already present - if (!propertyNames.contains("@out")) { - builder.putProperties("@out", convertToGrpcValue("@out", edge.getOut(), doc, docType)); + // If this is an Edge, include @out / @in for convenience (string rid or link) + if (dbRecord instanceof Edge edge) { + if (!builder.getPropertiesMap().containsKey("@out")) { + builder.putProperties("@out", toGrpcValue(edge.getOut().getIdentity())); } - if (!propertyNames.contains("@in")) { - builder.putProperties("@in", convertToGrpcValue("@in", edge.getIn(), doc, docType)); + if (!builder.getPropertiesMap().containsKey("@in")) { + builder.putProperties("@in", toGrpcValue(edge.getIn().getIdentity())); } } } - else { - - logger.debug("convertToGrpcRecord(): dbRecord = {} not a Document", dbRecord); - } - return builder.build(); } - + private GrpcValue convertPropToGrpcValue(String propName, Result result) { Object propValue = result.getProperty(propName); - GrpcValue.Builder builder = GrpcValue.newBuilder(); + return toGrpcValue(propValue); + } - GrpcValueType valueType = determineValueType(propValue); - String ofType = inferOfType(propValue); - - builder.setValueType(valueType); - builder.setValue(toProtoValue(propValue)); - - if (ofType != null) { - builder.setOfType(ofType); - } - - return builder.build(); - } - - private GrpcValue convertToGrpcValue(String propName, Object value, Document parentDoc, DocumentType docType) { - - GrpcValue.Builder builder = GrpcValue.newBuilder(); - - // Try to get schema information for this property - com.arcadedb.schema.Property schemaProp = null; - - if (docType != null) { - try { - schemaProp = docType.getProperty(propName); - } - catch (Exception e) { - // Property not defined in schema - } - } - - // Determine the value type - GrpcValueType valueType; - String ofType = null; - - if (schemaProp != null) { - // Use schema type if available - valueType = schemaTypeToGrpcType(schemaProp.getType()); - if (schemaProp.getOfType() != null) { - ofType = schemaProp.getOfType(); - } - } - else { - // Infer type from value - valueType = determineValueType(value); - ofType = inferOfType(value); - } - - builder.setValueType(valueType); - builder.setValue(toProtoValue(value)); - - if (ofType != null) { - builder.setOfType(ofType); - } - - return builder.build(); - } - - private GrpcValueType schemaTypeToGrpcType(com.arcadedb.schema.Type schemaType) { - - switch (schemaType) { - case BOOLEAN: - return GrpcValueType.BOOLEAN; - case BYTE: - return GrpcValueType.BYTE; - case SHORT: - return GrpcValueType.SHORT; - case INTEGER: - return GrpcValueType.INTEGER; - case LONG: - return GrpcValueType.LONG; - case FLOAT: - return GrpcValueType.FLOAT; - case DOUBLE: - return GrpcValueType.DOUBLE; - case DECIMAL: - return GrpcValueType.DECIMAL; - case STRING: - return GrpcValueType.STRING; - case BINARY: - return GrpcValueType.BINARY; - case DATE: - return GrpcValueType.DATE; - case DATETIME: - case DATETIME_MICROS: - case DATETIME_NANOS: - case DATETIME_SECOND: - return GrpcValueType.DATETIME; - case LINK: - return GrpcValueType.LINK; - case EMBEDDED: - return GrpcValueType.EMBEDDED; - case LIST: - return GrpcValueType.LIST; - case MAP: - return GrpcValueType.MAP; - default: - return GrpcValueType.STRING; - } - } - - private String inferOfType(Object value) { - if (value instanceof Document) { - return ((Document) value).getTypeName(); - } - if (value instanceof Identifiable) { - try { - Document linked = ((Identifiable) value).getRecord().asDocument(); - return linked.getTypeName(); - } - catch (Exception e) { - return null; - } - } - if (value instanceof List && !((List) value).isEmpty()) { - Object first = ((List) value).get(0); - if (first instanceof Document) { - return ((Document) first).getTypeName(); - } - } - return null; - } - - private GrpcValueType determineValueType(Object value) { - if (value == null) - return GrpcValueType.RECORD_VALUE_TYPE_UNSPECIFIED; - if (value instanceof Boolean) - return GrpcValueType.BOOLEAN; - if (value instanceof Byte) - return GrpcValueType.BYTE; - if (value instanceof Short) - return GrpcValueType.SHORT; - if (value instanceof Integer) - return GrpcValueType.INTEGER; - if (value instanceof Long) - return GrpcValueType.LONG; - if (value instanceof Float) - return GrpcValueType.FLOAT; - if (value instanceof Double) - return GrpcValueType.DOUBLE; - if (value instanceof String) - return GrpcValueType.STRING; - if (value instanceof java.math.BigDecimal) - return GrpcValueType.DECIMAL; - if (value instanceof byte[]) - return GrpcValueType.BINARY; - if (value instanceof java.util.Date || value instanceof java.time.LocalDate) - return GrpcValueType.DATE; - if (value instanceof java.time.Instant || value instanceof java.time.LocalDateTime) - return GrpcValueType.DATETIME; - if (value instanceof com.arcadedb.database.RID || value instanceof com.arcadedb.database.Identifiable) - return GrpcValueType.LINK; - if (value instanceof com.arcadedb.database.Document) - return GrpcValueType.EMBEDDED; - if (value instanceof java.util.Map) - return GrpcValueType.MAP; - if (value instanceof java.util.Collection || value.getClass().isArray()) - return GrpcValueType.LIST; - - return GrpcValueType.STRING; // fallback - } - - private String determineOfType(Object value, String propName, String parentTypeName) { - if (value instanceof com.arcadedb.database.Document) { - return ((Document) value).getTypeName(); - } - if (value instanceof com.arcadedb.database.Identifiable) { - // For links, return the target type if known - try { - Document linked = ((Identifiable) value).getRecord().asDocument(); - return linked.getTypeName(); - } - catch (Exception e) { - return null; - } - } - // For collections, you might need schema info to determine element type - return null; - } - - private com.google.protobuf.Value toProtoValue(Object o) { - if (o == null) - return com.google.protobuf.Value.newBuilder().setNullValue(com.google.protobuf.NullValue.NULL_VALUE).build(); - - if (o instanceof String s) - return com.google.protobuf.Value.newBuilder().setStringValue(s).build(); - if (o instanceof Boolean b) - return com.google.protobuf.Value.newBuilder().setBoolValue(b).build(); - if (o instanceof Number n) - return com.google.protobuf.Value.newBuilder().setNumberValue(n.doubleValue()).build(); - - if (o instanceof java.math.BigDecimal bd) - return com.google.protobuf.Value.newBuilder().setStringValue(bd.toPlainString()).build(); - - if (o instanceof byte[] bytes) - return com.google.protobuf.Value.newBuilder().setStringValue(java.util.Base64.getEncoder().encodeToString(bytes)).build(); - - // Date/Time → ISO-8601 - if (o instanceof java.util.Date d) - return com.google.protobuf.Value.newBuilder().setStringValue(d.toInstant().toString()).build(); - if (o instanceof java.time.Instant inst) - return com.google.protobuf.Value.newBuilder().setStringValue(inst.toString()).build(); - if (o instanceof java.time.LocalDate ld) - return com.google.protobuf.Value.newBuilder().setStringValue(ld.toString()).build(); - if (o instanceof java.time.LocalDateTime ldt) - return com.google.protobuf.Value.newBuilder().setStringValue(ldt.toString()).build(); - - // Embedded docs (mutable/immutable) - if (o instanceof com.arcadedb.database.Document doc) { - var sb = com.google.protobuf.Struct.newBuilder(); - for (String k : doc.getPropertyNames()) { - sb.putFields(k, toProtoValue(doc.get(k))); - } - return com.google.protobuf.Value.newBuilder().setStructValue(sb).build(); - } - - if (o instanceof java.util.Map m) { - var sb = com.google.protobuf.Struct.newBuilder(); - m.forEach((k, v) -> sb.putFields(String.valueOf(k), toProtoValue(v))); - return com.google.protobuf.Value.newBuilder().setStructValue(sb).build(); - } - - if (o instanceof java.util.Collection c) { - var lb = com.google.protobuf.ListValue.newBuilder(); - c.forEach(v -> lb.addValues(toProtoValue(v))); - return com.google.protobuf.Value.newBuilder().setListValue(lb).build(); - } - - if (o.getClass().isArray()) { - int len = java.lang.reflect.Array.getLength(o); - var lb = com.google.protobuf.ListValue.newBuilder(); - for (int i = 0; i < len; i++) { - lb.addValues(toProtoValue(java.lang.reflect.Array.get(o, i))); - } - return com.google.protobuf.Value.newBuilder().setListValue(lb).build(); - } - - // RID or Identifiable → string rid - if (o instanceof com.arcadedb.database.Identifiable id) - return com.google.protobuf.Value.newBuilder().setStringValue(id.getIdentity().toString()).build(); - if (o instanceof com.arcadedb.database.RID rid) - return com.google.protobuf.Value.newBuilder().setStringValue(rid.toString()).build(); - - // Fallback - return com.google.protobuf.Value.newBuilder().setStringValue(String.valueOf(o)).build(); - } - - // Apply properties from proto Record to a document/vertex/edge, schema-aware - // for EMBEDDED - private void applyGrpcRecord(MutableDocument d, GrpcRecord r, Database db, String targetClass) { - DocumentType dt = db.getSchema().getType(targetClass); - r.getPropertiesMap().forEach((k, val) -> d.set(k, toJavaForProperty(db, d, dt, k, val))); - } - - private void applyGrpcRecord(MutableVertex v, GrpcRecord r, Database db) { - DocumentType dt = db.getSchema().getType(v.getTypeName()); - r.getPropertiesMap().forEach((k, val) -> v.set(k, toJavaForProperty(db, v, dt, k, val))); - } - - private void applyGrpcRecord(MutableEdge e, GrpcRecord r, Database db, String edgeClass) { - DocumentType dt = db.getSchema().getType(edgeClass); - r.getPropertiesMap().forEach((k, val) -> e.set(k, toJavaForProperty(db, e, dt, k, val))); - } - - // Generic fallback (schema-agnostic) for nested values - private Object toJavaLoose(com.google.protobuf.Value v) { - return switch (v.getKindCase()) { - case NULL_VALUE -> null; - case STRING_VALUE -> v.getStringValue(); - case NUMBER_VALUE -> v.getNumberValue(); - case BOOL_VALUE -> v.getBoolValue(); - case LIST_VALUE -> v.getListValue().getValuesList().stream().map(this::toJavaLoose).toList(); - case STRUCT_VALUE -> { - var m = new java.util.LinkedHashMap(); - v.getStructValue().getFieldsMap().forEach((k, val) -> m.put(k, toJavaLoose(val))); - yield m; - } - case KIND_NOT_SET -> null; - }; - } - - private Object toJavaForProperty(final Database db, final MutableDocument parent, final DocumentType dtype, final String propName, - final GrpcValue grpcValue) { + private Object toJavaForProperty(final Database db, final MutableDocument parent, final DocumentType dtype, final String propName, + final GrpcValue grpcValue) { if (grpcValue == null) return null; - // Try to get schema information + // Try schema com.arcadedb.schema.Property prop = null; try { prop = (dtype != null) ? dtype.getProperty(propName) : null; } - catch (com.arcadedb.exception.SchemaException e) { - // Property not in schema, use the type hint from GrpcValue + catch (com.arcadedb.exception.SchemaException ignore) { } - // If we have schema info, use it; otherwise use the type from GrpcValue - if (prop != null) { + if (prop != null) return convertWithSchemaType(db, parent, prop, propName, grpcValue); - } - else { - // No schema, use the type hint from the GrpcValue - return fromGrpcValue(grpcValue); - } - } - - private Object convertWithSchemaType(Database db, MutableDocument parent, com.arcadedb.schema.Property prop, String propName, - GrpcValue grpcValue) { - Value v = grpcValue.getValue(); - com.arcadedb.schema.Type schemaType = prop.getType(); - - // Handle null values - if (v.hasNullValue()) { - return null; - } - - switch (schemaType) { - case BOOLEAN: - if (v.hasBoolValue()) - return v.getBoolValue(); - if (v.hasStringValue()) - return Boolean.parseBoolean(v.getStringValue()); - return null; - - case BYTE: - if (v.hasNumberValue()) - return (byte) v.getNumberValue(); - if (v.hasStringValue()) - return Byte.parseByte(v.getStringValue()); - return null; - - case SHORT: - if (v.hasNumberValue()) - return (short) v.getNumberValue(); - if (v.hasStringValue()) - return Short.parseShort(v.getStringValue()); - return null; - - case INTEGER: - if (v.hasNumberValue()) - return (int) v.getNumberValue(); - if (v.hasStringValue()) - return Integer.parseInt(v.getStringValue()); - return null; - - case LONG: - if (v.hasNumberValue()) - return (long) v.getNumberValue(); - if (v.hasStringValue()) - return Long.parseLong(v.getStringValue()); - return null; - - case FLOAT: - if (v.hasNumberValue()) - return (float) v.getNumberValue(); - if (v.hasStringValue()) - return Float.parseFloat(v.getStringValue()); - return null; - - case DOUBLE: - if (v.hasNumberValue()) - return v.getNumberValue(); - if (v.hasStringValue()) - return Double.parseDouble(v.getStringValue()); - return null; - - case DECIMAL: - if (v.hasStringValue()) - return new java.math.BigDecimal(v.getStringValue()); - if (v.hasNumberValue()) - return java.math.BigDecimal.valueOf(v.getNumberValue()); - return null; - - case STRING: - if (v.hasStringValue()) - return v.getStringValue(); - // Convert other types to string representation - if (v.hasNumberValue()) - return String.valueOf(v.getNumberValue()); - if (v.hasBoolValue()) - return String.valueOf(v.getBoolValue()); - return null; - - case BINARY: - if (v.hasStringValue()) { - // Expect base64 encoded string - return java.util.Base64.getDecoder().decode(v.getStringValue()); - } - return null; - - case DATE: - if (v.hasStringValue()) { - // Parse ISO date format (yyyy-MM-dd) - return java.sql.Date.valueOf(java.time.LocalDate.parse(v.getStringValue())); - } - if (v.hasNumberValue()) { - // Epoch millis - return new java.sql.Date((long) v.getNumberValue()); - } - return null; - - case DATETIME: - case DATETIME_MICROS: - case DATETIME_NANOS: - case DATETIME_SECOND: - if (v.hasStringValue()) { - // Parse ISO-8601 datetime - java.time.Instant instant = java.time.Instant.parse(v.getStringValue()); - return java.util.Date.from(instant); - } - if (v.hasNumberValue()) { - // Epoch millis - return new java.util.Date((long) v.getNumberValue()); - } - return null; - - case LINK: - if (v.hasStringValue()) { - // Parse RID string format (#cluster:position) - return new com.arcadedb.database.RID(v.getStringValue()); - } - return null; - case EMBEDDED: - if (v.hasStructValue()) { - String embeddedTypeName = grpcValue.hasOfType() ? grpcValue.getOfType() : prop.getOfType(); + // No schema → generic + return fromGrpcValue(grpcValue); + } + private Object convertWithSchemaType(Database db, MutableDocument parent, com.arcadedb.schema.Property prop, String propName, GrpcValue v) { + + var t = prop.getType(); + + switch (t) { + + case BOOLEAN: + return switch (v.getKindCase()) { + case BOOL_VALUE -> v.getBoolValue(); + case STRING_VALUE -> Boolean.parseBoolean(v.getStringValue()); + default -> null; + }; + + case BYTE: + return switch (v.getKindCase()) { + case INT32_VALUE, INT64_VALUE, DOUBLE_VALUE, FLOAT_VALUE -> (byte) (long) fromGrpcValue(v); + case STRING_VALUE -> Byte.parseByte(v.getStringValue()); + default -> null; + }; + + case SHORT: + return switch (v.getKindCase()) { + case INT32_VALUE, INT64_VALUE, DOUBLE_VALUE, FLOAT_VALUE -> (short) (long) fromGrpcValue(v); + case STRING_VALUE -> Short.parseShort(v.getStringValue()); + default -> null; + }; + + case INTEGER: + return switch (v.getKindCase()) { + case INT32_VALUE -> v.getInt32Value(); + case INT64_VALUE -> (int) v.getInt64Value(); + case DOUBLE_VALUE -> (int) v.getDoubleValue(); + case FLOAT_VALUE -> (int) v.getFloatValue(); + case STRING_VALUE -> Integer.parseInt(v.getStringValue()); + default -> null; + }; + + case LONG: + return switch (v.getKindCase()) { + case INT64_VALUE -> v.getInt64Value(); + case INT32_VALUE -> (long) v.getInt32Value(); + case DOUBLE_VALUE -> (long) v.getDoubleValue(); + case FLOAT_VALUE -> (long) v.getFloatValue(); + case STRING_VALUE -> Long.parseLong(v.getStringValue()); + default -> null; + }; + + case FLOAT: + return switch (v.getKindCase()) { + case FLOAT_VALUE -> v.getFloatValue(); + case DOUBLE_VALUE -> (float) v.getDoubleValue(); + case INT32_VALUE -> (float) v.getInt32Value(); + case INT64_VALUE -> (float) v.getInt64Value(); + case STRING_VALUE -> Float.parseFloat(v.getStringValue()); + default -> null; + }; + + case DOUBLE: + return switch (v.getKindCase()) { + case DOUBLE_VALUE -> v.getDoubleValue(); + case FLOAT_VALUE -> (double) v.getFloatValue(); + case INT32_VALUE -> (double) v.getInt32Value(); + case INT64_VALUE -> (double) v.getInt64Value(); + case STRING_VALUE -> Double.parseDouble(v.getStringValue()); + default -> null; + }; + + case STRING: + return switch (v.getKindCase()) { + case STRING_VALUE -> v.getStringValue(); + default -> String.valueOf(fromGrpcValue(v)); + }; + + case DECIMAL: + return switch (v.getKindCase()) { + case DECIMAL_VALUE -> { + var d = v.getDecimalValue(); + yield new java.math.BigDecimal(java.math.BigInteger.valueOf(d.getUnscaled()), d.getScale()); + } + case STRING_VALUE -> new java.math.BigDecimal(v.getStringValue()); + case DOUBLE_VALUE -> java.math.BigDecimal.valueOf(v.getDoubleValue()); + case INT32_VALUE -> java.math.BigDecimal.valueOf(v.getInt32Value()); + case INT64_VALUE -> java.math.BigDecimal.valueOf(v.getInt64Value()); + default -> null; + }; + + case DATE: + case DATETIME: + // Prefer timestamp_value; else accept epoch ms in int64; else parse string + return switch (v.getKindCase()) { + case TIMESTAMP_VALUE -> new java.util.Date(tsToMillis(v.getTimestampValue())); + case INT64_VALUE -> new java.util.Date(v.getInt64Value()); + case STRING_VALUE -> new java.util.Date(Long.parseLong(v.getStringValue())); // or parse ISO if you emit it + default -> null; + }; + + case BINARY: + return switch (v.getKindCase()) { + case BYTES_VALUE -> v.getBytesValue().toByteArray(); + case STRING_VALUE -> v.getStringValue().getBytes(java.nio.charset.StandardCharsets.UTF_8); + default -> null; + }; + + case LINK: + return switch (v.getKindCase()) { + case LINK_VALUE -> new com.arcadedb.database.RID(v.getLinkValue().getRid()); + case STRING_VALUE -> new com.arcadedb.database.RID(v.getStringValue()); + default -> null; + }; + + case EMBEDDED: { + // Use schema ofType if present; otherwise use embedded.type from the payload + String embeddedTypeName = (v.getKindCase() == GrpcValue.KindCase.EMBEDDED_VALUE && !v.getEmbeddedValue().getType().isEmpty()) + ? v.getEmbeddedValue().getType() + : prop.getOfType(); + + if (v.getKindCase() != GrpcValue.KindCase.EMBEDDED_VALUE || embeddedTypeName == null || embeddedTypeName.isEmpty()) { + // Fallback to a Map if we can't create a typed embedded doc + return fromGrpcValue(v); + } + MutableEmbeddedDocument ed = parent.newEmbeddedDocument(embeddedTypeName, propName); - - // Recursively convert struct fields - v.getStructValue().getFieldsMap().forEach((k, vv) -> { - // Try to get schema for embedded type - DocumentType embeddedType = null; - com.arcadedb.schema.Property embeddedProp = null; - - if (embeddedTypeName != null) { - try { - embeddedType = db.getSchema().getType(embeddedTypeName); - embeddedProp = embeddedType.getProperty(k); - } - catch (Exception e) { - // Property not in schema - } - } - - if (embeddedProp != null && embeddedType != null) { - // Create a simple GrpcValue wrapper for recursive conversion - GrpcValue.Builder rvBuilder = GrpcValue.newBuilder().setValue(vv) - .setValueType(schemaTypeToGrpcType(embeddedProp.getType())); - ed.set(k, convertWithSchemaType(db, ed, embeddedProp, k, rvBuilder.build())); - } - else { - // No schema, use generic conversion - ed.set(k, toJava(vv)); - } + DocumentType embeddedType = null; + try { + embeddedType = db.getSchema().getType(embeddedTypeName); + } + catch (Exception ignore) { + } + + final DocumentType embeddedTypeFinal = embeddedType; + + v.getEmbeddedValue().getFieldsMap().forEach((k, vv) -> { + Object j = (embeddedTypeFinal != null) ? toJavaForProperty(db, ed, embeddedTypeFinal, k, vv) : fromGrpcValue(vv); + ed.set(k, j); }); return ed; } - return null; - - case LIST: - case ARRAY_OF_SHORTS: - case ARRAY_OF_INTEGERS: - case ARRAY_OF_LONGS: - case ARRAY_OF_FLOATS: - case ARRAY_OF_DOUBLES: - if (v.hasListValue()) { - java.util.List list = new java.util.ArrayList<>(); - - // Determine element type - com.arcadedb.schema.Type elementType = getListElementType(schemaType); - - for (com.google.protobuf.Value item : v.getListValue().getValuesList()) { - if (elementType != null) { - // Create wrapper for typed conversion - GrpcValue.Builder rvBuilder = GrpcValue.newBuilder().setValue(item).setValueType(schemaTypeToGrpcType(elementType)); - - // Create a dummy property for element conversion - com.arcadedb.schema.Property elementProp = createVirtualProperty(elementType, prop.getOfType()); - list.add(convertWithSchemaType(db, parent, elementProp, propName + "[element]", rvBuilder.build())); - } - else { - // Generic conversion - list.add(toJava(item)); - } - } - - // Convert to appropriate array type if needed - if (schemaType == com.arcadedb.schema.Type.ARRAY_OF_SHORTS) { - return list.stream().mapToInt(o -> ((Number) o).shortValue()).toArray(); - } - else if (schemaType == com.arcadedb.schema.Type.ARRAY_OF_INTEGERS) { - return list.stream().mapToInt(o -> ((Number) o).intValue()).toArray(); - } - else if (schemaType == com.arcadedb.schema.Type.ARRAY_OF_LONGS) { - return list.stream().mapToLong(o -> ((Number) o).longValue()).toArray(); + + case MAP: + if (v.getKindCase() == GrpcValue.KindCase.MAP_VALUE) { + var m = new java.util.LinkedHashMap(); + v.getMapValue().getEntriesMap().forEach((k, vv) -> m.put(k, fromGrpcValue(vv))); + return m; } - else if (schemaType == com.arcadedb.schema.Type.ARRAY_OF_FLOATS) { - float[] array = new float[list.size()]; - for (int i = 0; i < list.size(); i++) { - array[i] = ((Number) list.get(i)).floatValue(); + return null; + + case LIST: + if (v.getKindCase() == GrpcValue.KindCase.LIST_VALUE) { + var list = new java.util.ArrayList<>(); + for (GrpcValue item : v.getListValue().getValuesList()) { + list.add(fromGrpcValue(item)); } - return array; + return list; } - else if (schemaType == com.arcadedb.schema.Type.ARRAY_OF_DOUBLES) { - return list.stream().mapToDouble(o -> ((Number) o).doubleValue()).toArray(); + return null; + case ARRAY_OF_SHORTS: { + if (v.getKindCase() == GrpcValue.KindCase.LIST_VALUE) { + var out = new java.util.ArrayList(); + for (GrpcValue item : v.getListValue().getValuesList()) { + Object o = fromGrpcValue(item); + if (o instanceof Number n) + out.add(n.shortValue()); + else if (o instanceof String s) + out.add(Short.parseShort(s)); + } + return out; } - - return list; - } - return null; - - case MAP: - if (v.hasStructValue()) { - java.util.Map map = new java.util.LinkedHashMap<>(); - v.getStructValue().getFieldsMap().forEach((k, vv) -> { - map.put(k, toJava(vv)); - }); - return map; - } - return null; - - default: - // Fallback to generic conversion - return toJava(v); - } - } - - private com.arcadedb.schema.Property createVirtualProperty(final com.arcadedb.schema.Type type, final String ofType) { - - return new com.arcadedb.schema.Property() { - private final Map customValues = new HashMap<>(); - private Object defaultValue = null; - private boolean readonly = false; - private boolean mandatory = false; - private boolean notNull = false; - private boolean hidden = false; - private String max = null; - private String min = null; - private String regexp = null; - - @Override - public Index createIndex(Schema.INDEX_TYPE indexType, boolean unique) { - // Virtual properties cannot create indexes - throw new UnsupportedOperationException("Cannot create index on virtual property"); - } - - @Override - public Index getOrCreateIndex(Schema.INDEX_TYPE indexType, boolean unique) { - // Virtual properties don't have indexes return null; } - - @Override - public String getName() { - return "__virtual_" + type.name().toLowerCase(); - } - - @Override - public Type getType() { - return type; - } - - @Override - public int getId() { - // Virtual properties don't have database IDs - return -1; - } - - @Override - public Object getDefaultValue() { - return defaultValue; - } - - @Override - public Property setDefaultValue(Object defaultValue) { - this.defaultValue = defaultValue; - return this; - } - - @Override - public String getOfType() { - return ofType; - } - - @Override - public Property setOfType(String ofType) { - // Cannot modify virtual property's ofType after creation - throw new UnsupportedOperationException("Cannot modify virtual property's ofType"); - } - - @Override - public Property setReadonly(boolean readonly) { - this.readonly = readonly; - return this; - } - - @Override - public boolean isReadonly() { - return readonly; - } - - @Override - public Property setMandatory(boolean mandatory) { - this.mandatory = mandatory; - return this; - } - - @Override - public boolean isMandatory() { - return mandatory; - } - - @Override - public Property setNotNull(boolean notNull) { - this.notNull = notNull; - return this; - } - - @Override - public boolean isNotNull() { - return notNull; - } - - @Override - public Property setHidden(boolean hidden) { - this.hidden = hidden; - return this; - } - - @Override - public boolean isHidden() { - return hidden; - } - - @Override - public Property setMax(String max) { - this.max = max; - return this; - } - - @Override - public String getMax() { - return max; - } - - @Override - public Property setMin(String min) { - this.min = min; - return this; - } - - @Override - public String getMin() { - return min; - } - - @Override - public Property setRegexp(String regexp) { - this.regexp = regexp; - return this; - } - - @Override - public String getRegexp() { - return regexp; - } - - @Override - public Set getCustomKeys() { - return customValues.keySet(); - } - - @Override - public Object getCustomValue(String key) { - return customValues.get(key); - } - - @Override - public Object setCustomValue(String key, Object value) { - return customValues.put(key, value); - } - - @Override - public JSONObject toJSON() { - JSONObject json = new JSONObject(); - json.put("name", getName()); - json.put("type", type.name()); - if (ofType != null) { - json.put("ofType", ofType); - } - if (defaultValue != null) { - json.put("default", defaultValue); - } - json.put("readonly", readonly); - json.put("mandatory", mandatory); - json.put("notNull", notNull); - json.put("hidden", hidden); - if (max != null) - json.put("max", max); - if (min != null) - json.put("min", min); - if (regexp != null) - json.put("regexp", regexp); - if (!customValues.isEmpty()) { - json.put("custom", new JSONObject(customValues)); + + case ARRAY_OF_INTEGERS: { + if (v.getKindCase() == GrpcValue.KindCase.LIST_VALUE) { + var out = new java.util.ArrayList(); + for (GrpcValue item : v.getListValue().getValuesList()) { + Object o = fromGrpcValue(item); + if (o instanceof Number n) + out.add(n.intValue()); + else if (o instanceof String s) + out.add(Integer.parseInt(s)); + } + return out; } - return json; + return null; } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (!(obj instanceof com.arcadedb.schema.Property)) - return false; - com.arcadedb.schema.Property other = (com.arcadedb.schema.Property) obj; - return type == other.getType() && Objects.equals(ofType, other.getOfType()); + + case ARRAY_OF_LONGS: { + if (v.getKindCase() == GrpcValue.KindCase.LIST_VALUE) { + var out = new java.util.ArrayList(); + for (GrpcValue item : v.getListValue().getValuesList()) { + Object o = fromGrpcValue(item); + if (o instanceof Number n) + out.add(n.longValue()); + else if (o instanceof String s) + out.add(Long.parseLong(s)); + } + return out; + } + return null; } - - @Override - public int hashCode() { - return Objects.hash(type, ofType); + + case ARRAY_OF_FLOATS: { + if (v.getKindCase() == GrpcValue.KindCase.LIST_VALUE) { + var out = new java.util.ArrayList(); + for (GrpcValue item : v.getListValue().getValuesList()) { + Object o = fromGrpcValue(item); + if (o instanceof Number n) + out.add(n.floatValue()); + else if (o instanceof String s) + out.add(Float.parseFloat(s)); + } + return out; + } + return null; } - - @Override - public String toString() { - return "VirtualProperty{type=" + type + (ofType != null ? ", ofType=" + ofType : "") + "}"; + + case ARRAY_OF_DOUBLES: { + if (v.getKindCase() == GrpcValue.KindCase.LIST_VALUE) { + var out = new java.util.ArrayList(); + for (GrpcValue item : v.getListValue().getValuesList()) { + Object o = fromGrpcValue(item); + if (o instanceof Number n) + out.add(n.doubleValue()); + else if (o instanceof String s) + out.add(Double.parseDouble(s)); + } + return out; + } + return null; } - }; - } - - // Helper method to determine element type for arrays/lists - private com.arcadedb.schema.Type getListElementType(com.arcadedb.schema.Type listType) { - switch (listType) { - case ARRAY_OF_SHORTS: - return com.arcadedb.schema.Type.SHORT; - case ARRAY_OF_INTEGERS: - return com.arcadedb.schema.Type.INTEGER; - case ARRAY_OF_LONGS: - return com.arcadedb.schema.Type.LONG; - case ARRAY_OF_FLOATS: - return com.arcadedb.schema.Type.FLOAT; - case ARRAY_OF_DOUBLES: - return com.arcadedb.schema.Type.DOUBLE; - default: - return null; - } + + case DATETIME_SECOND: + case DATETIME_MICROS: + case DATETIME_NANOS: { + // Same handling as DATETIME + return switch (v.getKindCase()) { + case TIMESTAMP_VALUE -> new java.util.Date(tsToMillis(v.getTimestampValue())); + case INT64_VALUE -> new java.util.Date(v.getInt64Value()); // epoch ms expected + case STRING_VALUE -> new java.util.Date(Long.parseLong(v.getStringValue())); + default -> null; + }; + } + } + + // default fallback + return fromGrpcValue(v); } private String generateTransactionId() { From f7f78d4c71c3059cdce0eb3493d0f9b4d1e67c0a Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Fri, 5 Sep 2025 06:24:41 -0700 Subject: [PATCH 24/35] Added debug logging --- .../server/grpc/ArcadeDbGrpcService.java | 134 +++++++++++++----- 1 file changed, 95 insertions(+), 39 deletions(-) diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java index 6805daf2f6..f8111b88d0 100644 --- a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java @@ -42,6 +42,7 @@ public class ArcadeDbGrpcService extends ArcadeDbServiceGrpc.ArcadeDbServiceImpl private static final Logger logger = LoggerFactory.getLogger(ArcadeDbGrpcService.class); + // Transaction management private final Map activeTransactions = new ConcurrentHashMap<>(); @@ -1576,7 +1577,9 @@ private void applyGrpcRecord(MutableDocument doc, GrpcRecord r) { // Skip system fields if (k.startsWith("@")) return; - doc.set(k, fromGrpcValue(grpcVal)); + Object javaVal = fromGrpcValue(grpcVal); + if (logger.isDebugEnabled()) logger.debug("APPLY-DOC {} <= {} -> {}", k, summarizeGrpc(grpcVal), summarizeJava(javaVal)); + doc.set(k, javaVal); }); } @@ -1585,7 +1588,9 @@ private void applyGrpcRecord(MutableVertex vertex, GrpcRecord r) { // Skip system fields if (k.startsWith("@")) return; - vertex.set(k, fromGrpcValue(grpcVal)); + Object javaVal = fromGrpcValue(grpcVal); + if (logger.isDebugEnabled()) logger.debug("APPLY-VERTEX {} <= {} -> {}", k, summarizeGrpc(grpcVal), summarizeJava(javaVal)); + vertex.set(k, javaVal); }); } @@ -1594,7 +1599,9 @@ private void applyGrpcRecord(MutableEdge edge, GrpcRecord r) { // Skip system fields if (k.startsWith("@")) return; - edge.set(k, fromGrpcValue(grpcVal)); + Object javaVal = fromGrpcValue(grpcVal); + if (logger.isDebugEnabled()) logger.debug("APPLY-EDGE {} <= {} -> {}", k, summarizeGrpc(grpcVal), summarizeJava(javaVal)); + edge.set(k, javaVal); }); } @@ -1604,55 +1611,55 @@ private static long tsToMillis(com.google.protobuf.Timestamp ts) { private Object fromGrpcValue(GrpcValue v) { if (v == null) - return null; + return dbgDec("fromGrpcValue", v, null, null); switch (v.getKindCase()) { case BOOL_VALUE: - return v.getBoolValue(); + return dbgDec("fromGrpcValue", v, v.getBoolValue(), null); case INT32_VALUE: - return v.getInt32Value(); + return dbgDec("fromGrpcValue", v, v.getInt32Value(), null); case INT64_VALUE: - return v.getInt64Value(); + return dbgDec("fromGrpcValue", v, v.getInt64Value(), null); case FLOAT_VALUE: - return v.getFloatValue(); + return dbgDec("fromGrpcValue", v, v.getFloatValue(), null); case DOUBLE_VALUE: - return v.getDoubleValue(); + return dbgDec("fromGrpcValue", v, v.getDoubleValue(), null); case STRING_VALUE: - return v.getStringValue(); + return dbgDec("fromGrpcValue", v, v.getStringValue(), null); case BYTES_VALUE: - return v.getBytesValue().toByteArray(); + return dbgDec("fromGrpcValue", v, v.getBytesValue().toByteArray(), null); case TIMESTAMP_VALUE: - return new java.util.Date(tsToMillis(v.getTimestampValue())); + return dbgDec("fromGrpcValue", v, new java.util.Date(tsToMillis(v.getTimestampValue())), null); case LINK_VALUE: - return new com.arcadedb.database.RID(v.getLinkValue().getRid()); + return dbgDec("fromGrpcValue", v, new com.arcadedb.database.RID(v.getLinkValue().getRid()), null); case DECIMAL_VALUE: { var d = v.getDecimalValue(); - return new java.math.BigDecimal(java.math.BigInteger.valueOf(d.getUnscaled()), d.getScale()); + return dbgDec("fromGrpcValue", v, new java.math.BigDecimal(java.math.BigInteger.valueOf(d.getUnscaled()), d.getScale()), null); } case LIST_VALUE: { var out = new java.util.ArrayList<>(); for (GrpcValue e : v.getListValue().getValuesList()) out.add(fromGrpcValue(e)); - return out; + return dbgDec("fromGrpcValue", v, out, null); } case MAP_VALUE: { var out = new java.util.LinkedHashMap(); v.getMapValue().getEntriesMap().forEach((k, vv) -> out.put(k, fromGrpcValue(vv))); - return out; + return dbgDec("fromGrpcValue", v, out, null); } case EMBEDDED_VALUE: { var out = new java.util.LinkedHashMap(); v.getEmbeddedValue().getFieldsMap().forEach((k, vv) -> out.put(k, fromGrpcValue(vv))); - return out; + return dbgDec("fromGrpcValue", v, out, null); } case KIND_NOT_SET: - return null; + return dbgDec("fromGrpcValue", v, null, null); } - return null; + return dbgDec("fromGrpcValue", v, null, null); } private static com.google.protobuf.Timestamp msToTimestamp(long ms) { @@ -1664,29 +1671,29 @@ private static com.google.protobuf.Timestamp msToTimestamp(long ms) { private GrpcValue toGrpcValue(Object o) { GrpcValue.Builder b = GrpcValue.newBuilder(); if (o == null) - return b.build(); + return dbgEnc("toGrpcValue", o, b.build(), null); if (o instanceof Boolean v) - return b.setBoolValue(v).build(); + return dbgEnc("toGrpcValue", o, b.setBoolValue(v).build(), null); if (o instanceof Integer v) - return b.setInt32Value(v).build(); + return dbgEnc("toGrpcValue", o, b.setInt32Value(v).build(), null); if (o instanceof Long v) - return b.setInt64Value(v).build(); + return dbgEnc("toGrpcValue", o, b.setInt64Value(v).build(), null); if (o instanceof Float v) - return b.setFloatValue(v).build(); + return dbgEnc("toGrpcValue", o, b.setFloatValue(v).build(), null); if (o instanceof Double v) - return b.setDoubleValue(v).build(); + return dbgEnc("toGrpcValue", o, b.setDoubleValue(v).build(), null); if (o instanceof CharSequence v) - return b.setStringValue(v.toString()).build(); + return dbgEnc("toGrpcValue", o, b.setStringValue(v.toString()).build(), null); if (o instanceof byte[] v) - return b.setBytesValue(com.google.protobuf.ByteString.copyFrom(v)).build(); + return dbgEnc("toGrpcValue", o, b.setBytesValue(com.google.protobuf.ByteString.copyFrom(v)).build(), null); if (o instanceof java.util.Date v) { - return GrpcValue.newBuilder().setTimestampValue(msToTimestamp(v.getTime())).setLogicalType("datetime").build(); + return dbgEnc("toGrpcValue", o, GrpcValue.newBuilder().setTimestampValue(msToTimestamp(v.getTime())).setLogicalType("datetime").build(), null); } if (o instanceof com.arcadedb.database.RID rid) { - return GrpcValue.newBuilder().setLinkValue(GrpcLink.newBuilder().setRid(rid.toString()).build()).setLogicalType("rid").build(); + return dbgEnc("toGrpcValue", o, GrpcValue.newBuilder().setLinkValue(GrpcLink.newBuilder().setRid(rid.toString()).build()).setLogicalType("rid").build(), null); } if (o instanceof java.math.BigDecimal v) { @@ -1698,7 +1705,7 @@ private GrpcValue toGrpcValue(Object o) { else { // if you need >64-bit unscaled, switch GrpcDecimal.unscaled to bytes in the // proto - return GrpcValue.newBuilder().setStringValue(v.toPlainString()).setLogicalType("decimal").build(); + return dbgEnc("toGrpcValue", o, GrpcValue.newBuilder().setStringValue(v.toPlainString()).setLogicalType("decimal").build(), null); } } @@ -1709,7 +1716,7 @@ private GrpcValue toGrpcValue(Object o) { for (String k : edoc.getPropertyNames()) { eb.putFields(k, toGrpcValue(edoc.get(k))); } - return GrpcValue.newBuilder().setEmbeddedValue(eb.build()).build(); + return dbgEnc("toGrpcValue", o, GrpcValue.newBuilder().setEmbeddedValue(eb.build()).build(), null); } if (o instanceof java.util.Map m) { @@ -1717,14 +1724,14 @@ private GrpcValue toGrpcValue(Object o) { for (var e : m.entrySet()) { mb.putEntries(String.valueOf(e.getKey()), toGrpcValue(e.getValue())); } - return GrpcValue.newBuilder().setMapValue(mb.build()).build(); + return dbgEnc("toGrpcValue", o, GrpcValue.newBuilder().setMapValue(mb.build()).build(), null); } if (o instanceof java.util.Collection c) { GrpcList.Builder lb = GrpcList.newBuilder(); for (Object e : c) lb.addValues(toGrpcValue(e)); - return GrpcValue.newBuilder().setListValue(lb.build()).build(); + return dbgEnc("toGrpcValue", o, GrpcValue.newBuilder().setListValue(lb.build()).build(), null); } if (o.getClass().isArray()) { @@ -1733,16 +1740,16 @@ private GrpcValue toGrpcValue(Object o) { for (int i = 0; i < len; i++) { lb.addValues(toGrpcValue(java.lang.reflect.Array.get(o, i))); } - return GrpcValue.newBuilder().setListValue(lb.build()).build(); + return dbgEnc("toGrpcValue", o, GrpcValue.newBuilder().setListValue(lb.build()).build(), null); } // RID/Identifiable string fallback if (o instanceof com.arcadedb.database.Identifiable id) - return GrpcValue.newBuilder().setLinkValue(GrpcLink.newBuilder().setRid(id.getIdentity().toString()).build()).setLogicalType("rid") - .build(); + return dbgEnc("toGrpcValue", o, GrpcValue.newBuilder().setLinkValue(GrpcLink.newBuilder().setRid(id.getIdentity().toString()).build()).setLogicalType("rid").build(), null); + // Fallback - return GrpcValue.newBuilder().setStringValue(String.valueOf(o)).build(); + return dbgEnc("toGrpcValue", o, GrpcValue.newBuilder().setStringValue(String.valueOf(o)).build(), null); } private InsertOptions defaults(InsertOptions in) { @@ -1944,8 +1951,11 @@ private GrpcRecord convertToGrpcRecord(com.arcadedb.database.Record dbRecord, Da // set all properties for (String propertyName : doc.getPropertyNames()) { Object value = doc.get(propertyName); - if (value != null) - builder.putProperties(propertyName, toGrpcValue(value)); + if (value != null) { + GrpcValue gv = toGrpcValue(value); + if (logger.isDebugEnabled()) logger.debug("ENC-REC {}.{}: {} -> {}", builder.getRid(), propertyName, summarizeJava(value), summarizeGrpc(gv)); + builder.putProperties(propertyName, gv); + } } // If this is an Edge, include @out / @in for convenience (string rid or link) @@ -1959,6 +1969,7 @@ private GrpcRecord convertToGrpcRecord(com.arcadedb.database.Record dbRecord, Da } } + if (logger.isDebugEnabled()) logger.debug("ENC-REC DONE rid={} type={} props={}", builder.getRid(), builder.getType(), builder.getPropertiesCount()); return builder.build(); } @@ -2240,4 +2251,49 @@ else if (o instanceof String s) private String generateTransactionId() { return "tx_" + System.nanoTime(); } + + // ---- Debug helpers ---- + private static String summarizeJava(Object o) { + if (o == null) return "null"; + try { + if (o instanceof CharSequence s) return "String(" + s.length() + ")=\"" + (s.length() > 120 ? s.subSequence(0,120) + "…" : s) + "\""; + if (o instanceof byte[] b) return "bytes[" + b.length + "]"; + if (o instanceof java.util.Collection c) return o.getClass().getSimpleName() + "[size=" + c.size() + "]"; + if (o instanceof java.util.Map m) return o.getClass().getSimpleName() + "[size=" + m.size() + "]"; + return o.getClass().getSimpleName() + "(" + String.valueOf(o) + ")"; + } catch (Exception e) { return o.getClass().getSimpleName(); } + } + + private static String summarizeGrpc(GrpcValue v) { + if (v == null) return "GrpcValue(null)"; + switch (v.getKindCase()) { + case BOOL_VALUE: return "BOOL(" + v.getBoolValue() + ")"; + case INT32_VALUE: return "INT32(" + v.getInt32Value() + ")"; + case INT64_VALUE: return "INT64(" + v.getInt64Value() + ")"; + case FLOAT_VALUE: return "FLOAT(" + v.getFloatValue() + ")"; + case DOUBLE_VALUE: return "DOUBLE(" + v.getDoubleValue() + ")"; + case STRING_VALUE: { + String s = v.getStringValue(); + return "STRING(" + s.length() + ")=\"" + (s.length() > 120 ? s.substring(0,120) + "…" : s) + "\""; + } + case BYTES_VALUE: return "BYTES[" + v.getBytesValue().size() + "]"; + case TIMESTAMP_VALUE: return "TIMESTAMP(" + v.getTimestampValue().getSeconds() + "." + v.getTimestampValue().getNanos() + ")"; + case LIST_VALUE: return "LIST[size=" + v.getListValue().getValuesCount() + "]"; + case MAP_VALUE: return "MAP[size=" + v.getMapValue().getEntriesCount() + "]"; + case EMBEDDED_VALUE: return "EMBEDDED[type=" + v.getEmbeddedValue().getType() + ", size=" + v.getEmbeddedValue().getFieldsCount() + "]"; + case LINK_VALUE: return "LINK(" + v.getLinkValue().getRid() + ")"; + case DECIMAL_VALUE: return "DECIMAL(unscaled=" + v.getDecimalValue().getUnscaled() + ", scale=" + v.getDecimalValue().getScale() + ")"; + case KIND_NOT_SET: default: return "GrpcValue(KIND_NOT_SET)"; + } + } + + private GrpcValue dbgEnc(String where, Object in, GrpcValue out, String ctx) { + if (logger.isDebugEnabled()) logger.debug("GRPC-ENC [{}]{} in={} -> out={}", where, (ctx==null?"" : " "+ctx), summarizeJava(in), summarizeGrpc(out)); + return out; + } + + private Object dbgDec(String where, GrpcValue in, Object out, String ctx) { + if (logger.isDebugEnabled()) logger.debug("GRPC-DEC [{}]{} in={} -> out={}", where, (ctx==null?"" : " "+ctx), summarizeGrpc(in), summarizeJava(out)); + return out; + } } \ No newline at end of file From 236d7140bcdd7cc70007150f9bc85b988107134a Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Fri, 5 Sep 2025 08:52:32 -0700 Subject: [PATCH 25/35] Revised GrpcValue structure and additional logging --- .../RemoteGrpcDatabase_with_debug.java | 1726 +++++++++++++++++ .../remote/grpc/RemoteGrpcDatabase.java | 543 +++--- .../remote/grpc/utils/ProtoUtils.java | 607 +++--- .../test/resources/arcadedb-log.properties | 1 + .../server/grpc/ArcadeDbGrpcService.java | 1 - 5 files changed, 2304 insertions(+), 574 deletions(-) create mode 100644 grpc-client/RemoteGrpcDatabase_with_debug.java diff --git a/grpc-client/RemoteGrpcDatabase_with_debug.java b/grpc-client/RemoteGrpcDatabase_with_debug.java new file mode 100644 index 0000000000..ae3b62592d --- /dev/null +++ b/grpc-client/RemoteGrpcDatabase_with_debug.java @@ -0,0 +1,1726 @@ +package com.arcadedb.remote.grpc; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; + +import com.arcadedb.ContextConfiguration; +import com.arcadedb.database.Database; +import com.arcadedb.database.MutableDocument; +import com.arcadedb.database.RID; +import com.arcadedb.database.Record; +import com.arcadedb.exception.ConcurrentModificationException; +import com.arcadedb.exception.DatabaseOperationException; +import com.arcadedb.exception.DuplicatedKeyException; +import com.arcadedb.exception.NeedRetryException; +import com.arcadedb.exception.RecordNotFoundException; +import com.arcadedb.exception.TimeoutException; +import com.arcadedb.exception.TransactionException; +import com.arcadedb.query.sql.executor.InternalResultSet; +import com.arcadedb.query.sql.executor.Result; +import com.arcadedb.query.sql.executor.ResultInternal; +import com.arcadedb.query.sql.executor.ResultSet; +import com.arcadedb.remote.RemoteDatabase; +import com.arcadedb.remote.RemoteException; +import com.arcadedb.remote.RemoteImmutableDocument; +import com.arcadedb.remote.RemoteImmutableEdge; +import com.arcadedb.remote.RemoteImmutableVertex; +import com.arcadedb.remote.RemoteSchema; +import com.arcadedb.remote.RemoteTransactionExplicitLock; +import com.arcadedb.remote.grpc.utils.ProtoUtils; +import com.arcadedb.server.grpc.ArcadeDbServiceGrpc; +import com.arcadedb.server.grpc.BatchAck; +import com.arcadedb.server.grpc.BeginTransactionRequest; +import com.arcadedb.server.grpc.BeginTransactionResponse; +import com.arcadedb.server.grpc.BulkInsertRequest; +import com.arcadedb.server.grpc.Commit; +import com.arcadedb.server.grpc.CommitTransactionRequest; +import com.arcadedb.server.grpc.CommitTransactionResponse; +import com.arcadedb.server.grpc.CreateRecordRequest; +import com.arcadedb.server.grpc.CreateRecordResponse; +import com.arcadedb.server.grpc.DatabaseCredentials; +import com.arcadedb.server.grpc.DeleteRecordRequest; +import com.arcadedb.server.grpc.DeleteRecordResponse; +import com.arcadedb.server.grpc.ExecuteCommandRequest; +import com.arcadedb.server.grpc.ExecuteCommandResponse; +import com.arcadedb.server.grpc.ExecuteQueryRequest; +import com.arcadedb.server.grpc.ExecuteQueryResponse; +import com.arcadedb.server.grpc.GrpcRecord; +import com.arcadedb.server.grpc.GrpcValue; +import com.arcadedb.server.grpc.InsertChunk; +import com.arcadedb.server.grpc.InsertOptions; +import com.arcadedb.server.grpc.InsertRequest; +import com.arcadedb.server.grpc.InsertResponse; +import com.arcadedb.server.grpc.InsertSummary; +import com.arcadedb.server.grpc.LookupByRidRequest; +import com.arcadedb.server.grpc.LookupByRidResponse; +import com.arcadedb.server.grpc.PropertiesUpdate; +import com.arcadedb.server.grpc.QueryResult; +import com.arcadedb.server.grpc.RollbackTransactionRequest; +import com.arcadedb.server.grpc.RollbackTransactionResponse; +import com.arcadedb.server.grpc.Start; +import com.arcadedb.server.grpc.StreamQueryRequest; +import com.arcadedb.server.grpc.TransactionContext; +import com.arcadedb.server.grpc.TransactionIsolation; +import com.arcadedb.server.grpc.UpdateRecordRequest; +import com.arcadedb.server.grpc.UpdateRecordResponse; + +import io.grpc.CallCredentials; +import io.grpc.CompressorRegistry; +import io.grpc.DecompressorRegistry; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.Metadata; +import io.grpc.Status; +import io.grpc.StatusException; +import io.grpc.StatusRuntimeException; +import io.grpc.stub.BlockingClientCall; +import io.grpc.stub.ClientCallStreamObserver; +import io.grpc.stub.ClientResponseObserver; +import io.grpc.stub.StreamObserver; + +/** + * Remote Database implementation using gRPC protocol instead of HTTP. Extends + * RemoteDatabase to maintain compatibility while overriding network operations. + * It's not thread safe. For multi-thread usage create one instance of + * RemoteGrpcDatabase per thread. + * + * @author Oleg Cohen (oleg.cohen@gmail.com) + */ +public class RemoteGrpcDatabase extends RemoteDatabase { + + +private static final Logger logger = LoggerFactory.getLogger(RemoteGrpcDatabase.class); + +// --- Debug helpers (client) --- +private static String summarize(Object o) { + if (o == null) return "null"; + try { + if (o instanceof CharSequence s) return "String(" + s.length() + ")"; + if (o instanceof byte[] b) return "bytes[" + b.length + "]"; + if (o instanceof java.util.Collection c) return o.getClass().getSimpleName() + "[size=" + c.size() + "]"; + if (o instanceof java.util.Map m) return o.getClass().getSimpleName() + "[size=" + m.size() + "]"; + return o.getClass().getSimpleName(); + } catch (Exception e) { return o.getClass().getSimpleName(); } +} +private static String summarize(GrpcValue v) { + if (v == null) return "GrpcValue(null)"; + return switch (v.getKindCase()) { + case BOOL_VALUE -> "BOOL"; + case INT32_VALUE -> "INT32"; + case INT64_VALUE -> "INT64"; + case FLOAT_VALUE -> "FLOAT"; + case DOUBLE_VALUE -> "DOUBLE"; + case STRING_VALUE -> "STRING(" + v.getStringValue().length() + ")"; + case BYTES_VALUE -> "BYTES[" + v.getBytesValue().size() + "]"; + case TIMESTAMP_VALUE -> "TIMESTAMP"; + case LIST_VALUE -> "LIST[" + v.getListValue().getValuesCount() + "]"; + case MAP_VALUE -> "MAP[" + v.getMapValue().getEntriesCount() + "]"; + case EMBEDDED_VALUE -> "EMBEDDED"; + case LINK_VALUE -> "LINK(" + v.getLinkValue().getRid() + ")"; + case DECIMAL_VALUE -> "DECIMAL(scale=" + v.getDecimalValue().getScale() + ")"; + case KIND_NOT_SET -> "KIND_NOT_SET"; + }; +} +private static String summarize(GrpcRecord r) { + if (r == null) return "GrpcRecord(null)"; + String rid = r.getRid(); + String ty = r.getType(); + int props = r.getPropertiesCount(); + return "GrpcRecord{rid=" + rid + ", type=" + ty + ", props=" + props + "}"; +} + + + private final ManagedChannel channel; + + private final ArcadeDbServiceGrpc.ArcadeDbServiceBlockingV2Stub blockingStub; + private final ArcadeDbServiceGrpc.ArcadeDbServiceStub asyncStub; + + private String transactionId; + + private final RemoteSchema schema; + + private final String userName; + private final String userPassword; + private String databaseName; + private RemoteTransactionExplicitLock explicitLock; + + public RemoteGrpcDatabase(final String server, final int grpcPort, final int httpPort, final String databaseName, final String userName, + final String userPassword) { + this(server, grpcPort, httpPort, databaseName, userName, userPassword, new ContextConfiguration()); + } + + public RemoteGrpcDatabase(final String server, final int grpcPort, final int httpPort, final String databaseName, final String userName, + final String userPassword, final ContextConfiguration configuration) { + + super(server, httpPort, databaseName, userName, userPassword, configuration); + + this.userName = userName; + this.userPassword = userPassword; + + this.databaseName = databaseName; + + // Create gRPC channel + this.channel = createChannel(server, grpcPort); + + this.blockingStub = createBlockingStub(channel); + + this.asyncStub = createAsyncStub(channel); + + // Create gRPC-specific schema + this.schema = new RemoteSchema(this); + } + + /** + * Creates the gRPC channel. Override this for custom channel configuration. + */ + protected ManagedChannel createChannel(String server, int port) { + return ManagedChannelBuilder.forAddress(server, port).usePlaintext() // No TLS by default + .build(); + } + + /** + * Creates call credentials for authentication + */ + protected CallCredentials createCallCredentials(String userName, String userPassword) { + return new CallCredentials() { + @Override + public void applyRequestMetadata(RequestInfo requestInfo, Executor appExecutor, MetadataApplier applier) { + Metadata headers = new Metadata(); + headers.put(Metadata.Key.of("username", Metadata.ASCII_STRING_MARSHALLER), userName); + headers.put(Metadata.Key.of("password", Metadata.ASCII_STRING_MARSHALLER), userPassword); + headers.put(Metadata.Key.of("x-arcade-user", Metadata.ASCII_STRING_MARSHALLER), userName); + headers.put(Metadata.Key.of("x-arcade-password", Metadata.ASCII_STRING_MARSHALLER), userPassword); + headers.put(Metadata.Key.of("x-arcade-database", Metadata.ASCII_STRING_MARSHALLER), databaseName); + applier.apply(headers); + } + + @Override + public void thisUsesUnstableApi() { + // Required by the interface + } + }; + } + + protected CallCredentials createCredentials() { + return createCallCredentials(userName, userPassword); + } + + /** + * Override this method to customize blocking stub creation + */ + protected ArcadeDbServiceGrpc.ArcadeDbServiceBlockingV2Stub createBlockingStub(ManagedChannel channel) { + return ArcadeDbServiceGrpc.newBlockingV2Stub(channel).withCallCredentials(createCredentials()); + } + + protected ArcadeDbServiceGrpc.ArcadeDbServiceStub createAsyncStub(ManagedChannel channel) { + return ArcadeDbServiceGrpc.newStub(channel).withCallCredentials(createCredentials()).withCompression("gzip"); + } + + @Override + public String getDatabasePath() { + return "grpc://" + channel.authority() + "/" + getName(); + } + + @Override + public RemoteSchema getSchema() { + return schema; + } + + @Override + public void close() { + if (transactionId != null) { + rollback(); + } + super.close(); + + // Shutdown gRPC channel + channel.shutdown(); + try { + if (!channel.awaitTermination(5, TimeUnit.SECONDS)) { + channel.shutdownNow(); + } + } catch (InterruptedException e) { + channel.shutdownNow(); + } + } + + @Override + public void begin(final Database.TRANSACTION_ISOLATION_LEVEL isolationLevel) { + checkDatabaseIsOpen(); + if (transactionId != null) + throw new TransactionException("Transaction already begun"); + + BeginTransactionRequest request = BeginTransactionRequest.newBuilder().setDatabase(getName()).setCredentials(buildCredentials()) + .setIsolation(mapIsolationLevel(isolationLevel)).build(); + + try { + BeginTransactionResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).beginTransaction(request); + transactionId = response.getTransactionId(); + // Store transaction ID in parent class session management + setSessionId(transactionId); + } catch (StatusRuntimeException e) { + throw new TransactionException("Error on transaction begin", e); + } catch (StatusException e) { + throw new TransactionException("Error on transaction begin", e); + } + } + + @Override + public void commit() { + checkDatabaseIsOpen(); + stats.txCommits.incrementAndGet(); + + if (transactionId == null) + throw new TransactionException("Transaction not begun"); + + CommitTransactionRequest request = CommitTransactionRequest.newBuilder() + .setTransaction(TransactionContext.newBuilder().setTransactionId(transactionId).setDatabase(getName()).build()) + .setCredentials(buildCredentials()).build(); + + try { + CommitTransactionResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).commitTransaction(request); + if (!response.getSuccess()) { + throw new TransactionException("Failed to commit transaction: " + response.getMessage()); + } + } catch (StatusRuntimeException e) { + handleGrpcException(e); + } catch (StatusException e) { + handleGrpcException(e); + } finally { + transactionId = null; + setSessionId(null); + } + } + + @Override + public void rollback() { + checkDatabaseIsOpen(); + stats.txRollbacks.incrementAndGet(); + + if (transactionId == null) + throw new TransactionException("Transaction not begun"); + + RollbackTransactionRequest request = RollbackTransactionRequest.newBuilder() + .setTransaction(TransactionContext.newBuilder().setTransactionId(transactionId).setDatabase(getName()).build()) + .setCredentials(buildCredentials()).build(); + + try { + RollbackTransactionResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .rollbackTransaction(request); + if (!response.getSuccess()) { + throw new TransactionException("Failed to rollback transaction: " + response.getMessage()); + } + } catch (StatusRuntimeException e) { + throw new TransactionException("Error on transaction rollback", e); + } catch (StatusException e) { + throw new TransactionException("Error on transaction rollback", e); + } finally { + transactionId = null; + setSessionId(null); + } + } + + @Override + public void deleteRecord(final Record record) { + checkDatabaseIsOpen(); + stats.deleteRecord.incrementAndGet(); + + if (record.getIdentity() == null) + throw new IllegalArgumentException("Cannot delete a non persistent record"); + + DeleteRecordRequest request = DeleteRecordRequest.newBuilder().setDatabase(getName()).setRid(record.getIdentity().toString()) + .setCredentials(buildCredentials()).build(); + + try { + DeleteRecordResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).deleteRecord(request); + if (!response.getSuccess()) { + throw new DatabaseOperationException("Failed to delete record: " + response.getMessage()); + } + } catch (StatusRuntimeException e) { + handleGrpcException(e); + } catch (StatusException e) { + handleGrpcException(e); + } + } + + @Override + public Iterator iterateType(final String typeName, final boolean polymorphic) { + String query = "select from `" + typeName + "`"; + if (!polymorphic) + query += " where @type = '" + typeName + "'"; + return streamQuery(query); + } + + @Override + public Iterator iterateBucket(final String bucketName) { + return streamQuery("select from bucket:`" + bucketName + "`"); + } + + @Override + public ResultSet query(final String language, final String query, final Map params) { + checkDatabaseIsOpen(); + stats.queries.incrementAndGet(); + + ExecuteQueryRequest.Builder requestBuilder = ExecuteQueryRequest.newBuilder().setDatabase(getName()).setQuery(query) + .setCredentials(buildCredentials()); + + if (transactionId != null) { + requestBuilder.setTransaction(TransactionContext.newBuilder().setTransactionId(transactionId).setDatabase(getName()).build()); + } + + if (params != null && !params.isEmpty()) { + requestBuilder.putAllParameters(convertParamsToGrpcValue(params)); + } + + try { + + if (logger.isDebugEnabled()) logger.debug("CLIENT executeQuery: db={}, tx={}, queryLen={}, params={}", getName(), (transactionId!=null), requestBuilder.getQuery().length(), requestBuilder.getParametersCount()); + + + ExecuteQueryResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .executeQuery(requestBuilder.build()); + + if (logger.isDebugEnabled()) { int _r=0; for (var qr : response.getResultsList()) _r+=qr.getRecordsCount(); logger.debug("CLIENT executeQuery: results={}", _r);} +return createGrpcResultSet(response); + } + catch (StatusRuntimeException e) { + handleGrpcException(e); + return new InternalResultSet(); + } + catch (StatusException e) { + handleGrpcException(e); + return new InternalResultSet(); + } + } + + @Override + public ResultSet query(final String language, final String query, final Object... args) { + checkDatabaseIsOpen(); + stats.queries.incrementAndGet(); + + final Map params = mapArgs(args); + + return query(language, query, params); + } + + @Override + public ResultSet command(final String language, final String command, final Map params) { + checkDatabaseIsOpen(); + stats.commands.incrementAndGet(); + + ExecuteCommandRequest.Builder requestBuilder = ExecuteCommandRequest.newBuilder().setDatabase(getName()).setCommand(command) + .setCredentials(buildCredentials()); + + if (transactionId != null) { + requestBuilder.setTransaction(TransactionContext.newBuilder().setTransactionId(transactionId).setDatabase(getName()).build()); + } + + if (params != null && !params.isEmpty()) { + requestBuilder.putAllParameters(convertParamsToGrpcValue(params)); + } + + try { + + if (logger.isDebugEnabled()) logger.debug("CLIENT executeCommand: db={}, tx={}, cmdLen={}, params={}", getName(), (transactionId!=null), requestBuilder.getCommand().length(), requestBuilder.getParametersCount()); + + + ExecuteCommandResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .executeCommand(requestBuilder.build()); + + if (logger.isDebugEnabled()) logger.debug("CLIENT executeCommand: status={}", response.getStatus()); +if (! response.getSuccess()) { + throw new DatabaseOperationException("Failed to execute command: " + response.getMessage()); + } + + // Create result set with command execution info + InternalResultSet resultSet = new InternalResultSet(); + + if (response.getAffectedRecords() > 0) { + Map result = new HashMap<>(); + result.put("affected", response.getAffectedRecords()); + result.put("executionTime", response.getExecutionTimeMs()); + resultSet.add(new ResultInternal(result)); + } + return resultSet; + } catch (StatusRuntimeException e) { + handleGrpcException(e); + return new InternalResultSet(); + } catch (StatusException e) { + handleGrpcException(e); + return new InternalResultSet(); + } + } + + public ExecuteCommandResponse execSql(String db, String sql, Map params, long timeoutMs) { + return executeCommand(db, "sql", sql, params, /* returnRows */ false, /* maxRows */ 0, txBeginCommit(), timeoutMs); + } + + public ExecuteCommandResponse execSql(String sql, Map params, long timeoutMs) { + return executeCommand(databaseName, "sql", sql, params, /* returnRows */ false, /* maxRows */ 0, txBeginCommit(), timeoutMs); + } + + public ExecuteCommandResponse executeCommand(String language, String command, Map params, + boolean returnRows, int maxRows, TransactionContext tx, long timeoutMs) { + + var reqB = ExecuteCommandRequest.newBuilder().setDatabase(databaseName).setCommand(command) + .putAllParameters(convertParamsToGrpcValue(params)).setLanguage(langOrDefault(language)).setReturnRows(returnRows) + .setMaxRows(maxRows > 0 ? maxRows : 0); + + if (tx != null) + reqB.setTransaction(tx); + // credentials: if your stub builds creds implicitly, set here if required + reqB.setCredentials(buildCredentials()); + + try { + return blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).executeCommand(reqB.build()); + } catch (StatusException e) { + throw new RuntimeException("Failed to execute command: " + e.getMessage(), e); + } + } + + public ExecuteCommandResponse executeCommand(String database, String language, String command, + Map params, boolean returnRows, int maxRows, TransactionContext tx, long timeoutMs) { + + var reqB = ExecuteCommandRequest.newBuilder().setDatabase(database).setCommand(command) + .putAllParameters(convertParamsToGrpcValue(params)).setLanguage(langOrDefault(language)).setReturnRows(returnRows) + .setMaxRows(maxRows > 0 ? maxRows : 0); + + if (tx != null) + reqB.setTransaction(tx); + // credentials: if your stub builds creds implicitly, set here if required + reqB.setCredentials(buildCredentials()); + + try { + return blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).executeCommand(reqB.build()); + } catch (StatusException e) { + throw new RuntimeException("Failed to execute command: " + e.getMessage(), e); + } + } + + @Override + protected RID saveRecord(final MutableDocument record) { + stats.createRecord.incrementAndGet(); + + final RID rid = record.getIdentity(); + + if (rid != null) { + // -------- UPDATE (partial) -------- + PropertiesUpdate partial = PropertiesUpdate.newBuilder() + .putAllProperties(convertParamsToGrpcValue(record.toMap(false))) + .build(); + + UpdateRecordRequest request = UpdateRecordRequest.newBuilder() + .setDatabase(getName()) + .setRid(rid.toString()) + .setPartial(partial) + .setDatabase(databaseName) + .setCredentials(buildCredentials()).build(); + + try { + + @SuppressWarnings("unused") + UpdateRecordResponse response = blockingStub + .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .updateRecord(request); + + // If your proto has flags, you can check response.getSuccess()/getUpdated() + // Otherwise, treat non-exception as success. + return rid; + } catch (StatusRuntimeException e) { + handleGrpcException(e); + return null; + } catch (StatusException e) { + handleGrpcException(e); + return null; + } + } else { + // -------- CREATE -------- + GrpcRecord recMsg = GrpcRecord.newBuilder() + .putAllProperties(convertParamsToGrpcValue(record.toMap(false))) + .build(); + + CreateRecordRequest request = CreateRecordRequest.newBuilder() + .setDatabase(getName()) + .setType(record.getTypeName()) + .setRecord(recMsg) // nested GrpcRecord payload + .setCredentials(buildCredentials()).build(); + + try { + CreateRecordResponse response = blockingStub + .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .createRecord(request); + + // Proto returns the newly created RID as a string + final String ridStr = response.getRid(); + if (ridStr == null || ridStr.isEmpty()) { + throw new DatabaseOperationException("Failed to create record (empty RID)"); + } + + // Construct a RID from the returned string + try { + return new RID(ridStr); + } catch (NoSuchMethodError | IllegalArgumentException ex) { + // Fallback for older APIs expecting (Database, String) + return new RID(this, ridStr); + } + } catch (StatusRuntimeException e) { + handleGrpcException(e); + return null; + } catch (StatusException e) { + handleGrpcException(e); + return null; + } + } + } + + // Convenience: default batch size stays 100, default mode = CURSOR + public Iterator queryStream(final String language, final String query) { + return queryStream(language, query, /* batchSize */100, StreamQueryRequest.RetrievalMode.CURSOR); + } + + public Iterator queryStream(final String language, final String query, final int batchSize) { + return queryStream(language, query, batchSize, StreamQueryRequest.RetrievalMode.CURSOR); + } + + // NEW: choose retrieval mode + public Iterator queryStream(final String language, final String query, final int batchSize, + final StreamQueryRequest.RetrievalMode mode) { + checkDatabaseIsOpen(); + stats.queries.incrementAndGet(); + + StreamQueryRequest request = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(query).setCredentials(buildCredentials()) + .setBatchSize(batchSize > 0 ? batchSize : 100).setRetrievalMode(mode) // <--- NEW + .build(); + + final BlockingClientCall responseIterator = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .streamQuery(request); + + return new Iterator() { + private Iterator currentBatch = Collections.emptyIterator(); + + @Override + public boolean hasNext() { + if (currentBatch.hasNext()) + return true; + + try { + while (responseIterator.hasNext()) { + QueryResult result = responseIterator.read(); + + // Defensive: skip any empty batches the server might send + if (result.getRecordsCount() == 0) { + if (result.getIsLastBatch()) + return false; + continue; + } + + currentBatch = result.getRecordsList().iterator(); + return true; + } + } + catch (InterruptedException e) { + throw new RuntimeException("Interrupted while waiting for stream", e); + } + catch (StatusException e) { + handleGrpcException(e); + } + + return false; + } + + @Override + public Record next() { + if (!hasNext()) + throw new NoSuchElementException(); + return grpcRecordToDBRecord(currentBatch.next()); + } + }; + } + + // PARAMETERIZED variant with retrieval mode + public Iterator queryStream(final String language, final String query, final Map params, final int batchSize, + final StreamQueryRequest.RetrievalMode mode) { + checkDatabaseIsOpen(); + stats.queries.incrementAndGet(); + + StreamQueryRequest.Builder b = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(query).setCredentials(buildCredentials()) + .setBatchSize(batchSize > 0 ? batchSize : 100).setRetrievalMode(mode); // <--- NEW + + if (params != null && !params.isEmpty()) { + b.putAllParameters(convertParamsToGrpcValue(params)); + } + + final BlockingClientCall responseIterator = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .streamQuery(b.build()); + + return new Iterator() { + private Iterator currentBatch = Collections.emptyIterator(); + + @Override + public boolean hasNext() { + if (currentBatch.hasNext()) + return true; + + try { + + while (responseIterator.hasNext()) { + + QueryResult result = responseIterator.read(); + + if (result.getRecordsCount() == 0) { + if (result.getIsLastBatch()) + return false; + continue; + } + + currentBatch = result.getRecordsList().iterator(); + return true; + } + + return false; + } + catch (Exception e) { + + throw new RuntimeException("Interrupted while waiting for stream", e); + } + } + + @Override + public Record next() { + if (!hasNext()) + throw new NoSuchElementException(); + return grpcRecordToDBRecord(currentBatch.next()); + } + }; + } + + // Keep the old signature working (defaults to CURSOR) + public Iterator queryStream(final String language, final String query, final Map params, final int batchSize) { + return queryStream(language, query, params, batchSize, StreamQueryRequest.RetrievalMode.CURSOR); + } + + public static final class QueryBatch { + private final List records; + private final int totalInBatch; + private final long runningTotal; + private final boolean lastBatch; + + public QueryBatch(List list, int totalInBatch, long runningTotal, boolean lastBatch) { + this.records = list; + this.totalInBatch = totalInBatch; + this.runningTotal = runningTotal; + this.lastBatch = lastBatch; + } + + public List records() { + return records; + } + + public int totalInBatch() { + return totalInBatch; + } + + public long runningTotal() { + return runningTotal; + } + + public boolean isLastBatch() { + return lastBatch; + } + } + + public Iterator queryStreamBatches(final String language, final String query, final Map params, + final int batchSize, final StreamQueryRequest.RetrievalMode mode) { + checkDatabaseIsOpen(); + stats.queries.incrementAndGet(); + + StreamQueryRequest.Builder b = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(query).setCredentials(buildCredentials()) + .setBatchSize(batchSize > 0 ? batchSize : 100).setRetrievalMode(mode); + + if (params != null && !params.isEmpty()) { + b.putAllParameters(convertParamsToGrpcValue(params)); + } + + final BlockingClientCall responseIterator = blockingStub.withWaitForReady() // optional, improves robustness + .withDeadlineAfter(/* e.g. */ 10, TimeUnit.MINUTES).streamQuery(b.build()); + + return new Iterator() { + private QueryBatch nextBatch = null; + private boolean drained = false; + + @Override + public boolean hasNext() { + if (nextBatch != null) + return true; + if (drained) + return false; + + try { + + while (responseIterator.hasNext()) { + + QueryResult qr = responseIterator.read(); + + int n = qr.getTotalRecordsInBatch(); // server-populated + // Guard: some servers could omit this; fallback to list size. + if (n == 0) + n = qr.getRecordsCount(); + + if (qr.getRecordsCount() == 0 && !qr.getIsLastBatch()) { + // skip empty non-terminal batch + continue; + } + + List converted = new ArrayList<>(qr.getRecordsCount()); + for (GrpcRecord gr : qr.getRecordsList()) { + converted.add(grpcRecordToDBRecord(gr)); + } + + nextBatch = new QueryBatch(converted, n, qr.getRunningTotalEmitted(), qr.getIsLastBatch()); + + if (qr.getIsLastBatch()) { + drained = true; // no more after this (even if server sent empty terminal) + } + return true; + } + + drained = true; + return false; + } + catch (Exception e) { + + throw new RuntimeException("Interrupted while waiting for stream", e); + } + } + + @Override + public QueryBatch next() { + if (!hasNext()) + throw new NoSuchElementException(); + QueryBatch out = nextBatch; + nextBatch = null; + return out; + } + }; + } + + public Iterator queryStream(String database, String sql, Map params, int batchSize, + StreamQueryRequest.RetrievalMode mode, TransactionContext tx, long timeoutMs) { + + var reqB = StreamQueryRequest.newBuilder().setDatabase(database).setQuery(sql) + .putAllParameters(convertParamsToGrpcValue(params)).setCredentials(buildCredentials()).setBatchSize(batchSize > 0 ? batchSize : 100) + .setRetrievalMode(mode); + + if (tx != null) + reqB.setTransaction(tx); + + final BlockingClientCall it = blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS) + .streamQuery(reqB.build()); + + return new Iterator<>() { + + private Iterator curr = Collections.emptyIterator(); + + public boolean hasNext() { + + if (curr.hasNext()) + return true; + + try { + + if (it.hasNext()) { + + curr = it.read().getRecordsList().iterator(); + return hasNext(); + } + } + catch (InterruptedException e) { + throw new RuntimeException("Interrupted while waiting for stream", e); + } + catch (StatusException e) { + throw new RuntimeException("Stream query failed: " + e.getStatus().getDescription()); + } + + return false; + } + + public GrpcRecord next() { + if (!hasNext()) + throw new NoSuchElementException(); + return curr.next(); + } + }; + } + + public Iterator queryStreamBatches(String passLabel, String sql, Map params, int batchSize, + StreamQueryRequest.RetrievalMode mode, TransactionContext tx, long timeoutMs) { + + var reqB = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(sql) + .putAllParameters(convertParamsToGrpcValue(params)).setCredentials(buildCredentials()).setBatchSize(batchSize > 0 ? batchSize : 100) + .setRetrievalMode(mode); + + if (tx != null) + reqB.setTransaction(tx); + + final BlockingClientCall respIter = blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS) + .streamQuery(reqB.build()); + + return new Iterator<>() { + + public boolean hasNext() { + + try { + return respIter.hasNext(); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + catch (StatusException e) { + throw new RuntimeException(e); + } + } + + public QueryBatch next() { + + QueryResult qr; + + try { + qr = respIter.read(); + List converted = new ArrayList<>(qr.getRecordsCount()); + + for (GrpcRecord gr : qr.getRecordsList()) { + converted.add(grpcRecordToDBRecord(gr)); + } + + return new QueryBatch(converted, qr.getTotalRecordsInBatch(), // int totalInBatch + qr.getRunningTotalEmitted(), // long runningTotal + qr.getIsLastBatch() // boolean lastBatch + ); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + catch (StatusException e) { + throw new RuntimeException(e); + } + } + }; + } + + public String createVertex(String cls, Map props, long timeoutMs) { + // Build the nested proto GrpcRecord payload + GrpcRecord recMsg = GrpcRecord.newBuilder().putAllProperties(convertParamsToGrpcValue(props)) + .build(); + + // Build the Create request + CreateRecordRequest req = CreateRecordRequest.newBuilder().setDatabase(getName()) + .setType(cls).setRecord(recMsg) // <<<< NESTED RECORD (not top-level properties) + .setCredentials(buildCredentials()).build(); + + // Call RPC + CreateRecordResponse res; + + try { + res = blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).createRecord(req); + // Response carries new RID string + return res.getRid(); // e.g., "#12:0" + } catch (StatusException e) { + throw new RuntimeException("Failed to create vertex", e); + } + } + + public boolean updateRecord(String rid, Map props, long timeoutMs) { + PropertiesUpdate partial = PropertiesUpdate.newBuilder() + .putAllProperties(convertParamsToGrpcValue(props)) + .build(); + + return updateRecord(rid, partial, timeoutMs); + } + + public boolean updateRecord(String rid, Record dbRecord, long timeoutMs) { + GrpcRecord record = ProtoUtils.toProtoRecord(dbRecord); + return updateRecordFull(rid, record, timeoutMs); + } + + private boolean updateRecord(String rid, PropertiesUpdate partial, long timeoutMs) { + // Build the Update request using setPartial for partial update + UpdateRecordRequest req = UpdateRecordRequest.newBuilder().setDatabase(getName()) + .setRid(rid).setPartial(partial) + .setTransaction(TransactionContext.newBuilder().setBegin(true).setCommit(true)) + .setCredentials(buildCredentials()).build(); + + // Call RPC + UpdateRecordResponse res; + + try { + res = blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).updateRecord(req); + // Choose the flag your proto defines. Most builds expose getSuccess(). + return res.getSuccess(); + } catch (StatusException e) { + throw new RuntimeException("Failed to update record", e); + } + } + + private boolean updateRecordFull(String rid, GrpcRecord record, long timeoutMs) { + // Build the Update request using setRecord for full replacement + UpdateRecordRequest req = UpdateRecordRequest.newBuilder().setDatabase(getName()) + .setRid(rid).setRecord(record) + .setTransaction(TransactionContext.newBuilder().setBegin(true).setCommit(true)) + .setCredentials(buildCredentials()).build(); + + // Call RPC + UpdateRecordResponse res; + + try { + res = blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).updateRecord(req); + return res.getSuccess(); + } catch (StatusException e) { + throw new RuntimeException("Failed to update record", e); + } + } + + public boolean deleteRecord(String rid, long timeoutMs) { + var req = DeleteRecordRequest.newBuilder().setDatabase(getName()).setRid(rid).setCredentials(buildCredentials()) + .build(); + DeleteRecordResponse res; + + try { + res = blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).deleteRecord(req); + return res.getDeleted(); + } catch (StatusException e) { + throw new RuntimeException("Failed to delete record", e); + } + } + + @Override + public RemoteTransactionExplicitLock acquireLock() { + // Need gRPC-specific implementation + if (explicitLock == null) + explicitLock = new RemoteGrpcTransactionExplicitLock(this); + + return explicitLock; + } + + @Override + public long countBucket(final String bucketName) { + checkDatabaseIsOpen(); + stats.countBucket.incrementAndGet(); + ResultSet result = query("sql", "select count(*) as count from bucket:" + bucketName); + if (result.hasNext()) { + Number count = result.next().getProperty("count"); + return count != null ? count.longValue() : 0; + } + return 0; + } + + @Override + public long countType(final String typeName, final boolean polymorphic) { + checkDatabaseIsOpen(); + stats.countType.incrementAndGet(); + final String appendix = polymorphic ? "" : " where @type = '" + typeName + "'"; + ResultSet result = query("sql", "select count(*) as count from " + typeName + appendix); + if (result.hasNext()) { + Number count = result.next().getProperty("count"); + return count != null ? count.longValue() : 0; + } + return 0; + } + + @Override + public Record lookupByRID(final RID rid) { + return lookupByRID(rid, true); + } + + @Override + public Record lookupByRID(final RID rid, final boolean loadContent) { + checkDatabaseIsOpen(); + stats.readRecord.incrementAndGet(); + + if (rid == null) + throw new IllegalArgumentException("Record is null"); + + LookupByRidRequest request = LookupByRidRequest.newBuilder().setDatabase(getName()).setRid(rid.toString()) + .setCredentials(buildCredentials()).build(); + + try { + LookupByRidResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).lookupByRid(request); + + if (!response.getFound()) { + throw new RecordNotFoundException("Record " + rid + " not found", rid); + } + return grpcRecordToDBRecord(response.getRecord()); + } + catch (StatusRuntimeException e) { + handleGrpcException(e); + return null; + } + catch (StatusException e) { + handleGrpcException(e); + return null; + } + } + + @Override + public boolean existsRecord(RID rid) { + stats.existsRecord.incrementAndGet(); + if (rid == null) + throw new IllegalArgumentException("Record is null"); + try { + return lookupByRID(rid, false) != null; + } catch (RecordNotFoundException e) { + return false; + } + } + + public InsertSummary insertBulk(final InsertOptions options, + final List protoRows, final long timeoutMs) { + + // Ensure options carry DB + credentials as the server expects + InsertOptions.Builder ob = options.toBuilder(); + + if (options.getDatabase() == null || options.getDatabase().isEmpty()) { + ob.setDatabase(getName()); // your wrapper's DB name + } + + boolean missingCreds = !options.hasCredentials() || options.getCredentials().getUsername().isEmpty(); + if (missingCreds) { + ob.setCredentials(buildCredentials()); // your existing helper used by queries + } + + InsertOptions newOptions = ob.build(); + + BulkInsertRequest req = BulkInsertRequest.newBuilder().setOptions(newOptions) + .addAllRows(protoRows).build(); + + try { + if (logger.isDebugEnabled()) logger.debug("CLIENT insertBulk: rows={}, timeoutMs={}", req.getRowsCount(), timeoutMs); + return blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).bulkInsert(req); + } catch (StatusException e) { + throw new RuntimeException("insertBulk() -> failed: " + e.getStatus()); + } + } + + // Convenience overload that accepts domain rows (convert first) + public InsertSummary insertBulkAsListOfMaps(final InsertOptions options, + final List> rows, final long timeoutMs) { + + List protoRows = rows.stream().map(this::toProtoRecordFromMap) // your converter + .collect(java.util.stream.Collectors.toList()); + + return insertBulk(options, protoRows, timeoutMs); + } + + public InsertSummary ingestStream(final InsertOptions options, + final List protoRows, final int chunkSize, final long timeoutMs) + throws InterruptedException { + + final java.util.concurrent.CountDownLatch done = new java.util.concurrent.CountDownLatch(1); + final java.util.concurrent.atomic.AtomicReference summaryRef = new java.util.concurrent.atomic.AtomicReference<>(); + + // Ensure options carry DB + credentials as the server expects + InsertOptions.Builder ob = options.toBuilder(); + + if (options.getDatabase() == null || options.getDatabase().isEmpty()) { + ob.setDatabase(getName()); // your wrapper's DB name + } + + boolean missingCreds = !options.hasCredentials() || options.getCredentials().getUsername().isEmpty(); + + if (missingCreds) { + ob.setCredentials(buildCredentials()); // your existing helper used by queries + } + + StreamObserver resp = new StreamObserver<>() { + @Override + public void onNext(InsertSummary value) { + summaryRef.set(value); + } + + @Override + public void onError(Throwable t) { + done.countDown(); + } + + @Override + public void onCompleted() { + done.countDown(); + } + }; + + // Use the write service async stub, per-call deadline + var stub = this.asyncStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS); + + final StreamObserver req = stub.insertStream(resp); + + // Stream chunks + final String sessionId = "sess-" + System.nanoTime(); + + long seq = 1; + + for (int i = 0; i < protoRows.size(); i += chunkSize) { + final int end = Math.min(i + chunkSize, protoRows.size()); + + final InsertChunk chunk = InsertChunk.newBuilder().setSessionId(sessionId) + .setOptions(ob.build()).setChunkSeq(seq++).addAllRows(protoRows.subList(i, end)).build(); + + req.onNext(chunk); + } + + req.onCompleted(); + + done.await(timeoutMs + 5_000, TimeUnit.MILLISECONDS); + + final InsertSummary s = summaryRef.get(); + return (s != null) ? s : InsertSummary.newBuilder().setReceived(protoRows.size()).build(); + } + + // Convenience overload + public InsertSummary ingestStreamAsListOfMaps(final InsertOptions options, + final List> rows, final int chunkSize, final long timeoutMs) throws InterruptedException { + + List protoRows = rows.stream().map(this::toProtoRecordFromMap) + .collect(java.util.stream.Collectors.toList()); + + return ingestStream(options, protoRows, chunkSize, timeoutMs); + } + + /** + * Core implementation of InsertBidirectional ingest + */ + private InsertSummary ingestBidiCore(final List rows, + final InsertOptions options, final int chunkSize, final int maxInflight, final long timeoutMs, + final java.util.function.Function mapper) throws InterruptedException { + + // 1) Ensure options carry DB + credentials the server expects + final InsertOptions.Builder ob = options.toBuilder(); + + if (options.getDatabase() == null || options.getDatabase().isEmpty()) { + ob.setDatabase(getName()); + } + + final boolean missingCreds = !options.hasCredentials() || options.getCredentials().getUsername().isEmpty(); + + if (missingCreds) { + ob.setCredentials(buildCredentials()); + } + + final InsertOptions effectiveOpts = ob.build(); + + // 2) Pre-map rows → proto once (outside the observer) + final List protoRows = rows.stream().map(mapper) + .collect(java.util.stream.Collectors.toList()); + + if (logger.isDebugEnabled()) logger.debug("CLIENT ingestBidi start: rows={}, chunkSize={}, maxInflight={}, timeoutMs={}", protoRows.size(), chunkSize, maxInflight, timeoutMs); +// 3) Streaming state + final String sessionId = "sess-" + System.nanoTime(); + final java.util.concurrent.CountDownLatch done = new java.util.concurrent.CountDownLatch(1); + final java.util.concurrent.atomic.AtomicLong seq = new java.util.concurrent.atomic.AtomicLong(1); + final java.util.concurrent.atomic.AtomicInteger cursor = new java.util.concurrent.atomic.AtomicInteger(0); + final java.util.concurrent.atomic.AtomicInteger sent = new java.util.concurrent.atomic.AtomicInteger(0); + final java.util.concurrent.atomic.AtomicInteger acked = new java.util.concurrent.atomic.AtomicInteger(0); + final java.util.concurrent.atomic.AtomicReference committed = new java.util.concurrent.atomic.AtomicReference<>(); + final List acks = java.util.Collections.synchronizedList(new ArrayList<>()); + + // small holder to access req inside helpers + final java.util.concurrent.atomic.AtomicReference> observerRef = + new java.util.concurrent.atomic.AtomicReference<>(); + + final java.util.concurrent.atomic.AtomicBoolean commitSent = new java.util.concurrent.atomic.AtomicBoolean(false); + final java.util.concurrent.ScheduledExecutorService scheduler = java.util.concurrent.Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "grpc-ack-grace-timer"); + t.setDaemon(true); + return t; + }); + final long ackGraceMillis = Math.min(Math.max(timeoutMs / 10, 1_000L), 10_000L); // e.g., 1s..10s window + final Object timerLock = new Object(); + final java.util.concurrent.atomic.AtomicReference> ackGraceFuture = new java.util.concurrent.atomic.AtomicReference<>(); + + // helper: schedule COMMIT if still waiting + final Runnable sendCommitIfNeeded = () -> { + if (commitSent.compareAndSet(false, true)) { + try { + if (observerRef.get() != null) { + observerRef.get().onNext(InsertRequest.newBuilder() + .setCommit(Commit.newBuilder().setSessionId(sessionId)).build()); + observerRef.get().onCompleted(); + } + } catch (Throwable ignore) { + /* best effort */ } + } + }; + + // helper: (re)arm the grace timer + final Runnable armAckGraceTimer = () -> { + synchronized (timerLock) { + var prev = ackGraceFuture.getAndSet(null); + if (prev != null) + prev.cancel(false); + // only arm if we've sent all chunks but acks are still pending + if (cursor.get() >= protoRows.size() && acked.get() < sent.get() && !commitSent.get()) { + var fut = scheduler.schedule(sendCommitIfNeeded, ackGraceMillis, TimeUnit.MILLISECONDS); + ackGraceFuture.set(fut); + } + } + }; + + // helper: cancel timer (e.g., once all ACKed or after COMMIT observed) + final Runnable cancelAckGraceTimer = () -> { + synchronized (timerLock) { + var prev = ackGraceFuture.getAndSet(null); + if (prev != null) + prev.cancel(false); + } + }; + + final ClientResponseObserver observer = new ClientResponseObserver<>() { + + ClientCallStreamObserver req; + volatile boolean started = false; + + @Override + public void beforeStart(ClientCallStreamObserver r) { + this.req = r; + observerRef.set(r); // <-- make req visible to helper + r.disableAutoInboundFlowControl(); + r.setOnReadyHandler(this::drain); + } + + private void drain() { + if (!req.isReady()) + return; + + if (!started) { + req.onNext(InsertRequest.newBuilder() + .setStart(Start.newBuilder().setOptions(effectiveOpts)).build()); + + started = true; + + // Now that the call is started, it's safe to pull the first response + req.request(1); + } + + while (req.isReady() && (sent.get() - acked.get()) < maxInflight) { + final int start = cursor.get(); + if (start >= protoRows.size()) + break; + + final int end = Math.min(start + chunkSize, protoRows.size()); + final var slice = protoRows.subList(start, end); + + final var chunk = InsertChunk.newBuilder().setSessionId(sessionId) + .setChunkSeq(seq.getAndIncrement()).addAllRows(slice).build(); + + req.onNext(InsertRequest.newBuilder().setChunk(chunk).build()); + cursor.set(end); + sent.incrementAndGet(); + } + + // If all chunks sent: + if (cursor.get() >= protoRows.size()) { + if (acked.get() >= sent.get()) { + // all acked → commit immediately + sendCommitIfNeeded.run(); + } else { + // not all acked → (re)arm grace timer; if last ACK never arrives, timer will COMMIT + armAckGraceTimer.run(); + } + } + } + + @Override + public void onNext(InsertResponse v) { + switch (v.getMsgCase()) { + case STARTED -> { + /* ok */ } + case BATCH_ACK -> { + acks.add(v.getBatchAck()); + acked.incrementAndGet(); + // Each ACK may free capacity: push more & manage timer + drain(); + if (cursor.get() >= protoRows.size()) { + if (acked.get() >= sent.get()) { + cancelAckGraceTimer.run(); + sendCommitIfNeeded.run(); + } else { + armAckGraceTimer.run(); + } + } + } + case COMMITTED -> { + committed.set(v.getCommitted().getSummary()); + cancelAckGraceTimer.run(); + } + case MSG_NOT_SET -> { + /* ignore */ + } + case ERROR -> { + /* TBD */ + } + } + + req.request(1); // continue pulling responses + } + + @Override + public void onError(Throwable t) { + cancelAckGraceTimer.run(); + done.countDown(); + } + + @Override + public void onCompleted() { + cancelAckGraceTimer.run(); + done.countDown(); + } + }; + // 4) Kick off the bidi call with deadline + + try { + if (logger.isDebugEnabled()) logger.debug("CLIENT ingestBidi opening stream: timeoutMs={}", timeoutMs); + asyncStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).insertBidirectional(observer); + @SuppressWarnings("unused") + boolean finished = done.await(timeoutMs + 5_000, TimeUnit.MILLISECONDS); + if (logger.isDebugEnabled()) try { logger.debug("CLIENT ingestBidi finished: finished={}, sent={}, acked={}", finished, sent.get(), acked.get()); } catch (Throwable ignore) {} +} finally { + scheduler.shutdownNow(); + } + + // 6) Prefer COMMITTED summary if present; otherwise aggregate ACKs + final InsertSummary finalSummary = committed.get(); + if (finalSummary != null) + return finalSummary; + + final long ins = acks.stream().mapToLong(BatchAck::getInserted).sum(); + final long upd = acks.stream().mapToLong(BatchAck::getUpdated).sum(); + final long ign = acks.stream().mapToLong(BatchAck::getIgnored).sum(); + final long fail = acks.stream().mapToLong(BatchAck::getFailed).sum(); + + return InsertSummary.newBuilder().setReceived(protoRows.size()).setInserted(ins).setUpdated(upd).setIgnored(ign) + .setFailed(fail).build(); + } + + /** + * Pushes domain {@code com.arcadedb.database.Record} rows via InsertBidirectional with per-batch ACKs. + */ + public InsertSummary ingestBidi(final List rows, + final InsertOptions opts, final int chunkSize, final int maxInflight, final long timeoutMs) + throws InterruptedException { + + return ingestBidiCore(rows, opts, chunkSize, maxInflight, timeoutMs, + (Object o) -> toProtoRecordFromDbRecord((com.arcadedb.database.Record) o)); + } + + public InsertSummary ingestBidi(final List rows, + final InsertOptions opts, final int chunkSize, final int maxInflight) throws InterruptedException { + + return ingestBidiCore(rows, opts, chunkSize, maxInflight, /* timeoutMs */ 5 * 60_000L, + this::toProtoRecordFromDbRecord); + } + + /** + * Pushes map-shaped rows (property map per row) via InsertBidirectional with per-batch ACKs. + */ + public InsertSummary ingestBidi(final InsertOptions options, + final List> rows, final int chunkSize, final int maxInflight) throws InterruptedException { + + return ingestBidiCore(rows, options, chunkSize, maxInflight, /* timeoutMs */ 5 * 60_000L, + this::toProtoRecordFromMap); + } + + public InsertSummary ingestBidi(final InsertOptions options, + final List> rows, final int chunkSize, final int maxInflight, final long timeoutMs) + throws InterruptedException { + + return ingestBidiCore(rows, options, chunkSize, maxInflight, timeoutMs, + this::toProtoRecordFromMap); + } + + // Map -> GrpcRecord + private GrpcRecord toProtoRecordFromMap(Map row) { + GrpcRecord.Builder b = GrpcRecord.newBuilder(); + row.forEach((k, v) -> b.putProperties(k, objectToGrpcValue(v))); + GrpcRecord rec = b.build(); + if (logger.isDebugEnabled()) logger.debug("CLIENT toProtoRecordFromMap: {}", summarize(rec)); + return rec; + } + + // Domain Record (storage) -> GrpcRecord + private GrpcRecord toProtoRecordFromDbRecord(com.arcadedb.database.Record rec) { + // Use ProtoUtils for proper conversion + GrpcRecord out = ProtoUtils.toProtoRecord(rec); + if (logger.isDebugEnabled()) logger.debug("CLIENT toProtoRecordFromDbRecord: {}", summarize(out)); + return out; + } + + // Convert Java object -> GrpcValue (extend as needed) + private GrpcValue objectToGrpcValue(Object v) { + return ProtoUtils.toGrpcValue(v); + } + + public DatabaseCredentials buildCredentials() { + return DatabaseCredentials.newBuilder().setUsername(getUserName()).setPassword(getUserPassword()).build(); + } + + private TransactionIsolation mapIsolationLevel(Database.TRANSACTION_ISOLATION_LEVEL level) { + switch (level) { +// case READ_UNCOMMITTED: +// return TransactionIsolation.READ_UNCOMMITTED; + case READ_COMMITTED: + return TransactionIsolation.READ_COMMITTED; + case REPEATABLE_READ: + return TransactionIsolation.REPEATABLE_READ; +// case SERIALIZABLE: +// return TransactionIsolation.SERIALIZABLE; + default: + return TransactionIsolation.READ_COMMITTED; + } + } + + // ---- TX helpers ------------------------------------------------------------- + private static TransactionContext txBeginCommit() { + return TransactionContext.newBuilder().setBegin(true).setCommit(true).build(); + } + + private static TransactionContext txNone() { + return TransactionContext.getDefaultInstance(); + } + + // Optional: language defaulting (server defaults to "sql" too) + private static String langOrDefault(String language) { + return (language == null || language.isEmpty()) ? "sql" : language; + } + + private Iterator streamQuery(final String query) { + StreamQueryRequest request = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(query).setCredentials(buildCredentials()) + .setBatchSize(100).build(); + + final BlockingClientCall responseIterator = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .streamQuery(request); + + return new Iterator() { + private Iterator currentBatch = Collections.emptyIterator(); + + @Override + public boolean hasNext() { + + if (currentBatch.hasNext()) { + return true; + } + + try { + if (responseIterator.hasNext()) { + + QueryResult result; + result = responseIterator.read(); + currentBatch = result.getRecordsList().iterator(); + return currentBatch.hasNext(); + } + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + catch (StatusException e) { + throw new RuntimeException(e); + } + + return false; + } + + @Override + public Record next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return grpcRecordToDBRecord(currentBatch.next()); + } + }; + } + + private ResultSet createGrpcResultSet(ExecuteQueryResponse response) { + InternalResultSet resultSet = new InternalResultSet(); + for (QueryResult queryResult : response.getResultsList()) { + for (GrpcRecord record : queryResult.getRecordsList()) { + resultSet.add(grpcRecordToResult(record)); + } + } + return resultSet; + } + + private Result grpcRecordToResult(GrpcRecord grpcRecord) { + + Record record = grpcRecordToDBRecord(grpcRecord); + + if (record == null) { + + Map properties = new HashMap<>(); + grpcRecord.getPropertiesMap().forEach((k, v) -> properties.put(k, grpcValueToObject(v))); + return new ResultInternal(properties); + } + + return new ResultInternal(record); + } + + private Record grpcRecordToDBRecord(GrpcRecord grpcRecord) { + + Map map = new HashMap<>(); + + // Convert properties + grpcRecord.getPropertiesMap().forEach((k, v) -> map.put(k, grpcValueToObject(v))); + + // Add metadata + map.put("@rid", grpcRecord.getRid()); + map.put("@type", grpcRecord.getType()); + map.put("@cat", mapRecordType(grpcRecord)); + + String cat = (String) map.get("@cat"); + + if (cat == null) { + + return null; + } + + switch (cat) { + case "d": + return new RemoteImmutableDocument(this, map); + case "v": + return new RemoteImmutableVertex(this, map); + case "e": + return new RemoteImmutableEdge(this, map); + default: + return null; + } + } + + private String mapRecordType(GrpcRecord grpcRecord) { + + // Determine record category from type name + String typeName = grpcRecord.getType(); + + // Check schema to determine actual type + try { + + if (getSchema().existsType(typeName)) { + + Object type = getSchema().getType(typeName); + + if (type instanceof com.arcadedb.schema.VertexType) { + return "v"; + } + else if (type instanceof com.arcadedb.schema.EdgeType) { + return "e"; + } + else if (type instanceof com.arcadedb.schema.DocumentType) { + return "d"; + } + else { + return null; + } + } + else { + + return null; + } + } + catch (Exception e) { + // Fall back to name-based detection + } + + return null; + } + + private Map convertParamsToGrpcValue(Map params) { + Map grpcParams = new HashMap<>(); + + for (Map.Entry entry : params.entrySet()) { + GrpcValue value = objectToGrpcValue(entry.getValue()); + grpcParams.put(entry.getKey(), value); + } + + return grpcParams; + } + + + private Object grpcValueToObject(GrpcValue grpcValue) { + Object out = ProtoUtils.fromGrpcValue(grpcValue); + if (logger.isDebugEnabled()) logger.debug("CLIENT decode grpcValueToObject: {} -> {}", summarize(grpcValue), summarize(out)); + return out; + } + + + private void handleGrpcException(StatusRuntimeException e) { + Status status = e.getStatus(); + switch (status.getCode()) { + case NOT_FOUND: + throw new RecordNotFoundException(status.getDescription(), null); + case ALREADY_EXISTS: + throw new DuplicatedKeyException("", "", null); + case ABORTED: + throw new ConcurrentModificationException(status.getDescription()); + case DEADLINE_EXCEEDED: + throw new TimeoutException(status.getDescription()); + case PERMISSION_DENIED: + throw new SecurityException(status.getDescription()); + case UNAVAILABLE: + throw new NeedRetryException(status.getDescription()); + default: + throw new RemoteException("gRPC error: " + status.getDescription(), e); + } + } + + private void handleGrpcException(StatusException e) { + Status status = e.getStatus(); + switch (status.getCode()) { + case NOT_FOUND: + throw new RecordNotFoundException(status.getDescription(), null); + case ALREADY_EXISTS: + throw new DuplicatedKeyException("", "", null); + case ABORTED: + throw new ConcurrentModificationException(status.getDescription()); + case DEADLINE_EXCEEDED: + throw new TimeoutException(status.getDescription()); + case PERMISSION_DENIED: + throw new SecurityException(status.getDescription()); + case UNAVAILABLE: + throw new NeedRetryException(status.getDescription()); + default: + throw new RemoteException("gRPC error: " + status.getDescription(), e); + } + } + + /** + * Extended version with compression support + */ + public static class RemoteGrpcDatabaseWithCompression extends RemoteGrpcDatabase { + + public RemoteGrpcDatabaseWithCompression(String server, int grpcPort, int httpPort, String databaseName, String userName, + String userPassword, ContextConfiguration configuration) { + super(server, grpcPort, httpPort, databaseName, userName, userPassword, configuration); + } + + @Override + protected ManagedChannel createChannel(String server, int port) { + + return ManagedChannelBuilder.forAddress(server, port).usePlaintext() // No TLS/SSL + .compressorRegistry(CompressorRegistry.getDefaultInstance()).decompressorRegistry(DecompressorRegistry.getDefaultInstance()) + .maxInboundMessageSize(100 * 1024 * 1024) // 100MB max message size + .keepAliveTime(30, TimeUnit.SECONDS) // Keep-alive configuration + .keepAliveTimeout(10, TimeUnit.SECONDS).keepAliveWithoutCalls(true).build(); + } + } + + // Add the missing RemoteGrpcTransactionExplicitLock class reference + private static class RemoteGrpcTransactionExplicitLock extends RemoteTransactionExplicitLock { + public RemoteGrpcTransactionExplicitLock(RemoteGrpcDatabase database) { + super(database); + } + } +} \ No newline at end of file diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java index 4264be198a..cd13013a75 100644 --- a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java @@ -10,6 +10,9 @@ import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.arcadedb.ContextConfiguration; import com.arcadedb.database.Database; import com.arcadedb.database.MutableDocument; @@ -70,7 +73,6 @@ import com.arcadedb.server.grpc.TransactionIsolation; import com.arcadedb.server.grpc.UpdateRecordRequest; import com.arcadedb.server.grpc.UpdateRecordResponse; -import com.google.protobuf.Value; import io.grpc.CallCredentials; import io.grpc.CompressorRegistry; @@ -96,6 +98,58 @@ */ public class RemoteGrpcDatabase extends RemoteDatabase { + private static final Logger logger = LoggerFactory.getLogger(RemoteGrpcDatabase.class); + +// --- Debug helpers (client) --- + private static String summarize(Object o) { + if (o == null) + return "null"; + try { + if (o instanceof CharSequence s) + return "String(" + s.length() + ")"; + if (o instanceof byte[] b) + return "bytes[" + b.length + "]"; + if (o instanceof java.util.Collection c) + return o.getClass().getSimpleName() + "[size=" + c.size() + "]"; + if (o instanceof java.util.Map m) + return o.getClass().getSimpleName() + "[size=" + m.size() + "]"; + return o.getClass().getSimpleName(); + } + catch (Exception e) { + return o.getClass().getSimpleName(); + } + } + + private static String summarize(GrpcValue v) { + if (v == null) + return "GrpcValue(null)"; + return switch (v.getKindCase()) { + case BOOL_VALUE -> "BOOL"; + case INT32_VALUE -> "INT32"; + case INT64_VALUE -> "INT64"; + case FLOAT_VALUE -> "FLOAT"; + case DOUBLE_VALUE -> "DOUBLE"; + case STRING_VALUE -> "STRING(" + v.getStringValue().length() + ")"; + case BYTES_VALUE -> "BYTES[" + v.getBytesValue().size() + "]"; + case TIMESTAMP_VALUE -> "TIMESTAMP"; + case LIST_VALUE -> "LIST[" + v.getListValue().getValuesCount() + "]"; + case MAP_VALUE -> "MAP[" + v.getMapValue().getEntriesCount() + "]"; + case EMBEDDED_VALUE -> "EMBEDDED"; + case LINK_VALUE -> "LINK(" + v.getLinkValue().getRid() + ")"; + case DECIMAL_VALUE -> "DECIMAL(scale=" + v.getDecimalValue().getScale() + ")"; + case KIND_NOT_SET -> "KIND_NOT_SET"; + }; + } + + private static String summarize(GrpcRecord r) { + if (r == null) + return "GrpcRecord(null)"; + String rid = r.getRid(); + String ty = r.getType(); + int props = r.getPropertiesCount(); + return "GrpcRecord{rid=" + rid + ", type=" + ty + ", props=" + props + "}"; + } + private final ManagedChannel channel; private final ArcadeDbServiceGrpc.ArcadeDbServiceBlockingV2Stub blockingStub; @@ -128,9 +182,6 @@ public RemoteGrpcDatabase(final String server, final int grpcPort, final int htt // Create gRPC channel this.channel = createChannel(server, grpcPort); - // Create stubs with authentication - CallCredentials credentials = createCallCredentials(userName, userPassword); - this.blockingStub = createBlockingStub(channel); this.asyncStub = createAsyncStub(channel); @@ -208,7 +259,8 @@ public void close() { if (!channel.awaitTermination(5, TimeUnit.SECONDS)) { channel.shutdownNow(); } - } catch (InterruptedException e) { + } + catch (InterruptedException e) { channel.shutdownNow(); } } @@ -227,9 +279,11 @@ public void begin(final Database.TRANSACTION_ISOLATION_LEVEL isolationLevel) { transactionId = response.getTransactionId(); // Store transaction ID in parent class session management setSessionId(transactionId); - } catch (StatusRuntimeException e) { + } + catch (StatusRuntimeException e) { throw new TransactionException("Error on transaction begin", e); - } catch (StatusException e) { + } + catch (StatusException e) { throw new TransactionException("Error on transaction begin", e); } } @@ -251,11 +305,14 @@ public void commit() { if (!response.getSuccess()) { throw new TransactionException("Failed to commit transaction: " + response.getMessage()); } - } catch (StatusRuntimeException e) { + } + catch (StatusRuntimeException e) { handleGrpcException(e); - } catch (StatusException e) { + } + catch (StatusException e) { handleGrpcException(e); - } finally { + } + finally { transactionId = null; setSessionId(null); } @@ -279,11 +336,14 @@ public void rollback() { if (!response.getSuccess()) { throw new TransactionException("Failed to rollback transaction: " + response.getMessage()); } - } catch (StatusRuntimeException e) { + } + catch (StatusRuntimeException e) { throw new TransactionException("Error on transaction rollback", e); - } catch (StatusException e) { + } + catch (StatusException e) { throw new TransactionException("Error on transaction rollback", e); - } finally { + } + finally { transactionId = null; setSessionId(null); } @@ -305,9 +365,11 @@ public void deleteRecord(final Record record) { if (!response.getSuccess()) { throw new DatabaseOperationException("Failed to delete record: " + response.getMessage()); } - } catch (StatusRuntimeException e) { + } + catch (StatusRuntimeException e) { handleGrpcException(e); - } catch (StatusException e) { + } + catch (StatusException e) { handleGrpcException(e); } } @@ -342,16 +404,26 @@ public ResultSet query(final String language, final String query, final Map 0) { Map result = new HashMap<>(); result.put("affected", response.getAffectedRecords()); @@ -403,10 +482,12 @@ public ResultSet command(final String language, final String command, final Map< resultSet.add(new ResultInternal(result)); } return resultSet; - } catch (StatusRuntimeException e) { + } + catch (StatusRuntimeException e) { handleGrpcException(e); return new InternalResultSet(); - } catch (StatusException e) { + } + catch (StatusException e) { handleGrpcException(e); return new InternalResultSet(); } @@ -420,8 +501,8 @@ public ExecuteCommandResponse execSql(String sql, Map params, lo return executeCommand(databaseName, "sql", sql, params, /* returnRows */ false, /* maxRows */ 0, txBeginCommit(), timeoutMs); } - public ExecuteCommandResponse executeCommand(String language, String command, Map params, - boolean returnRows, int maxRows, TransactionContext tx, long timeoutMs) { + public ExecuteCommandResponse executeCommand(String language, String command, Map params, boolean returnRows, int maxRows, + TransactionContext tx, long timeoutMs) { var reqB = ExecuteCommandRequest.newBuilder().setDatabase(databaseName).setCommand(command) .putAllParameters(convertParamsToGrpcValue(params)).setLanguage(langOrDefault(language)).setReturnRows(returnRows) @@ -434,13 +515,14 @@ public ExecuteCommandResponse executeCommand(String language, String command, Ma try { return blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).executeCommand(reqB.build()); - } catch (StatusException e) { + } + catch (StatusException e) { throw new RuntimeException("Failed to execute command: " + e.getMessage(), e); } } - public ExecuteCommandResponse executeCommand(String database, String language, String command, - Map params, boolean returnRows, int maxRows, TransactionContext tx, long timeoutMs) { + public ExecuteCommandResponse executeCommand(String database, String language, String command, Map params, + boolean returnRows, int maxRows, TransactionContext tx, long timeoutMs) { var reqB = ExecuteCommandRequest.newBuilder().setDatabase(database).setCommand(command) .putAllParameters(convertParamsToGrpcValue(params)).setLanguage(langOrDefault(language)).setReturnRows(returnRows) @@ -453,7 +535,8 @@ public ExecuteCommandResponse executeCommand(String database, String language, S try { return blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).executeCommand(reqB.build()); - } catch (StatusException e) { + } + catch (StatusException e) { throw new RuntimeException("Failed to execute command: " + e.getMessage(), e); } } @@ -466,48 +549,40 @@ protected RID saveRecord(final MutableDocument record) { if (rid != null) { // -------- UPDATE (partial) -------- - PropertiesUpdate partial = PropertiesUpdate.newBuilder() - .putAllProperties(convertParamsToGrpcValue(record.toMap(false))) - .build(); - - UpdateRecordRequest request = UpdateRecordRequest.newBuilder() - .setDatabase(getName()) - .setRid(rid.toString()) - .setPartial(partial) - .setDatabase(databaseName) - .setCredentials(buildCredentials()).build(); + PropertiesUpdate partial = PropertiesUpdate.newBuilder().putAllProperties(convertParamsToGrpcValue(record.toMap(false))).build(); + + UpdateRecordRequest request = UpdateRecordRequest.newBuilder().setDatabase(getName()).setRid(rid.toString()).setPartial(partial) + .setDatabase(databaseName).setCredentials(buildCredentials()).build(); try { - UpdateRecordResponse response = blockingStub - .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) - .updateRecord(request); + + @SuppressWarnings("unused") + UpdateRecordResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).updateRecord(request); // If your proto has flags, you can check response.getSuccess()/getUpdated() // Otherwise, treat non-exception as success. return rid; - } catch (StatusRuntimeException e) { + } + catch (StatusRuntimeException e) { handleGrpcException(e); return null; - } catch (StatusException e) { + } + catch (StatusException e) { handleGrpcException(e); return null; } - } else { + } + else { // -------- CREATE -------- - GrpcRecord recMsg = GrpcRecord.newBuilder() - .putAllProperties(convertParamsToGrpcValue(record.toMap(false))) - .build(); - - CreateRecordRequest request = CreateRecordRequest.newBuilder() - .setDatabase(getName()) - .setType(record.getTypeName()) - .setRecord(recMsg) // nested GrpcRecord payload + GrpcRecord recMsg = GrpcRecord.newBuilder().putAllProperties(convertParamsToGrpcValue(record.toMap(false))).build(); + + CreateRecordRequest request = CreateRecordRequest.newBuilder().setDatabase(getName()).setType(record.getTypeName()).setRecord(recMsg) // nested + // GrpcRecord + // payload .setCredentials(buildCredentials()).build(); try { - CreateRecordResponse response = blockingStub - .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) - .createRecord(request); + CreateRecordResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).createRecord(request); // Proto returns the newly created RID as a string final String ridStr = response.getRid(); @@ -518,14 +593,17 @@ protected RID saveRecord(final MutableDocument record) { // Construct a RID from the returned string try { return new RID(ridStr); - } catch (NoSuchMethodError | IllegalArgumentException ex) { + } + catch (NoSuchMethodError | IllegalArgumentException ex) { // Fallback for older APIs expecting (Database, String) return new RID(this, ridStr); } - } catch (StatusRuntimeException e) { + } + catch (StatusRuntimeException e) { handleGrpcException(e); return null; - } catch (StatusException e) { + } + catch (StatusException e) { handleGrpcException(e); return null; } @@ -621,9 +699,9 @@ public boolean hasNext() { return true; try { - + while (responseIterator.hasNext()) { - + QueryResult result = responseIterator.read(); if (result.getRecordsCount() == 0) { @@ -639,9 +717,9 @@ public boolean hasNext() { return false; } catch (Exception e) { - + throw new RuntimeException("Interrupted while waiting for stream", e); - } + } } @Override @@ -713,13 +791,13 @@ public boolean hasNext() { return true; if (drained) return false; - + try { - + while (responseIterator.hasNext()) { - + QueryResult qr = responseIterator.read(); - + int n = qr.getTotalRecordsInBatch(); // server-populated // Guard: some servers could omit this; fallback to list size. if (n == 0) @@ -747,9 +825,9 @@ public boolean hasNext() { return false; } catch (Exception e) { - + throw new RuntimeException("Interrupted while waiting for stream", e); - } + } } @Override @@ -766,29 +844,27 @@ public QueryBatch next() { public Iterator queryStream(String database, String sql, Map params, int batchSize, StreamQueryRequest.RetrievalMode mode, TransactionContext tx, long timeoutMs) { - var reqB = StreamQueryRequest.newBuilder().setDatabase(database).setQuery(sql) - .putAllParameters(convertParamsToGrpcValue(params)).setCredentials(buildCredentials()).setBatchSize(batchSize > 0 ? batchSize : 100) - .setRetrievalMode(mode); + var reqB = StreamQueryRequest.newBuilder().setDatabase(database).setQuery(sql).putAllParameters(convertParamsToGrpcValue(params)) + .setCredentials(buildCredentials()).setBatchSize(batchSize > 0 ? batchSize : 100).setRetrievalMode(mode); if (tx != null) reqB.setTransaction(tx); - final BlockingClientCall it = blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS) - .streamQuery(reqB.build()); + final BlockingClientCall it = blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).streamQuery(reqB.build()); return new Iterator<>() { - + private Iterator curr = Collections.emptyIterator(); public boolean hasNext() { - + if (curr.hasNext()) return true; try { - + if (it.hasNext()) { - + curr = it.read().getRecordsList().iterator(); return hasNext(); } @@ -814,9 +890,8 @@ public GrpcRecord next() { public Iterator queryStreamBatches(String passLabel, String sql, Map params, int batchSize, StreamQueryRequest.RetrievalMode mode, TransactionContext tx, long timeoutMs) { - var reqB = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(sql) - .putAllParameters(convertParamsToGrpcValue(params)).setCredentials(buildCredentials()).setBatchSize(batchSize > 0 ? batchSize : 100) - .setRetrievalMode(mode); + var reqB = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(sql).putAllParameters(convertParamsToGrpcValue(params)) + .setCredentials(buildCredentials()).setBatchSize(batchSize > 0 ? batchSize : 100).setRetrievalMode(mode); if (tx != null) reqB.setTransaction(tx); @@ -825,9 +900,9 @@ public Iterator queryStreamBatches(String passLabel, String sql, Map .streamQuery(reqB.build()); return new Iterator<>() { - + public boolean hasNext() { - + try { return respIter.hasNext(); } @@ -835,18 +910,18 @@ public boolean hasNext() { throw new RuntimeException(e); } catch (StatusException e) { - throw new RuntimeException(e); + throw new RuntimeException(e); } } public QueryBatch next() { QueryResult qr; - + try { qr = respIter.read(); List converted = new ArrayList<>(qr.getRecordsCount()); - + for (GrpcRecord gr : qr.getRecordsList()) { converted.add(grpcRecordToDBRecord(gr)); } @@ -860,20 +935,20 @@ public QueryBatch next() { throw new RuntimeException(e); } catch (StatusException e) { - throw new RuntimeException(e); - } + throw new RuntimeException(e); + } } }; } public String createVertex(String cls, Map props, long timeoutMs) { // Build the nested proto GrpcRecord payload - GrpcRecord recMsg = GrpcRecord.newBuilder().putAllProperties(convertParamsToGrpcValue(props)) - .build(); + GrpcRecord recMsg = GrpcRecord.newBuilder().putAllProperties(convertParamsToGrpcValue(props)).build(); // Build the Create request - CreateRecordRequest req = CreateRecordRequest.newBuilder().setDatabase(getName()) - .setType(cls).setRecord(recMsg) // <<<< NESTED RECORD (not top-level properties) + CreateRecordRequest req = CreateRecordRequest.newBuilder().setDatabase(getName()).setType(cls).setRecord(recMsg) // <<<< NESTED RECORD + // (not top-level + // properties) .setCredentials(buildCredentials()).build(); // Call RPC @@ -883,15 +958,14 @@ public String createVertex(String cls, Map props, long timeoutMs res = blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).createRecord(req); // Response carries new RID string return res.getRid(); // e.g., "#12:0" - } catch (StatusException e) { + } + catch (StatusException e) { throw new RuntimeException("Failed to create vertex", e); } } public boolean updateRecord(String rid, Map props, long timeoutMs) { - PropertiesUpdate partial = PropertiesUpdate.newBuilder() - .putAllProperties(convertParamsToGrpcValue(props)) - .build(); + PropertiesUpdate partial = PropertiesUpdate.newBuilder().putAllProperties(convertParamsToGrpcValue(props)).build(); return updateRecord(rid, partial, timeoutMs); } @@ -903,10 +977,8 @@ public boolean updateRecord(String rid, Record dbRecord, long timeoutMs) { private boolean updateRecord(String rid, PropertiesUpdate partial, long timeoutMs) { // Build the Update request using setPartial for partial update - UpdateRecordRequest req = UpdateRecordRequest.newBuilder().setDatabase(getName()) - .setRid(rid).setPartial(partial) - .setTransaction(TransactionContext.newBuilder().setBegin(true).setCommit(true)) - .setCredentials(buildCredentials()).build(); + UpdateRecordRequest req = UpdateRecordRequest.newBuilder().setDatabase(getName()).setRid(rid).setPartial(partial) + .setTransaction(TransactionContext.newBuilder().setBegin(true).setCommit(true)).setCredentials(buildCredentials()).build(); // Call RPC UpdateRecordResponse res; @@ -915,17 +987,16 @@ private boolean updateRecord(String rid, PropertiesUpdate partial, long timeoutM res = blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).updateRecord(req); // Choose the flag your proto defines. Most builds expose getSuccess(). return res.getSuccess(); - } catch (StatusException e) { + } + catch (StatusException e) { throw new RuntimeException("Failed to update record", e); } } private boolean updateRecordFull(String rid, GrpcRecord record, long timeoutMs) { // Build the Update request using setRecord for full replacement - UpdateRecordRequest req = UpdateRecordRequest.newBuilder().setDatabase(getName()) - .setRid(rid).setRecord(record) - .setTransaction(TransactionContext.newBuilder().setBegin(true).setCommit(true)) - .setCredentials(buildCredentials()).build(); + UpdateRecordRequest req = UpdateRecordRequest.newBuilder().setDatabase(getName()).setRid(rid).setRecord(record) + .setTransaction(TransactionContext.newBuilder().setBegin(true).setCommit(true)).setCredentials(buildCredentials()).build(); // Call RPC UpdateRecordResponse res; @@ -933,20 +1004,21 @@ private boolean updateRecordFull(String rid, GrpcRecord record, long timeoutMs) try { res = blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).updateRecord(req); return res.getSuccess(); - } catch (StatusException e) { + } + catch (StatusException e) { throw new RuntimeException("Failed to update record", e); } } - + public boolean deleteRecord(String rid, long timeoutMs) { - var req = DeleteRecordRequest.newBuilder().setDatabase(getName()).setRid(rid).setCredentials(buildCredentials()) - .build(); + var req = DeleteRecordRequest.newBuilder().setDatabase(getName()).setRid(rid).setCredentials(buildCredentials()).build(); DeleteRecordResponse res; try { res = blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).deleteRecord(req); return res.getDeleted(); - } catch (StatusException e) { + } + catch (StatusException e) { throw new RuntimeException("Failed to delete record", e); } } @@ -989,7 +1061,7 @@ public long countType(final String typeName, final boolean polymorphic) { public Record lookupByRID(final RID rid) { return lookupByRID(rid, true); } - + @Override public Record lookupByRID(final RID rid, final boolean loadContent) { checkDatabaseIsOpen(); @@ -1008,17 +1080,17 @@ public Record lookupByRID(final RID rid, final boolean loadContent) { throw new RecordNotFoundException("Record " + rid + " not found", rid); } return grpcRecordToDBRecord(response.getRecord()); - } + } catch (StatusRuntimeException e) { handleGrpcException(e); return null; - } + } catch (StatusException e) { handleGrpcException(e); return null; } } - + @Override public boolean existsRecord(RID rid) { stats.existsRecord.incrementAndGet(); @@ -1026,13 +1098,13 @@ public boolean existsRecord(RID rid) { throw new IllegalArgumentException("Record is null"); try { return lookupByRID(rid, false) != null; - } catch (RecordNotFoundException e) { + } + catch (RecordNotFoundException e) { return false; } } - public InsertSummary insertBulk(final InsertOptions options, - final List protoRows, final long timeoutMs) { + public InsertSummary insertBulk(final InsertOptions options, final List protoRows, final long timeoutMs) { // Ensure options carry DB + credentials as the server expects InsertOptions.Builder ob = options.toBuilder(); @@ -1048,19 +1120,20 @@ public InsertSummary insertBulk(final InsertOptions options, InsertOptions newOptions = ob.build(); - BulkInsertRequest req = BulkInsertRequest.newBuilder().setOptions(newOptions) - .addAllRows(protoRows).build(); + BulkInsertRequest req = BulkInsertRequest.newBuilder().setOptions(newOptions).addAllRows(protoRows).build(); try { + if (logger.isDebugEnabled()) + logger.debug("CLIENT insertBulk: rows={}, timeoutMs={}", req.getRowsCount(), timeoutMs); return blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).bulkInsert(req); - } catch (StatusException e) { + } + catch (StatusException e) { throw new RuntimeException("insertBulk() -> failed: " + e.getStatus()); } } // Convenience overload that accepts domain rows (convert first) - public InsertSummary insertBulkAsListOfMaps(final InsertOptions options, - final List> rows, final long timeoutMs) { + public InsertSummary insertBulkAsListOfMaps(final InsertOptions options, final List> rows, final long timeoutMs) { List protoRows = rows.stream().map(this::toProtoRecordFromMap) // your converter .collect(java.util.stream.Collectors.toList()); @@ -1068,8 +1141,7 @@ public InsertSummary insertBulkAsListOfMaps(final InsertOptions options, return insertBulk(options, protoRows, timeoutMs); } - public InsertSummary ingestStream(final InsertOptions options, - final List protoRows, final int chunkSize, final long timeoutMs) + public InsertSummary ingestStream(final InsertOptions options, final List protoRows, final int chunkSize, final long timeoutMs) throws InterruptedException { final java.util.concurrent.CountDownLatch done = new java.util.concurrent.CountDownLatch(1); @@ -1118,8 +1190,8 @@ public void onCompleted() { for (int i = 0; i < protoRows.size(); i += chunkSize) { final int end = Math.min(i + chunkSize, protoRows.size()); - final InsertChunk chunk = InsertChunk.newBuilder().setSessionId(sessionId) - .setOptions(ob.build()).setChunkSeq(seq++).addAllRows(protoRows.subList(i, end)).build(); + final InsertChunk chunk = InsertChunk.newBuilder().setSessionId(sessionId).setOptions(ob.build()).setChunkSeq(seq++) + .addAllRows(protoRows.subList(i, end)).build(); req.onNext(chunk); } @@ -1133,11 +1205,10 @@ public void onCompleted() { } // Convenience overload - public InsertSummary ingestStreamAsListOfMaps(final InsertOptions options, - final List> rows, final int chunkSize, final long timeoutMs) throws InterruptedException { + public InsertSummary ingestStreamAsListOfMaps(final InsertOptions options, final List> rows, final int chunkSize, + final long timeoutMs) throws InterruptedException { - List protoRows = rows.stream().map(this::toProtoRecordFromMap) - .collect(java.util.stream.Collectors.toList()); + List protoRows = rows.stream().map(this::toProtoRecordFromMap).collect(java.util.stream.Collectors.toList()); return ingestStream(options, protoRows, chunkSize, timeoutMs); } @@ -1145,9 +1216,8 @@ public InsertSummary ingestStreamAsListOfMaps(final InsertOptions options, /** * Core implementation of InsertBidirectional ingest */ - private InsertSummary ingestBidiCore(final List rows, - final InsertOptions options, final int chunkSize, final int maxInflight, final long timeoutMs, - final java.util.function.Function mapper) throws InterruptedException { + private InsertSummary ingestBidiCore(final List rows, final InsertOptions options, final int chunkSize, final int maxInflight, + final long timeoutMs, final java.util.function.Function mapper) throws InterruptedException { // 1) Ensure options carry DB + credentials the server expects final InsertOptions.Builder ob = options.toBuilder(); @@ -1165,10 +1235,12 @@ private InsertSummary ingestBidiCore(final List rows, final InsertOptions effectiveOpts = ob.build(); // 2) Pre-map rows → proto once (outside the observer) - final List protoRows = rows.stream().map(mapper) - .collect(java.util.stream.Collectors.toList()); + final List protoRows = rows.stream().map(mapper).collect(java.util.stream.Collectors.toList()); - // 3) Streaming state + if (logger.isDebugEnabled()) + logger.debug("CLIENT ingestBidi start: rows={}, chunkSize={}, maxInflight={}, timeoutMs={}", protoRows.size(), chunkSize, + maxInflight, timeoutMs); +// 3) Streaming state final String sessionId = "sess-" + System.nanoTime(); final java.util.concurrent.CountDownLatch done = new java.util.concurrent.CountDownLatch(1); final java.util.concurrent.atomic.AtomicLong seq = new java.util.concurrent.atomic.AtomicLong(1); @@ -1179,8 +1251,7 @@ private InsertSummary ingestBidiCore(final List rows, final List acks = java.util.Collections.synchronizedList(new ArrayList<>()); // small holder to access req inside helpers - final java.util.concurrent.atomic.AtomicReference> observerRef = - new java.util.concurrent.atomic.AtomicReference<>(); + final java.util.concurrent.atomic.AtomicReference> observerRef = new java.util.concurrent.atomic.AtomicReference<>(); final java.util.concurrent.atomic.AtomicBoolean commitSent = new java.util.concurrent.atomic.AtomicBoolean(false); final java.util.concurrent.ScheduledExecutorService scheduler = java.util.concurrent.Executors.newSingleThreadScheduledExecutor(r -> { @@ -1197,11 +1268,11 @@ private InsertSummary ingestBidiCore(final List rows, if (commitSent.compareAndSet(false, true)) { try { if (observerRef.get() != null) { - observerRef.get().onNext(InsertRequest.newBuilder() - .setCommit(Commit.newBuilder().setSessionId(sessionId)).build()); + observerRef.get().onNext(InsertRequest.newBuilder().setCommit(Commit.newBuilder().setSessionId(sessionId)).build()); observerRef.get().onCompleted(); } - } catch (Throwable ignore) { + } + catch (Throwable ignore) { /* best effort */ } } }; @@ -1247,11 +1318,10 @@ private void drain() { return; if (!started) { - req.onNext(InsertRequest.newBuilder() - .setStart(Start.newBuilder().setOptions(effectiveOpts)).build()); - + req.onNext(InsertRequest.newBuilder().setStart(Start.newBuilder().setOptions(effectiveOpts)).build()); + started = true; - + // Now that the call is started, it's safe to pull the first response req.request(1); } @@ -1264,8 +1334,8 @@ private void drain() { final int end = Math.min(start + chunkSize, protoRows.size()); final var slice = protoRows.subList(start, end); - final var chunk = InsertChunk.newBuilder().setSessionId(sessionId) - .setChunkSeq(seq.getAndIncrement()).addAllRows(slice).build(); + final var chunk = InsertChunk.newBuilder().setSessionId(sessionId).setChunkSeq(seq.getAndIncrement()).addAllRows(slice) + .build(); req.onNext(InsertRequest.newBuilder().setChunk(chunk).build()); cursor.set(end); @@ -1277,8 +1347,10 @@ private void drain() { if (acked.get() >= sent.get()) { // all acked → commit immediately sendCommitIfNeeded.run(); - } else { - // not all acked → (re)arm grace timer; if last ACK never arrives, timer will COMMIT + } + else { + // not all acked → (re)arm grace timer; if last ACK never arrives, timer will + // COMMIT armAckGraceTimer.run(); } } @@ -1298,7 +1370,8 @@ public void onNext(InsertResponse v) { if (acked.get() >= sent.get()) { cancelAckGraceTimer.run(); sendCommitIfNeeded.run(); - } else { + } + else { armAckGraceTimer.run(); } } @@ -1308,8 +1381,13 @@ public void onNext(InsertResponse v) { cancelAckGraceTimer.run(); } case MSG_NOT_SET -> { - /* ignore */ } + /* ignore */ } + case ERROR -> { + /* TBD */ + } + } + req.request(1); // continue pulling responses } @@ -1328,9 +1406,19 @@ public void onCompleted() { // 4) Kick off the bidi call with deadline try { - asyncStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).insertBidirectional(observer); + if (logger.isDebugEnabled()) + logger.debug("CLIENT ingestBidi opening stream: timeoutMs={}", timeoutMs); + asyncStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).insertBidirectional(observer); + @SuppressWarnings("unused") boolean finished = done.await(timeoutMs + 5_000, TimeUnit.MILLISECONDS); - } finally { + if (logger.isDebugEnabled()) + try { + logger.debug("CLIENT ingestBidi finished: finished={}, sent={}, acked={}", finished, sent.get(), acked.get()); + } + catch (Throwable ignore) { + } + } + finally { scheduler.shutdownNow(); } @@ -1344,67 +1432,59 @@ public void onCompleted() { final long ign = acks.stream().mapToLong(BatchAck::getIgnored).sum(); final long fail = acks.stream().mapToLong(BatchAck::getFailed).sum(); - return InsertSummary.newBuilder().setReceived(protoRows.size()).setInserted(ins).setUpdated(upd).setIgnored(ign) - .setFailed(fail).build(); + return InsertSummary.newBuilder().setReceived(protoRows.size()).setInserted(ins).setUpdated(upd).setIgnored(ign).setFailed(fail).build(); } /** - * Pushes domain {@code com.arcadedb.database.Record} rows via InsertBidirectional with per-batch ACKs. + * Pushes domain {@code com.arcadedb.database.Record} rows via + * InsertBidirectional with per-batch ACKs. */ - public InsertSummary ingestBidi(final List rows, - final InsertOptions opts, final int chunkSize, final int maxInflight, final long timeoutMs) - throws InterruptedException { + public InsertSummary ingestBidi(final List rows, final InsertOptions opts, final int chunkSize, + final int maxInflight, final long timeoutMs) throws InterruptedException { return ingestBidiCore(rows, opts, chunkSize, maxInflight, timeoutMs, (Object o) -> toProtoRecordFromDbRecord((com.arcadedb.database.Record) o)); } - public InsertSummary ingestBidi(final List rows, - final InsertOptions opts, final int chunkSize, final int maxInflight) throws InterruptedException { + public InsertSummary ingestBidi(final List rows, final InsertOptions opts, final int chunkSize, + final int maxInflight) throws InterruptedException { - return ingestBidiCore(rows, opts, chunkSize, maxInflight, /* timeoutMs */ 5 * 60_000L, - (Object o) -> toProtoRecordFromDbRecord((com.arcadedb.database.Record) o)); + return ingestBidiCore(rows, opts, chunkSize, maxInflight, /* timeoutMs */ 5 * 60_000L, this::toProtoRecordFromDbRecord); } /** - * Pushes map-shaped rows (property map per row) via InsertBidirectional with per-batch ACKs. + * Pushes map-shaped rows (property map per row) via InsertBidirectional with + * per-batch ACKs. */ - public InsertSummary ingestBidi(final InsertOptions options, - final List> rows, final int chunkSize, final int maxInflight) throws InterruptedException { + public InsertSummary ingestBidi(final InsertOptions options, final List> rows, final int chunkSize, + final int maxInflight) throws InterruptedException { - return ingestBidiCore(rows, options, chunkSize, maxInflight, /* timeoutMs */ 5 * 60_000L, - (Object o) -> toProtoRecordFromMap((Map) o)); + return ingestBidiCore(rows, options, chunkSize, maxInflight, /* timeoutMs */ 5 * 60_000L, this::toProtoRecordFromMap); } - public InsertSummary ingestBidi(final InsertOptions options, - final List> rows, final int chunkSize, final int maxInflight, final long timeoutMs) - throws InterruptedException { + public InsertSummary ingestBidi(final InsertOptions options, final List> rows, final int chunkSize, + final int maxInflight, final long timeoutMs) throws InterruptedException { - return ingestBidiCore(rows, options, chunkSize, maxInflight, timeoutMs, - (Object o) -> toProtoRecordFromMap((Map) o)); + return ingestBidiCore(rows, options, chunkSize, maxInflight, timeoutMs, this::toProtoRecordFromMap); } // Map -> GrpcRecord private GrpcRecord toProtoRecordFromMap(Map row) { GrpcRecord.Builder b = GrpcRecord.newBuilder(); row.forEach((k, v) -> b.putProperties(k, objectToGrpcValue(v))); - return b.build(); + GrpcRecord rec = b.build(); + if (logger.isDebugEnabled()) + logger.debug("CLIENT toProtoRecordFromMap: {}", summarize(rec)); + return rec; } // Domain Record (storage) -> GrpcRecord private GrpcRecord toProtoRecordFromDbRecord(com.arcadedb.database.Record rec) { // Use ProtoUtils for proper conversion - return ProtoUtils.toProtoRecord(rec); - } - - // Query Result -> GrpcRecord - private GrpcRecord toProtoRecordFromResult(Result res) { - GrpcRecord.Builder b = GrpcRecord.newBuilder(); - for (String name : res.getPropertyNames()) { - Object v = res.getProperty(name); - b.putProperties(name, objectToGrpcValue(v)); - } - return b.build(); + GrpcRecord out = ProtoUtils.toProtoRecord(rec); + if (logger.isDebugEnabled()) + logger.debug("CLIENT toProtoRecordFromDbRecord: {}", summarize(out)); + return out; } // Convert Java object -> GrpcValue (extend as needed) @@ -1412,37 +1492,6 @@ private GrpcValue objectToGrpcValue(Object v) { return ProtoUtils.toGrpcValue(v); } - // google.protobuf.Value -> Java (mirror of your toProtoValue) - private static Object fromProtoValue(Value v) { - if (v == null) - return null; - switch (v.getKindCase()) { - case NULL_VALUE: - return null; - case BOOL_VALUE: - return v.getBoolValue(); - case NUMBER_VALUE: - return v.getNumberValue(); - case STRING_VALUE: - return v.getStringValue(); - case LIST_VALUE: { - List out = new ArrayList<>(); - for (Value item : v.getListValue().getValuesList()) { - out.add(fromProtoValue(item)); - } - return out; - } - case STRUCT_VALUE: { - Map m = new java.util.LinkedHashMap<>(); - v.getStructValue().getFieldsMap().forEach((k, vv) -> m.put(k, fromProtoValue(vv))); - return m; - } - case KIND_NOT_SET: - default: - return null; - } - } - public DatabaseCredentials buildCredentials() { return DatabaseCredentials.newBuilder().setUsername(getUserName()).setPassword(getUserPassword()).build(); } @@ -1488,7 +1537,7 @@ private Iterator streamQuery(final String query) { @Override public boolean hasNext() { - + if (currentBatch.hasNext()) { return true; } @@ -1497,9 +1546,9 @@ public boolean hasNext() { if (responseIterator.hasNext()) { QueryResult result; - result = responseIterator.read(); - currentBatch = result.getRecordsList().iterator(); - return currentBatch.hasNext(); + result = responseIterator.read(); + currentBatch = result.getRecordsList().iterator(); + return currentBatch.hasNext(); } } catch (InterruptedException e) { @@ -1533,17 +1582,21 @@ private ResultSet createGrpcResultSet(ExecuteQueryResponse response) { } private Result grpcRecordToResult(GrpcRecord grpcRecord) { + Record record = grpcRecordToDBRecord(grpcRecord); + if (record == null) { + Map properties = new HashMap<>(); grpcRecord.getPropertiesMap().forEach((k, v) -> properties.put(k, grpcValueToObject(v))); return new ResultInternal(properties); } + return new ResultInternal(record); } private Record grpcRecordToDBRecord(GrpcRecord grpcRecord) { - + Map map = new HashMap<>(); // Convert properties @@ -1555,12 +1608,12 @@ private Record grpcRecordToDBRecord(GrpcRecord grpcRecord) { map.put("@cat", mapRecordType(grpcRecord)); String cat = (String) map.get("@cat"); - + if (cat == null) { - + return null; } - + switch (cat) { case "d": return new RemoteImmutableDocument(this, map); @@ -1574,20 +1627,20 @@ private Record grpcRecordToDBRecord(GrpcRecord grpcRecord) { } private String mapRecordType(GrpcRecord grpcRecord) { - + // Determine record category from type name String typeName = grpcRecord.getType(); // Check schema to determine actual type try { - + if (getSchema().existsType(typeName)) { - + Object type = getSchema().getType(typeName); - - if (type instanceof com.arcadedb.schema.VertexType) { + + if (type instanceof com.arcadedb.schema.VertexType) { return "v"; - } + } else if (type instanceof com.arcadedb.schema.EdgeType) { return "e"; } @@ -1595,43 +1648,37 @@ else if (type instanceof com.arcadedb.schema.DocumentType) { return "d"; } else { - return null; - } + return null; + } } else { - + return null; } - } + } catch (Exception e) { // Fall back to name-based detection } return null; - -// // Fall back to name-based detection -// if (typeName.contains("Vertex") || typeName.startsWith("V_")) { -// return "v"; -// } else if (typeName.contains("Edge") || typeName.startsWith("E_")) { -// return "e"; -// } else { -// return "d"; -// } } private Map convertParamsToGrpcValue(Map params) { Map grpcParams = new HashMap<>(); - + for (Map.Entry entry : params.entrySet()) { GrpcValue value = objectToGrpcValue(entry.getValue()); grpcParams.put(entry.getKey(), value); } - + return grpcParams; } private Object grpcValueToObject(GrpcValue grpcValue) { - return ProtoUtils.fromGrpcValue(grpcValue); + Object out = ProtoUtils.fromGrpcValue(grpcValue); + if (logger.isDebugEnabled()) + logger.debug("CLIENT decode grpcValueToObject: {} -> {}", summarize(grpcValue), summarize(out)); + return out; } private void handleGrpcException(StatusRuntimeException e) { diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/utils/ProtoUtils.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/utils/ProtoUtils.java index f49641efc0..45648761d4 100644 --- a/grpc-client/src/main/java/com/arcadedb/remote/grpc/utils/ProtoUtils.java +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/utils/ProtoUtils.java @@ -1,353 +1,310 @@ package com.arcadedb.remote.grpc.utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import java.util.Map; -import java.util.HashMap; -import java.util.List; -import java.util.ArrayList; import java.util.Collection; +import java.util.Map; import com.arcadedb.database.Document; -import com.arcadedb.database.Record; import com.arcadedb.database.RID; -import com.arcadedb.graph.Vertex; +import com.arcadedb.database.Record; import com.arcadedb.graph.Edge; +import com.arcadedb.graph.Vertex; import com.arcadedb.server.grpc.GrpcRecord; import com.arcadedb.server.grpc.GrpcValue; -import com.arcadedb.server.grpc.GrpcValueType; -import com.google.protobuf.Value; -import com.google.protobuf.Struct; -import com.google.protobuf.ListValue; -import com.google.protobuf.NullValue; public class ProtoUtils { - /** - * Converts an ArcadeDB Record into a gRPC GrpcRecord message. - * - * @param rec ArcadeDB record (Document, Vertex, or Edge) - * @return Proto GrpcRecord message - */ - public static GrpcRecord toProtoRecord(Record rec) { - if (rec == null) { - throw new IllegalArgumentException("Record cannot be null"); - } - - GrpcRecord.Builder builder = GrpcRecord.newBuilder(); - - // Set RID if available - if (rec.getIdentity() != null) { - builder.setRid(rec.getIdentity().toString()); - } - - // Determine logical type/class name - String typeName = null; - - if (rec instanceof Vertex) { - Vertex vertex = rec.asVertex(); - if (vertex != null && vertex.getType() != null) { - typeName = vertex.getType().getName(); - } - } else if (rec instanceof Edge) { - Edge edge = rec.asEdge(); - if (edge != null && edge.getType() != null) { - typeName = edge.getType().getName(); - } - } else if (rec instanceof Document) { - Document doc = rec.asDocument(); - if (doc != null && doc.getType() != null) { - typeName = doc.getType().getName(); - } - } - - if (typeName != null && !typeName.isEmpty()) { - builder.setType(typeName); - } + private static final Logger logger = LoggerFactory.getLogger(ProtoUtils.class); - // Convert properties - if (rec instanceof Document) { - Document doc = (Document) rec; - for (String propName : doc.getPropertyNames()) { - Object value = doc.get(propName); - builder.putProperties(propName, toGrpcValue(value)); - } - } + // Logging config via system properties (optional) + private static final int LOG_STR_PREVIEW = Integer.getInteger("arcadedb.grpc.log.preview", 96); + private static final boolean LOG_SHOW_STR_AT_DEBUG = Boolean.getBoolean("arcadedb.grpc.log.showString"); - return builder.build(); + // Optional per-thread context for logs + private static final ThreadLocal LOG_CTX = new ThreadLocal<>(); + public static AutoCloseable pushLogContext(String ctx) { + final String prev = LOG_CTX.get(); + LOG_CTX.set(ctx); + return () -> LOG_CTX.set(prev); } - - /** - * Converts a Map into a gRPC GrpcRecord message. - * - * @param map Map with properties - * @param rid Optional RID - * @param type Optional type name - * @return Proto GrpcRecord message - */ - public static GrpcRecord mapToProtoRecord(Map map, String rid, String type) { - GrpcRecord.Builder builder = GrpcRecord.newBuilder(); - - if (rid != null && !rid.isEmpty()) { - builder.setRid(rid); - } - - if (type != null && !type.isEmpty()) { - builder.setType(type); - } - - for (Map.Entry entry : map.entrySet()) { - builder.putProperties(entry.getKey(), toGrpcValue(entry.getValue())); - } - - return builder.build(); + private static String ctx() { + String c = LOG_CTX.get(); + return (c == null || c.isEmpty()) ? "" : " " + c; } - /** - * Converts a Java Object into a GrpcValue for GrpcRecord.properties map. - * - * Supports primitives, strings, numbers, booleans, lists, maps, dates, and RIDs. - * - * @param value Object value - * @return GrpcValue - */ - public static GrpcValue toGrpcValue(Object value) { - GrpcValue.Builder builder = GrpcValue.newBuilder(); - Value.Builder valueBuilder = Value.newBuilder(); - - if (value == null) { - valueBuilder.setNullValue(NullValue.NULL_VALUE); - builder.setValueType(GrpcValueType.RECORD_VALUE_TYPE_UNSPECIFIED); - } else if (value instanceof Boolean) { - valueBuilder.setBoolValue((Boolean) value); - builder.setValueType(GrpcValueType.BOOLEAN); - } else if (value instanceof Byte) { - valueBuilder.setNumberValue(((Byte) value).doubleValue()); - builder.setValueType(GrpcValueType.BYTE); - } else if (value instanceof Short) { - valueBuilder.setNumberValue(((Short) value).doubleValue()); - builder.setValueType(GrpcValueType.SHORT); - } else if (value instanceof Integer) { - valueBuilder.setNumberValue(((Integer) value).doubleValue()); - builder.setValueType(GrpcValueType.INTEGER); - } else if (value instanceof Long) { - valueBuilder.setNumberValue(((Long) value).doubleValue()); - builder.setValueType(GrpcValueType.LONG); - } else if (value instanceof Float) { - valueBuilder.setNumberValue(((Float) value).doubleValue()); - builder.setValueType(GrpcValueType.FLOAT); - } else if (value instanceof Double) { - valueBuilder.setNumberValue((Double) value); - builder.setValueType(GrpcValueType.DOUBLE); - } else if (value instanceof String) { - valueBuilder.setStringValue((String) value); - builder.setValueType(GrpcValueType.STRING); - } else if (value instanceof RID) { - valueBuilder.setStringValue(value.toString()); - builder.setValueType(GrpcValueType.LINK); - } else if (value instanceof java.util.Date) { - // Serialize date as ISO8601 string - valueBuilder.setStringValue(((java.util.Date) value).toInstant().toString()); - builder.setValueType(GrpcValueType.DATETIME); - } else if (value instanceof java.time.LocalDate) { - valueBuilder.setStringValue(value.toString()); - builder.setValueType(GrpcValueType.DATE); - } else if (value instanceof java.time.LocalDateTime) { - valueBuilder.setStringValue(value.toString()); - builder.setValueType(GrpcValueType.DATETIME); - } else if (value instanceof java.time.Instant) { - valueBuilder.setStringValue(value.toString()); - builder.setValueType(GrpcValueType.DATETIME); - } else if (value instanceof byte[]) { - // For binary data, encode as base64 string - valueBuilder.setStringValue(java.util.Base64.getEncoder().encodeToString((byte[]) value)); - builder.setValueType(GrpcValueType.BINARY); - } else if (value instanceof Map) { - Struct.Builder structBuilder = Struct.newBuilder(); - Map map = (Map) value; - - // Determine if this is an embedded document or a regular map - boolean isEmbedded = map.containsKey("@type") || map.containsKey("@class"); - - for (Map.Entry entry : map.entrySet()) { - GrpcValue nestedValue = toGrpcValue(entry.getValue()); - structBuilder.putFields(String.valueOf(entry.getKey()), nestedValue.getValue()); - } - - valueBuilder.setStructValue(structBuilder.build()); - builder.setValueType(isEmbedded ? GrpcValueType.EMBEDDED : GrpcValueType.MAP); - - // Set ofType if this is an embedded document with a known type - if (isEmbedded && map.containsKey("@type")) { - builder.setOfType(String.valueOf(map.get("@type"))); - } - } else if (value instanceof Collection) { - ListValue.Builder listBuilder = ListValue.newBuilder(); - Collection collection = (Collection) value; - - String elementType = null; - for (Object item : collection) { - GrpcValue nestedValue = toGrpcValue(item); - listBuilder.addValues(nestedValue.getValue()); - - // Try to infer element type from first non-null element - if (elementType == null && item != null) { - elementType = inferTypeString(item); + private static String summarizeJava(Object o) { + if (o == null) return "null"; + try { + if (o instanceof CharSequence s) { + if (logger.isTraceEnabled() || LOG_SHOW_STR_AT_DEBUG) { + String txt = s.toString(); + return "String(" + s.length() + ")=\"" + (txt.length() > LOG_STR_PREVIEW ? txt.substring(0, LOG_STR_PREVIEW) + "…" : txt) + "\""; } + return "String(" + s.length() + ")"; } - - valueBuilder.setListValue(listBuilder.build()); - builder.setValueType(GrpcValueType.LIST); - - if (elementType != null) { - builder.setOfType(elementType); - } - } else if (value instanceof Document) { - // Handle embedded documents - Document doc = (Document) value; - Map docMap = new HashMap<>(); - - for (String propName : doc.getPropertyNames()) { - docMap.put(propName, doc.get(propName)); - } - - if (doc.getType() != null) { - docMap.put("@type", doc.getType().getName()); - } - - // Recursive call to handle as map - return toGrpcValue(docMap); - } else { - // Fallback to string representation - valueBuilder.setStringValue(value.toString()); - builder.setValueType(GrpcValueType.STRING); - } - - builder.setValue(valueBuilder.build()); - return builder.build(); + if (o instanceof byte[] b) return "bytes[" + b.length + "]"; + if (o instanceof java.util.Collection c) return o.getClass().getSimpleName() + "[size=" + c.size() + "]"; + if (o instanceof java.util.Map m) return o.getClass().getSimpleName() + "[size=" + m.size() + "]"; + return o.getClass().getSimpleName() + "(" + String.valueOf(o) + ")"; + } catch (Exception e) { return o.getClass().getSimpleName(); } } - /** - * Converts a GrpcValue back to a Java Object. - * - * @param grpcValue GrpcValue to convert - * @return Java Object - */ - public static Object fromGrpcValue(GrpcValue grpcValue) { - if (grpcValue == null) { - return null; - } - - Value value = grpcValue.getValue(); - GrpcValueType type = grpcValue.getValueType(); - - switch (value.getKindCase()) { - case NULL_VALUE: - return null; - - case BOOL_VALUE: - return value.getBoolValue(); - - case NUMBER_VALUE: - double numValue = value.getNumberValue(); - // Use the type hint to determine the specific numeric type - switch (type) { - case BYTE: - return (byte) numValue; - case SHORT: - return (short) numValue; - case INTEGER: - return (int) numValue; - case LONG: - return (long) numValue; - case FLOAT: - return (float) numValue; - case DOUBLE: - case DECIMAL: - default: - return numValue; - } - - case STRING_VALUE: - String strValue = value.getStringValue(); - // Check if this is a special string type - switch (type) { - case LINK: - // Parse as RID if it looks like one - if (strValue.startsWith("#")) { - try { - return new RID(strValue); - } catch (Exception e) { - return strValue; - } + private static String summarizeGrpc(GrpcValue v) { + if (v == null) return "GrpcValue(null)"; + switch (v.getKindCase()) { + case BOOL_VALUE: return "BOOL(" + v.getBoolValue() + ")"; + case INT32_VALUE: return "INT32(" + v.getInt32Value() + ")"; + case INT64_VALUE: return "INT64(" + v.getInt64Value() + ")"; + case FLOAT_VALUE: return "FLOAT(" + v.getFloatValue() + ")"; + case DOUBLE_VALUE: return "DOUBLE(" + v.getDoubleValue() + ")"; + case STRING_VALUE: { + String s = v.getStringValue(); + if (logger.isTraceEnabled() || LOG_SHOW_STR_AT_DEBUG) { + String txt = s; + return "STRING(" + s.length() + ")=\"" + (txt.length() > LOG_STR_PREVIEW ? txt.substring(0, LOG_STR_PREVIEW) + "…" : txt) + "\""; } - return strValue; - case BINARY: - // Decode base64 string back to byte array - try { - return java.util.Base64.getDecoder().decode(strValue); - } catch (Exception e) { - return strValue; - } - case DATE: - case DATETIME: - // Could parse as date/time if needed - // For now, return as string and let the caller handle parsing - return strValue; - default: - return strValue; + return "STRING(" + s.length() + ")"; } - - case STRUCT_VALUE: - Map map = new HashMap<>(); - value.getStructValue().getFieldsMap().forEach((k, v) -> { - // Create a GrpcValue wrapper for recursive conversion - // We don't have type info for nested values, so use UNSPECIFIED - GrpcValue nestedValue = GrpcValue.newBuilder() - .setValue(v) - .setValueType(GrpcValueType.RECORD_VALUE_TYPE_UNSPECIFIED) - .build(); - map.put(k, fromGrpcValue(nestedValue)); - }); - return map; - - case LIST_VALUE: - List list = new ArrayList<>(); - value.getListValue().getValuesList().forEach(v -> { - // Create a GrpcValue wrapper for recursive conversion - GrpcValue nestedValue = GrpcValue.newBuilder() - .setValue(v) - .setValueType(GrpcValueType.RECORD_VALUE_TYPE_UNSPECIFIED) - .build(); - list.add(fromGrpcValue(nestedValue)); - }); - return list; - - default: - return null; + case BYTES_VALUE: return "BYTES[" + v.getBytesValue().size() + "]"; + case TIMESTAMP_VALUE: return "TIMESTAMP(" + v.getTimestampValue().getSeconds() + "." + v.getTimestampValue().getNanos() + ")"; + case LIST_VALUE: return "LIST[size=" + v.getListValue().getValuesCount() + "]"; + case MAP_VALUE: return "MAP[size=" + v.getMapValue().getEntriesCount() + "]"; + case EMBEDDED_VALUE: return "EMBEDDED[type=" + v.getEmbeddedValue().getType() + ", size=" + v.getEmbeddedValue().getFieldsCount() + "]"; + case LINK_VALUE: return "LINK(" + v.getLinkValue().getRid() + ")"; + case DECIMAL_VALUE: return "DECIMAL(unscaled=" + v.getDecimalValue().getUnscaled() + ", scale=" + v.getDecimalValue().getScale() + ")"; + case KIND_NOT_SET: default: return "GrpcValue(KIND_NOT_SET)"; } } - /** - * Helper method to infer type string from an object. - * Used for setting the ofType field in collections. - */ - private static String inferTypeString(Object obj) { - if (obj == null) return null; - if (obj instanceof String) return "String"; - if (obj instanceof Integer) return "Integer"; - if (obj instanceof Long) return "Long"; - if (obj instanceof Double) return "Double"; - if (obj instanceof Float) return "Float"; - if (obj instanceof Boolean) return "Boolean"; - if (obj instanceof java.util.Date) return "Date"; - if (obj instanceof RID) return "Link"; - if (obj instanceof Map) { - Map map = (Map) obj; - if (map.containsKey("@type")) { - return String.valueOf(map.get("@type")); - } - return "Map"; - } - if (obj instanceof Collection) return "List"; - return obj.getClass().getSimpleName(); + private static GrpcValue dbgEnc(String where, Object in, GrpcValue out) { + if (logger.isDebugEnabled()) logger.debug("CLIENT GRPC-ENC [{}]{} in={} -> out={}", where, ctx(), summarizeJava(in), summarizeGrpc(out)); + return out; + } + + private static Object dbgDec(String where, GrpcValue in, Object out) { + if (logger.isDebugEnabled()) logger.debug("CLIENT GRPC-DEC [{}]{} in={} -> out={}", where, ctx(), summarizeGrpc(in), summarizeJava(out)); + return out; } + + + /** + * Converts an ArcadeDB Record into a gRPC GrpcRecord message. + * + * @param rec ArcadeDB record (Document, Vertex, or Edge) + * @return Proto GrpcRecord message + */ + public static GrpcRecord toProtoRecord(Record rec) { + + if (rec == null) { + throw new IllegalArgumentException("Record cannot be null"); + } + + GrpcRecord.Builder builder = GrpcRecord.newBuilder(); + + // Set RID if available + if (rec.getIdentity() != null) { + builder.setRid(rec.getIdentity().toString()); + } + + // Determine logical type/class name + String typeName = null; + + if (rec instanceof Vertex) { + Vertex vertex = rec.asVertex(); + if (vertex != null && vertex.getType() != null) { + typeName = vertex.getType().getName(); + } + } + else if (rec instanceof Edge) { + Edge edge = rec.asEdge(); + if (edge != null && edge.getType() != null) { + typeName = edge.getType().getName(); + } + } + else if (rec instanceof Document) { + Document doc = rec.asDocument(); + if (doc != null && doc.getType() != null) { + typeName = doc.getType().getName(); + } + } + + if (typeName != null && !typeName.isEmpty()) { + builder.setType(typeName); + } + + // Convert properties + if (rec instanceof Document) { + Document doc = (Document) rec; + for (String propName : doc.getPropertyNames()) { + Object value = doc.get(propName); + + System.out.print("toProtoRecord: " + propName + ": " + value); + + builder.putProperties(propName, toGrpcValue(value)); + } + } + + return builder.build(); + } + + /** + * Converts a Map into a gRPC GrpcRecord message. + * + * @param map Map with properties + * @param rid Optional RID + * @param type Optional type name + * @return Proto GrpcRecord message + */ + public static GrpcRecord mapToProtoRecord(Map map, String rid, String type) { + GrpcRecord.Builder builder = GrpcRecord.newBuilder(); + + if (rid != null && !rid.isEmpty()) { + builder.setRid(rid); + } + + if (type != null && !type.isEmpty()) { + builder.setType(type); + } + + for (Map.Entry entry : map.entrySet()) { + builder.putProperties(entry.getKey(), toGrpcValue(entry.getValue())); + } + + return builder.build(); + } + + /** + * Converts a Java Object into a GrpcValue for GrpcRecord.properties map. + * + * Supports primitives, strings, numbers, booleans, lists, maps, dates, and + * RIDs. + * + * @param value Object value + * @return GrpcValue + */ + +public static GrpcValue toGrpcValue(Object value) { + GrpcValue.Builder b = GrpcValue.newBuilder(); + if (value == null) return dbgEnc("toGrpcValue", value, b.build()); + + if (value instanceof Boolean v) return dbgEnc("toGrpcValue", value, b.setBoolValue(v).build()); + if (value instanceof Integer v) return dbgEnc("toGrpcValue", value, b.setInt32Value(v).build()); + if (value instanceof Long v) return dbgEnc("toGrpcValue", value, b.setInt64Value(v).build()); + if (value instanceof Float v) return dbgEnc("toGrpcValue", value, b.setFloatValue(v).build()); + if (value instanceof Double v) return dbgEnc("toGrpcValue", value, b.setDoubleValue(v).build()); + if (value instanceof CharSequence v)return dbgEnc("toGrpcValue", value, b.setStringValue(v.toString()).build()); + if (value instanceof byte[] v) return dbgEnc("toGrpcValue", value, b.setBytesValue(com.google.protobuf.ByteString.copyFrom(v)).build()); + + if (value instanceof java.util.Date v) { + return dbgEnc("toGrpcValue", value, GrpcValue.newBuilder() + .setTimestampValue(com.google.protobuf.Timestamp.newBuilder() + .setSeconds(Math.floorDiv(v.getTime(), 1000L)) + .setNanos((int)Math.floorMod(v.getTime(), 1000L)*1_000_000) + .build()) + .setLogicalType("datetime") + .build()); + } + + if (value instanceof RID rid) { + return dbgEnc("toGrpcValue", value, GrpcValue.newBuilder() + .setLinkValue(com.arcadedb.server.grpc.GrpcLink.newBuilder().setRid(rid.toString()).build()) + .setLogicalType("rid") + .build()); + } + + if (value instanceof java.math.BigDecimal v) { + java.math.BigInteger unscaled = v.unscaledValue(); + if (unscaled.bitLength() <= 63) { + return dbgEnc("toGrpcValue", value, GrpcValue.newBuilder() + .setDecimalValue(com.arcadedb.server.grpc.GrpcDecimal.newBuilder() + .setUnscaled(unscaled.longValue()) + .setScale(v.scale()) + .build()) + .setLogicalType("decimal") + .build()); + } else { + return dbgEnc("toGrpcValue", value, GrpcValue.newBuilder().setStringValue(v.toPlainString()).setLogicalType("decimal").build()); + } + } + + if (value instanceof Document edoc && (edoc.getIdentity() == null)) { + com.arcadedb.server.grpc.GrpcEmbedded.Builder eb = com.arcadedb.server.grpc.GrpcEmbedded.newBuilder(); + if (edoc.getType() != null) eb.setType(edoc.getType().getName()); + for (String k : edoc.getPropertyNames()) { + eb.putFields(k, toGrpcValue(edoc.get(k))); + } + return dbgEnc("toGrpcValue", value, GrpcValue.newBuilder().setEmbeddedValue(eb.build()).build()); + } + + if (value instanceof Map m) { + com.arcadedb.server.grpc.GrpcMap.Builder mb = com.arcadedb.server.grpc.GrpcMap.newBuilder(); + for (var e : m.entrySet()) { + mb.putEntries(String.valueOf(e.getKey()), toGrpcValue(e.getValue())); + } + return dbgEnc("toGrpcValue", value, GrpcValue.newBuilder().setMapValue(mb.build()).build()); + } + + if (value instanceof Collection c) { + com.arcadedb.server.grpc.GrpcList.Builder lb = com.arcadedb.server.grpc.GrpcList.newBuilder(); + for (Object e : c) lb.addValues(toGrpcValue(e)); + return dbgEnc("toGrpcValue", value, GrpcValue.newBuilder().setListValue(lb.build()).build()); + } + + return dbgEnc("toGrpcValue", value, GrpcValue.newBuilder().setStringValue(String.valueOf(value)).build()); +} + + + public static Object fromGrpcValue(GrpcValue v) { + + switch (v.getKindCase()) { + case BOOL_VALUE: + return dbgDec("fromGrpcValue", v, v.getBoolValue()); + case INT32_VALUE: + return dbgDec("fromGrpcValue", v, v.getInt32Value()); + case INT64_VALUE: + return dbgDec("fromGrpcValue", v, v.getInt64Value()); + case FLOAT_VALUE: + return dbgDec("fromGrpcValue", v, v.getFloatValue()); + case DOUBLE_VALUE: + return dbgDec("fromGrpcValue", v, v.getDoubleValue()); + case STRING_VALUE: + return dbgDec("fromGrpcValue", v, v.getStringValue()); + case BYTES_VALUE: + return dbgDec("fromGrpcValue", v, v.getBytesValue().toByteArray()); + case TIMESTAMP_VALUE: + return dbgDec("fromGrpcValue", v, tsToMillis(v.getTimestampValue())); // or Instant + case LIST_VALUE: { + var out = new java.util.ArrayList<>(); + for (GrpcValue e : v.getListValue().getValuesList()) + out.add(fromGrpcValue(e)); + return dbgDec("fromGrpcValue", v, out); + } + case MAP_VALUE: { + var out = new java.util.LinkedHashMap(); + v.getMapValue().getEntriesMap().forEach((k, vv) -> out.put(k, fromGrpcValue(vv))); + return dbgDec("fromGrpcValue", v, out); + } + case EMBEDDED_VALUE: { + var out = new java.util.LinkedHashMap(); + v.getEmbeddedValue().getFieldsMap().forEach((k, vv) -> out.put(k, fromGrpcValue(vv))); + // v.getEmbeddedValue().getType() is available if you want to instantiate a + // typed DTO + return dbgDec("fromGrpcValue", v, out); + } + case LINK_VALUE: + return dbgDec("fromGrpcValue", v, v.getLinkValue().getRid()); // or a Link object + case DECIMAL_VALUE: { + var d = v.getDecimalValue(); + return dbgDec("fromGrpcValue", v, new java.math.BigDecimal(java.math.BigInteger.valueOf(d.getUnscaled()), d.getScale())); + } + case KIND_NOT_SET: + return dbgDec("fromGrpcValue", v, null); + } + return dbgDec("fromGrpcValue", v, null); + } + + static long tsToMillis(com.google.protobuf.Timestamp ts) { + return ts.getSeconds() * 1000L + ts.getNanos() / 1_000_000L; + } } \ No newline at end of file diff --git a/grpc-client/src/test/resources/arcadedb-log.properties b/grpc-client/src/test/resources/arcadedb-log.properties index 536f217457..29b52b4421 100644 --- a/grpc-client/src/test/resources/arcadedb-log.properties +++ b/grpc-client/src/test/resources/arcadedb-log.properties @@ -25,6 +25,7 @@ handlers=java.util.logging.ConsoleHandler # Set the default logging level for the root logger .level=WARNING com.arcadedb.level=WARNING +com.arcadedb.remote.grpc.level=FINE # Set the default logging level for new ConsoleHandler instances java.util.logging.ConsoleHandler.level=WARNING # Set the default formatter for new ConsoleHandler instances diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java index f8111b88d0..0c810135ec 100644 --- a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java @@ -42,7 +42,6 @@ public class ArcadeDbGrpcService extends ArcadeDbServiceGrpc.ArcadeDbServiceImpl private static final Logger logger = LoggerFactory.getLogger(ArcadeDbGrpcService.class); - // Transaction management private final Map activeTransactions = new ConcurrentHashMap<>(); From 3769678517febb22431c676860f7597844c5e9ea Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Fri, 5 Sep 2025 09:54:41 -0700 Subject: [PATCH 26/35] Adding more advanced JSON processing --- .../RemoteGrpcDatabase_with_debug.java | 1726 ----------------- grpc-client/docs/Readme.txt | 56 + .../server/grpc/ArcadeDbGrpcService.java | 674 ++++--- .../grpc/GrpcCompressionInterceptor.java | 156 +- 4 files changed, 543 insertions(+), 2069 deletions(-) delete mode 100644 grpc-client/RemoteGrpcDatabase_with_debug.java diff --git a/grpc-client/RemoteGrpcDatabase_with_debug.java b/grpc-client/RemoteGrpcDatabase_with_debug.java deleted file mode 100644 index ae3b62592d..0000000000 --- a/grpc-client/RemoteGrpcDatabase_with_debug.java +++ /dev/null @@ -1,1726 +0,0 @@ -package com.arcadedb.remote.grpc; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.concurrent.Executor; -import java.util.concurrent.TimeUnit; - -import com.arcadedb.ContextConfiguration; -import com.arcadedb.database.Database; -import com.arcadedb.database.MutableDocument; -import com.arcadedb.database.RID; -import com.arcadedb.database.Record; -import com.arcadedb.exception.ConcurrentModificationException; -import com.arcadedb.exception.DatabaseOperationException; -import com.arcadedb.exception.DuplicatedKeyException; -import com.arcadedb.exception.NeedRetryException; -import com.arcadedb.exception.RecordNotFoundException; -import com.arcadedb.exception.TimeoutException; -import com.arcadedb.exception.TransactionException; -import com.arcadedb.query.sql.executor.InternalResultSet; -import com.arcadedb.query.sql.executor.Result; -import com.arcadedb.query.sql.executor.ResultInternal; -import com.arcadedb.query.sql.executor.ResultSet; -import com.arcadedb.remote.RemoteDatabase; -import com.arcadedb.remote.RemoteException; -import com.arcadedb.remote.RemoteImmutableDocument; -import com.arcadedb.remote.RemoteImmutableEdge; -import com.arcadedb.remote.RemoteImmutableVertex; -import com.arcadedb.remote.RemoteSchema; -import com.arcadedb.remote.RemoteTransactionExplicitLock; -import com.arcadedb.remote.grpc.utils.ProtoUtils; -import com.arcadedb.server.grpc.ArcadeDbServiceGrpc; -import com.arcadedb.server.grpc.BatchAck; -import com.arcadedb.server.grpc.BeginTransactionRequest; -import com.arcadedb.server.grpc.BeginTransactionResponse; -import com.arcadedb.server.grpc.BulkInsertRequest; -import com.arcadedb.server.grpc.Commit; -import com.arcadedb.server.grpc.CommitTransactionRequest; -import com.arcadedb.server.grpc.CommitTransactionResponse; -import com.arcadedb.server.grpc.CreateRecordRequest; -import com.arcadedb.server.grpc.CreateRecordResponse; -import com.arcadedb.server.grpc.DatabaseCredentials; -import com.arcadedb.server.grpc.DeleteRecordRequest; -import com.arcadedb.server.grpc.DeleteRecordResponse; -import com.arcadedb.server.grpc.ExecuteCommandRequest; -import com.arcadedb.server.grpc.ExecuteCommandResponse; -import com.arcadedb.server.grpc.ExecuteQueryRequest; -import com.arcadedb.server.grpc.ExecuteQueryResponse; -import com.arcadedb.server.grpc.GrpcRecord; -import com.arcadedb.server.grpc.GrpcValue; -import com.arcadedb.server.grpc.InsertChunk; -import com.arcadedb.server.grpc.InsertOptions; -import com.arcadedb.server.grpc.InsertRequest; -import com.arcadedb.server.grpc.InsertResponse; -import com.arcadedb.server.grpc.InsertSummary; -import com.arcadedb.server.grpc.LookupByRidRequest; -import com.arcadedb.server.grpc.LookupByRidResponse; -import com.arcadedb.server.grpc.PropertiesUpdate; -import com.arcadedb.server.grpc.QueryResult; -import com.arcadedb.server.grpc.RollbackTransactionRequest; -import com.arcadedb.server.grpc.RollbackTransactionResponse; -import com.arcadedb.server.grpc.Start; -import com.arcadedb.server.grpc.StreamQueryRequest; -import com.arcadedb.server.grpc.TransactionContext; -import com.arcadedb.server.grpc.TransactionIsolation; -import com.arcadedb.server.grpc.UpdateRecordRequest; -import com.arcadedb.server.grpc.UpdateRecordResponse; - -import io.grpc.CallCredentials; -import io.grpc.CompressorRegistry; -import io.grpc.DecompressorRegistry; -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; -import io.grpc.Metadata; -import io.grpc.Status; -import io.grpc.StatusException; -import io.grpc.StatusRuntimeException; -import io.grpc.stub.BlockingClientCall; -import io.grpc.stub.ClientCallStreamObserver; -import io.grpc.stub.ClientResponseObserver; -import io.grpc.stub.StreamObserver; - -/** - * Remote Database implementation using gRPC protocol instead of HTTP. Extends - * RemoteDatabase to maintain compatibility while overriding network operations. - * It's not thread safe. For multi-thread usage create one instance of - * RemoteGrpcDatabase per thread. - * - * @author Oleg Cohen (oleg.cohen@gmail.com) - */ -public class RemoteGrpcDatabase extends RemoteDatabase { - - -private static final Logger logger = LoggerFactory.getLogger(RemoteGrpcDatabase.class); - -// --- Debug helpers (client) --- -private static String summarize(Object o) { - if (o == null) return "null"; - try { - if (o instanceof CharSequence s) return "String(" + s.length() + ")"; - if (o instanceof byte[] b) return "bytes[" + b.length + "]"; - if (o instanceof java.util.Collection c) return o.getClass().getSimpleName() + "[size=" + c.size() + "]"; - if (o instanceof java.util.Map m) return o.getClass().getSimpleName() + "[size=" + m.size() + "]"; - return o.getClass().getSimpleName(); - } catch (Exception e) { return o.getClass().getSimpleName(); } -} -private static String summarize(GrpcValue v) { - if (v == null) return "GrpcValue(null)"; - return switch (v.getKindCase()) { - case BOOL_VALUE -> "BOOL"; - case INT32_VALUE -> "INT32"; - case INT64_VALUE -> "INT64"; - case FLOAT_VALUE -> "FLOAT"; - case DOUBLE_VALUE -> "DOUBLE"; - case STRING_VALUE -> "STRING(" + v.getStringValue().length() + ")"; - case BYTES_VALUE -> "BYTES[" + v.getBytesValue().size() + "]"; - case TIMESTAMP_VALUE -> "TIMESTAMP"; - case LIST_VALUE -> "LIST[" + v.getListValue().getValuesCount() + "]"; - case MAP_VALUE -> "MAP[" + v.getMapValue().getEntriesCount() + "]"; - case EMBEDDED_VALUE -> "EMBEDDED"; - case LINK_VALUE -> "LINK(" + v.getLinkValue().getRid() + ")"; - case DECIMAL_VALUE -> "DECIMAL(scale=" + v.getDecimalValue().getScale() + ")"; - case KIND_NOT_SET -> "KIND_NOT_SET"; - }; -} -private static String summarize(GrpcRecord r) { - if (r == null) return "GrpcRecord(null)"; - String rid = r.getRid(); - String ty = r.getType(); - int props = r.getPropertiesCount(); - return "GrpcRecord{rid=" + rid + ", type=" + ty + ", props=" + props + "}"; -} - - - private final ManagedChannel channel; - - private final ArcadeDbServiceGrpc.ArcadeDbServiceBlockingV2Stub blockingStub; - private final ArcadeDbServiceGrpc.ArcadeDbServiceStub asyncStub; - - private String transactionId; - - private final RemoteSchema schema; - - private final String userName; - private final String userPassword; - private String databaseName; - private RemoteTransactionExplicitLock explicitLock; - - public RemoteGrpcDatabase(final String server, final int grpcPort, final int httpPort, final String databaseName, final String userName, - final String userPassword) { - this(server, grpcPort, httpPort, databaseName, userName, userPassword, new ContextConfiguration()); - } - - public RemoteGrpcDatabase(final String server, final int grpcPort, final int httpPort, final String databaseName, final String userName, - final String userPassword, final ContextConfiguration configuration) { - - super(server, httpPort, databaseName, userName, userPassword, configuration); - - this.userName = userName; - this.userPassword = userPassword; - - this.databaseName = databaseName; - - // Create gRPC channel - this.channel = createChannel(server, grpcPort); - - this.blockingStub = createBlockingStub(channel); - - this.asyncStub = createAsyncStub(channel); - - // Create gRPC-specific schema - this.schema = new RemoteSchema(this); - } - - /** - * Creates the gRPC channel. Override this for custom channel configuration. - */ - protected ManagedChannel createChannel(String server, int port) { - return ManagedChannelBuilder.forAddress(server, port).usePlaintext() // No TLS by default - .build(); - } - - /** - * Creates call credentials for authentication - */ - protected CallCredentials createCallCredentials(String userName, String userPassword) { - return new CallCredentials() { - @Override - public void applyRequestMetadata(RequestInfo requestInfo, Executor appExecutor, MetadataApplier applier) { - Metadata headers = new Metadata(); - headers.put(Metadata.Key.of("username", Metadata.ASCII_STRING_MARSHALLER), userName); - headers.put(Metadata.Key.of("password", Metadata.ASCII_STRING_MARSHALLER), userPassword); - headers.put(Metadata.Key.of("x-arcade-user", Metadata.ASCII_STRING_MARSHALLER), userName); - headers.put(Metadata.Key.of("x-arcade-password", Metadata.ASCII_STRING_MARSHALLER), userPassword); - headers.put(Metadata.Key.of("x-arcade-database", Metadata.ASCII_STRING_MARSHALLER), databaseName); - applier.apply(headers); - } - - @Override - public void thisUsesUnstableApi() { - // Required by the interface - } - }; - } - - protected CallCredentials createCredentials() { - return createCallCredentials(userName, userPassword); - } - - /** - * Override this method to customize blocking stub creation - */ - protected ArcadeDbServiceGrpc.ArcadeDbServiceBlockingV2Stub createBlockingStub(ManagedChannel channel) { - return ArcadeDbServiceGrpc.newBlockingV2Stub(channel).withCallCredentials(createCredentials()); - } - - protected ArcadeDbServiceGrpc.ArcadeDbServiceStub createAsyncStub(ManagedChannel channel) { - return ArcadeDbServiceGrpc.newStub(channel).withCallCredentials(createCredentials()).withCompression("gzip"); - } - - @Override - public String getDatabasePath() { - return "grpc://" + channel.authority() + "/" + getName(); - } - - @Override - public RemoteSchema getSchema() { - return schema; - } - - @Override - public void close() { - if (transactionId != null) { - rollback(); - } - super.close(); - - // Shutdown gRPC channel - channel.shutdown(); - try { - if (!channel.awaitTermination(5, TimeUnit.SECONDS)) { - channel.shutdownNow(); - } - } catch (InterruptedException e) { - channel.shutdownNow(); - } - } - - @Override - public void begin(final Database.TRANSACTION_ISOLATION_LEVEL isolationLevel) { - checkDatabaseIsOpen(); - if (transactionId != null) - throw new TransactionException("Transaction already begun"); - - BeginTransactionRequest request = BeginTransactionRequest.newBuilder().setDatabase(getName()).setCredentials(buildCredentials()) - .setIsolation(mapIsolationLevel(isolationLevel)).build(); - - try { - BeginTransactionResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).beginTransaction(request); - transactionId = response.getTransactionId(); - // Store transaction ID in parent class session management - setSessionId(transactionId); - } catch (StatusRuntimeException e) { - throw new TransactionException("Error on transaction begin", e); - } catch (StatusException e) { - throw new TransactionException("Error on transaction begin", e); - } - } - - @Override - public void commit() { - checkDatabaseIsOpen(); - stats.txCommits.incrementAndGet(); - - if (transactionId == null) - throw new TransactionException("Transaction not begun"); - - CommitTransactionRequest request = CommitTransactionRequest.newBuilder() - .setTransaction(TransactionContext.newBuilder().setTransactionId(transactionId).setDatabase(getName()).build()) - .setCredentials(buildCredentials()).build(); - - try { - CommitTransactionResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).commitTransaction(request); - if (!response.getSuccess()) { - throw new TransactionException("Failed to commit transaction: " + response.getMessage()); - } - } catch (StatusRuntimeException e) { - handleGrpcException(e); - } catch (StatusException e) { - handleGrpcException(e); - } finally { - transactionId = null; - setSessionId(null); - } - } - - @Override - public void rollback() { - checkDatabaseIsOpen(); - stats.txRollbacks.incrementAndGet(); - - if (transactionId == null) - throw new TransactionException("Transaction not begun"); - - RollbackTransactionRequest request = RollbackTransactionRequest.newBuilder() - .setTransaction(TransactionContext.newBuilder().setTransactionId(transactionId).setDatabase(getName()).build()) - .setCredentials(buildCredentials()).build(); - - try { - RollbackTransactionResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) - .rollbackTransaction(request); - if (!response.getSuccess()) { - throw new TransactionException("Failed to rollback transaction: " + response.getMessage()); - } - } catch (StatusRuntimeException e) { - throw new TransactionException("Error on transaction rollback", e); - } catch (StatusException e) { - throw new TransactionException("Error on transaction rollback", e); - } finally { - transactionId = null; - setSessionId(null); - } - } - - @Override - public void deleteRecord(final Record record) { - checkDatabaseIsOpen(); - stats.deleteRecord.incrementAndGet(); - - if (record.getIdentity() == null) - throw new IllegalArgumentException("Cannot delete a non persistent record"); - - DeleteRecordRequest request = DeleteRecordRequest.newBuilder().setDatabase(getName()).setRid(record.getIdentity().toString()) - .setCredentials(buildCredentials()).build(); - - try { - DeleteRecordResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).deleteRecord(request); - if (!response.getSuccess()) { - throw new DatabaseOperationException("Failed to delete record: " + response.getMessage()); - } - } catch (StatusRuntimeException e) { - handleGrpcException(e); - } catch (StatusException e) { - handleGrpcException(e); - } - } - - @Override - public Iterator iterateType(final String typeName, final boolean polymorphic) { - String query = "select from `" + typeName + "`"; - if (!polymorphic) - query += " where @type = '" + typeName + "'"; - return streamQuery(query); - } - - @Override - public Iterator iterateBucket(final String bucketName) { - return streamQuery("select from bucket:`" + bucketName + "`"); - } - - @Override - public ResultSet query(final String language, final String query, final Map params) { - checkDatabaseIsOpen(); - stats.queries.incrementAndGet(); - - ExecuteQueryRequest.Builder requestBuilder = ExecuteQueryRequest.newBuilder().setDatabase(getName()).setQuery(query) - .setCredentials(buildCredentials()); - - if (transactionId != null) { - requestBuilder.setTransaction(TransactionContext.newBuilder().setTransactionId(transactionId).setDatabase(getName()).build()); - } - - if (params != null && !params.isEmpty()) { - requestBuilder.putAllParameters(convertParamsToGrpcValue(params)); - } - - try { - - if (logger.isDebugEnabled()) logger.debug("CLIENT executeQuery: db={}, tx={}, queryLen={}, params={}", getName(), (transactionId!=null), requestBuilder.getQuery().length(), requestBuilder.getParametersCount()); - - - ExecuteQueryResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) - .executeQuery(requestBuilder.build()); - - if (logger.isDebugEnabled()) { int _r=0; for (var qr : response.getResultsList()) _r+=qr.getRecordsCount(); logger.debug("CLIENT executeQuery: results={}", _r);} -return createGrpcResultSet(response); - } - catch (StatusRuntimeException e) { - handleGrpcException(e); - return new InternalResultSet(); - } - catch (StatusException e) { - handleGrpcException(e); - return new InternalResultSet(); - } - } - - @Override - public ResultSet query(final String language, final String query, final Object... args) { - checkDatabaseIsOpen(); - stats.queries.incrementAndGet(); - - final Map params = mapArgs(args); - - return query(language, query, params); - } - - @Override - public ResultSet command(final String language, final String command, final Map params) { - checkDatabaseIsOpen(); - stats.commands.incrementAndGet(); - - ExecuteCommandRequest.Builder requestBuilder = ExecuteCommandRequest.newBuilder().setDatabase(getName()).setCommand(command) - .setCredentials(buildCredentials()); - - if (transactionId != null) { - requestBuilder.setTransaction(TransactionContext.newBuilder().setTransactionId(transactionId).setDatabase(getName()).build()); - } - - if (params != null && !params.isEmpty()) { - requestBuilder.putAllParameters(convertParamsToGrpcValue(params)); - } - - try { - - if (logger.isDebugEnabled()) logger.debug("CLIENT executeCommand: db={}, tx={}, cmdLen={}, params={}", getName(), (transactionId!=null), requestBuilder.getCommand().length(), requestBuilder.getParametersCount()); - - - ExecuteCommandResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) - .executeCommand(requestBuilder.build()); - - if (logger.isDebugEnabled()) logger.debug("CLIENT executeCommand: status={}", response.getStatus()); -if (! response.getSuccess()) { - throw new DatabaseOperationException("Failed to execute command: " + response.getMessage()); - } - - // Create result set with command execution info - InternalResultSet resultSet = new InternalResultSet(); - - if (response.getAffectedRecords() > 0) { - Map result = new HashMap<>(); - result.put("affected", response.getAffectedRecords()); - result.put("executionTime", response.getExecutionTimeMs()); - resultSet.add(new ResultInternal(result)); - } - return resultSet; - } catch (StatusRuntimeException e) { - handleGrpcException(e); - return new InternalResultSet(); - } catch (StatusException e) { - handleGrpcException(e); - return new InternalResultSet(); - } - } - - public ExecuteCommandResponse execSql(String db, String sql, Map params, long timeoutMs) { - return executeCommand(db, "sql", sql, params, /* returnRows */ false, /* maxRows */ 0, txBeginCommit(), timeoutMs); - } - - public ExecuteCommandResponse execSql(String sql, Map params, long timeoutMs) { - return executeCommand(databaseName, "sql", sql, params, /* returnRows */ false, /* maxRows */ 0, txBeginCommit(), timeoutMs); - } - - public ExecuteCommandResponse executeCommand(String language, String command, Map params, - boolean returnRows, int maxRows, TransactionContext tx, long timeoutMs) { - - var reqB = ExecuteCommandRequest.newBuilder().setDatabase(databaseName).setCommand(command) - .putAllParameters(convertParamsToGrpcValue(params)).setLanguage(langOrDefault(language)).setReturnRows(returnRows) - .setMaxRows(maxRows > 0 ? maxRows : 0); - - if (tx != null) - reqB.setTransaction(tx); - // credentials: if your stub builds creds implicitly, set here if required - reqB.setCredentials(buildCredentials()); - - try { - return blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).executeCommand(reqB.build()); - } catch (StatusException e) { - throw new RuntimeException("Failed to execute command: " + e.getMessage(), e); - } - } - - public ExecuteCommandResponse executeCommand(String database, String language, String command, - Map params, boolean returnRows, int maxRows, TransactionContext tx, long timeoutMs) { - - var reqB = ExecuteCommandRequest.newBuilder().setDatabase(database).setCommand(command) - .putAllParameters(convertParamsToGrpcValue(params)).setLanguage(langOrDefault(language)).setReturnRows(returnRows) - .setMaxRows(maxRows > 0 ? maxRows : 0); - - if (tx != null) - reqB.setTransaction(tx); - // credentials: if your stub builds creds implicitly, set here if required - reqB.setCredentials(buildCredentials()); - - try { - return blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).executeCommand(reqB.build()); - } catch (StatusException e) { - throw new RuntimeException("Failed to execute command: " + e.getMessage(), e); - } - } - - @Override - protected RID saveRecord(final MutableDocument record) { - stats.createRecord.incrementAndGet(); - - final RID rid = record.getIdentity(); - - if (rid != null) { - // -------- UPDATE (partial) -------- - PropertiesUpdate partial = PropertiesUpdate.newBuilder() - .putAllProperties(convertParamsToGrpcValue(record.toMap(false))) - .build(); - - UpdateRecordRequest request = UpdateRecordRequest.newBuilder() - .setDatabase(getName()) - .setRid(rid.toString()) - .setPartial(partial) - .setDatabase(databaseName) - .setCredentials(buildCredentials()).build(); - - try { - - @SuppressWarnings("unused") - UpdateRecordResponse response = blockingStub - .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) - .updateRecord(request); - - // If your proto has flags, you can check response.getSuccess()/getUpdated() - // Otherwise, treat non-exception as success. - return rid; - } catch (StatusRuntimeException e) { - handleGrpcException(e); - return null; - } catch (StatusException e) { - handleGrpcException(e); - return null; - } - } else { - // -------- CREATE -------- - GrpcRecord recMsg = GrpcRecord.newBuilder() - .putAllProperties(convertParamsToGrpcValue(record.toMap(false))) - .build(); - - CreateRecordRequest request = CreateRecordRequest.newBuilder() - .setDatabase(getName()) - .setType(record.getTypeName()) - .setRecord(recMsg) // nested GrpcRecord payload - .setCredentials(buildCredentials()).build(); - - try { - CreateRecordResponse response = blockingStub - .withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) - .createRecord(request); - - // Proto returns the newly created RID as a string - final String ridStr = response.getRid(); - if (ridStr == null || ridStr.isEmpty()) { - throw new DatabaseOperationException("Failed to create record (empty RID)"); - } - - // Construct a RID from the returned string - try { - return new RID(ridStr); - } catch (NoSuchMethodError | IllegalArgumentException ex) { - // Fallback for older APIs expecting (Database, String) - return new RID(this, ridStr); - } - } catch (StatusRuntimeException e) { - handleGrpcException(e); - return null; - } catch (StatusException e) { - handleGrpcException(e); - return null; - } - } - } - - // Convenience: default batch size stays 100, default mode = CURSOR - public Iterator queryStream(final String language, final String query) { - return queryStream(language, query, /* batchSize */100, StreamQueryRequest.RetrievalMode.CURSOR); - } - - public Iterator queryStream(final String language, final String query, final int batchSize) { - return queryStream(language, query, batchSize, StreamQueryRequest.RetrievalMode.CURSOR); - } - - // NEW: choose retrieval mode - public Iterator queryStream(final String language, final String query, final int batchSize, - final StreamQueryRequest.RetrievalMode mode) { - checkDatabaseIsOpen(); - stats.queries.incrementAndGet(); - - StreamQueryRequest request = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(query).setCredentials(buildCredentials()) - .setBatchSize(batchSize > 0 ? batchSize : 100).setRetrievalMode(mode) // <--- NEW - .build(); - - final BlockingClientCall responseIterator = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) - .streamQuery(request); - - return new Iterator() { - private Iterator currentBatch = Collections.emptyIterator(); - - @Override - public boolean hasNext() { - if (currentBatch.hasNext()) - return true; - - try { - while (responseIterator.hasNext()) { - QueryResult result = responseIterator.read(); - - // Defensive: skip any empty batches the server might send - if (result.getRecordsCount() == 0) { - if (result.getIsLastBatch()) - return false; - continue; - } - - currentBatch = result.getRecordsList().iterator(); - return true; - } - } - catch (InterruptedException e) { - throw new RuntimeException("Interrupted while waiting for stream", e); - } - catch (StatusException e) { - handleGrpcException(e); - } - - return false; - } - - @Override - public Record next() { - if (!hasNext()) - throw new NoSuchElementException(); - return grpcRecordToDBRecord(currentBatch.next()); - } - }; - } - - // PARAMETERIZED variant with retrieval mode - public Iterator queryStream(final String language, final String query, final Map params, final int batchSize, - final StreamQueryRequest.RetrievalMode mode) { - checkDatabaseIsOpen(); - stats.queries.incrementAndGet(); - - StreamQueryRequest.Builder b = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(query).setCredentials(buildCredentials()) - .setBatchSize(batchSize > 0 ? batchSize : 100).setRetrievalMode(mode); // <--- NEW - - if (params != null && !params.isEmpty()) { - b.putAllParameters(convertParamsToGrpcValue(params)); - } - - final BlockingClientCall responseIterator = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) - .streamQuery(b.build()); - - return new Iterator() { - private Iterator currentBatch = Collections.emptyIterator(); - - @Override - public boolean hasNext() { - if (currentBatch.hasNext()) - return true; - - try { - - while (responseIterator.hasNext()) { - - QueryResult result = responseIterator.read(); - - if (result.getRecordsCount() == 0) { - if (result.getIsLastBatch()) - return false; - continue; - } - - currentBatch = result.getRecordsList().iterator(); - return true; - } - - return false; - } - catch (Exception e) { - - throw new RuntimeException("Interrupted while waiting for stream", e); - } - } - - @Override - public Record next() { - if (!hasNext()) - throw new NoSuchElementException(); - return grpcRecordToDBRecord(currentBatch.next()); - } - }; - } - - // Keep the old signature working (defaults to CURSOR) - public Iterator queryStream(final String language, final String query, final Map params, final int batchSize) { - return queryStream(language, query, params, batchSize, StreamQueryRequest.RetrievalMode.CURSOR); - } - - public static final class QueryBatch { - private final List records; - private final int totalInBatch; - private final long runningTotal; - private final boolean lastBatch; - - public QueryBatch(List list, int totalInBatch, long runningTotal, boolean lastBatch) { - this.records = list; - this.totalInBatch = totalInBatch; - this.runningTotal = runningTotal; - this.lastBatch = lastBatch; - } - - public List records() { - return records; - } - - public int totalInBatch() { - return totalInBatch; - } - - public long runningTotal() { - return runningTotal; - } - - public boolean isLastBatch() { - return lastBatch; - } - } - - public Iterator queryStreamBatches(final String language, final String query, final Map params, - final int batchSize, final StreamQueryRequest.RetrievalMode mode) { - checkDatabaseIsOpen(); - stats.queries.incrementAndGet(); - - StreamQueryRequest.Builder b = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(query).setCredentials(buildCredentials()) - .setBatchSize(batchSize > 0 ? batchSize : 100).setRetrievalMode(mode); - - if (params != null && !params.isEmpty()) { - b.putAllParameters(convertParamsToGrpcValue(params)); - } - - final BlockingClientCall responseIterator = blockingStub.withWaitForReady() // optional, improves robustness - .withDeadlineAfter(/* e.g. */ 10, TimeUnit.MINUTES).streamQuery(b.build()); - - return new Iterator() { - private QueryBatch nextBatch = null; - private boolean drained = false; - - @Override - public boolean hasNext() { - if (nextBatch != null) - return true; - if (drained) - return false; - - try { - - while (responseIterator.hasNext()) { - - QueryResult qr = responseIterator.read(); - - int n = qr.getTotalRecordsInBatch(); // server-populated - // Guard: some servers could omit this; fallback to list size. - if (n == 0) - n = qr.getRecordsCount(); - - if (qr.getRecordsCount() == 0 && !qr.getIsLastBatch()) { - // skip empty non-terminal batch - continue; - } - - List converted = new ArrayList<>(qr.getRecordsCount()); - for (GrpcRecord gr : qr.getRecordsList()) { - converted.add(grpcRecordToDBRecord(gr)); - } - - nextBatch = new QueryBatch(converted, n, qr.getRunningTotalEmitted(), qr.getIsLastBatch()); - - if (qr.getIsLastBatch()) { - drained = true; // no more after this (even if server sent empty terminal) - } - return true; - } - - drained = true; - return false; - } - catch (Exception e) { - - throw new RuntimeException("Interrupted while waiting for stream", e); - } - } - - @Override - public QueryBatch next() { - if (!hasNext()) - throw new NoSuchElementException(); - QueryBatch out = nextBatch; - nextBatch = null; - return out; - } - }; - } - - public Iterator queryStream(String database, String sql, Map params, int batchSize, - StreamQueryRequest.RetrievalMode mode, TransactionContext tx, long timeoutMs) { - - var reqB = StreamQueryRequest.newBuilder().setDatabase(database).setQuery(sql) - .putAllParameters(convertParamsToGrpcValue(params)).setCredentials(buildCredentials()).setBatchSize(batchSize > 0 ? batchSize : 100) - .setRetrievalMode(mode); - - if (tx != null) - reqB.setTransaction(tx); - - final BlockingClientCall it = blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS) - .streamQuery(reqB.build()); - - return new Iterator<>() { - - private Iterator curr = Collections.emptyIterator(); - - public boolean hasNext() { - - if (curr.hasNext()) - return true; - - try { - - if (it.hasNext()) { - - curr = it.read().getRecordsList().iterator(); - return hasNext(); - } - } - catch (InterruptedException e) { - throw new RuntimeException("Interrupted while waiting for stream", e); - } - catch (StatusException e) { - throw new RuntimeException("Stream query failed: " + e.getStatus().getDescription()); - } - - return false; - } - - public GrpcRecord next() { - if (!hasNext()) - throw new NoSuchElementException(); - return curr.next(); - } - }; - } - - public Iterator queryStreamBatches(String passLabel, String sql, Map params, int batchSize, - StreamQueryRequest.RetrievalMode mode, TransactionContext tx, long timeoutMs) { - - var reqB = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(sql) - .putAllParameters(convertParamsToGrpcValue(params)).setCredentials(buildCredentials()).setBatchSize(batchSize > 0 ? batchSize : 100) - .setRetrievalMode(mode); - - if (tx != null) - reqB.setTransaction(tx); - - final BlockingClientCall respIter = blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS) - .streamQuery(reqB.build()); - - return new Iterator<>() { - - public boolean hasNext() { - - try { - return respIter.hasNext(); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } - catch (StatusException e) { - throw new RuntimeException(e); - } - } - - public QueryBatch next() { - - QueryResult qr; - - try { - qr = respIter.read(); - List converted = new ArrayList<>(qr.getRecordsCount()); - - for (GrpcRecord gr : qr.getRecordsList()) { - converted.add(grpcRecordToDBRecord(gr)); - } - - return new QueryBatch(converted, qr.getTotalRecordsInBatch(), // int totalInBatch - qr.getRunningTotalEmitted(), // long runningTotal - qr.getIsLastBatch() // boolean lastBatch - ); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } - catch (StatusException e) { - throw new RuntimeException(e); - } - } - }; - } - - public String createVertex(String cls, Map props, long timeoutMs) { - // Build the nested proto GrpcRecord payload - GrpcRecord recMsg = GrpcRecord.newBuilder().putAllProperties(convertParamsToGrpcValue(props)) - .build(); - - // Build the Create request - CreateRecordRequest req = CreateRecordRequest.newBuilder().setDatabase(getName()) - .setType(cls).setRecord(recMsg) // <<<< NESTED RECORD (not top-level properties) - .setCredentials(buildCredentials()).build(); - - // Call RPC - CreateRecordResponse res; - - try { - res = blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).createRecord(req); - // Response carries new RID string - return res.getRid(); // e.g., "#12:0" - } catch (StatusException e) { - throw new RuntimeException("Failed to create vertex", e); - } - } - - public boolean updateRecord(String rid, Map props, long timeoutMs) { - PropertiesUpdate partial = PropertiesUpdate.newBuilder() - .putAllProperties(convertParamsToGrpcValue(props)) - .build(); - - return updateRecord(rid, partial, timeoutMs); - } - - public boolean updateRecord(String rid, Record dbRecord, long timeoutMs) { - GrpcRecord record = ProtoUtils.toProtoRecord(dbRecord); - return updateRecordFull(rid, record, timeoutMs); - } - - private boolean updateRecord(String rid, PropertiesUpdate partial, long timeoutMs) { - // Build the Update request using setPartial for partial update - UpdateRecordRequest req = UpdateRecordRequest.newBuilder().setDatabase(getName()) - .setRid(rid).setPartial(partial) - .setTransaction(TransactionContext.newBuilder().setBegin(true).setCommit(true)) - .setCredentials(buildCredentials()).build(); - - // Call RPC - UpdateRecordResponse res; - - try { - res = blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).updateRecord(req); - // Choose the flag your proto defines. Most builds expose getSuccess(). - return res.getSuccess(); - } catch (StatusException e) { - throw new RuntimeException("Failed to update record", e); - } - } - - private boolean updateRecordFull(String rid, GrpcRecord record, long timeoutMs) { - // Build the Update request using setRecord for full replacement - UpdateRecordRequest req = UpdateRecordRequest.newBuilder().setDatabase(getName()) - .setRid(rid).setRecord(record) - .setTransaction(TransactionContext.newBuilder().setBegin(true).setCommit(true)) - .setCredentials(buildCredentials()).build(); - - // Call RPC - UpdateRecordResponse res; - - try { - res = blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).updateRecord(req); - return res.getSuccess(); - } catch (StatusException e) { - throw new RuntimeException("Failed to update record", e); - } - } - - public boolean deleteRecord(String rid, long timeoutMs) { - var req = DeleteRecordRequest.newBuilder().setDatabase(getName()).setRid(rid).setCredentials(buildCredentials()) - .build(); - DeleteRecordResponse res; - - try { - res = blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).deleteRecord(req); - return res.getDeleted(); - } catch (StatusException e) { - throw new RuntimeException("Failed to delete record", e); - } - } - - @Override - public RemoteTransactionExplicitLock acquireLock() { - // Need gRPC-specific implementation - if (explicitLock == null) - explicitLock = new RemoteGrpcTransactionExplicitLock(this); - - return explicitLock; - } - - @Override - public long countBucket(final String bucketName) { - checkDatabaseIsOpen(); - stats.countBucket.incrementAndGet(); - ResultSet result = query("sql", "select count(*) as count from bucket:" + bucketName); - if (result.hasNext()) { - Number count = result.next().getProperty("count"); - return count != null ? count.longValue() : 0; - } - return 0; - } - - @Override - public long countType(final String typeName, final boolean polymorphic) { - checkDatabaseIsOpen(); - stats.countType.incrementAndGet(); - final String appendix = polymorphic ? "" : " where @type = '" + typeName + "'"; - ResultSet result = query("sql", "select count(*) as count from " + typeName + appendix); - if (result.hasNext()) { - Number count = result.next().getProperty("count"); - return count != null ? count.longValue() : 0; - } - return 0; - } - - @Override - public Record lookupByRID(final RID rid) { - return lookupByRID(rid, true); - } - - @Override - public Record lookupByRID(final RID rid, final boolean loadContent) { - checkDatabaseIsOpen(); - stats.readRecord.incrementAndGet(); - - if (rid == null) - throw new IllegalArgumentException("Record is null"); - - LookupByRidRequest request = LookupByRidRequest.newBuilder().setDatabase(getName()).setRid(rid.toString()) - .setCredentials(buildCredentials()).build(); - - try { - LookupByRidResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).lookupByRid(request); - - if (!response.getFound()) { - throw new RecordNotFoundException("Record " + rid + " not found", rid); - } - return grpcRecordToDBRecord(response.getRecord()); - } - catch (StatusRuntimeException e) { - handleGrpcException(e); - return null; - } - catch (StatusException e) { - handleGrpcException(e); - return null; - } - } - - @Override - public boolean existsRecord(RID rid) { - stats.existsRecord.incrementAndGet(); - if (rid == null) - throw new IllegalArgumentException("Record is null"); - try { - return lookupByRID(rid, false) != null; - } catch (RecordNotFoundException e) { - return false; - } - } - - public InsertSummary insertBulk(final InsertOptions options, - final List protoRows, final long timeoutMs) { - - // Ensure options carry DB + credentials as the server expects - InsertOptions.Builder ob = options.toBuilder(); - - if (options.getDatabase() == null || options.getDatabase().isEmpty()) { - ob.setDatabase(getName()); // your wrapper's DB name - } - - boolean missingCreds = !options.hasCredentials() || options.getCredentials().getUsername().isEmpty(); - if (missingCreds) { - ob.setCredentials(buildCredentials()); // your existing helper used by queries - } - - InsertOptions newOptions = ob.build(); - - BulkInsertRequest req = BulkInsertRequest.newBuilder().setOptions(newOptions) - .addAllRows(protoRows).build(); - - try { - if (logger.isDebugEnabled()) logger.debug("CLIENT insertBulk: rows={}, timeoutMs={}", req.getRowsCount(), timeoutMs); - return blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).bulkInsert(req); - } catch (StatusException e) { - throw new RuntimeException("insertBulk() -> failed: " + e.getStatus()); - } - } - - // Convenience overload that accepts domain rows (convert first) - public InsertSummary insertBulkAsListOfMaps(final InsertOptions options, - final List> rows, final long timeoutMs) { - - List protoRows = rows.stream().map(this::toProtoRecordFromMap) // your converter - .collect(java.util.stream.Collectors.toList()); - - return insertBulk(options, protoRows, timeoutMs); - } - - public InsertSummary ingestStream(final InsertOptions options, - final List protoRows, final int chunkSize, final long timeoutMs) - throws InterruptedException { - - final java.util.concurrent.CountDownLatch done = new java.util.concurrent.CountDownLatch(1); - final java.util.concurrent.atomic.AtomicReference summaryRef = new java.util.concurrent.atomic.AtomicReference<>(); - - // Ensure options carry DB + credentials as the server expects - InsertOptions.Builder ob = options.toBuilder(); - - if (options.getDatabase() == null || options.getDatabase().isEmpty()) { - ob.setDatabase(getName()); // your wrapper's DB name - } - - boolean missingCreds = !options.hasCredentials() || options.getCredentials().getUsername().isEmpty(); - - if (missingCreds) { - ob.setCredentials(buildCredentials()); // your existing helper used by queries - } - - StreamObserver resp = new StreamObserver<>() { - @Override - public void onNext(InsertSummary value) { - summaryRef.set(value); - } - - @Override - public void onError(Throwable t) { - done.countDown(); - } - - @Override - public void onCompleted() { - done.countDown(); - } - }; - - // Use the write service async stub, per-call deadline - var stub = this.asyncStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS); - - final StreamObserver req = stub.insertStream(resp); - - // Stream chunks - final String sessionId = "sess-" + System.nanoTime(); - - long seq = 1; - - for (int i = 0; i < protoRows.size(); i += chunkSize) { - final int end = Math.min(i + chunkSize, protoRows.size()); - - final InsertChunk chunk = InsertChunk.newBuilder().setSessionId(sessionId) - .setOptions(ob.build()).setChunkSeq(seq++).addAllRows(protoRows.subList(i, end)).build(); - - req.onNext(chunk); - } - - req.onCompleted(); - - done.await(timeoutMs + 5_000, TimeUnit.MILLISECONDS); - - final InsertSummary s = summaryRef.get(); - return (s != null) ? s : InsertSummary.newBuilder().setReceived(protoRows.size()).build(); - } - - // Convenience overload - public InsertSummary ingestStreamAsListOfMaps(final InsertOptions options, - final List> rows, final int chunkSize, final long timeoutMs) throws InterruptedException { - - List protoRows = rows.stream().map(this::toProtoRecordFromMap) - .collect(java.util.stream.Collectors.toList()); - - return ingestStream(options, protoRows, chunkSize, timeoutMs); - } - - /** - * Core implementation of InsertBidirectional ingest - */ - private InsertSummary ingestBidiCore(final List rows, - final InsertOptions options, final int chunkSize, final int maxInflight, final long timeoutMs, - final java.util.function.Function mapper) throws InterruptedException { - - // 1) Ensure options carry DB + credentials the server expects - final InsertOptions.Builder ob = options.toBuilder(); - - if (options.getDatabase() == null || options.getDatabase().isEmpty()) { - ob.setDatabase(getName()); - } - - final boolean missingCreds = !options.hasCredentials() || options.getCredentials().getUsername().isEmpty(); - - if (missingCreds) { - ob.setCredentials(buildCredentials()); - } - - final InsertOptions effectiveOpts = ob.build(); - - // 2) Pre-map rows → proto once (outside the observer) - final List protoRows = rows.stream().map(mapper) - .collect(java.util.stream.Collectors.toList()); - - if (logger.isDebugEnabled()) logger.debug("CLIENT ingestBidi start: rows={}, chunkSize={}, maxInflight={}, timeoutMs={}", protoRows.size(), chunkSize, maxInflight, timeoutMs); -// 3) Streaming state - final String sessionId = "sess-" + System.nanoTime(); - final java.util.concurrent.CountDownLatch done = new java.util.concurrent.CountDownLatch(1); - final java.util.concurrent.atomic.AtomicLong seq = new java.util.concurrent.atomic.AtomicLong(1); - final java.util.concurrent.atomic.AtomicInteger cursor = new java.util.concurrent.atomic.AtomicInteger(0); - final java.util.concurrent.atomic.AtomicInteger sent = new java.util.concurrent.atomic.AtomicInteger(0); - final java.util.concurrent.atomic.AtomicInteger acked = new java.util.concurrent.atomic.AtomicInteger(0); - final java.util.concurrent.atomic.AtomicReference committed = new java.util.concurrent.atomic.AtomicReference<>(); - final List acks = java.util.Collections.synchronizedList(new ArrayList<>()); - - // small holder to access req inside helpers - final java.util.concurrent.atomic.AtomicReference> observerRef = - new java.util.concurrent.atomic.AtomicReference<>(); - - final java.util.concurrent.atomic.AtomicBoolean commitSent = new java.util.concurrent.atomic.AtomicBoolean(false); - final java.util.concurrent.ScheduledExecutorService scheduler = java.util.concurrent.Executors.newSingleThreadScheduledExecutor(r -> { - Thread t = new Thread(r, "grpc-ack-grace-timer"); - t.setDaemon(true); - return t; - }); - final long ackGraceMillis = Math.min(Math.max(timeoutMs / 10, 1_000L), 10_000L); // e.g., 1s..10s window - final Object timerLock = new Object(); - final java.util.concurrent.atomic.AtomicReference> ackGraceFuture = new java.util.concurrent.atomic.AtomicReference<>(); - - // helper: schedule COMMIT if still waiting - final Runnable sendCommitIfNeeded = () -> { - if (commitSent.compareAndSet(false, true)) { - try { - if (observerRef.get() != null) { - observerRef.get().onNext(InsertRequest.newBuilder() - .setCommit(Commit.newBuilder().setSessionId(sessionId)).build()); - observerRef.get().onCompleted(); - } - } catch (Throwable ignore) { - /* best effort */ } - } - }; - - // helper: (re)arm the grace timer - final Runnable armAckGraceTimer = () -> { - synchronized (timerLock) { - var prev = ackGraceFuture.getAndSet(null); - if (prev != null) - prev.cancel(false); - // only arm if we've sent all chunks but acks are still pending - if (cursor.get() >= protoRows.size() && acked.get() < sent.get() && !commitSent.get()) { - var fut = scheduler.schedule(sendCommitIfNeeded, ackGraceMillis, TimeUnit.MILLISECONDS); - ackGraceFuture.set(fut); - } - } - }; - - // helper: cancel timer (e.g., once all ACKed or after COMMIT observed) - final Runnable cancelAckGraceTimer = () -> { - synchronized (timerLock) { - var prev = ackGraceFuture.getAndSet(null); - if (prev != null) - prev.cancel(false); - } - }; - - final ClientResponseObserver observer = new ClientResponseObserver<>() { - - ClientCallStreamObserver req; - volatile boolean started = false; - - @Override - public void beforeStart(ClientCallStreamObserver r) { - this.req = r; - observerRef.set(r); // <-- make req visible to helper - r.disableAutoInboundFlowControl(); - r.setOnReadyHandler(this::drain); - } - - private void drain() { - if (!req.isReady()) - return; - - if (!started) { - req.onNext(InsertRequest.newBuilder() - .setStart(Start.newBuilder().setOptions(effectiveOpts)).build()); - - started = true; - - // Now that the call is started, it's safe to pull the first response - req.request(1); - } - - while (req.isReady() && (sent.get() - acked.get()) < maxInflight) { - final int start = cursor.get(); - if (start >= protoRows.size()) - break; - - final int end = Math.min(start + chunkSize, protoRows.size()); - final var slice = protoRows.subList(start, end); - - final var chunk = InsertChunk.newBuilder().setSessionId(sessionId) - .setChunkSeq(seq.getAndIncrement()).addAllRows(slice).build(); - - req.onNext(InsertRequest.newBuilder().setChunk(chunk).build()); - cursor.set(end); - sent.incrementAndGet(); - } - - // If all chunks sent: - if (cursor.get() >= protoRows.size()) { - if (acked.get() >= sent.get()) { - // all acked → commit immediately - sendCommitIfNeeded.run(); - } else { - // not all acked → (re)arm grace timer; if last ACK never arrives, timer will COMMIT - armAckGraceTimer.run(); - } - } - } - - @Override - public void onNext(InsertResponse v) { - switch (v.getMsgCase()) { - case STARTED -> { - /* ok */ } - case BATCH_ACK -> { - acks.add(v.getBatchAck()); - acked.incrementAndGet(); - // Each ACK may free capacity: push more & manage timer - drain(); - if (cursor.get() >= protoRows.size()) { - if (acked.get() >= sent.get()) { - cancelAckGraceTimer.run(); - sendCommitIfNeeded.run(); - } else { - armAckGraceTimer.run(); - } - } - } - case COMMITTED -> { - committed.set(v.getCommitted().getSummary()); - cancelAckGraceTimer.run(); - } - case MSG_NOT_SET -> { - /* ignore */ - } - case ERROR -> { - /* TBD */ - } - } - - req.request(1); // continue pulling responses - } - - @Override - public void onError(Throwable t) { - cancelAckGraceTimer.run(); - done.countDown(); - } - - @Override - public void onCompleted() { - cancelAckGraceTimer.run(); - done.countDown(); - } - }; - // 4) Kick off the bidi call with deadline - - try { - if (logger.isDebugEnabled()) logger.debug("CLIENT ingestBidi opening stream: timeoutMs={}", timeoutMs); - asyncStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).insertBidirectional(observer); - @SuppressWarnings("unused") - boolean finished = done.await(timeoutMs + 5_000, TimeUnit.MILLISECONDS); - if (logger.isDebugEnabled()) try { logger.debug("CLIENT ingestBidi finished: finished={}, sent={}, acked={}", finished, sent.get(), acked.get()); } catch (Throwable ignore) {} -} finally { - scheduler.shutdownNow(); - } - - // 6) Prefer COMMITTED summary if present; otherwise aggregate ACKs - final InsertSummary finalSummary = committed.get(); - if (finalSummary != null) - return finalSummary; - - final long ins = acks.stream().mapToLong(BatchAck::getInserted).sum(); - final long upd = acks.stream().mapToLong(BatchAck::getUpdated).sum(); - final long ign = acks.stream().mapToLong(BatchAck::getIgnored).sum(); - final long fail = acks.stream().mapToLong(BatchAck::getFailed).sum(); - - return InsertSummary.newBuilder().setReceived(protoRows.size()).setInserted(ins).setUpdated(upd).setIgnored(ign) - .setFailed(fail).build(); - } - - /** - * Pushes domain {@code com.arcadedb.database.Record} rows via InsertBidirectional with per-batch ACKs. - */ - public InsertSummary ingestBidi(final List rows, - final InsertOptions opts, final int chunkSize, final int maxInflight, final long timeoutMs) - throws InterruptedException { - - return ingestBidiCore(rows, opts, chunkSize, maxInflight, timeoutMs, - (Object o) -> toProtoRecordFromDbRecord((com.arcadedb.database.Record) o)); - } - - public InsertSummary ingestBidi(final List rows, - final InsertOptions opts, final int chunkSize, final int maxInflight) throws InterruptedException { - - return ingestBidiCore(rows, opts, chunkSize, maxInflight, /* timeoutMs */ 5 * 60_000L, - this::toProtoRecordFromDbRecord); - } - - /** - * Pushes map-shaped rows (property map per row) via InsertBidirectional with per-batch ACKs. - */ - public InsertSummary ingestBidi(final InsertOptions options, - final List> rows, final int chunkSize, final int maxInflight) throws InterruptedException { - - return ingestBidiCore(rows, options, chunkSize, maxInflight, /* timeoutMs */ 5 * 60_000L, - this::toProtoRecordFromMap); - } - - public InsertSummary ingestBidi(final InsertOptions options, - final List> rows, final int chunkSize, final int maxInflight, final long timeoutMs) - throws InterruptedException { - - return ingestBidiCore(rows, options, chunkSize, maxInflight, timeoutMs, - this::toProtoRecordFromMap); - } - - // Map -> GrpcRecord - private GrpcRecord toProtoRecordFromMap(Map row) { - GrpcRecord.Builder b = GrpcRecord.newBuilder(); - row.forEach((k, v) -> b.putProperties(k, objectToGrpcValue(v))); - GrpcRecord rec = b.build(); - if (logger.isDebugEnabled()) logger.debug("CLIENT toProtoRecordFromMap: {}", summarize(rec)); - return rec; - } - - // Domain Record (storage) -> GrpcRecord - private GrpcRecord toProtoRecordFromDbRecord(com.arcadedb.database.Record rec) { - // Use ProtoUtils for proper conversion - GrpcRecord out = ProtoUtils.toProtoRecord(rec); - if (logger.isDebugEnabled()) logger.debug("CLIENT toProtoRecordFromDbRecord: {}", summarize(out)); - return out; - } - - // Convert Java object -> GrpcValue (extend as needed) - private GrpcValue objectToGrpcValue(Object v) { - return ProtoUtils.toGrpcValue(v); - } - - public DatabaseCredentials buildCredentials() { - return DatabaseCredentials.newBuilder().setUsername(getUserName()).setPassword(getUserPassword()).build(); - } - - private TransactionIsolation mapIsolationLevel(Database.TRANSACTION_ISOLATION_LEVEL level) { - switch (level) { -// case READ_UNCOMMITTED: -// return TransactionIsolation.READ_UNCOMMITTED; - case READ_COMMITTED: - return TransactionIsolation.READ_COMMITTED; - case REPEATABLE_READ: - return TransactionIsolation.REPEATABLE_READ; -// case SERIALIZABLE: -// return TransactionIsolation.SERIALIZABLE; - default: - return TransactionIsolation.READ_COMMITTED; - } - } - - // ---- TX helpers ------------------------------------------------------------- - private static TransactionContext txBeginCommit() { - return TransactionContext.newBuilder().setBegin(true).setCommit(true).build(); - } - - private static TransactionContext txNone() { - return TransactionContext.getDefaultInstance(); - } - - // Optional: language defaulting (server defaults to "sql" too) - private static String langOrDefault(String language) { - return (language == null || language.isEmpty()) ? "sql" : language; - } - - private Iterator streamQuery(final String query) { - StreamQueryRequest request = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(query).setCredentials(buildCredentials()) - .setBatchSize(100).build(); - - final BlockingClientCall responseIterator = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) - .streamQuery(request); - - return new Iterator() { - private Iterator currentBatch = Collections.emptyIterator(); - - @Override - public boolean hasNext() { - - if (currentBatch.hasNext()) { - return true; - } - - try { - if (responseIterator.hasNext()) { - - QueryResult result; - result = responseIterator.read(); - currentBatch = result.getRecordsList().iterator(); - return currentBatch.hasNext(); - } - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } - catch (StatusException e) { - throw new RuntimeException(e); - } - - return false; - } - - @Override - public Record next() { - if (!hasNext()) { - throw new NoSuchElementException(); - } - return grpcRecordToDBRecord(currentBatch.next()); - } - }; - } - - private ResultSet createGrpcResultSet(ExecuteQueryResponse response) { - InternalResultSet resultSet = new InternalResultSet(); - for (QueryResult queryResult : response.getResultsList()) { - for (GrpcRecord record : queryResult.getRecordsList()) { - resultSet.add(grpcRecordToResult(record)); - } - } - return resultSet; - } - - private Result grpcRecordToResult(GrpcRecord grpcRecord) { - - Record record = grpcRecordToDBRecord(grpcRecord); - - if (record == null) { - - Map properties = new HashMap<>(); - grpcRecord.getPropertiesMap().forEach((k, v) -> properties.put(k, grpcValueToObject(v))); - return new ResultInternal(properties); - } - - return new ResultInternal(record); - } - - private Record grpcRecordToDBRecord(GrpcRecord grpcRecord) { - - Map map = new HashMap<>(); - - // Convert properties - grpcRecord.getPropertiesMap().forEach((k, v) -> map.put(k, grpcValueToObject(v))); - - // Add metadata - map.put("@rid", grpcRecord.getRid()); - map.put("@type", grpcRecord.getType()); - map.put("@cat", mapRecordType(grpcRecord)); - - String cat = (String) map.get("@cat"); - - if (cat == null) { - - return null; - } - - switch (cat) { - case "d": - return new RemoteImmutableDocument(this, map); - case "v": - return new RemoteImmutableVertex(this, map); - case "e": - return new RemoteImmutableEdge(this, map); - default: - return null; - } - } - - private String mapRecordType(GrpcRecord grpcRecord) { - - // Determine record category from type name - String typeName = grpcRecord.getType(); - - // Check schema to determine actual type - try { - - if (getSchema().existsType(typeName)) { - - Object type = getSchema().getType(typeName); - - if (type instanceof com.arcadedb.schema.VertexType) { - return "v"; - } - else if (type instanceof com.arcadedb.schema.EdgeType) { - return "e"; - } - else if (type instanceof com.arcadedb.schema.DocumentType) { - return "d"; - } - else { - return null; - } - } - else { - - return null; - } - } - catch (Exception e) { - // Fall back to name-based detection - } - - return null; - } - - private Map convertParamsToGrpcValue(Map params) { - Map grpcParams = new HashMap<>(); - - for (Map.Entry entry : params.entrySet()) { - GrpcValue value = objectToGrpcValue(entry.getValue()); - grpcParams.put(entry.getKey(), value); - } - - return grpcParams; - } - - - private Object grpcValueToObject(GrpcValue grpcValue) { - Object out = ProtoUtils.fromGrpcValue(grpcValue); - if (logger.isDebugEnabled()) logger.debug("CLIENT decode grpcValueToObject: {} -> {}", summarize(grpcValue), summarize(out)); - return out; - } - - - private void handleGrpcException(StatusRuntimeException e) { - Status status = e.getStatus(); - switch (status.getCode()) { - case NOT_FOUND: - throw new RecordNotFoundException(status.getDescription(), null); - case ALREADY_EXISTS: - throw new DuplicatedKeyException("", "", null); - case ABORTED: - throw new ConcurrentModificationException(status.getDescription()); - case DEADLINE_EXCEEDED: - throw new TimeoutException(status.getDescription()); - case PERMISSION_DENIED: - throw new SecurityException(status.getDescription()); - case UNAVAILABLE: - throw new NeedRetryException(status.getDescription()); - default: - throw new RemoteException("gRPC error: " + status.getDescription(), e); - } - } - - private void handleGrpcException(StatusException e) { - Status status = e.getStatus(); - switch (status.getCode()) { - case NOT_FOUND: - throw new RecordNotFoundException(status.getDescription(), null); - case ALREADY_EXISTS: - throw new DuplicatedKeyException("", "", null); - case ABORTED: - throw new ConcurrentModificationException(status.getDescription()); - case DEADLINE_EXCEEDED: - throw new TimeoutException(status.getDescription()); - case PERMISSION_DENIED: - throw new SecurityException(status.getDescription()); - case UNAVAILABLE: - throw new NeedRetryException(status.getDescription()); - default: - throw new RemoteException("gRPC error: " + status.getDescription(), e); - } - } - - /** - * Extended version with compression support - */ - public static class RemoteGrpcDatabaseWithCompression extends RemoteGrpcDatabase { - - public RemoteGrpcDatabaseWithCompression(String server, int grpcPort, int httpPort, String databaseName, String userName, - String userPassword, ContextConfiguration configuration) { - super(server, grpcPort, httpPort, databaseName, userName, userPassword, configuration); - } - - @Override - protected ManagedChannel createChannel(String server, int port) { - - return ManagedChannelBuilder.forAddress(server, port).usePlaintext() // No TLS/SSL - .compressorRegistry(CompressorRegistry.getDefaultInstance()).decompressorRegistry(DecompressorRegistry.getDefaultInstance()) - .maxInboundMessageSize(100 * 1024 * 1024) // 100MB max message size - .keepAliveTime(30, TimeUnit.SECONDS) // Keep-alive configuration - .keepAliveTimeout(10, TimeUnit.SECONDS).keepAliveWithoutCalls(true).build(); - } - } - - // Add the missing RemoteGrpcTransactionExplicitLock class reference - private static class RemoteGrpcTransactionExplicitLock extends RemoteTransactionExplicitLock { - public RemoteGrpcTransactionExplicitLock(RemoteGrpcDatabase database) { - super(database); - } - } -} \ No newline at end of file diff --git a/grpc-client/docs/Readme.txt b/grpc-client/docs/Readme.txt index 051f316d04..dd57c6d14e 100644 --- a/grpc-client/docs/Readme.txt +++ b/grpc-client/docs/Readme.txt @@ -12,3 +12,59 @@ GTPC: 50059 root root1234 + +Logging: + +d exec -it arcadedb1-vulcan sh + + +/home/arcadedb/config/arcadedb-log.properties + + +# +# Copyright 2021-present Arcade Data Ltd +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Specify the handlers to create in the root logger +# (all loggers are children of the root logger) +# The following creates two handlers +handlers = java.util.logging.ConsoleHandler, java.util.logging.FileHandler + +# Set the default logging level for the root logger +.level = INFO +com.arcadedb.level = INFO +com.arcadedb.server.grpc.level = FINE + +# Set the default logging level for new ConsoleHandler instances +java.util.logging.ConsoleHandler.level = INFO +# Set the default formatter for new ConsoleHandler instances +java.util.logging.ConsoleHandler.formatter = com.arcadedb.utility.AnsiLogFormatter + +# Set the default logging level for new FileHandler instances +java.util.logging.FileHandler.level = FINE +# Naming style for the output file +java.util.logging.FileHandler.pattern=./log/arcadedb.log +# Set the default formatter for new FileHandler instances +java.util.logging.FileHandler.formatter = com.arcadedb.log.LogFormatter +# Limiting size of output file in bytes: +java.util.logging.FileHandler.limit=100000000 +# Number of output files to cycle through, by appending an +# integer to the base file name: +java.util.logging.FileHandler.count=10 \ No newline at end of file diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java index 0c810135ec..f905d5dd8d 100644 --- a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java @@ -1,5 +1,11 @@ package com.arcadedb.server.grpc; +import com.google.gson.JsonElement; +import com.google.gson.JsonPrimitive; +import com.google.gson.JsonObject; +import com.google.gson.JsonArray; +import com.google.gson.JsonNull; + import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -1125,11 +1131,11 @@ public void onError(Throwable t) { @Override public void onCompleted() { - + try { InsertContext ctx = ctxRef.get(); - + if (ctx == null) { // Client closed without sending a first chunk → nothing to do resp.onNext(InsertSummary.newBuilder().setReceived(0).setInserted(0).setUpdated(0).setIgnored(0).setFailed(0) @@ -1463,9 +1469,9 @@ void err(long rowIndex, String code, String msg, String field) { } private Counts insertRows(InsertContext ctx, java.util.Iterator it) { - + Counts c = new Counts(); - + int inBatch = 0; Schema schema = ctx.db.getSchema(); @@ -1577,8 +1583,9 @@ private void applyGrpcRecord(MutableDocument doc, GrpcRecord r) { if (k.startsWith("@")) return; Object javaVal = fromGrpcValue(grpcVal); - if (logger.isDebugEnabled()) logger.debug("APPLY-DOC {} <= {} -> {}", k, summarizeGrpc(grpcVal), summarizeJava(javaVal)); - doc.set(k, javaVal); + if (logger.isDebugEnabled()) + logger.debug("APPLY-DOC {} <= {} -> {}", k, summarizeGrpc(grpcVal), summarizeJava(javaVal)); + doc.set(k, javaVal); }); } @@ -1588,8 +1595,9 @@ private void applyGrpcRecord(MutableVertex vertex, GrpcRecord r) { if (k.startsWith("@")) return; Object javaVal = fromGrpcValue(grpcVal); - if (logger.isDebugEnabled()) logger.debug("APPLY-VERTEX {} <= {} -> {}", k, summarizeGrpc(grpcVal), summarizeJava(javaVal)); - vertex.set(k, javaVal); + if (logger.isDebugEnabled()) + logger.debug("APPLY-VERTEX {} <= {} -> {}", k, summarizeGrpc(grpcVal), summarizeJava(javaVal)); + vertex.set(k, javaVal); }); } @@ -1599,8 +1607,9 @@ private void applyGrpcRecord(MutableEdge edge, GrpcRecord r) { if (k.startsWith("@")) return; Object javaVal = fromGrpcValue(grpcVal); - if (logger.isDebugEnabled()) logger.debug("APPLY-EDGE {} <= {} -> {}", k, summarizeGrpc(grpcVal), summarizeJava(javaVal)); - edge.set(k, javaVal); + if (logger.isDebugEnabled()) + logger.debug("APPLY-EDGE {} <= {} -> {}", k, summarizeGrpc(grpcVal), summarizeJava(javaVal)); + edge.set(k, javaVal); }); } @@ -1668,6 +1677,11 @@ private static com.google.protobuf.Timestamp msToTimestamp(long ms) { } private GrpcValue toGrpcValue(Object o) { + + if (o instanceof JsonElement je) { + return gsonToGrpc(je); + } + GrpcValue.Builder b = GrpcValue.newBuilder(); if (o == null) return dbgEnc("toGrpcValue", o, b.build(), null); @@ -1688,11 +1702,14 @@ private GrpcValue toGrpcValue(Object o) { return dbgEnc("toGrpcValue", o, b.setBytesValue(com.google.protobuf.ByteString.copyFrom(v)).build(), null); if (o instanceof java.util.Date v) { - return dbgEnc("toGrpcValue", o, GrpcValue.newBuilder().setTimestampValue(msToTimestamp(v.getTime())).setLogicalType("datetime").build(), null); + return dbgEnc("toGrpcValue", o, + GrpcValue.newBuilder().setTimestampValue(msToTimestamp(v.getTime())).setLogicalType("datetime").build(), null); } if (o instanceof com.arcadedb.database.RID rid) { - return dbgEnc("toGrpcValue", o, GrpcValue.newBuilder().setLinkValue(GrpcLink.newBuilder().setRid(rid.toString()).build()).setLogicalType("rid").build(), null); + return dbgEnc("toGrpcValue", o, + GrpcValue.newBuilder().setLinkValue(GrpcLink.newBuilder().setRid(rid.toString()).build()).setLogicalType("rid").build(), + null); } if (o instanceof java.math.BigDecimal v) { @@ -1704,7 +1721,8 @@ private GrpcValue toGrpcValue(Object o) { else { // if you need >64-bit unscaled, switch GrpcDecimal.unscaled to bytes in the // proto - return dbgEnc("toGrpcValue", o, GrpcValue.newBuilder().setStringValue(v.toPlainString()).setLogicalType("decimal").build(), null); + return dbgEnc("toGrpcValue", o, GrpcValue.newBuilder().setStringValue(v.toPlainString()).setLogicalType("decimal").build(), + null); } } @@ -1744,8 +1762,8 @@ private GrpcValue toGrpcValue(Object o) { // RID/Identifiable string fallback if (o instanceof com.arcadedb.database.Identifiable id) - return dbgEnc("toGrpcValue", o, GrpcValue.newBuilder().setLinkValue(GrpcLink.newBuilder().setRid(id.getIdentity().toString()).build()).setLogicalType("rid").build(), null); - + return dbgEnc("toGrpcValue", o, GrpcValue.newBuilder() + .setLinkValue(GrpcLink.newBuilder().setRid(id.getIdentity().toString()).build()).setLogicalType("rid").build(), null); // Fallback return dbgEnc("toGrpcValue", o, GrpcValue.newBuilder().setStringValue(String.valueOf(o)).build(), null); @@ -1828,9 +1846,9 @@ void flushCommit(boolean end) { } InsertSummary summary(Counts c, long startedAtMs) { - + long now = System.currentTimeMillis(); - + return InsertSummary.newBuilder().setReceived(c.received).setInserted(c.inserted).setUpdated(c.updated).setIgnored(c.ignored) .setFailed(c.failed).addAllErrors(c.errors).setStartedAt(ts(startedAtMs)).setFinishedAt(ts(now)).build(); } @@ -1952,7 +1970,8 @@ private GrpcRecord convertToGrpcRecord(com.arcadedb.database.Record dbRecord, Da Object value = doc.get(propertyName); if (value != null) { GrpcValue gv = toGrpcValue(value); - if (logger.isDebugEnabled()) logger.debug("ENC-REC {}.{}: {} -> {}", builder.getRid(), propertyName, summarizeJava(value), summarizeGrpc(gv)); + if (logger.isDebugEnabled()) + logger.debug("ENC-REC {}.{}: {} -> {}", builder.getRid(), propertyName, summarizeJava(value), summarizeGrpc(gv)); builder.putProperties(propertyName, gv); } } @@ -1968,7 +1987,8 @@ private GrpcRecord convertToGrpcRecord(com.arcadedb.database.Record dbRecord, Da } } - if (logger.isDebugEnabled()) logger.debug("ENC-REC DONE rid={} type={} props={}", builder.getRid(), builder.getType(), builder.getPropertiesCount()); + if (logger.isDebugEnabled()) + logger.debug("ENC-REC DONE rid={} type={} props={}", builder.getRid(), builder.getType(), builder.getPropertiesCount()); return builder.build(); } @@ -2001,246 +2021,246 @@ private Object toJavaForProperty(final Database db, final MutableDocument parent } private Object convertWithSchemaType(Database db, MutableDocument parent, com.arcadedb.schema.Property prop, String propName, GrpcValue v) { - + var t = prop.getType(); - + switch (t) { - - case BOOLEAN: - return switch (v.getKindCase()) { - case BOOL_VALUE -> v.getBoolValue(); - case STRING_VALUE -> Boolean.parseBoolean(v.getStringValue()); - default -> null; - }; - - case BYTE: - return switch (v.getKindCase()) { - case INT32_VALUE, INT64_VALUE, DOUBLE_VALUE, FLOAT_VALUE -> (byte) (long) fromGrpcValue(v); - case STRING_VALUE -> Byte.parseByte(v.getStringValue()); - default -> null; - }; - - case SHORT: - return switch (v.getKindCase()) { - case INT32_VALUE, INT64_VALUE, DOUBLE_VALUE, FLOAT_VALUE -> (short) (long) fromGrpcValue(v); - case STRING_VALUE -> Short.parseShort(v.getStringValue()); - default -> null; - }; - - case INTEGER: - return switch (v.getKindCase()) { - case INT32_VALUE -> v.getInt32Value(); - case INT64_VALUE -> (int) v.getInt64Value(); - case DOUBLE_VALUE -> (int) v.getDoubleValue(); - case FLOAT_VALUE -> (int) v.getFloatValue(); - case STRING_VALUE -> Integer.parseInt(v.getStringValue()); - default -> null; - }; - - case LONG: - return switch (v.getKindCase()) { - case INT64_VALUE -> v.getInt64Value(); - case INT32_VALUE -> (long) v.getInt32Value(); - case DOUBLE_VALUE -> (long) v.getDoubleValue(); - case FLOAT_VALUE -> (long) v.getFloatValue(); - case STRING_VALUE -> Long.parseLong(v.getStringValue()); - default -> null; - }; - - case FLOAT: - return switch (v.getKindCase()) { - case FLOAT_VALUE -> v.getFloatValue(); - case DOUBLE_VALUE -> (float) v.getDoubleValue(); - case INT32_VALUE -> (float) v.getInt32Value(); - case INT64_VALUE -> (float) v.getInt64Value(); - case STRING_VALUE -> Float.parseFloat(v.getStringValue()); - default -> null; - }; - - case DOUBLE: - return switch (v.getKindCase()) { - case DOUBLE_VALUE -> v.getDoubleValue(); - case FLOAT_VALUE -> (double) v.getFloatValue(); - case INT32_VALUE -> (double) v.getInt32Value(); - case INT64_VALUE -> (double) v.getInt64Value(); - case STRING_VALUE -> Double.parseDouble(v.getStringValue()); - default -> null; - }; - - case STRING: - return switch (v.getKindCase()) { - case STRING_VALUE -> v.getStringValue(); - default -> String.valueOf(fromGrpcValue(v)); - }; - - case DECIMAL: - return switch (v.getKindCase()) { - case DECIMAL_VALUE -> { - var d = v.getDecimalValue(); - yield new java.math.BigDecimal(java.math.BigInteger.valueOf(d.getUnscaled()), d.getScale()); - } - case STRING_VALUE -> new java.math.BigDecimal(v.getStringValue()); - case DOUBLE_VALUE -> java.math.BigDecimal.valueOf(v.getDoubleValue()); - case INT32_VALUE -> java.math.BigDecimal.valueOf(v.getInt32Value()); - case INT64_VALUE -> java.math.BigDecimal.valueOf(v.getInt64Value()); - default -> null; - }; - - case DATE: - case DATETIME: - // Prefer timestamp_value; else accept epoch ms in int64; else parse string - return switch (v.getKindCase()) { - case TIMESTAMP_VALUE -> new java.util.Date(tsToMillis(v.getTimestampValue())); - case INT64_VALUE -> new java.util.Date(v.getInt64Value()); - case STRING_VALUE -> new java.util.Date(Long.parseLong(v.getStringValue())); // or parse ISO if you emit it - default -> null; - }; - - case BINARY: - return switch (v.getKindCase()) { - case BYTES_VALUE -> v.getBytesValue().toByteArray(); - case STRING_VALUE -> v.getStringValue().getBytes(java.nio.charset.StandardCharsets.UTF_8); - default -> null; - }; - - case LINK: - return switch (v.getKindCase()) { - case LINK_VALUE -> new com.arcadedb.database.RID(v.getLinkValue().getRid()); - case STRING_VALUE -> new com.arcadedb.database.RID(v.getStringValue()); - default -> null; - }; - - case EMBEDDED: { - // Use schema ofType if present; otherwise use embedded.type from the payload - String embeddedTypeName = (v.getKindCase() == GrpcValue.KindCase.EMBEDDED_VALUE && !v.getEmbeddedValue().getType().isEmpty()) - ? v.getEmbeddedValue().getType() - : prop.getOfType(); - - if (v.getKindCase() != GrpcValue.KindCase.EMBEDDED_VALUE || embeddedTypeName == null || embeddedTypeName.isEmpty()) { - // Fallback to a Map if we can't create a typed embedded doc - return fromGrpcValue(v); - } - - MutableEmbeddedDocument ed = parent.newEmbeddedDocument(embeddedTypeName, propName); - DocumentType embeddedType = null; - try { - embeddedType = db.getSchema().getType(embeddedTypeName); - } - catch (Exception ignore) { - } - - final DocumentType embeddedTypeFinal = embeddedType; - - v.getEmbeddedValue().getFieldsMap().forEach((k, vv) -> { - Object j = (embeddedTypeFinal != null) ? toJavaForProperty(db, ed, embeddedTypeFinal, k, vv) : fromGrpcValue(vv); - ed.set(k, j); - }); - return ed; - } - - case MAP: - if (v.getKindCase() == GrpcValue.KindCase.MAP_VALUE) { - var m = new java.util.LinkedHashMap(); - v.getMapValue().getEntriesMap().forEach((k, vv) -> m.put(k, fromGrpcValue(vv))); - return m; - } - return null; - - case LIST: - if (v.getKindCase() == GrpcValue.KindCase.LIST_VALUE) { - var list = new java.util.ArrayList<>(); - for (GrpcValue item : v.getListValue().getValuesList()) { - list.add(fromGrpcValue(item)); - } - return list; + + case BOOLEAN: + return switch (v.getKindCase()) { + case BOOL_VALUE -> v.getBoolValue(); + case STRING_VALUE -> Boolean.parseBoolean(v.getStringValue()); + default -> null; + }; + + case BYTE: + return switch (v.getKindCase()) { + case INT32_VALUE, INT64_VALUE, DOUBLE_VALUE, FLOAT_VALUE -> (byte) (long) fromGrpcValue(v); + case STRING_VALUE -> Byte.parseByte(v.getStringValue()); + default -> null; + }; + + case SHORT: + return switch (v.getKindCase()) { + case INT32_VALUE, INT64_VALUE, DOUBLE_VALUE, FLOAT_VALUE -> (short) (long) fromGrpcValue(v); + case STRING_VALUE -> Short.parseShort(v.getStringValue()); + default -> null; + }; + + case INTEGER: + return switch (v.getKindCase()) { + case INT32_VALUE -> v.getInt32Value(); + case INT64_VALUE -> (int) v.getInt64Value(); + case DOUBLE_VALUE -> (int) v.getDoubleValue(); + case FLOAT_VALUE -> (int) v.getFloatValue(); + case STRING_VALUE -> Integer.parseInt(v.getStringValue()); + default -> null; + }; + + case LONG: + return switch (v.getKindCase()) { + case INT64_VALUE -> v.getInt64Value(); + case INT32_VALUE -> (long) v.getInt32Value(); + case DOUBLE_VALUE -> (long) v.getDoubleValue(); + case FLOAT_VALUE -> (long) v.getFloatValue(); + case STRING_VALUE -> Long.parseLong(v.getStringValue()); + default -> null; + }; + + case FLOAT: + return switch (v.getKindCase()) { + case FLOAT_VALUE -> v.getFloatValue(); + case DOUBLE_VALUE -> (float) v.getDoubleValue(); + case INT32_VALUE -> (float) v.getInt32Value(); + case INT64_VALUE -> (float) v.getInt64Value(); + case STRING_VALUE -> Float.parseFloat(v.getStringValue()); + default -> null; + }; + + case DOUBLE: + return switch (v.getKindCase()) { + case DOUBLE_VALUE -> v.getDoubleValue(); + case FLOAT_VALUE -> (double) v.getFloatValue(); + case INT32_VALUE -> (double) v.getInt32Value(); + case INT64_VALUE -> (double) v.getInt64Value(); + case STRING_VALUE -> Double.parseDouble(v.getStringValue()); + default -> null; + }; + + case STRING: + return switch (v.getKindCase()) { + case STRING_VALUE -> v.getStringValue(); + default -> String.valueOf(fromGrpcValue(v)); + }; + + case DECIMAL: + return switch (v.getKindCase()) { + case DECIMAL_VALUE -> { + var d = v.getDecimalValue(); + yield new java.math.BigDecimal(java.math.BigInteger.valueOf(d.getUnscaled()), d.getScale()); + } + case STRING_VALUE -> new java.math.BigDecimal(v.getStringValue()); + case DOUBLE_VALUE -> java.math.BigDecimal.valueOf(v.getDoubleValue()); + case INT32_VALUE -> java.math.BigDecimal.valueOf(v.getInt32Value()); + case INT64_VALUE -> java.math.BigDecimal.valueOf(v.getInt64Value()); + default -> null; + }; + + case DATE: + case DATETIME: + // Prefer timestamp_value; else accept epoch ms in int64; else parse string + return switch (v.getKindCase()) { + case TIMESTAMP_VALUE -> new java.util.Date(tsToMillis(v.getTimestampValue())); + case INT64_VALUE -> new java.util.Date(v.getInt64Value()); + case STRING_VALUE -> new java.util.Date(Long.parseLong(v.getStringValue())); // or parse ISO if you emit it + default -> null; + }; + + case BINARY: + return switch (v.getKindCase()) { + case BYTES_VALUE -> v.getBytesValue().toByteArray(); + case STRING_VALUE -> v.getStringValue().getBytes(java.nio.charset.StandardCharsets.UTF_8); + default -> null; + }; + + case LINK: + return switch (v.getKindCase()) { + case LINK_VALUE -> new com.arcadedb.database.RID(v.getLinkValue().getRid()); + case STRING_VALUE -> new com.arcadedb.database.RID(v.getStringValue()); + default -> null; + }; + + case EMBEDDED: { + // Use schema ofType if present; otherwise use embedded.type from the payload + String embeddedTypeName = (v.getKindCase() == GrpcValue.KindCase.EMBEDDED_VALUE && !v.getEmbeddedValue().getType().isEmpty()) + ? v.getEmbeddedValue().getType() + : prop.getOfType(); + + if (v.getKindCase() != GrpcValue.KindCase.EMBEDDED_VALUE || embeddedTypeName == null || embeddedTypeName.isEmpty()) { + // Fallback to a Map if we can't create a typed embedded doc + return fromGrpcValue(v); + } + + MutableEmbeddedDocument ed = parent.newEmbeddedDocument(embeddedTypeName, propName); + DocumentType embeddedType = null; + try { + embeddedType = db.getSchema().getType(embeddedTypeName); + } + catch (Exception ignore) { + } + + final DocumentType embeddedTypeFinal = embeddedType; + + v.getEmbeddedValue().getFieldsMap().forEach((k, vv) -> { + Object j = (embeddedTypeFinal != null) ? toJavaForProperty(db, ed, embeddedTypeFinal, k, vv) : fromGrpcValue(vv); + ed.set(k, j); + }); + return ed; + } + + case MAP: + if (v.getKindCase() == GrpcValue.KindCase.MAP_VALUE) { + var m = new java.util.LinkedHashMap(); + v.getMapValue().getEntriesMap().forEach((k, vv) -> m.put(k, fromGrpcValue(vv))); + return m; + } + return null; + + case LIST: + if (v.getKindCase() == GrpcValue.KindCase.LIST_VALUE) { + var list = new java.util.ArrayList<>(); + for (GrpcValue item : v.getListValue().getValuesList()) { + list.add(fromGrpcValue(item)); } - return null; - case ARRAY_OF_SHORTS: { - if (v.getKindCase() == GrpcValue.KindCase.LIST_VALUE) { - var out = new java.util.ArrayList(); - for (GrpcValue item : v.getListValue().getValuesList()) { - Object o = fromGrpcValue(item); - if (o instanceof Number n) - out.add(n.shortValue()); - else if (o instanceof String s) - out.add(Short.parseShort(s)); - } - return out; + return list; + } + return null; + case ARRAY_OF_SHORTS: { + if (v.getKindCase() == GrpcValue.KindCase.LIST_VALUE) { + var out = new java.util.ArrayList(); + for (GrpcValue item : v.getListValue().getValuesList()) { + Object o = fromGrpcValue(item); + if (o instanceof Number n) + out.add(n.shortValue()); + else if (o instanceof String s) + out.add(Short.parseShort(s)); } - return null; - } - - case ARRAY_OF_INTEGERS: { - if (v.getKindCase() == GrpcValue.KindCase.LIST_VALUE) { - var out = new java.util.ArrayList(); - for (GrpcValue item : v.getListValue().getValuesList()) { - Object o = fromGrpcValue(item); - if (o instanceof Number n) - out.add(n.intValue()); - else if (o instanceof String s) - out.add(Integer.parseInt(s)); - } - return out; + return out; + } + return null; + } + + case ARRAY_OF_INTEGERS: { + if (v.getKindCase() == GrpcValue.KindCase.LIST_VALUE) { + var out = new java.util.ArrayList(); + for (GrpcValue item : v.getListValue().getValuesList()) { + Object o = fromGrpcValue(item); + if (o instanceof Number n) + out.add(n.intValue()); + else if (o instanceof String s) + out.add(Integer.parseInt(s)); } - return null; - } - - case ARRAY_OF_LONGS: { - if (v.getKindCase() == GrpcValue.KindCase.LIST_VALUE) { - var out = new java.util.ArrayList(); - for (GrpcValue item : v.getListValue().getValuesList()) { - Object o = fromGrpcValue(item); - if (o instanceof Number n) - out.add(n.longValue()); - else if (o instanceof String s) - out.add(Long.parseLong(s)); - } - return out; + return out; + } + return null; + } + + case ARRAY_OF_LONGS: { + if (v.getKindCase() == GrpcValue.KindCase.LIST_VALUE) { + var out = new java.util.ArrayList(); + for (GrpcValue item : v.getListValue().getValuesList()) { + Object o = fromGrpcValue(item); + if (o instanceof Number n) + out.add(n.longValue()); + else if (o instanceof String s) + out.add(Long.parseLong(s)); } - return null; - } - - case ARRAY_OF_FLOATS: { - if (v.getKindCase() == GrpcValue.KindCase.LIST_VALUE) { - var out = new java.util.ArrayList(); - for (GrpcValue item : v.getListValue().getValuesList()) { - Object o = fromGrpcValue(item); - if (o instanceof Number n) - out.add(n.floatValue()); - else if (o instanceof String s) - out.add(Float.parseFloat(s)); - } - return out; + return out; + } + return null; + } + + case ARRAY_OF_FLOATS: { + if (v.getKindCase() == GrpcValue.KindCase.LIST_VALUE) { + var out = new java.util.ArrayList(); + for (GrpcValue item : v.getListValue().getValuesList()) { + Object o = fromGrpcValue(item); + if (o instanceof Number n) + out.add(n.floatValue()); + else if (o instanceof String s) + out.add(Float.parseFloat(s)); } - return null; - } - - case ARRAY_OF_DOUBLES: { - if (v.getKindCase() == GrpcValue.KindCase.LIST_VALUE) { - var out = new java.util.ArrayList(); - for (GrpcValue item : v.getListValue().getValuesList()) { - Object o = fromGrpcValue(item); - if (o instanceof Number n) - out.add(n.doubleValue()); - else if (o instanceof String s) - out.add(Double.parseDouble(s)); - } - return out; + return out; + } + return null; + } + + case ARRAY_OF_DOUBLES: { + if (v.getKindCase() == GrpcValue.KindCase.LIST_VALUE) { + var out = new java.util.ArrayList(); + for (GrpcValue item : v.getListValue().getValuesList()) { + Object o = fromGrpcValue(item); + if (o instanceof Number n) + out.add(n.doubleValue()); + else if (o instanceof String s) + out.add(Double.parseDouble(s)); } - return null; - } - - case DATETIME_SECOND: - case DATETIME_MICROS: - case DATETIME_NANOS: { - // Same handling as DATETIME - return switch (v.getKindCase()) { - case TIMESTAMP_VALUE -> new java.util.Date(tsToMillis(v.getTimestampValue())); - case INT64_VALUE -> new java.util.Date(v.getInt64Value()); // epoch ms expected - case STRING_VALUE -> new java.util.Date(Long.parseLong(v.getStringValue())); - default -> null; - }; - } + return out; + } + return null; + } + + case DATETIME_SECOND: + case DATETIME_MICROS: + case DATETIME_NANOS: { + // Same handling as DATETIME + return switch (v.getKindCase()) { + case TIMESTAMP_VALUE -> new java.util.Date(tsToMillis(v.getTimestampValue())); + case INT64_VALUE -> new java.util.Date(v.getInt64Value()); // epoch ms expected + case STRING_VALUE -> new java.util.Date(Long.parseLong(v.getStringValue())); + default -> null; + }; + } } // default fallback @@ -2250,49 +2270,147 @@ else if (o instanceof String s) private String generateTransactionId() { return "tx_" + System.nanoTime(); } - + // ---- Debug helpers ---- private static String summarizeJava(Object o) { - if (o == null) return "null"; + if (o == null) + return "null"; try { - if (o instanceof CharSequence s) return "String(" + s.length() + ")=\"" + (s.length() > 120 ? s.subSequence(0,120) + "…" : s) + "\""; - if (o instanceof byte[] b) return "bytes[" + b.length + "]"; - if (o instanceof java.util.Collection c) return o.getClass().getSimpleName() + "[size=" + c.size() + "]"; - if (o instanceof java.util.Map m) return o.getClass().getSimpleName() + "[size=" + m.size() + "]"; + if (o instanceof CharSequence s) + return "String(" + s.length() + ")=\"" + (s.length() > 120 ? s.subSequence(0, 120) + "…" : s) + "\""; + if (o instanceof byte[] b) + return "bytes[" + b.length + "]"; + if (o instanceof java.util.Collection c) + return o.getClass().getSimpleName() + "[size=" + c.size() + "]"; + if (o instanceof java.util.Map m) + return o.getClass().getSimpleName() + "[size=" + m.size() + "]"; return o.getClass().getSimpleName() + "(" + String.valueOf(o) + ")"; - } catch (Exception e) { return o.getClass().getSimpleName(); } + } + catch (Exception e) { + return o.getClass().getSimpleName(); + } } private static String summarizeGrpc(GrpcValue v) { - if (v == null) return "GrpcValue(null)"; + if (v == null) + return "GrpcValue(null)"; switch (v.getKindCase()) { - case BOOL_VALUE: return "BOOL(" + v.getBoolValue() + ")"; - case INT32_VALUE: return "INT32(" + v.getInt32Value() + ")"; - case INT64_VALUE: return "INT64(" + v.getInt64Value() + ")"; - case FLOAT_VALUE: return "FLOAT(" + v.getFloatValue() + ")"; - case DOUBLE_VALUE: return "DOUBLE(" + v.getDoubleValue() + ")"; - case STRING_VALUE: { - String s = v.getStringValue(); - return "STRING(" + s.length() + ")=\"" + (s.length() > 120 ? s.substring(0,120) + "…" : s) + "\""; - } - case BYTES_VALUE: return "BYTES[" + v.getBytesValue().size() + "]"; - case TIMESTAMP_VALUE: return "TIMESTAMP(" + v.getTimestampValue().getSeconds() + "." + v.getTimestampValue().getNanos() + ")"; - case LIST_VALUE: return "LIST[size=" + v.getListValue().getValuesCount() + "]"; - case MAP_VALUE: return "MAP[size=" + v.getMapValue().getEntriesCount() + "]"; - case EMBEDDED_VALUE: return "EMBEDDED[type=" + v.getEmbeddedValue().getType() + ", size=" + v.getEmbeddedValue().getFieldsCount() + "]"; - case LINK_VALUE: return "LINK(" + v.getLinkValue().getRid() + ")"; - case DECIMAL_VALUE: return "DECIMAL(unscaled=" + v.getDecimalValue().getUnscaled() + ", scale=" + v.getDecimalValue().getScale() + ")"; - case KIND_NOT_SET: default: return "GrpcValue(KIND_NOT_SET)"; + case BOOL_VALUE: + return "BOOL(" + v.getBoolValue() + ")"; + case INT32_VALUE: + return "INT32(" + v.getInt32Value() + ")"; + case INT64_VALUE: + return "INT64(" + v.getInt64Value() + ")"; + case FLOAT_VALUE: + return "FLOAT(" + v.getFloatValue() + ")"; + case DOUBLE_VALUE: + return "DOUBLE(" + v.getDoubleValue() + ")"; + case STRING_VALUE: { + String s = v.getStringValue(); + return "STRING(" + s.length() + ")=\"" + (s.length() > 120 ? s.substring(0, 120) + "…" : s) + "\""; + } + case BYTES_VALUE: + return "BYTES[" + v.getBytesValue().size() + "]"; + case TIMESTAMP_VALUE: + return "TIMESTAMP(" + v.getTimestampValue().getSeconds() + "." + v.getTimestampValue().getNanos() + ")"; + case LIST_VALUE: + return "LIST[size=" + v.getListValue().getValuesCount() + "]"; + case MAP_VALUE: + return "MAP[size=" + v.getMapValue().getEntriesCount() + "]"; + case EMBEDDED_VALUE: + return "EMBEDDED[type=" + v.getEmbeddedValue().getType() + ", size=" + v.getEmbeddedValue().getFieldsCount() + "]"; + case LINK_VALUE: + return "LINK(" + v.getLinkValue().getRid() + ")"; + case DECIMAL_VALUE: + return "DECIMAL(unscaled=" + v.getDecimalValue().getUnscaled() + ", scale=" + v.getDecimalValue().getScale() + ")"; + case KIND_NOT_SET: + default: + return "GrpcValue(KIND_NOT_SET)"; } } private GrpcValue dbgEnc(String where, Object in, GrpcValue out, String ctx) { - if (logger.isDebugEnabled()) logger.debug("GRPC-ENC [{}]{} in={} -> out={}", where, (ctx==null?"" : " "+ctx), summarizeJava(in), summarizeGrpc(out)); + if (logger.isDebugEnabled()) + logger.debug("GRPC-ENC [{}]{} in={} -> out={}", where, (ctx == null ? "" : " " + ctx), summarizeJava(in), summarizeGrpc(out)); return out; } private Object dbgDec(String where, GrpcValue in, Object out, String ctx) { - if (logger.isDebugEnabled()) logger.debug("GRPC-DEC [{}]{} in={} -> out={}", where, (ctx==null?"" : " "+ctx), summarizeGrpc(in), summarizeJava(out)); + if (logger.isDebugEnabled()) + logger.debug("GRPC-DEC [{}]{} in={} -> out={}", where, (ctx == null ? "" : " " + ctx), summarizeGrpc(in), summarizeJava(out)); return out; } + + // --- Gson structural encoder: JsonElement -> GrpcValue --- + private GrpcValue gsonToGrpc(JsonElement n) { + GrpcValue.Builder b = GrpcValue.newBuilder(); + if (n == null || n.isJsonNull()) { + return b.build(); // KIND_NOT_SET => null on the client + } + + if (n.isJsonPrimitive()) { + JsonPrimitive p = n.getAsJsonPrimitive(); + if (p.isBoolean()) + return b.setBoolValue(p.getAsBoolean()).build(); + if (p.isNumber()) { + java.math.BigDecimal bd; + try { + bd = p.getAsBigDecimal(); + } + catch (Exception e) { + return b.setDoubleValue(p.getAsDouble()).build(); + } + if (bd.scale() == 0) { + try { + int i = bd.intValueExact(); + return b.setInt32Value(i).build(); + } + catch (ArithmeticException ignore) { + } + try { + long l = bd.longValueExact(); + return b.setInt64Value(l).build(); + } + catch (ArithmeticException ignore) { + } + java.math.BigInteger unscaled = bd.unscaledValue(); + if (unscaled.bitLength() <= 63) { + return b.setDecimalValue(GrpcDecimal.newBuilder().setUnscaled(unscaled.longValue()).setScale(bd.scale()).build()) + .build(); + } + return b.setStringValue(bd.toPlainString()).build(); + } + else { + java.math.BigInteger unscaled = bd.unscaledValue(); + if (unscaled.bitLength() <= 63) { + return b.setDecimalValue(GrpcDecimal.newBuilder().setUnscaled(unscaled.longValue()).setScale(bd.scale()).build()) + .build(); + } + return b.setStringValue(bd.toPlainString()).build(); + } + } + if (p.isString()) + return b.setStringValue(p.getAsString()).build(); + return b.setStringValue(p.getAsString()).build(); + } + + if (n.isJsonArray()) { + GrpcList.Builder lb = GrpcList.newBuilder(); + for (JsonElement e : n.getAsJsonArray()) { + lb.addValues(gsonToGrpc(e)); + } + return b.setListValue(lb.build()).build(); + } + + if (n.isJsonObject()) { + GrpcMap.Builder mb = GrpcMap.newBuilder(); + JsonObject obj = n.getAsJsonObject(); + for (var entry : obj.entrySet()) { + mb.putEntries(entry.getKey(), gsonToGrpc(entry.getValue())); + } + return b.setMapValue(mb.build()).build(); + } + + return b.setStringValue(n.toString()).build(); + } } \ No newline at end of file diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcCompressionInterceptor.java b/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcCompressionInterceptor.java index c1221db2f9..d40fff0e2f 100644 --- a/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcCompressionInterceptor.java +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcCompressionInterceptor.java @@ -15,69 +15,95 @@ * Compression interceptor that can force compression based on configuration */ class GrpcCompressionInterceptor implements ServerInterceptor { - private static final Logger logger = LoggerFactory.getLogger(GrpcCompressionInterceptor.class); - - // Context key to store compression info - public static final Context.Key COMPRESSION_KEY = Context.key("compression-info"); - - private final boolean forceCompression; - private final String compressionType; - private final int minMessageSizeForCompression; - - public GrpcCompressionInterceptor(boolean forceCompression, String compressionType, int minMessageSizeBytes) { - this.forceCompression = forceCompression; - this.compressionType = compressionType != null ? compressionType : "gzip"; - this.minMessageSizeForCompression = minMessageSizeBytes; - } - - @Override - public ServerCall.Listener interceptCall( - ServerCall call, - Metadata headers, - ServerCallHandler next) { - - String methodName = call.getMethodDescriptor().getFullMethodName(); - - // Check if request is compressed - String requestEncoding = headers.get(Metadata.Key.of("grpc-encoding", Metadata.ASCII_STRING_MARSHALLER)); - boolean requestCompressed = requestEncoding != null && !requestEncoding.equals("identity"); - - // Store compression info in context - CompressionInfo compressionInfo = new CompressionInfo(requestCompressed, requestEncoding); - Context context = Context.current().withValue(COMPRESSION_KEY, compressionInfo); - - // Wrap the call to control compression - ServerCall compressedCall = new ForwardingServerCall.SimpleForwardingServerCall(call) { - private boolean compressionSet = false; - - @Override - public void sendMessage(RespT message) { - if (!compressionSet && forceCompression) { - // Force compression for this response - if (this.getMethodDescriptor().getType().serverSendsOneMessage()) { - // For unary calls, we can set compression - setMessageCompression(true); - compressionSet = true; - logger.debug("Forced {} compression for method: {}", compressionType, methodName); - } - } - super.sendMessage(message); - } - }; - - return Contexts.interceptCall(context, compressedCall, headers, next); - } - - /** - * Helper class to store compression information - */ - public static class CompressionInfo { - public final boolean requestCompressed; - public final String requestEncoding; - - public CompressionInfo(boolean requestCompressed, String requestEncoding) { - this.requestCompressed = requestCompressed; - this.requestEncoding = requestEncoding; - } - } + private static final Logger logger = LoggerFactory.getLogger(GrpcCompressionInterceptor.class); + + // Context key to store compression info + public static final Context.Key COMPRESSION_KEY = Context.key("compression-info"); + + private static final Metadata.Key GRPC_ACCEPT_ENCODING = Metadata.Key.of("grpc-accept-encoding", Metadata.ASCII_STRING_MARSHALLER); + private static final Metadata.Key GRPC_ENCODING = Metadata.Key.of("grpc-encoding", Metadata.ASCII_STRING_MARSHALLER); + + private final boolean forceCompression; + private final String compressionType; + + private final int minMessageSizeForCompression; + + public GrpcCompressionInterceptor(boolean forceCompression, String compressionType, int minMessageSizeBytes) { + this.forceCompression = forceCompression; + this.compressionType = compressionType != null ? compressionType : "gzip"; + this.minMessageSizeForCompression = minMessageSizeBytes; + } + + @Override + public ServerCall.Listener interceptCall(ServerCall call, Metadata headers, + ServerCallHandler next) { + + String methodName = call.getMethodDescriptor().getFullMethodName(); + + // Check if request is compressed + String requestEncoding = headers.get(Metadata.Key.of("grpc-encoding", Metadata.ASCII_STRING_MARSHALLER)); + boolean requestCompressed = requestEncoding != null && !requestEncoding.equals("identity"); + + final boolean clientAcceptsGzip = headers.get(GRPC_ACCEPT_ENCODING) != null && headers.get(GRPC_ACCEPT_ENCODING).contains("gzip"); + + // Store compression info in context + CompressionInfo compressionInfo = new CompressionInfo(requestCompressed, requestEncoding); + Context context = Context.current().withValue(COMPRESSION_KEY, compressionInfo); + + // Wrap the call to control compression + ServerCall compressedCall = new ForwardingServerCall.SimpleForwardingServerCall(call) { + private boolean compressionSet = false; + + @Override + public void sendHeaders(Metadata responseHeaders) { + + if (clientAcceptsGzip && !compressionSet) { + + // Pick the compressor for this call before headers go out + super.setCompression("gzip"); + setMessageCompression(true); + + logger.debug("Forced {} compression for method: {}", compressionType, methodName); + + compressionSet = true; + } + + super.sendHeaders(responseHeaders); + } + + @Override + public void sendMessage(RespT message) { + + if (!compressionSet && forceCompression) { + + // Force compression for this response + + if (this.getMethodDescriptor().getType().serverSendsOneMessage()) { + // For unary calls, we can set compression + setMessageCompression(true); + compressionSet = true; + + logger.debug("Forced {} compression for method: {}", compressionType, methodName); + } + } + + super.sendMessage(message); + } + }; + + return Contexts.interceptCall(context, compressedCall, headers, next); + } + + /** + * Helper class to store compression information + */ + public static class CompressionInfo { + public final boolean requestCompressed; + public final String requestEncoding; + + public CompressionInfo(boolean requestCompressed, String requestEncoding) { + this.requestCompressed = requestCompressed; + this.requestEncoding = requestEncoding; + } + } } \ No newline at end of file From 7850fbb4dea9abc3d94d9bec606bb3b7008709bb Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Fri, 5 Sep 2025 11:52:42 -0700 Subject: [PATCH 27/35] Checkpoint Commit --- grpc-client/docs/Readme.txt | 42 ++--------- .../RemoteGrpcDatabaseRegressionTest.java | 72 +++++++++++++++++-- .../server/grpc/ArcadeDbGrpcService.java | 36 ++++++++-- 3 files changed, 104 insertions(+), 46 deletions(-) diff --git a/grpc-client/docs/Readme.txt b/grpc-client/docs/Readme.txt index dd57c6d14e..7a9fd9c556 100644 --- a/grpc-client/docs/Readme.txt +++ b/grpc-client/docs/Readme.txt @@ -21,50 +21,22 @@ d exec -it arcadedb1-vulcan sh /home/arcadedb/config/arcadedb-log.properties -# -# Copyright 2021-present Arcade Data Ltd -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Specify the handlers to create in the root logger -# (all loggers are children of the root logger) -# The following creates two handlers handlers = java.util.logging.ConsoleHandler, java.util.logging.FileHandler -# Set the default logging level for the root logger .level = INFO com.arcadedb.level = INFO com.arcadedb.server.grpc.level = FINE -# Set the default logging level for new ConsoleHandler instances -java.util.logging.ConsoleHandler.level = INFO -# Set the default formatter for new ConsoleHandler instances +java.util.logging.ConsoleHandler.level = FINE java.util.logging.ConsoleHandler.formatter = com.arcadedb.utility.AnsiLogFormatter -# Set the default logging level for new FileHandler instances java.util.logging.FileHandler.level = FINE -# Naming style for the output file java.util.logging.FileHandler.pattern=./log/arcadedb.log -# Set the default formatter for new FileHandler instances java.util.logging.FileHandler.formatter = com.arcadedb.log.LogFormatter -# Limiting size of output file in bytes: java.util.logging.FileHandler.limit=100000000 -# Number of output files to cycle through, by appending an -# integer to the base file name: -java.util.logging.FileHandler.count=10 \ No newline at end of file +java.util.logging.FileHandler.count=10 + + + + +tail -f /home/arcadedb/log/arcadedb.log. diff --git a/grpc-client/src/test/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseRegressionTest.java b/grpc-client/src/test/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseRegressionTest.java index 23615c23aa..dcf7f57bd6 100644 --- a/grpc-client/src/test/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseRegressionTest.java +++ b/grpc-client/src/test/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseRegressionTest.java @@ -108,14 +108,14 @@ private InsertOptions defaultInsertOptions(final String targetClass, final List< } private long countAll(String type) { - + String sql = "SELECT count(*) AS c FROM " + type; - + try (ResultSet rs = grpc.query("sql", sql, Map.of())) { long c = 0; while (rs.hasNext()) { Result r = rs.next(); - //System.out.println("Count all: r = " + r); + // System.out.println("Count all: r = " + r); Number n = r.getProperty("c"); c = (n == null) ? 0 : n.longValue(); } @@ -191,7 +191,7 @@ void basicCrudViaCommand() { grpc.begin(); grpc.command("sql", "UPDATE `" + TYPE + "` SET name = :name, n = :n WHERE id = :id", Map.of("name", "ONE!", "n", 11, "id", "x1")); grpc.commit(); - + try (ResultSet rs = grpc.query("sql", "SELECT from `" + TYPE + "` WHERE id = :id", Map.of("id", "x1"))) { Result r = rs.next(); assertEquals("ONE!", r.getProperty("name")); @@ -199,7 +199,7 @@ void basicCrudViaCommand() { } // Delete - grpc.command("sql", "DELETE FROM `" + TYPE + "` WHERE id = :id", Map.of("id", "x2")); + grpc.command("sql", "DELETE FROM `" + TYPE + "` WHERE id = :id", Map.of("id", "x2")); assertEquals(1, countAll(TYPE), "one row remains after delete"); } @@ -220,4 +220,66 @@ void transactionsRollbackAndCommit() { grpc.commit(); assertEquals(before + 1, countAll(TYPE), "commit must persist insert"); } + + @Test + @DisplayName("schema:types → properties decoded as List>") + void schemaTypesPropertiesDecodedAsMaps() { + + try (ResultSet rs = grpc.query("sql", "SELECT FROM schema:types WHERE name = :name", Map.of("name", TYPE))) { + + assertTrue(rs.hasNext(), "schema:types should contain our test type"); + + Result r = rs.next(); + + Object propsObj = r.getProperty("properties"); + + assertTrue(propsObj instanceof java.util.List, "properties must be a List"); + + java.util.List props = (java.util.List) propsObj; + assertTrue(!props.isEmpty(), "properties list should not be empty"); + + Object first = props.get(0); + assertTrue(first instanceof java.util.Map, "each property is expected to be a Map"); + + java.util.Map p0 = (java.util.Map) first; + + // Spot-check expected keys + assertTrue(p0.containsKey("name"), "property map must have 'name'"); + assertTrue(p0.containsKey("type"), "property map must have 'type'"); + } + } + + @Test + @DisplayName("Embedded _auditMetadata map round-trips with Long timestamps") + void embeddedAuditMetadataRoundTrip() { + + final String recId = "audit1"; + final java.util.Map audit = new java.util.LinkedHashMap<>(); + + audit.put("createdDate", 1720225210408L); + audit.put("createdByUser", "service-account-empower-platform-admin"); + audit.put("lastModifiedDate", 1741795459718L); + audit.put("lastModifiedByUser", "service-account-empower-platform-admin"); + + grpc.command("sql", "INSERT INTO `" + TYPE + "` SET id = :id, name = :name, n = :n, _auditMetadata = :audit", + Map.of("id", recId, "name", "with-audit", "n", 1, "audit", audit)); + + try (ResultSet rs = grpc.query("sql", "SELECT FROM `" + TYPE + "` WHERE id = :id", Map.of("id", recId))) { + assertTrue(rs.hasNext(), "inserted record should be queriable"); + Result r = rs.next(); + Object auditObj = r.getProperty("_auditMetadata"); + assertTrue(auditObj instanceof java.util.Map, "_auditMetadata must be a Map"); + @SuppressWarnings("unchecked") + java.util.Map m = (java.util.Map) auditObj; + + Object cd = m.get("createdDate"); + Object md = m.get("lastModifiedDate"); + assertTrue(cd instanceof Number, "createdDate must be numeric"); + assertTrue(md instanceof Number, "lastModifiedDate must be numeric"); + assertEquals(1720225210408L, ((Number) cd).longValue(), "createdDate must be precise long"); + assertEquals(1741795459718L, ((Number) md).longValue(), "lastModifiedDate must be precise long"); + assertEquals("service-account-empower-platform-admin", m.get("createdByUser")); + assertEquals("service-account-empower-platform-admin", m.get("lastModifiedByUser")); + } + } } diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java index f905d5dd8d..0e2b562c9c 100644 --- a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java @@ -1,11 +1,5 @@ package com.arcadedb.server.grpc; -import com.google.gson.JsonElement; -import com.google.gson.JsonPrimitive; -import com.google.gson.JsonObject; -import com.google.gson.JsonArray; -import com.google.gson.JsonNull; - import java.util.HashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -34,6 +28,9 @@ import com.arcadedb.server.ArcadeDBServer; import com.arcadedb.server.grpc.InsertOptions.ConflictMode; import com.arcadedb.server.grpc.InsertOptions.TransactionMode; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; import com.google.protobuf.Timestamp; import io.grpc.Status; @@ -1678,6 +1675,11 @@ private static com.google.protobuf.Timestamp msToTimestamp(long ms) { private GrpcValue toGrpcValue(Object o) { + if (logger.isDebugEnabled()) { + logger.debug("toGrpcValue(): Converting\n value ={}\n class = {}", o, o.getClass()); + } + + // class com.arcadedb.query.sql.executor.ResultInternal if (o instanceof JsonElement je) { return gsonToGrpc(je); } @@ -1765,6 +1767,18 @@ private GrpcValue toGrpcValue(Object o) { return dbgEnc("toGrpcValue", o, GrpcValue.newBuilder() .setLinkValue(GrpcLink.newBuilder().setRid(id.getIdentity().toString()).build()).setLogicalType("rid").build(), null); + // Support ArcadeDB Result/ResultInternal as structural MAP + if (o instanceof com.arcadedb.query.sql.executor.Result res) { + GrpcMap.Builder mb = GrpcMap.newBuilder(); + try { + for (String k : res.getPropertyNames()) { + mb.putEntries(k, toGrpcValue(res.getProperty(k))); + } + } catch (Throwable ignore) { } + + return dbgEnc("toGrpcValue", o, GrpcValue.newBuilder().setMapValue(mb.build()).build(), null); + } + // Fallback return dbgEnc("toGrpcValue", o, GrpcValue.newBuilder().setStringValue(String.valueOf(o)).build(), null); } @@ -1969,9 +1983,13 @@ private GrpcRecord convertToGrpcRecord(com.arcadedb.database.Record dbRecord, Da for (String propertyName : doc.getPropertyNames()) { Object value = doc.get(propertyName); if (value != null) { + if (logger.isDebugEnabled()) { + logger.debug("convertToGrpcRecord(): Converting {}\n value = {}\n class = {}", propertyName, value, value.getClass()); + } GrpcValue gv = toGrpcValue(value); if (logger.isDebugEnabled()) logger.debug("ENC-REC {}.{}: {} -> {}", builder.getRid(), propertyName, summarizeJava(value), summarizeGrpc(gv)); + builder.putProperties(propertyName, gv); } } @@ -1996,6 +2014,10 @@ private GrpcValue convertPropToGrpcValue(String propName, Result result) { Object propValue = result.getProperty(propName); + if (logger.isDebugEnabled()) { + logger.debug("convertPropToGrpcValue(): Converting {}\n value = {}\n class = {}", propName, propValue, propValue.getClass()); + } + return toGrpcValue(propValue); } @@ -2343,7 +2365,9 @@ private Object dbgDec(String where, GrpcValue in, Object out, String ctx) { // --- Gson structural encoder: JsonElement -> GrpcValue --- private GrpcValue gsonToGrpc(JsonElement n) { + GrpcValue.Builder b = GrpcValue.newBuilder(); + if (n == null || n.isJsonNull()) { return b.build(); // KIND_NOT_SET => null on the client } From 511cbd122f53b21bbb8bef8b894fd6699fa1f4cd Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Fri, 5 Sep 2025 22:43:48 -0700 Subject: [PATCH 28/35] Support Schema Types retrieval using gRPC --- .../remote/grpc/RemoteGrpcDatabase.java | 203 +++++++++++++----- .../server/grpc/ArcadeDbGrpcService.java | 20 ++ .../arcadedb/remote/RemoteDocumentType.java | 18 +- 3 files changed, 184 insertions(+), 57 deletions(-) diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java index cd13013a75..bf886df485 100644 --- a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java @@ -157,7 +157,7 @@ private static String summarize(GrpcRecord r) { private String transactionId; - private final RemoteSchema schema; + private RemoteSchema schema; private final String userName; private final String userPassword; @@ -386,14 +386,60 @@ public Iterator iterateType(final String typeName, final boolean polymor public Iterator iterateBucket(final String bucketName) { return streamQuery("select from bucket:`" + bucketName + "`"); } + +// @Override +// public ResultSet command(final String language, final String command) { +// checkDatabaseIsOpen(); +// stats.commands.incrementAndGet(); +// +// final Map params = new HashMap(); +// +// return commandInternal(language, command, params); +// } +// @Override - public ResultSet query(final String language, final String query, final Map params) { + public ResultSet command(final String language, final String command, final ContextConfiguration configuration, final Object... args) { + checkDatabaseIsOpen(); - stats.queries.incrementAndGet(); + stats.commands.incrementAndGet(); - ExecuteQueryRequest.Builder requestBuilder = ExecuteQueryRequest.newBuilder().setDatabase(getName()).setQuery(query) - .setCredentials(buildCredentials()); + final Map params = mapArgs(args); + + return commandInternal(language, command, params); + } + + @Override + public ResultSet command(final String language, final String command, final ContextConfiguration configuration, + final Map params) { + checkDatabaseIsOpen(); + stats.commands.incrementAndGet(); + + return commandInternal(language, command, params); + } + + @Override + public ResultSet command(final String language, final String command, final Object... args) { + checkDatabaseIsOpen(); + stats.commands.incrementAndGet(); + + final Map params = mapArgs(args); + + return commandInternal(language, command, params); + } + + @Override + public ResultSet command(final String language, final String command, final Map params) { + checkDatabaseIsOpen(); + stats.commands.incrementAndGet(); + + return commandInternal(language, command, params); + } + + private ResultSet commandInternal(final String language, final String command, final Map params) { + + ExecuteCommandRequest.Builder requestBuilder = ExecuteCommandRequest.newBuilder().setDatabase(getName()).setCommand(command) + .setLanguage(language).setCredentials(buildCredentials()); if (transactionId != null) { requestBuilder.setTransaction(TransactionContext.newBuilder().setTransactionId(transactionId).setDatabase(getName()).build()); @@ -402,23 +448,47 @@ public ResultSet query(final String language, final String query, final Map 0) { + + Map result = new HashMap<>(); + result.put("affected", response.getAffectedRecords()); + result.put("executionTime", response.getExecutionTimeMs()); + + ((InternalResultSet)resultSet).add(new ResultInternal(result)); + } + } + + return resultSet; } catch (StatusRuntimeException e) { handleGrpcException(e); @@ -429,23 +499,13 @@ public ResultSet query(final String language, final String query, final Map params) { checkDatabaseIsOpen(); stats.queries.incrementAndGet(); - final Map params = mapArgs(args); - - return query(language, query, params); - } - - @Override - public ResultSet command(final String language, final String command, final Map params) { - checkDatabaseIsOpen(); - stats.commands.incrementAndGet(); - - ExecuteCommandRequest.Builder requestBuilder = ExecuteCommandRequest.newBuilder().setDatabase(getName()).setCommand(command) + ExecuteQueryRequest.Builder requestBuilder = ExecuteQueryRequest.newBuilder().setDatabase(getName()).setQuery(query) .setCredentials(buildCredentials()); if (transactionId != null) { @@ -459,29 +519,19 @@ public ResultSet command(final String language, final String command, final Map< try { if (logger.isDebugEnabled()) - logger.debug("CLIENT executeCommand: db={}, tx={}, cmdLen={}, params={}", getName(), (transactionId != null), - requestBuilder.getCommand().length(), requestBuilder.getParametersCount()); - - ExecuteCommandResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) - .executeCommand(requestBuilder.build()); - - if (logger.isDebugEnabled()) - logger.debug("CLIENT executeCommand: success = {}", response.getSuccess()); - - if (!response.getSuccess()) { - throw new DatabaseOperationException("Failed to execute command: " + response.getMessage()); - } + logger.debug("CLIENT executeQuery: db={}, tx={}, queryLen={}, params={}", getName(), (transactionId != null), + requestBuilder.getQuery().length(), requestBuilder.getParametersCount()); - // Create result set with command execution info - InternalResultSet resultSet = new InternalResultSet(); + ExecuteQueryResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .executeQuery(requestBuilder.build()); - if (response.getAffectedRecords() > 0) { - Map result = new HashMap<>(); - result.put("affected", response.getAffectedRecords()); - result.put("executionTime", response.getExecutionTimeMs()); - resultSet.add(new ResultInternal(result)); + if (logger.isDebugEnabled()) { + int _r = 0; + for (var qr : response.getResultsList()) + _r += qr.getRecordsCount(); + logger.debug("CLIENT executeQuery: results={}", _r); } - return resultSet; + return createGrpcResultSet(response); } catch (StatusRuntimeException e) { handleGrpcException(e); @@ -493,6 +543,17 @@ public ResultSet command(final String language, final String command, final Map< } } + @Override + public ResultSet query(final String language, final String query, final Object... args) { + checkDatabaseIsOpen(); + stats.queries.incrementAndGet(); + + final Map params = mapArgs(args); + + return query(language, query, params); + } + + public ExecuteCommandResponse execSql(String db, String sql, Map params, long timeoutMs) { return executeCommand(db, "sql", sql, params, /* returnRows */ false, /* maxRows */ 0, txBeginCommit(), timeoutMs); } @@ -1581,6 +1642,14 @@ private ResultSet createGrpcResultSet(ExecuteQueryResponse response) { return resultSet; } + private ResultSet createGrpcResultSet(ExecuteCommandResponse response) { + InternalResultSet resultSet = new InternalResultSet(); + for (GrpcRecord record : response.getRecordsList()) { + resultSet.add(grpcRecordToResult(record)); + } + return resultSet; + } + private Result grpcRecordToResult(GrpcRecord grpcRecord) { Record record = grpcRecordToDBRecord(grpcRecord); @@ -1605,9 +1674,24 @@ private Record grpcRecordToDBRecord(GrpcRecord grpcRecord) { // Add metadata map.put("@rid", grpcRecord.getRid()); map.put("@type", grpcRecord.getType()); - map.put("@cat", mapRecordType(grpcRecord)); - String cat = (String) map.get("@cat"); + GrpcValue catFromGrpcRecord = grpcRecord.getPropertiesMap().get("@cat"); + + String cat = null; + + if (catFromGrpcRecord != null) { + + cat = catFromGrpcRecord.getStringValue(); + } + else { + + cat = mapRecordType(grpcRecord); + } + + if (cat != null) { + + map.put("@cat", cat); + } if (cat == null) { @@ -1616,11 +1700,17 @@ private Record grpcRecordToDBRecord(GrpcRecord grpcRecord) { switch (cat) { case "d": + return new RemoteImmutableDocument(this, map); + case "v": + return new RemoteImmutableVertex(this, map); + case "e": + return new RemoteImmutableEdge(this, map); + default: return null; } @@ -1634,20 +1724,24 @@ private String mapRecordType(GrpcRecord grpcRecord) { // Check schema to determine actual type try { - if (getSchema().existsType(typeName)) { + if (typeName != null && !typeName.isBlank() && getSchema().existsType(typeName)) { Object type = getSchema().getType(typeName); if (type instanceof com.arcadedb.schema.VertexType) { + return "v"; } else if (type instanceof com.arcadedb.schema.EdgeType) { + return "e"; } else if (type instanceof com.arcadedb.schema.DocumentType) { + return "d"; } else { + return null; } } @@ -1657,10 +1751,9 @@ else if (type instanceof com.arcadedb.schema.DocumentType) { } } catch (Exception e) { - // Fall back to name-based detection - } - return null; + throw new RuntimeException(e); + } } private Map convertParamsToGrpcValue(Map params) { diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java index 0e2b562c9c..c8bc2632bc 100644 --- a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java @@ -630,9 +630,15 @@ public void executeQuery(ExecuteQueryRequest request, StreamObserver @rid = {}", dbRecord.asDocument().get("@rid")); GrpcRecord grpcRecord = convertToGrpcRecord(dbRecord, database); + + logger.debug("executeQuery(): grpcRecord -> @rid = {}", grpcRecord.getRid()); + resultBuilder.addRecords(grpcRecord); + count++; // Apply limit if specified @@ -1973,20 +1979,30 @@ private Map convertParameters(Map protoParams } private GrpcRecord convertToGrpcRecord(com.arcadedb.database.Record dbRecord, Database db) { + GrpcRecord.Builder builder = GrpcRecord.newBuilder().setRid(dbRecord.getIdentity().toString()); if (dbRecord instanceof Document doc) { + if (doc.getType() != null) builder.setType(doc.getTypeName()); + if (doc.getIdentity() != null) + builder.setRid(doc.getIdentity().toString()); + // set all properties for (String propertyName : doc.getPropertyNames()) { + Object value = doc.get(propertyName); + if (value != null) { + if (logger.isDebugEnabled()) { logger.debug("convertToGrpcRecord(): Converting {}\n value = {}\n class = {}", propertyName, value, value.getClass()); } + GrpcValue gv = toGrpcValue(value); + if (logger.isDebugEnabled()) logger.debug("ENC-REC {}.{}: {} -> {}", builder.getRid(), propertyName, summarizeJava(value), summarizeGrpc(gv)); @@ -1996,9 +2012,12 @@ private GrpcRecord convertToGrpcRecord(com.arcadedb.database.Record dbRecord, Da // If this is an Edge, include @out / @in for convenience (string rid or link) if (dbRecord instanceof Edge edge) { + if (!builder.getPropertiesMap().containsKey("@out")) { builder.putProperties("@out", toGrpcValue(edge.getOut().getIdentity())); } + + if (!builder.getPropertiesMap().containsKey("@in")) { builder.putProperties("@in", toGrpcValue(edge.getIn().getIdentity())); } @@ -2007,6 +2026,7 @@ private GrpcRecord convertToGrpcRecord(com.arcadedb.database.Record dbRecord, Da if (logger.isDebugEnabled()) logger.debug("ENC-REC DONE rid={} type={} props={}", builder.getRid(), builder.getType(), builder.getPropertiesCount()); + return builder.build(); } diff --git a/network/src/main/java/com/arcadedb/remote/RemoteDocumentType.java b/network/src/main/java/com/arcadedb/remote/RemoteDocumentType.java index b1a9517568..98c3d7d280 100644 --- a/network/src/main/java/com/arcadedb/remote/RemoteDocumentType.java +++ b/network/src/main/java/com/arcadedb/remote/RemoteDocumentType.java @@ -64,8 +64,22 @@ public class RemoteDocumentType implements DocumentType { } void reload(final Result record) { - count = record.getProperty("records"); - buckets = record.getProperty("buckets"); + + Object recordsAsObject = record.getProperty("records"); + + if (recordsAsObject != null) { + + if (recordsAsObject instanceof Integer recordsAsInt) { + count = recordsAsInt; + } + else if (recordsAsObject instanceof Long recordsAsLong) { + count = recordsAsLong.intValue(); + } + } + + // count = record.getProperty("records"); + + buckets = record.getProperty("buckets"); bucketSelectionStrategy = record.getProperty("bucketSelectionStrategy"); parentTypes = record.getProperty("parentTypes"); From 99dc6d2095fd3951674914b50313e20dbf808bb7 Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Sun, 7 Sep 2025 11:52:10 -0700 Subject: [PATCH 29/35] Added Project Serialization --- grpc-client/docs/Readme.txt | 11 +- .../remote/grpc/RemoteGrpcDatabase.java | 183 +++---- .../RemoteGrpcDatabaseWithCompression.java | 31 +- .../remote/grpc/RemoteGrpcServer.java | 161 ++++-- .../remote/grpc/utils/ProtoUtils.java | 310 ++++++----- .../remote/grpc/ArcadeDbHTTPTvsGRPCBench.java | 17 +- .../RemoteGrpcDatabaseRegressionTest.java | 20 +- grpc/src/main/proto/arcadedb-server.proto | 15 +- .../server/grpc/ArcadeDbGrpcService.java | 489 ++++++++++++++++-- .../server/grpc/GrpcServerPlugin.java | 13 +- 10 files changed, 869 insertions(+), 381 deletions(-) diff --git a/grpc-client/docs/Readme.txt b/grpc-client/docs/Readme.txt index 7a9fd9c556..9ea64f2f92 100644 --- a/grpc-client/docs/Readme.txt +++ b/grpc-client/docs/Readme.txt @@ -18,7 +18,7 @@ Logging: d exec -it arcadedb1-vulcan sh -/home/arcadedb/config/arcadedb-log.properties +vi /home/arcadedb/config/arcadedb-log.properties handlers = java.util.logging.ConsoleHandler, java.util.logging.FileHandler @@ -27,7 +27,7 @@ handlers = java.util.logging.ConsoleHandler, java.util.logging.FileHandler com.arcadedb.level = INFO com.arcadedb.server.grpc.level = FINE -java.util.logging.ConsoleHandler.level = FINE +java.util.logging.ConsoleHandler.level = INFO java.util.logging.ConsoleHandler.formatter = com.arcadedb.utility.AnsiLogFormatter java.util.logging.FileHandler.level = FINE @@ -39,4 +39,9 @@ java.util.logging.FileHandler.count=10 -tail -f /home/arcadedb/log/arcadedb.log. +tail -f /home/arcadedb/log/arcadedb.log.0 + + + + +d exec -it arcadedb1-vulcan sh \ No newline at end of file diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java index bf886df485..e3f9ab9f21 100644 --- a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java @@ -53,6 +53,7 @@ import com.arcadedb.server.grpc.ExecuteCommandRequest; import com.arcadedb.server.grpc.ExecuteCommandResponse; import com.arcadedb.server.grpc.ExecuteQueryRequest; +import com.arcadedb.server.grpc.ExecuteQueryRequest.ProjectionEncoding; import com.arcadedb.server.grpc.ExecuteQueryResponse; import com.arcadedb.server.grpc.GrpcRecord; import com.arcadedb.server.grpc.GrpcValue; @@ -73,12 +74,9 @@ import com.arcadedb.server.grpc.TransactionIsolation; import com.arcadedb.server.grpc.UpdateRecordRequest; import com.arcadedb.server.grpc.UpdateRecordResponse; +import com.google.protobuf.Int32Value; import io.grpc.CallCredentials; -import io.grpc.CompressorRegistry; -import io.grpc.DecompressorRegistry; -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; import io.grpc.Metadata; import io.grpc.Status; import io.grpc.StatusException; @@ -100,58 +98,6 @@ public class RemoteGrpcDatabase extends RemoteDatabase { private static final Logger logger = LoggerFactory.getLogger(RemoteGrpcDatabase.class); -// --- Debug helpers (client) --- - private static String summarize(Object o) { - if (o == null) - return "null"; - try { - if (o instanceof CharSequence s) - return "String(" + s.length() + ")"; - if (o instanceof byte[] b) - return "bytes[" + b.length + "]"; - if (o instanceof java.util.Collection c) - return o.getClass().getSimpleName() + "[size=" + c.size() + "]"; - if (o instanceof java.util.Map m) - return o.getClass().getSimpleName() + "[size=" + m.size() + "]"; - return o.getClass().getSimpleName(); - } - catch (Exception e) { - return o.getClass().getSimpleName(); - } - } - - private static String summarize(GrpcValue v) { - if (v == null) - return "GrpcValue(null)"; - return switch (v.getKindCase()) { - case BOOL_VALUE -> "BOOL"; - case INT32_VALUE -> "INT32"; - case INT64_VALUE -> "INT64"; - case FLOAT_VALUE -> "FLOAT"; - case DOUBLE_VALUE -> "DOUBLE"; - case STRING_VALUE -> "STRING(" + v.getStringValue().length() + ")"; - case BYTES_VALUE -> "BYTES[" + v.getBytesValue().size() + "]"; - case TIMESTAMP_VALUE -> "TIMESTAMP"; - case LIST_VALUE -> "LIST[" + v.getListValue().getValuesCount() + "]"; - case MAP_VALUE -> "MAP[" + v.getMapValue().getEntriesCount() + "]"; - case EMBEDDED_VALUE -> "EMBEDDED"; - case LINK_VALUE -> "LINK(" + v.getLinkValue().getRid() + ")"; - case DECIMAL_VALUE -> "DECIMAL(scale=" + v.getDecimalValue().getScale() + ")"; - case KIND_NOT_SET -> "KIND_NOT_SET"; - }; - } - - private static String summarize(GrpcRecord r) { - if (r == null) - return "GrpcRecord(null)"; - String rid = r.getRid(); - String ty = r.getType(); - int props = r.getPropertiesCount(); - return "GrpcRecord{rid=" + rid + ", type=" + ty + ", props=" + props + "}"; - } - - private final ManagedChannel channel; - private final ArcadeDbServiceGrpc.ArcadeDbServiceBlockingV2Stub blockingStub; private final ArcadeDbServiceGrpc.ArcadeDbServiceStub asyncStub; @@ -163,41 +109,33 @@ private static String summarize(GrpcRecord r) { private final String userPassword; private String databaseName; private RemoteTransactionExplicitLock explicitLock; + + protected RemoteGrpcServer remoteGrpcServer; - public RemoteGrpcDatabase(final String server, final int grpcPort, final int httpPort, final String databaseName, final String userName, + public RemoteGrpcDatabase(final RemoteGrpcServer remoteGrpcServer, final String server, final int grpcPort, final int httpPort, final String databaseName, final String userName, final String userPassword) { - this(server, grpcPort, httpPort, databaseName, userName, userPassword, new ContextConfiguration()); + this(remoteGrpcServer, server, grpcPort, httpPort, databaseName, userName, userPassword, new ContextConfiguration()); } - public RemoteGrpcDatabase(final String server, final int grpcPort, final int httpPort, final String databaseName, final String userName, + public RemoteGrpcDatabase(final RemoteGrpcServer remoteGrpcServer, final String host, final int grpcPort, final int httpPort, final String databaseName, final String userName, final String userPassword, final ContextConfiguration configuration) { - super(server, httpPort, databaseName, userName, userPassword, configuration); + super(host, httpPort, databaseName, userName, userPassword, configuration); + + this.remoteGrpcServer = remoteGrpcServer; this.userName = userName; this.userPassword = userPassword; this.databaseName = databaseName; - // Create gRPC channel - this.channel = createChannel(server, grpcPort); + this.blockingStub = createBlockingStub(); - this.blockingStub = createBlockingStub(channel); + this.asyncStub = createAsyncStub(); - this.asyncStub = createAsyncStub(channel); - - // Create gRPC-specific schema this.schema = new RemoteSchema(this); } - /** - * Creates the gRPC channel. Override this for custom channel configuration. - */ - protected ManagedChannel createChannel(String server, int port) { - return ManagedChannelBuilder.forAddress(server, port).usePlaintext() // No TLS by default - .build(); - } - /** * Creates call credentials for authentication */ @@ -228,17 +166,17 @@ protected CallCredentials createCredentials() { /** * Override this method to customize blocking stub creation */ - protected ArcadeDbServiceGrpc.ArcadeDbServiceBlockingV2Stub createBlockingStub(ManagedChannel channel) { - return ArcadeDbServiceGrpc.newBlockingV2Stub(channel).withCallCredentials(createCredentials()); + protected ArcadeDbServiceGrpc.ArcadeDbServiceBlockingV2Stub createBlockingStub() { + return this.remoteGrpcServer.newBlockingStub(getTimeout()); } - protected ArcadeDbServiceGrpc.ArcadeDbServiceStub createAsyncStub(ManagedChannel channel) { - return ArcadeDbServiceGrpc.newStub(channel).withCallCredentials(createCredentials()).withCompression("gzip"); + protected ArcadeDbServiceGrpc.ArcadeDbServiceStub createAsyncStub() { + return this.remoteGrpcServer.newAsyncStub(getTimeout()); } @Override public String getDatabasePath() { - return "grpc://" + channel.authority() + "/" + getName(); + return "grpc://" + this.remoteGrpcServer.channel().authority() + "/" + getName(); } @Override @@ -248,21 +186,11 @@ public RemoteSchema getSchema() { @Override public void close() { + if (transactionId != null) { rollback(); } super.close(); - - // Shutdown gRPC channel - channel.shutdown(); - try { - if (!channel.awaitTermination(5, TimeUnit.SECONDS)) { - channel.shutdownNow(); - } - } - catch (InterruptedException e) { - channel.shutdownNow(); - } } @Override @@ -515,6 +443,10 @@ public ResultSet query(final String language, final String query, final Map c) + return o.getClass().getSimpleName() + "[size=" + c.size() + "]"; + if (o instanceof java.util.Map m) + return o.getClass().getSimpleName() + "[size=" + m.size() + "]"; + return o.getClass().getSimpleName(); + } + catch (Exception e) { + return o.getClass().getSimpleName(); + } + } + + private static String summarize(GrpcValue v) { + if (v == null) + return "GrpcValue(null)"; + return switch (v.getKindCase()) { + case BOOL_VALUE -> "BOOL"; + case INT32_VALUE -> "INT32"; + case INT64_VALUE -> "INT64"; + case FLOAT_VALUE -> "FLOAT"; + case DOUBLE_VALUE -> "DOUBLE"; + case STRING_VALUE -> "STRING(" + v.getStringValue().length() + ")"; + case BYTES_VALUE -> "BYTES[" + v.getBytesValue().size() + "]"; + case TIMESTAMP_VALUE -> "TIMESTAMP"; + case LIST_VALUE -> "LIST[" + v.getListValue().getValuesCount() + "]"; + case MAP_VALUE -> "MAP[" + v.getMapValue().getEntriesCount() + "]"; + case EMBEDDED_VALUE -> "EMBEDDED"; + case LINK_VALUE -> "LINK(" + v.getLinkValue().getRid() + ")"; + case DECIMAL_VALUE -> "DECIMAL(scale=" + v.getDecimalValue().getScale() + ")"; + case KIND_NOT_SET -> "KIND_NOT_SET"; + }; + } + + private static String summarize(GrpcRecord r) { + if (r == null) + return "GrpcRecord(null)"; + String rid = r.getRid(); + String ty = r.getType(); + int props = r.getPropertiesCount(); + return "GrpcRecord{rid=" + rid + ", type=" + ty + ", props=" + props + "}"; + } } \ No newline at end of file diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseWithCompression.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseWithCompression.java index 220d0c637e..e985407250 100644 --- a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseWithCompression.java +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseWithCompression.java @@ -1,41 +1,14 @@ package com.arcadedb.remote.grpc; -import java.util.concurrent.TimeUnit; - import com.arcadedb.ContextConfiguration; -import com.arcadedb.server.grpc.ArcadeDbServiceGrpc; - -import io.grpc.ManagedChannel; -import io.grpc.ManagedChannelBuilder; /** * Extended RemoteGrpcDatabase with compression support */ public class RemoteGrpcDatabaseWithCompression extends RemoteGrpcDatabase { - public RemoteGrpcDatabaseWithCompression(String server, int grpcPort, int httpPort, String databaseName, String userName, + public RemoteGrpcDatabaseWithCompression(RemoteGrpcServer remoteGrpcServer, String host, int grpcPort, int httpPort, String databaseName, String userName, String userPassword, ContextConfiguration configuration) { - super(server, grpcPort, httpPort, databaseName, userName, userPassword, configuration); - } - - @Override - protected ManagedChannel createChannel(String server, int port) { - - return ManagedChannelBuilder.forAddress(server, port).usePlaintext() // No TLS/SSL - .maxInboundMessageSize(100 * 1024 * 1024) // 100MB max message size - .keepAliveTime(30, TimeUnit.SECONDS) // Keep-alive configuration - .keepAliveTimeout(10, TimeUnit.SECONDS).keepAliveWithoutCalls(true).build(); - } - - @Override - protected ArcadeDbServiceGrpc.ArcadeDbServiceBlockingV2Stub createBlockingStub(ManagedChannel channel) { - return ArcadeDbServiceGrpc.newBlockingV2Stub(channel).withCompression("gzip") // Enable GZIP compression - .withCallCredentials(createCredentials()).withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS); - } - - @Override - protected ArcadeDbServiceGrpc.ArcadeDbServiceStub createAsyncStub(ManagedChannel channel) { - return ArcadeDbServiceGrpc.newStub(channel).withCompression("gzip") // Enable GZIP compression - .withCallCredentials(createCredentials()).withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS); + super(remoteGrpcServer, host, grpcPort, httpPort, databaseName, userName, userPassword, configuration); } } diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcServer.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcServer.java index f5093dea7c..74b8d9689b 100644 --- a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcServer.java +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcServer.java @@ -5,7 +5,10 @@ import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; +import javax.annotation.PreDestroy; + import com.arcadedb.server.grpc.ArcadeDbAdminServiceGrpc; +import com.arcadedb.server.grpc.ArcadeDbServiceGrpc; import com.arcadedb.server.grpc.CreateDatabaseRequest; import com.arcadedb.server.grpc.DatabaseCredentials; import com.arcadedb.server.grpc.DropDatabaseRequest; @@ -13,10 +16,19 @@ import com.arcadedb.server.grpc.ListDatabasesResponse; import io.grpc.CallCredentials; +import io.grpc.Channel; +import io.grpc.ClientInterceptor; +import io.grpc.ClientInterceptors; +import io.grpc.CompressorRegistry; +import io.grpc.DecompressorRegistry; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import io.grpc.Metadata; import io.grpc.StatusException; +import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder; +import io.grpc.netty.shaded.io.netty.channel.EventLoopGroup; +import io.grpc.netty.shaded.io.netty.channel.nio.NioEventLoopGroup; +import io.grpc.netty.shaded.io.netty.channel.socket.nio.NioSocketChannel; import io.grpc.stub.AbstractStub; /** @@ -39,40 +51,124 @@ public class RemoteGrpcServer implements AutoCloseable { private final String userPassword; private final long defaultTimeoutMs; + + private final List interceptors; + private final boolean plaintext; - private final ManagedChannel channel; - private final ArcadeDbAdminServiceGrpc.ArcadeDbAdminServiceBlockingV2Stub blockingV2Stub; + private ManagedChannel channel; + + private ArcadeDbAdminServiceGrpc.ArcadeDbAdminServiceBlockingV2Stub adminServiceBlockingV2Stub; - public RemoteGrpcServer(final String host, final int port, final String user, final String pass) { - this(host, port, user, pass, 30_000); + public RemoteGrpcServer(final String host, final int port, final String user, final String pass, boolean plaintext, List interceptors) { + this(host, port, user, pass, plaintext, interceptors, 30_000); } - public RemoteGrpcServer(final String host, final int port, final String user, final String pass, final long defaultTimeoutMs) { - + public RemoteGrpcServer(final String host, final int port, final String user, final String pass, boolean plaintext, List interceptors, final long defaultTimeoutMs) { + this.host = Objects.requireNonNull(host, "host"); - + this.port = port; - + + this.plaintext = plaintext; + this.interceptors = interceptors == null ? List.of() : List.copyOf(interceptors); + this.userName = Objects.requireNonNull(user, "user"); - + this.userPassword = Objects.requireNonNull(pass, "pass"); - + this.defaultTimeoutMs = defaultTimeoutMs > 0 ? defaultTimeoutMs : 30_000; + } - this.channel = ManagedChannelBuilder.forAddress(this.host, this.port).usePlaintext() // switch to TLS if enabled server-side - .build(); + + public synchronized void start() { + + if (channel != null) return; + +// NettyChannelBuilder b = NettyChannelBuilder.forAddress(host, port) +// .maxInboundMessageSize(100 * 1024 * 1024) // 100MB max message size +// .keepAliveTime(30, TimeUnit.SECONDS) // Keep-alive configuration +// .keepAliveTimeout(10, TimeUnit.SECONDS) +// .keepAliveWithoutCalls(true) +// .decompressorRegistry(DecompressorRegistry.getDefaultInstance()) +// .compressorRegistry(CompressorRegistry.getDefaultInstance()); + + + EventLoopGroup elg = new NioEventLoopGroup(); + + NettyChannelBuilder chBuilder = NettyChannelBuilder.forAddress(host, port) + .eventLoopGroup(elg) // share across clients + .channelType(NioSocketChannel.class) + .maxInboundMessageSize(150 * 1024 * 1024) + .maxInboundMetadataSize(32 * 1024 * 1024) + .keepAliveTime(30, TimeUnit.SECONDS) + .keepAliveTimeout(10, TimeUnit.SECONDS) + .keepAliveWithoutCalls(true) + .decompressorRegistry(DecompressorRegistry.getDefaultInstance()) + .compressorRegistry(CompressorRegistry.getDefaultInstance()) + .flowControlWindow(8 * 1024 * 1024); + + // .proxyDetector(myProxyDetector) + + if (plaintext) chBuilder.usePlaintext(); + + channel = chBuilder.build(); + } + + /** Returns a Channel (wrapped with interceptors if provided). */ + public Channel channel() { + + if (channel == null) { + start(); + } + + return interceptors.isEmpty() ? channel : ClientInterceptors.intercept(channel, interceptors); + } - this.blockingV2Stub = ArcadeDbAdminServiceGrpc.newBlockingV2Stub(channel).withCallCredentials(createCallCredentials(userName, userPassword)); + public ArcadeDbServiceGrpc.ArcadeDbServiceBlockingV2Stub newBlockingStub(int timeout) { + + return ArcadeDbServiceGrpc.newBlockingV2Stub(channel()) + .withCallCredentials(createCredentials()) + .withDeadlineAfter(timeout, TimeUnit.MILLISECONDS) + .withCompression("gzip"); } + public ArcadeDbServiceGrpc.ArcadeDbServiceStub newAsyncStub(int timeout) { + + return ArcadeDbServiceGrpc.newStub(channel()) + .withCallCredentials(createCredentials()) + .withDeadlineAfter(timeout, TimeUnit.MILLISECONDS) + .withCompression("gzip"); + } + + private ArcadeDbAdminServiceGrpc.ArcadeDbAdminServiceBlockingV2Stub adminServiceBlockingV2Stub() { + + if (this.adminServiceBlockingV2Stub == null) { + + this.adminServiceBlockingV2Stub = ArcadeDbAdminServiceGrpc.newBlockingV2Stub(channel()) + .withCallCredentials(createCallCredentials(userName, userPassword)); + } + + return this.adminServiceBlockingV2Stub; + } + + @PreDestroy @Override public void close() { - channel.shutdown(); + if (channel == null) + return; try { - channel.awaitTermination(5, TimeUnit.SECONDS); + channel.shutdown(); + if (!channel.awaitTermination(5, TimeUnit.SECONDS)) { + channel.shutdownNow(); + channel.awaitTermination(2, TimeUnit.SECONDS); + } } - catch (InterruptedException e) { + catch (InterruptedException ie) { Thread.currentThread().interrupt(); + channel.shutdownNow(); + } + finally { + // log.info("gRPC channel to {}:{} closed.", host, port); } } @@ -81,25 +177,26 @@ private > S withDeadline(S stub, long timeoutMs) { return stub.withDeadlineAfter(t, TimeUnit.MILLISECONDS); } - public DatabaseCredentials buildCredentials() { - return DatabaseCredentials.newBuilder().setUsername(userName == null ? "" : userName).setPassword(userPassword == null ? "" : userPassword).build(); + public DatabaseCredentials buildCredentials() { + return DatabaseCredentials.newBuilder().setUsername(userName == null ? "" : userName) + .setPassword(userPassword == null ? "" : userPassword).build(); } /** Returns the list of database names from the server. */ public List listDatabases() { - + ListDatabasesResponse resp = null; - + try { - resp = withDeadline(blockingV2Stub, defaultTimeoutMs) + resp = withDeadline(adminServiceBlockingV2Stub(), defaultTimeoutMs) .listDatabases(ListDatabasesRequest.newBuilder().setCredentials(buildCredentials()).build()); return resp.getDatabasesList(); } catch (StatusException e) { - + throw new RuntimeException("Failed to list databases: " + e.getMessage(), e); - } + } } /** Checks existence by listing (no ExistsDatabaseRequest needed). */ @@ -109,11 +206,10 @@ public boolean existsDatabase(final String database) { /** Creates a database with type "graph" or "document". */ public void createDatabase(final String database) { - + try { - withDeadline(blockingV2Stub, defaultTimeoutMs).createDatabase(CreateDatabaseRequest.newBuilder() - .setName(database) - .setCredentials(buildCredentials()).build()); + withDeadline(adminServiceBlockingV2Stub(), defaultTimeoutMs) + .createDatabase(CreateDatabaseRequest.newBuilder().setName(database).setCredentials(buildCredentials()).build()); } catch (StatusException e) { throw new RuntimeException("Failed to create database: " + e.getMessage(), e); @@ -129,9 +225,9 @@ public void createDatabaseIfMissing(final String database) { /** Drops a database. If your proto supports 'force', add it here. */ public void dropDatabase(final String database) { - + try { - withDeadline(blockingV2Stub, defaultTimeoutMs) + withDeadline(adminServiceBlockingV2Stub(), defaultTimeoutMs) .dropDatabase(DropDatabaseRequest.newBuilder().setName(database).setCredentials(buildCredentials()).build()); } catch (StatusException e) { @@ -147,7 +243,7 @@ public String endpoint() { public String toString() { return "RemoteGrpcServer{endpoint=" + endpoint() + ", userName='" + userName + "'}"; } - + /** * Creates call credentials for authentication */ @@ -182,11 +278,12 @@ public void applyRequestMetadata(RequestInfo requestInfo, Executor appExecutor, applier.apply(headers); } - // x-arcade-user: root" -H "x-arcade-password: oY9uU2uJ8nD8iY7t" -H "x-arcade-database: local_shakeiq_curonix_poc-app" + // x-arcade-user: root" -H "x-arcade-password: oY9uU2uJ8nD8iY7t" -H + // "x-arcade-database: local_shakeiq_curonix_poc-app" @Override public void thisUsesUnstableApi() { // Required by the interface } }; - } + } } \ No newline at end of file diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/utils/ProtoUtils.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/utils/ProtoUtils.java index 45648761d4..0ff069ab44 100644 --- a/grpc-client/src/main/java/com/arcadedb/remote/grpc/utils/ProtoUtils.java +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/utils/ProtoUtils.java @@ -1,4 +1,5 @@ package com.arcadedb.remote.grpc.utils; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -12,81 +13,113 @@ import com.arcadedb.graph.Vertex; import com.arcadedb.server.grpc.GrpcRecord; import com.arcadedb.server.grpc.GrpcValue; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.ToNumberPolicy; public class ProtoUtils { - private static final Logger logger = LoggerFactory.getLogger(ProtoUtils.class); - - // Logging config via system properties (optional) - private static final int LOG_STR_PREVIEW = Integer.getInteger("arcadedb.grpc.log.preview", 96); - private static final boolean LOG_SHOW_STR_AT_DEBUG = Boolean.getBoolean("arcadedb.grpc.log.showString"); - - // Optional per-thread context for logs - private static final ThreadLocal LOG_CTX = new ThreadLocal<>(); - public static AutoCloseable pushLogContext(String ctx) { - final String prev = LOG_CTX.get(); - LOG_CTX.set(ctx); - return () -> LOG_CTX.set(prev); - } - private static String ctx() { - String c = LOG_CTX.get(); - return (c == null || c.isEmpty()) ? "" : " " + c; - } - - private static String summarizeJava(Object o) { - if (o == null) return "null"; - try { - if (o instanceof CharSequence s) { - if (logger.isTraceEnabled() || LOG_SHOW_STR_AT_DEBUG) { - String txt = s.toString(); - return "String(" + s.length() + ")=\"" + (txt.length() > LOG_STR_PREVIEW ? txt.substring(0, LOG_STR_PREVIEW) + "…" : txt) + "\""; - } - return "String(" + s.length() + ")"; - } - if (o instanceof byte[] b) return "bytes[" + b.length + "]"; - if (o instanceof java.util.Collection c) return o.getClass().getSimpleName() + "[size=" + c.size() + "]"; - if (o instanceof java.util.Map m) return o.getClass().getSimpleName() + "[size=" + m.size() + "]"; - return o.getClass().getSimpleName() + "(" + String.valueOf(o) + ")"; - } catch (Exception e) { return o.getClass().getSimpleName(); } - } - - private static String summarizeGrpc(GrpcValue v) { - if (v == null) return "GrpcValue(null)"; - switch (v.getKindCase()) { - case BOOL_VALUE: return "BOOL(" + v.getBoolValue() + ")"; - case INT32_VALUE: return "INT32(" + v.getInt32Value() + ")"; - case INT64_VALUE: return "INT64(" + v.getInt64Value() + ")"; - case FLOAT_VALUE: return "FLOAT(" + v.getFloatValue() + ")"; - case DOUBLE_VALUE: return "DOUBLE(" + v.getDoubleValue() + ")"; - case STRING_VALUE: { - String s = v.getStringValue(); - if (logger.isTraceEnabled() || LOG_SHOW_STR_AT_DEBUG) { - String txt = s; - return "STRING(" + s.length() + ")=\"" + (txt.length() > LOG_STR_PREVIEW ? txt.substring(0, LOG_STR_PREVIEW) + "…" : txt) + "\""; - } - return "STRING(" + s.length() + ")"; - } - case BYTES_VALUE: return "BYTES[" + v.getBytesValue().size() + "]"; - case TIMESTAMP_VALUE: return "TIMESTAMP(" + v.getTimestampValue().getSeconds() + "." + v.getTimestampValue().getNanos() + ")"; - case LIST_VALUE: return "LIST[size=" + v.getListValue().getValuesCount() + "]"; - case MAP_VALUE: return "MAP[size=" + v.getMapValue().getEntriesCount() + "]"; - case EMBEDDED_VALUE: return "EMBEDDED[type=" + v.getEmbeddedValue().getType() + ", size=" + v.getEmbeddedValue().getFieldsCount() + "]"; - case LINK_VALUE: return "LINK(" + v.getLinkValue().getRid() + ")"; - case DECIMAL_VALUE: return "DECIMAL(unscaled=" + v.getDecimalValue().getUnscaled() + ", scale=" + v.getDecimalValue().getScale() + ")"; - case KIND_NOT_SET: default: return "GrpcValue(KIND_NOT_SET)"; - } - } - - private static GrpcValue dbgEnc(String where, Object in, GrpcValue out) { - if (logger.isDebugEnabled()) logger.debug("CLIENT GRPC-ENC [{}]{} in={} -> out={}", where, ctx(), summarizeJava(in), summarizeGrpc(out)); - return out; - } - - private static Object dbgDec(String where, GrpcValue in, Object out) { - if (logger.isDebugEnabled()) logger.debug("CLIENT GRPC-DEC [{}]{} in={} -> out={}", where, ctx(), summarizeGrpc(in), summarizeJava(out)); - return out; - } + private static final Logger logger = LoggerFactory.getLogger(ProtoUtils.class); + + private static final Gson GSON = new GsonBuilder().disableHtmlEscaping().serializeNulls(). + setObjectToNumberStrategy(ToNumberPolicy.LONG_OR_DOUBLE).create(); + + // Logging config via system properties (optional) + private static final int LOG_STR_PREVIEW = Integer.getInteger("arcadedb.grpc.log.preview", 96); + private static final boolean LOG_SHOW_STR_AT_DEBUG = Boolean.getBoolean("arcadedb.grpc.log.showString"); + + // Optional per-thread context for logs + private static final ThreadLocal LOG_CTX = new ThreadLocal<>(); + + public static AutoCloseable pushLogContext(String ctx) { + final String prev = LOG_CTX.get(); + LOG_CTX.set(ctx); + return () -> LOG_CTX.set(prev); + } + + private static String ctx() { + String c = LOG_CTX.get(); + return (c == null || c.isEmpty()) ? "" : " " + c; + } + + private static String summarizeJava(Object o) { + if (o == null) + return "null"; + try { + if (o instanceof CharSequence s) { + if (logger.isTraceEnabled() || LOG_SHOW_STR_AT_DEBUG) { + String txt = s.toString(); + return "String(" + s.length() + ")=\"" + (txt.length() > LOG_STR_PREVIEW ? txt.substring(0, LOG_STR_PREVIEW) + "…" : txt) + + "\""; + } + return "String(" + s.length() + ")"; + } + if (o instanceof byte[] b) + return "bytes[" + b.length + "]"; + if (o instanceof java.util.Collection c) + return o.getClass().getSimpleName() + "[size=" + c.size() + "]"; + if (o instanceof java.util.Map m) + return o.getClass().getSimpleName() + "[size=" + m.size() + "]"; + return o.getClass().getSimpleName() + "(" + String.valueOf(o) + ")"; + } + catch (Exception e) { + return o.getClass().getSimpleName(); + } + } + + private static String summarizeGrpc(GrpcValue v) { + if (v == null) + return "GrpcValue(null)"; + switch (v.getKindCase()) { + case BOOL_VALUE: + return "BOOL(" + v.getBoolValue() + ")"; + case INT32_VALUE: + return "INT32(" + v.getInt32Value() + ")"; + case INT64_VALUE: + return "INT64(" + v.getInt64Value() + ")"; + case FLOAT_VALUE: + return "FLOAT(" + v.getFloatValue() + ")"; + case DOUBLE_VALUE: + return "DOUBLE(" + v.getDoubleValue() + ")"; + case STRING_VALUE: { + String s = v.getStringValue(); + if (logger.isTraceEnabled() || LOG_SHOW_STR_AT_DEBUG) { + String txt = s; + return "STRING(" + s.length() + ")=\"" + (txt.length() > LOG_STR_PREVIEW ? txt.substring(0, LOG_STR_PREVIEW) + "…" : txt) + "\""; + } + return "STRING(" + s.length() + ")"; + } + case BYTES_VALUE: + return "BYTES[" + v.getBytesValue().size() + "]"; + case TIMESTAMP_VALUE: + return "TIMESTAMP(" + v.getTimestampValue().getSeconds() + "." + v.getTimestampValue().getNanos() + ")"; + case LIST_VALUE: + return "LIST[size=" + v.getListValue().getValuesCount() + "]"; + case MAP_VALUE: + return "MAP[size=" + v.getMapValue().getEntriesCount() + "]"; + case EMBEDDED_VALUE: + return "EMBEDDED[type=" + v.getEmbeddedValue().getType() + ", size=" + v.getEmbeddedValue().getFieldsCount() + "]"; + case LINK_VALUE: + return "LINK(" + v.getLinkValue().getRid() + ")"; + case DECIMAL_VALUE: + return "DECIMAL(unscaled=" + v.getDecimalValue().getUnscaled() + ", scale=" + v.getDecimalValue().getScale() + ")"; + case KIND_NOT_SET: + default: + return "GrpcValue(KIND_NOT_SET)"; + } + } + + private static GrpcValue dbgEnc(String where, Object in, GrpcValue out) { + if (logger.isDebugEnabled()) + logger.debug("CLIENT GRPC-ENC [{}]{} in={} -> out={}", where, ctx(), summarizeJava(in), summarizeGrpc(out)); + return out; + } + private static Object dbgDec(String where, GrpcValue in, Object out) { + if (logger.isDebugEnabled()) + logger.debug("CLIENT GRPC-DEC [{}]{} in={} -> out={}", where, ctx(), summarizeGrpc(in), summarizeJava(out)); + return out; + } /** * Converts an ArcadeDB Record into a gRPC GrpcRecord message. @@ -183,80 +216,82 @@ public static GrpcRecord mapToProtoRecord(Map map, String rid, S * @param value Object value * @return GrpcValue */ - -public static GrpcValue toGrpcValue(Object value) { - GrpcValue.Builder b = GrpcValue.newBuilder(); - if (value == null) return dbgEnc("toGrpcValue", value, b.build()); - - if (value instanceof Boolean v) return dbgEnc("toGrpcValue", value, b.setBoolValue(v).build()); - if (value instanceof Integer v) return dbgEnc("toGrpcValue", value, b.setInt32Value(v).build()); - if (value instanceof Long v) return dbgEnc("toGrpcValue", value, b.setInt64Value(v).build()); - if (value instanceof Float v) return dbgEnc("toGrpcValue", value, b.setFloatValue(v).build()); - if (value instanceof Double v) return dbgEnc("toGrpcValue", value, b.setDoubleValue(v).build()); - if (value instanceof CharSequence v)return dbgEnc("toGrpcValue", value, b.setStringValue(v.toString()).build()); - if (value instanceof byte[] v) return dbgEnc("toGrpcValue", value, b.setBytesValue(com.google.protobuf.ByteString.copyFrom(v)).build()); - - if (value instanceof java.util.Date v) { - return dbgEnc("toGrpcValue", value, GrpcValue.newBuilder() - .setTimestampValue(com.google.protobuf.Timestamp.newBuilder() - .setSeconds(Math.floorDiv(v.getTime(), 1000L)) - .setNanos((int)Math.floorMod(v.getTime(), 1000L)*1_000_000) - .build()) - .setLogicalType("datetime") - .build()); - } - if (value instanceof RID rid) { - return dbgEnc("toGrpcValue", value, GrpcValue.newBuilder() - .setLinkValue(com.arcadedb.server.grpc.GrpcLink.newBuilder().setRid(rid.toString()).build()) - .setLogicalType("rid") - .build()); - } + public static GrpcValue toGrpcValue(Object value) { + GrpcValue.Builder b = GrpcValue.newBuilder(); + if (value == null) + return dbgEnc("toGrpcValue", value, b.build()); + + if (value instanceof Boolean v) + return dbgEnc("toGrpcValue", value, b.setBoolValue(v).build()); + if (value instanceof Integer v) + return dbgEnc("toGrpcValue", value, b.setInt32Value(v).build()); + if (value instanceof Long v) + return dbgEnc("toGrpcValue", value, b.setInt64Value(v).build()); + if (value instanceof Float v) + return dbgEnc("toGrpcValue", value, b.setFloatValue(v).build()); + if (value instanceof Double v) + return dbgEnc("toGrpcValue", value, b.setDoubleValue(v).build()); + if (value instanceof CharSequence v) + return dbgEnc("toGrpcValue", value, b.setStringValue(v.toString()).build()); + if (value instanceof byte[] v) + return dbgEnc("toGrpcValue", value, b.setBytesValue(com.google.protobuf.ByteString.copyFrom(v)).build()); + + if (value instanceof java.util.Date v) { + return dbgEnc("toGrpcValue", value, + GrpcValue.newBuilder().setTimestampValue(com.google.protobuf.Timestamp.newBuilder() + .setSeconds(Math.floorDiv(v.getTime(), 1000L)).setNanos((int) Math.floorMod(v.getTime(), 1000L) * 1_000_000).build()) + .setLogicalType("datetime").build()); + } - if (value instanceof java.math.BigDecimal v) { - java.math.BigInteger unscaled = v.unscaledValue(); - if (unscaled.bitLength() <= 63) { + if (value instanceof RID rid) { return dbgEnc("toGrpcValue", value, GrpcValue.newBuilder() - .setDecimalValue(com.arcadedb.server.grpc.GrpcDecimal.newBuilder() - .setUnscaled(unscaled.longValue()) - .setScale(v.scale()) - .build()) - .setLogicalType("decimal") - .build()); - } else { - return dbgEnc("toGrpcValue", value, GrpcValue.newBuilder().setStringValue(v.toPlainString()).setLogicalType("decimal").build()); + .setLinkValue(com.arcadedb.server.grpc.GrpcLink.newBuilder().setRid(rid.toString()).build()).setLogicalType("rid").build()); } - } - if (value instanceof Document edoc && (edoc.getIdentity() == null)) { - com.arcadedb.server.grpc.GrpcEmbedded.Builder eb = com.arcadedb.server.grpc.GrpcEmbedded.newBuilder(); - if (edoc.getType() != null) eb.setType(edoc.getType().getName()); - for (String k : edoc.getPropertyNames()) { - eb.putFields(k, toGrpcValue(edoc.get(k))); + if (value instanceof java.math.BigDecimal v) { + java.math.BigInteger unscaled = v.unscaledValue(); + if (unscaled.bitLength() <= 63) { + return dbgEnc("toGrpcValue", value, + GrpcValue.newBuilder().setDecimalValue( + com.arcadedb.server.grpc.GrpcDecimal.newBuilder().setUnscaled(unscaled.longValue()).setScale(v.scale()).build()) + .setLogicalType("decimal").build()); + } + else { + return dbgEnc("toGrpcValue", value, GrpcValue.newBuilder().setStringValue(v.toPlainString()).setLogicalType("decimal").build()); + } } - return dbgEnc("toGrpcValue", value, GrpcValue.newBuilder().setEmbeddedValue(eb.build()).build()); - } - if (value instanceof Map m) { - com.arcadedb.server.grpc.GrpcMap.Builder mb = com.arcadedb.server.grpc.GrpcMap.newBuilder(); - for (var e : m.entrySet()) { - mb.putEntries(String.valueOf(e.getKey()), toGrpcValue(e.getValue())); + if (value instanceof Document edoc && (edoc.getIdentity() == null)) { + com.arcadedb.server.grpc.GrpcEmbedded.Builder eb = com.arcadedb.server.grpc.GrpcEmbedded.newBuilder(); + if (edoc.getType() != null) + eb.setType(edoc.getType().getName()); + for (String k : edoc.getPropertyNames()) { + eb.putFields(k, toGrpcValue(edoc.get(k))); + } + return dbgEnc("toGrpcValue", value, GrpcValue.newBuilder().setEmbeddedValue(eb.build()).build()); } - return dbgEnc("toGrpcValue", value, GrpcValue.newBuilder().setMapValue(mb.build()).build()); - } - if (value instanceof Collection c) { - com.arcadedb.server.grpc.GrpcList.Builder lb = com.arcadedb.server.grpc.GrpcList.newBuilder(); - for (Object e : c) lb.addValues(toGrpcValue(e)); - return dbgEnc("toGrpcValue", value, GrpcValue.newBuilder().setListValue(lb.build()).build()); - } + if (value instanceof Map m) { + com.arcadedb.server.grpc.GrpcMap.Builder mb = com.arcadedb.server.grpc.GrpcMap.newBuilder(); + for (var e : m.entrySet()) { + mb.putEntries(String.valueOf(e.getKey()), toGrpcValue(e.getValue())); + } + return dbgEnc("toGrpcValue", value, GrpcValue.newBuilder().setMapValue(mb.build()).build()); + } - return dbgEnc("toGrpcValue", value, GrpcValue.newBuilder().setStringValue(String.valueOf(value)).build()); -} + if (value instanceof Collection c) { + com.arcadedb.server.grpc.GrpcList.Builder lb = com.arcadedb.server.grpc.GrpcList.newBuilder(); + for (Object e : c) + lb.addValues(toGrpcValue(e)); + return dbgEnc("toGrpcValue", value, GrpcValue.newBuilder().setListValue(lb.build()).build()); + } + return dbgEnc("toGrpcValue", value, GrpcValue.newBuilder().setStringValue(String.valueOf(value)).build()); + } public static Object fromGrpcValue(GrpcValue v) { - + switch (v.getKindCase()) { case BOOL_VALUE: return dbgDec("fromGrpcValue", v, v.getBoolValue()); @@ -271,7 +306,17 @@ public static Object fromGrpcValue(GrpcValue v) { case STRING_VALUE: return dbgDec("fromGrpcValue", v, v.getStringValue()); case BYTES_VALUE: + + if ("json".equalsIgnoreCase(v.getLogicalType())) { + String json = v.getBytesValue().toStringUtf8(); + + @SuppressWarnings("unchecked") + java.util.Map map = GSON.fromJson(json, java.util.Map.class); + + return dbgDec("fromGrpcValue", v, map); + } return dbgDec("fromGrpcValue", v, v.getBytesValue().toByteArray()); + case TIMESTAMP_VALUE: return dbgDec("fromGrpcValue", v, tsToMillis(v.getTimestampValue())); // or Instant case LIST_VALUE: { @@ -301,6 +346,7 @@ public static Object fromGrpcValue(GrpcValue v) { case KIND_NOT_SET: return dbgDec("fromGrpcValue", v, null); } + return dbgDec("fromGrpcValue", v, null); } diff --git a/grpc-client/src/test/java/com/arcadedb/remote/grpc/ArcadeDbHTTPTvsGRPCBench.java b/grpc-client/src/test/java/com/arcadedb/remote/grpc/ArcadeDbHTTPTvsGRPCBench.java index a1f669dfd6..abbe71d7c2 100644 --- a/grpc-client/src/test/java/com/arcadedb/remote/grpc/ArcadeDbHTTPTvsGRPCBench.java +++ b/grpc-client/src/test/java/com/arcadedb/remote/grpc/ArcadeDbHTTPTvsGRPCBench.java @@ -58,23 +58,22 @@ else if (a.startsWith("--pass=")) System.out.printf("Benchmarking DB='%s' HTTP=%s:%d gRPC=%s:%d as %s%n", DB_NAME, HTTP_HOST, HTTP_PORT, GRPC_HOST, GRPC_PORT, USER); - try (RemoteGrpcServer grpcServer = new RemoteGrpcServer(GRPC_HOST, GRPC_PORT, USER, PASS)) { + RemoteGrpcServer grpcServer = new RemoteGrpcServer(GRPC_HOST, GRPC_PORT, USER, PASS, true, List.of()); - if (grpcServer.existsDatabase(DB_NAME)) { + if (grpcServer.existsDatabase(DB_NAME)) { - System.out.printf("Database %s exists%n", DB_NAME); - } - else { + System.out.printf("Database %s exists%n", DB_NAME); + } + else { - System.out.printf("Creating database %s%n", DB_NAME); + System.out.printf("Creating database %s%n", DB_NAME); - grpcServer.createDatabase(DB_NAME); - } + grpcServer.createDatabase(DB_NAME); } // ---------- Open both clients ---------- try (RemoteDatabase http = new RemoteDatabase(HTTP_HOST, HTTP_PORT, DB_NAME, USER, PASS); - RemoteGrpcDatabase grpc = new RemoteGrpcDatabase(GRPC_HOST, GRPC_PORT, HTTP_PORT, DB_NAME, USER, PASS)) { + RemoteGrpcDatabase grpc = new RemoteGrpcDatabase(grpcServer, GRPC_HOST, GRPC_PORT, HTTP_PORT, DB_NAME, USER, PASS)) { // ensure schema aligned prepareSchemaHTTP(http); diff --git a/grpc-client/src/test/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseRegressionTest.java b/grpc-client/src/test/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseRegressionTest.java index dcf7f57bd6..5a6512d769 100644 --- a/grpc-client/src/test/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseRegressionTest.java +++ b/grpc-client/src/test/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseRegressionTest.java @@ -53,26 +53,26 @@ public class RemoteGrpcDatabaseRegressionTest { // Test type & props static final String TYPE = "RG_Feedback"; + private RemoteGrpcServer grpcServer; private RemoteGrpcDatabase grpc; @BeforeAll void ensureDatabaseExists() { + + this.grpcServer = new RemoteGrpcServer(GRPC_HOST, GRPC_PORT, USER, PASS, true, List.of()); + + // Prefer using the gRPC admin helper if available (same package). - try (RemoteGrpcServer admin = new RemoteGrpcServer(GRPC_HOST, GRPC_PORT, USER, PASS)) { - if (!admin.existsDatabase(DB_NAME)) { - admin.createDatabase(DB_NAME); - } - } - catch (Throwable t) { - // If admin tooling isn't on the classpath, tests may still pass provided DB - // exists. - System.err.println("[WARN] Could not verify/create DB via RemoteGrpcServer: " + t.getMessage()); + if (!grpcServer.existsDatabase(DB_NAME)) { + grpcServer.createDatabase(DB_NAME); } } @BeforeEach void open() { - grpc = new RemoteGrpcDatabase(GRPC_HOST, GRPC_PORT, HTTP_PORT, DB_NAME, USER, PASS); + + grpc = new RemoteGrpcDatabase(this.grpcServer, GRPC_HOST, GRPC_PORT, HTTP_PORT, DB_NAME, USER, PASS); + // Create isolated schema for these tests (id unique, name string, n integer) grpc.command("sql", "CREATE VERTEX TYPE `" + TYPE + "` IF NOT EXISTS", Map.of()); grpc.command("sql", "CREATE PROPERTY `" + TYPE + "`.id IF NOT EXISTS STRING", Map.of()); diff --git a/grpc/src/main/proto/arcadedb-server.proto b/grpc/src/main/proto/arcadedb-server.proto index d459520699..7cee7d312d 100644 --- a/grpc/src/main/proto/arcadedb-server.proto +++ b/grpc/src/main/proto/arcadedb-server.proto @@ -9,6 +9,7 @@ option java_outer_classname = "ArcadeDbProto"; import "google/protobuf/empty.proto"; import "google/protobuf/struct.proto"; import "google/protobuf/timestamp.proto"; +import "google/protobuf/wrappers.proto"; // ----------------------------------------------------------------------------- // Common types @@ -241,16 +242,26 @@ message ExecuteCommandResponse { // ----------------------------------------------------------------------------- message ExecuteQueryRequest { - string database = 1; string query = 2; - map parameters = 3; DatabaseCredentials credentials = 4; TransactionContext transaction = 5; int32 limit = 6; int32 timeout_ms = 7; + + // NEW + bool include_projections = 8; // default: false (RID only in results) + enum ProjectionEncoding { + PROJECTION_AS_LINK = 0; // RID only (GrpcValue.link_value) + PROJECTION_AS_MAP = 1; // Fully typed GrpcValue map/embedded tree + PROJECTION_AS_JSON = 2; // UTF-8 JSON bytes (GrpcValue.bytes_value, logical_type="json") + } + ProjectionEncoding projection_encoding = 9; // default: PROJECTION_AS_LINK + + // (optional, helps avoid giant messages) + google.protobuf.Int32Value projection_soft_limit_bytes = 10;// e.g., 32768; server may downshift encoding if exceeded } message ExecuteQueryResponse { diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java index c8bc2632bc..866b2bf3ac 100644 --- a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java @@ -45,6 +45,18 @@ public class ArcadeDbGrpcService extends ArcadeDbServiceGrpc.ArcadeDbServiceImpl private static final Logger logger = LoggerFactory.getLogger(ArcadeDbGrpcService.class); + private static final java.util.Base64.Encoder B64 = java.util.Base64.getEncoder(); + + private static final com.google.gson.Gson GSON = new com.google.gson.GsonBuilder().disableHtmlEscaping().serializeNulls() + .registerTypeHierarchyAdapter(java.nio.ByteBuffer.class, + (com.google.gson.JsonSerializer) (src, t, + ctx) -> new com.google.gson.JsonPrimitive(B64.encodeToString(toBytes(src)))) + // Optional: make byte[] consistent too + .registerTypeHierarchyAdapter(byte[].class, + (com.google.gson.JsonSerializer) (src, t, ctx) -> new com.google.gson.JsonPrimitive(B64.encodeToString(src))) + + .create(); + // Transaction management private final Map activeTransactions = new ConcurrentHashMap<>(); @@ -398,6 +410,7 @@ public void updateRecord(UpdateRecordRequest req, StreamObserver @rid = {}", dbRecord.asDocument().get("@rid")); GrpcRecord grpcRecord = convertToGrpcRecord(dbRecord, database); - + logger.debug("executeQuery(): grpcRecord -> @rid = {}", grpcRecord.getRid()); - + resultBuilder.addRecords(grpcRecord); - + count++; // Apply limit if specified @@ -656,7 +676,7 @@ public void executeQuery(ExecuteQueryRequest request, StreamObserver LINK rid={}", doc.getIdentity()); + } + return GrpcValue.newBuilder().setLinkValue(GrpcLink.newBuilder().setRid(doc.getIdentity().toString()).build()) + .setLogicalType("rid").build(); + } + // No identity → treat as EMBEDDED-like MAP + if (logger.isDebugEnabled()) { + logger.debug("GRPC-ENC [toGrpcValue] DOC-NON-PROJECTION (no rid) -> MAP"); + } + GrpcMap.Builder mb = GrpcMap.newBuilder(); + if (doc.getType() != null) + mb.putEntries("@type", GrpcValue.newBuilder().setStringValue(doc.getTypeName()).build()); + for (String k : doc.getPropertyNames()) { + mb.putEntries(k, toGrpcValue(doc.get(k), pc)); + } + return GrpcValue.newBuilder().setMapValue(mb.build()).build(); + } + + // In a PROJECTION row: obey encoding mode + ExecuteQueryRequest.ProjectionEncoding enc = pc.enc; + + // 3.1 LINK mode + if (enc == ExecuteQueryRequest.ProjectionEncoding.PROJECTION_AS_LINK) { + if (doc.getIdentity() != null && doc.getIdentity().isValid()) { + if (logger.isDebugEnabled()) { + logger.debug("GRPC-ENC [toGrpcValue] PROJECTION -> LINK rid={}", doc.getIdentity()); + } + return GrpcValue.newBuilder().setLinkValue(GrpcLink.newBuilder().setRid(doc.getIdentity().toString()).build()) + .setLogicalType("rid").build(); + } + // No rid: fall back to MAP + if (logger.isDebugEnabled()) { + logger.debug("GRPC-ENC [toGrpcValue] PROJECTION LINK fallback -> MAP (no rid)"); + } + enc = ExecuteQueryRequest.ProjectionEncoding.PROJECTION_AS_MAP; + } + + // 3.2 MAP mode + if (enc == ExecuteQueryRequest.ProjectionEncoding.PROJECTION_AS_MAP) { + if (logger.isDebugEnabled()) { + logger.debug("GRPC-ENC [toGrpcValue] PROJECTION -> MAP rid={} type={}", + (doc.getIdentity() != null ? doc.getIdentity() : "null"), (doc.getType() != null ? doc.getTypeName() : "null")); + } + GrpcMap.Builder mb = GrpcMap.newBuilder(); + + // meta + if (doc.getIdentity() != null && doc.getIdentity().isValid()) { + GrpcValue ridVal = GrpcValue.newBuilder().setLinkValue(GrpcLink.newBuilder().setRid(doc.getIdentity().toString()).build()) + .setLogicalType("rid").build(); + // soft limit accounting (precise, using serialized sizes) + if (!pc.wouldExceed(bytesOf("@rid") + ridVal.getSerializedSize())) { + mb.putEntries("@rid", ridVal); + } + else { + pc.truncated = true; + } + } + if (doc.getType() != null) { + GrpcValue typeVal = GrpcValue.newBuilder().setStringValue(doc.getTypeName()).build(); + if (!pc.wouldExceed(bytesOf("@type") + typeVal.getSerializedSize())) { + mb.putEntries("@type", typeVal); + } + else { + pc.truncated = true; + } + } + + for (String k : doc.getPropertyNames()) { + GrpcValue child = toGrpcValue(doc.get(k), pc); // recurse with config + int add = bytesOf(k) + child.getSerializedSize(); + if (pc.wouldExceed(add)) { + if (logger.isDebugEnabled()) { + logger.debug("GRPC-ENC [toGrpcValue] PROJECTION MAP soft-limit hit; skipping '{}' (limit={}, used~{})", k, + pc.softLimitBytes, pc.used.get()); + } + pc.truncated = true; + break; + } + mb.putEntries(k, child); + } + + return GrpcValue.newBuilder().setMapValue(mb.build()).build(); + } + + // 3.3 JSON mode + if (enc == ExecuteQueryRequest.ProjectionEncoding.PROJECTION_AS_JSON) { + + if (logger.isDebugEnabled()) { + logger.debug("GRPC-ENC [toGrpcValue] PROJECTION -> JSON rid={} type={}", + (doc.getIdentity() != null ? doc.getIdentity() : "null"), (doc.getType() != null ? doc.getTypeName() : "null")); + } + + // Build a simple Map and serialize via your existing Gson + java.util.Map m = new java.util.LinkedHashMap<>(); + + if (doc.getIdentity() != null && doc.getIdentity().isValid()) + m.put("@rid", doc.getIdentity().toString()); + + if (doc.getType() != null) + m.put("@type", doc.getTypeName()); + + for (String k : doc.getPropertyNames()) + m.put(k, doc.get(k)); + + + Object sanitized = sanitizeForJson(m); + byte[] jsonBytes = GSON.toJson(sanitized).getBytes(java.nio.charset.StandardCharsets.UTF_8); + + if (pc.softLimitBytes > 0 && jsonBytes.length > pc.softLimitBytes) { + + pc.truncated = true; + + if (logger.isDebugEnabled()) { + logger.debug("GRPC-ENC [toGrpcValue] PROJECTION JSON soft-limit hit; size={} limit={}", jsonBytes.length, + pc.softLimitBytes); + } + + // Choose your policy: truncate, or fall back to LINK if possible + if (doc.getIdentity() != null && doc.getIdentity().isValid()) { + return GrpcValue.newBuilder().setLinkValue(GrpcLink.newBuilder().setRid(doc.getIdentity().toString()).build()) + .setLogicalType("rid").build(); + } + else { + // Send a tiny marker instead of giant JSON + jsonBytes = "{\"__truncated\":true}".getBytes(java.nio.charset.StandardCharsets.UTF_8); + } + } + else { + pc.used.addAndGet(jsonBytes.length); + } + + return GrpcValue.newBuilder().setBytesValue(com.google.protobuf.ByteString.copyFrom(jsonBytes)).setLogicalType("json").build(); + } + + // Shouldn’t get here, but fall back + if (logger.isDebugEnabled()) { + logger.debug("GRPC-ENC [toGrpcValue] PROJECTION unknown encoding {}; falling back to LINK/STRING", enc.name()); + } + if (doc.getIdentity() != null && doc.getIdentity().isValid()) { + return GrpcValue.newBuilder().setLinkValue(GrpcLink.newBuilder().setRid(doc.getIdentity().toString()).build()) + .setLogicalType("rid").build(); + } + + // else small MAP fallback + GrpcMap.Builder mb = GrpcMap.newBuilder(); + for (String k : doc.getPropertyNames()) + mb.putEntries(k, toGrpcValue(doc.get(k), pc)); + return GrpcValue.newBuilder().setMapValue(mb.build()).build(); + } + if (o instanceof java.util.Map m) { GrpcMap.Builder mb = GrpcMap.newBuilder(); for (var e : m.entrySet()) { @@ -1775,20 +1959,30 @@ private GrpcValue toGrpcValue(Object o) { // Support ArcadeDB Result/ResultInternal as structural MAP if (o instanceof com.arcadedb.query.sql.executor.Result res) { - GrpcMap.Builder mb = GrpcMap.newBuilder(); - try { - for (String k : res.getPropertyNames()) { - mb.putEntries(k, toGrpcValue(res.getProperty(k))); - } - } catch (Throwable ignore) { } - - return dbgEnc("toGrpcValue", o, GrpcValue.newBuilder().setMapValue(mb.build()).build(), null); - } - + GrpcMap.Builder mb = GrpcMap.newBuilder(); + try { + for (String k : res.getPropertyNames()) { + mb.putEntries(k, toGrpcValue(res.getProperty(k))); + } + } + catch (Throwable ignore) { + } + + return dbgEnc("toGrpcValue", o, GrpcValue.newBuilder().setMapValue(mb.build()).build(), null); + } + // Fallback + if (logger.isDebugEnabled()) { + logger.debug("GRPC-ENC [toGrpcValue] FALLBACK-TO-STRING for class={} value={}", o.getClass().getName(), String.valueOf(o)); + } + return dbgEnc("toGrpcValue", o, GrpcValue.newBuilder().setStringValue(String.valueOf(o)).build(), null); } + private static int bytesOf(String s) { + return s == null ? 0 : s.getBytes(java.nio.charset.StandardCharsets.UTF_8).length; + } + private InsertOptions defaults(InsertOptions in) { InsertOptions.Builder b = in.toBuilder(); if (in.getServerBatchSize() == 0) @@ -1979,11 +2173,11 @@ private Map convertParameters(Map protoParams } private GrpcRecord convertToGrpcRecord(com.arcadedb.database.Record dbRecord, Database db) { - + GrpcRecord.Builder builder = GrpcRecord.newBuilder().setRid(dbRecord.getIdentity().toString()); if (dbRecord instanceof Document doc) { - + if (doc.getType() != null) builder.setType(doc.getTypeName()); @@ -1994,30 +2188,29 @@ private GrpcRecord convertToGrpcRecord(com.arcadedb.database.Record dbRecord, Da for (String propertyName : doc.getPropertyNames()) { Object value = doc.get(propertyName); - + if (value != null) { - - if (logger.isDebugEnabled()) { + + if (logger.isDebugEnabled()) { logger.debug("convertToGrpcRecord(): Converting {}\n value = {}\n class = {}", propertyName, value, value.getClass()); } - + GrpcValue gv = toGrpcValue(value); - + if (logger.isDebugEnabled()) logger.debug("ENC-REC {}.{}: {} -> {}", builder.getRid(), propertyName, summarizeJava(value), summarizeGrpc(gv)); - + builder.putProperties(propertyName, gv); } } // If this is an Edge, include @out / @in for convenience (string rid or link) if (dbRecord instanceof Edge edge) { - + if (!builder.getPropertiesMap().containsKey("@out")) { builder.putProperties("@out", toGrpcValue(edge.getOut().getIdentity())); } - - + if (!builder.getPropertiesMap().containsKey("@in")) { builder.putProperties("@in", toGrpcValue(edge.getIn().getIdentity())); } @@ -2026,18 +2219,29 @@ private GrpcRecord convertToGrpcRecord(com.arcadedb.database.Record dbRecord, Da if (logger.isDebugEnabled()) logger.debug("ENC-REC DONE rid={} type={} props={}", builder.getRid(), builder.getType(), builder.getPropertiesCount()); - + return builder.build(); } + private GrpcValue convertPropToGrpcValue(String propName, Result result, ProjectionConfig pc) { + + Object propValue = result.getProperty(propName); + + if (logger.isDebugEnabled()) { + logger.debug("convertPropToGrpcValue(): Converting {}\n value = {}\n class = {}", propName, propValue, propValue.getClass()); + } + + return toGrpcValue(propValue, pc); + } + private GrpcValue convertPropToGrpcValue(String propName, Result result) { Object propValue = result.getProperty(propName); - if (logger.isDebugEnabled()) { + if (logger.isDebugEnabled()) { logger.debug("convertPropToGrpcValue(): Converting {}\n value = {}\n class = {}", propName, propValue, propValue.getClass()); } - + return toGrpcValue(propValue); } @@ -2173,30 +2377,99 @@ private Object convertWithSchemaType(Database db, MutableDocument parent, com.ar }; case EMBEDDED: { + // Use schema ofType if present; otherwise use embedded.type from the payload + if (logger.isDebugEnabled()) { + logger.debug("EMBEDDED: prop='{}', incoming kind={}, schema ofType='{}'", propName, v.getKindCase(), prop.getOfType()); + } + String embeddedTypeName = (v.getKindCase() == GrpcValue.KindCase.EMBEDDED_VALUE && !v.getEmbeddedValue().getType().isEmpty()) ? v.getEmbeddedValue().getType() : prop.getOfType(); - if (v.getKindCase() != GrpcValue.KindCase.EMBEDDED_VALUE || embeddedTypeName == null || embeddedTypeName.isEmpty()) { - // Fallback to a Map if we can't create a typed embedded doc + // If still unknown, try to discover from MAP payload keys + // (type/@type/empowerType) + if ((embeddedTypeName == null || embeddedTypeName.isEmpty()) && v.getKindCase() == GrpcValue.KindCase.MAP_VALUE) { + var entries = v.getMapValue().getEntriesMap(); + GrpcValue tv = entries.getOrDefault("type", entries.getOrDefault("@type", entries.get("empowerType"))); + if (tv != null && tv.getKindCase() == GrpcValue.KindCase.STRING_VALUE) { + embeddedTypeName = tv.getStringValue(); + if (logger.isDebugEnabled()) { + logger.debug("EMBEDDED: discovered embeddedTypeName='{}' from MAP payload", embeddedTypeName); + } + } + } + + // If we still don't know the type and payload isn't EMBEDDED, fall back to Map + if ((embeddedTypeName == null || embeddedTypeName.isEmpty()) && v.getKindCase() != GrpcValue.KindCase.EMBEDDED_VALUE) { + + if (logger.isDebugEnabled()) { + logger.debug("EMBEDDED: fallback to Map for prop='{}' (kind={}, embeddedTypeName='{}')", propName, v.getKindCase(), + embeddedTypeName); + } return fromGrpcValue(v); } + if (logger.isDebugEnabled()) { + final String source = (v.getKindCase() == GrpcValue.KindCase.EMBEDDED_VALUE && !v.getEmbeddedValue().getType().isEmpty()) + ? "payload" + : "schema/discovered"; + int fields = (v.getKindCase() == GrpcValue.KindCase.EMBEDDED_VALUE) ? v.getEmbeddedValue().getFieldsCount() + : (v.getKindCase() == GrpcValue.KindCase.MAP_VALUE) ? v.getMapValue().getEntriesCount() : 0; + logger.debug("EMBEDDED: resolved embeddedTypeName='{}' (source={}), fields={}", embeddedTypeName, source, fields); + } + + // Build typed embedded document MutableEmbeddedDocument ed = parent.newEmbeddedDocument(embeddedTypeName, propName); DocumentType embeddedType = null; try { embeddedType = db.getSchema().getType(embeddedTypeName); + if (logger.isDebugEnabled()) { + logger.debug("EMBEDDED: schema type lookup '{}' -> {}", embeddedTypeName, (embeddedType != null ? "FOUND" : "NOT FOUND")); + } } - catch (Exception ignore) { + catch (Exception e) { + if (logger.isDebugEnabled()) { + logger.debug("EMBEDDED: schema type lookup failed for '{}': {}", embeddedTypeName, e.toString()); + } } - final DocumentType embeddedTypeFinal = embeddedType; - v.getEmbeddedValue().getFieldsMap().forEach((k, vv) -> { - Object j = (embeddedTypeFinal != null) ? toJavaForProperty(db, ed, embeddedTypeFinal, k, vv) : fromGrpcValue(vv); - ed.set(k, j); - }); + // Populate from either EMBEDDED_VALUE.fields or MAP_VALUE.entries + switch (v.getKindCase()) { + case EMBEDDED_VALUE: + v.getEmbeddedValue().getFieldsMap().forEach((k, vv) -> { + if (logger.isDebugEnabled()) + logger.debug("EMBEDDED: field '{}' raw kind={}", k, vv.getKindCase()); + Object j = (embeddedTypeFinal != null) ? toJavaForProperty(db, ed, embeddedTypeFinal, k, vv) : fromGrpcValue(vv); + if (logger.isDebugEnabled()) + logger.debug("EMBEDDED: field '{}' converted -> {}", k, (j == null ? "null" : j.getClass().getSimpleName())); + ed.set(k, j); + }); + break; + + case MAP_VALUE: + v.getMapValue().getEntriesMap().forEach((k, vv) -> { + if (logger.isDebugEnabled()) + logger.debug("EMBEDDED: field '{}' raw kind={}", k, vv.getKindCase()); + Object j = (embeddedTypeFinal != null) ? toJavaForProperty(db, ed, embeddedTypeFinal, k, vv) : fromGrpcValue(vv); + if (logger.isDebugEnabled()) + logger.debug("EMBEDDED: field '{}' converted -> {}", k, (j == null ? "null" : j.getClass().getSimpleName())); + ed.set(k, j); + }); + break; + + default: + // Shouldn't happen; be safe + if (logger.isDebugEnabled()) { + logger.debug("EMBEDDED: unexpected kind={}, falling back to Map for prop='{}'", v.getKindCase(), propName); + } + return fromGrpcValue(v); + } + + if (logger.isDebugEnabled()) { + logger.debug("EMBEDDED: built embedded doc type='{}' for prop='{}'", embeddedTypeName, propName); + } return ed; } @@ -2385,9 +2658,9 @@ private Object dbgDec(String where, GrpcValue in, Object out, String ctx) { // --- Gson structural encoder: JsonElement -> GrpcValue --- private GrpcValue gsonToGrpc(JsonElement n) { - + GrpcValue.Builder b = GrpcValue.newBuilder(); - + if (n == null || n.isJsonNull()) { return b.build(); // KIND_NOT_SET => null on the client } @@ -2457,4 +2730,122 @@ private GrpcValue gsonToGrpc(JsonElement n) { return b.setStringValue(n.toString()).build(); } + + private static byte[] toBytes(java.nio.ByteBuffer buf) { + java.nio.ByteBuffer dup = buf.asReadOnlyBuffer(); + byte[] out = new byte[dup.remaining()]; + dup.get(out); + return out; + } + + /** + * Deep-scrub arbitrary objects into a JSON-safe graph + * (Map/List/primitive/String). + */ + private Object sanitizeForJson(Object x) { + if (x == null) + return null; + + // Primitives & simple types + if (x instanceof String || x instanceof Number || x instanceof Boolean) + return x; + + // Common binary holders -> base64 string + if (x instanceof byte[] arr) + return B64.encodeToString(arr); + if (x instanceof java.nio.ByteBuffer bb) + return B64.encodeToString(toBytes(bb)); + + // Arcade IDs/links/dates + if (x instanceof com.arcadedb.database.RID rid) + return rid.toString(); + if (x instanceof com.arcadedb.database.Identifiable id) + return id.getIdentity() != null ? id.getIdentity().toString() : null; + if (x instanceof java.util.Date d) + return java.time.Instant.ofEpochMilli(d.getTime()).toString(); + + // Arcade Result -> Map projection + if (x instanceof com.arcadedb.query.sql.executor.Result res) { + var m = new java.util.LinkedHashMap(); + try { + for (String k : res.getPropertyNames()) { + m.put(k, sanitizeForJson(res.getProperty(k))); + } + } + catch (Throwable ignore) { + } + return m; + } + + // Embedded / Document + if (x instanceof com.arcadedb.database.Document doc) { + var m = new java.util.LinkedHashMap(); + if (doc.getIdentity() != null && doc.getIdentity().isValid()) + m.put("@rid", doc.getIdentity().toString()); + if (doc.getType() != null) + m.put("@type", doc.getTypeName()); + for (String k : doc.getPropertyNames()) { + m.put(k, sanitizeForJson(doc.get(k))); + } + return m; + } + + // Collections/arrays + if (x instanceof java.util.Map in) { + var m = new java.util.LinkedHashMap(); + in.forEach((k, v) -> m.put(String.valueOf(k), sanitizeForJson(v))); + return m; + } + if (x instanceof java.util.Collection c) { + var out = new java.util.ArrayList<>(); + for (Object e : c) + out.add(sanitizeForJson(e)); + return out; + } + if (x.getClass().isArray()) { + int n = java.lang.reflect.Array.getLength(x); + var out = new java.util.ArrayList<>(n); + for (int i = 0; i < n; i++) + out.add(sanitizeForJson(java.lang.reflect.Array.get(x, i))); + return out; + } + + // FINAL GUARD: any JDK “handle-ish” or unknown classes -> describe as String + String pkg = x.getClass().getPackageName(); + if (pkg.startsWith("java.io") || pkg.startsWith("java.nio") || pkg.startsWith("sun.") || pkg.startsWith("jdk.")) { + if (logger.isDebugEnabled()) { + logger.debug("GRPC-ENC [toGrpcValue] JSON scrubbed {} via toString()", x.getClass().getName()); + } + return String.valueOf(x); + } + + // Default safe fallback + return String.valueOf(x); + } + + static final class ProjectionConfig { + final boolean include; + final ExecuteQueryRequest.ProjectionEncoding enc; + final int softLimitBytes; // 0 = no soft limit + + // Running counter for MAP/JSON builds: + final java.util.concurrent.atomic.AtomicInteger used = new java.util.concurrent.atomic.AtomicInteger(0); + volatile boolean truncated = false; + + ProjectionConfig(boolean include, ExecuteQueryRequest.ProjectionEncoding enc, int softLimitBytes) { + this.include = include; + this.enc = enc == null ? ExecuteQueryRequest.ProjectionEncoding.PROJECTION_AS_LINK : enc; + this.softLimitBytes = Math.max(softLimitBytes, 0); + } + + boolean wouldExceed(int add) { + if (softLimitBytes <= 0) + return false; + int after = used.addAndGet(add); + if (after > softLimitBytes) { + truncated = true; + } + return after > softLimitBytes; + } + } } \ No newline at end of file diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcServerPlugin.java b/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcServerPlugin.java index d3d5855aa9..d6fc8317dc 100644 --- a/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcServerPlugin.java +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcServerPlugin.java @@ -113,7 +113,8 @@ public void startService() { } private void startStandardServer(ContextConfiguration config) throws IOException { - int port = getConfigInt(config, CONFIG_PORT, 50051); + + int port = getConfigInt(config, CONFIG_PORT, 50051); String host = getConfigString(config, CONFIG_HOST, "0.0.0.0"); ServerBuilder serverBuilder; @@ -143,8 +144,10 @@ private void startStandardServer(ContextConfiguration config) throws IOException } if (getConfigBoolean(config, CONFIG_COMPRESSION_ENABLED, true)) { - status.append(", compression: "); - if (getConfigBoolean(config, CONFIG_COMPRESSION_FORCE, false)) { + + status.append(", compression: "); + + if (getConfigBoolean(config, CONFIG_COMPRESSION_FORCE, false)) { status.append("forced-").append(getConfigString(config, CONFIG_COMPRESSION_TYPE, "gzip")); } else { status.append("available"); @@ -211,6 +214,7 @@ private void configureServer(ServerBuilder serverBuilder, ContextConfiguratio // Configure max message size int maxMessageSizeMB = getConfigInt(config, CONFIG_MAX_MESSAGE_SIZE, 100); + serverBuilder.maxInboundMessageSize(maxMessageSizeMB * 1024 * 1024); // Add interceptors for logging, metrics, auth, etc. @@ -364,7 +368,8 @@ private String getConfigString(ContextConfiguration config, String key, String d } private int getConfigInt(ContextConfiguration config, String key, int defaultValue) { - String value = getConfigString(config, key, null); + + String value = getConfigString(config, key, null); if (value != null) { try { return Integer.parseInt(value); From 0beb1a403c438b5b398c196745048bd260796ca2 Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Sun, 7 Sep 2025 15:30:15 -0700 Subject: [PATCH 30/35] Checkpoint Commit --- .../remote/grpc/RemoteGrpcConfig.java | 41 ++++ .../remote/grpc/RemoteGrpcDatabase.java | 143 +++++++------ grpc/src/main/proto/arcadedb-server.proto | 30 ++- .../server/grpc/ArcadeDbGrpcService.java | 202 +++++++++++++----- 4 files changed, 283 insertions(+), 133 deletions(-) create mode 100644 grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcConfig.java diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcConfig.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcConfig.java new file mode 100644 index 0000000000..f055e10a1e --- /dev/null +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcConfig.java @@ -0,0 +1,41 @@ +package com.arcadedb.remote.grpc; + +import com.arcadedb.server.grpc.ProjectionSettings.ProjectionEncoding; + +public class RemoteGrpcConfig { + + private boolean includeProjections; + private ProjectionEncoding projectionEncoding; + private int softLimitBytes; + + public RemoteGrpcConfig(boolean includeProjections, ProjectionEncoding projectionEncoding, int softLimitBytes) { + this.includeProjections = includeProjections; + this.projectionEncoding = projectionEncoding; + this.softLimitBytes = softLimitBytes; + } + + public boolean isIncludeProjections() { + return includeProjections; + } + + public void setIncludeProjections(boolean includeProjections) { + this.includeProjections = includeProjections; + } + + public ProjectionEncoding getProjectionEncoding() { + return projectionEncoding; + } + + public void setProjectionEncoding(ProjectionEncoding projectionEncoding) { + this.projectionEncoding = projectionEncoding; + } + + public int getSoftLimitBytes() { + return softLimitBytes; + } + + public void setSoftLimitBytes(int softLimitBytes) { + this.softLimitBytes = softLimitBytes; + } + +} diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java index e3f9ab9f21..61c06cc099 100644 --- a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java @@ -53,7 +53,6 @@ import com.arcadedb.server.grpc.ExecuteCommandRequest; import com.arcadedb.server.grpc.ExecuteCommandResponse; import com.arcadedb.server.grpc.ExecuteQueryRequest; -import com.arcadedb.server.grpc.ExecuteQueryRequest.ProjectionEncoding; import com.arcadedb.server.grpc.ExecuteQueryResponse; import com.arcadedb.server.grpc.GrpcRecord; import com.arcadedb.server.grpc.GrpcValue; @@ -64,6 +63,8 @@ import com.arcadedb.server.grpc.InsertSummary; import com.arcadedb.server.grpc.LookupByRidRequest; import com.arcadedb.server.grpc.LookupByRidResponse; +import com.arcadedb.server.grpc.ProjectionSettings; +import com.arcadedb.server.grpc.ProjectionSettings.ProjectionEncoding; import com.arcadedb.server.grpc.PropertiesUpdate; import com.arcadedb.server.grpc.QueryResult; import com.arcadedb.server.grpc.RollbackTransactionRequest; @@ -428,8 +429,37 @@ private ResultSet commandInternal(final String language, final String command, f } } + @Override + public ResultSet query(final String language, final String query, final Object... args) { + checkDatabaseIsOpen(); + stats.queries.incrementAndGet(); + + final Map params = mapArgs(args); + + RemoteGrpcConfig remoteGrpcConfig = new RemoteGrpcConfig(true, ProjectionEncoding.PROJECTION_AS_JSON, 0); + + return query(language, query, remoteGrpcConfig, params); + } + + @Override public ResultSet query(final String language, final String query, final Map params) { + + RemoteGrpcConfig remoteGrpcConfig = new RemoteGrpcConfig(true, ProjectionEncoding.PROJECTION_AS_JSON, 0); + + return query(language, query, remoteGrpcConfig, params); + } + + + public ResultSet query(final String language, final String query, RemoteGrpcConfig remoteGrpcConfig, final Object... args) { + + final Map params = mapArgs(args); + + return query(language, query, remoteGrpcConfig, params); + } + + public ResultSet query(final String language, final String query, RemoteGrpcConfig remoteGrpcConfig, final Map params) { + checkDatabaseIsOpen(); stats.queries.incrementAndGet(); @@ -444,9 +474,13 @@ public ResultSet query(final String language, final String query, final Map params = mapArgs(args); - - return query(language, query, params); - } - - + + public ExecuteCommandResponse execSql(String db, String sql, Map params, long timeoutMs) { return executeCommand(db, "sql", sql, params, /* returnRows */ false, /* maxRows */ 0, txBeginCommit(), timeoutMs); } @@ -603,73 +627,43 @@ protected RID saveRecord(final MutableDocument record) { } } + private RemoteGrpcConfig getDefaultRemoteGrpcConfig() { + + return new RemoteGrpcConfig(true, ProjectionEncoding.PROJECTION_AS_JSON, 0); + } + // Convenience: default batch size stays 100, default mode = CURSOR + public Iterator queryStream(final String language, final String query) { - return queryStream(language, query, /* batchSize */100, StreamQueryRequest.RetrievalMode.CURSOR); + return queryStream(language, query, getDefaultRemoteGrpcConfig(), /* batchSize */100, StreamQueryRequest.RetrievalMode.CURSOR); } + public Iterator queryStream(final String language, final String query, final RemoteGrpcConfig config) { + return queryStream(language, query, config, /* batchSize */100, StreamQueryRequest.RetrievalMode.CURSOR); + } + public Iterator queryStream(final String language, final String query, final int batchSize) { - return queryStream(language, query, batchSize, StreamQueryRequest.RetrievalMode.CURSOR); + return queryStream(language, query, getDefaultRemoteGrpcConfig(), batchSize, StreamQueryRequest.RetrievalMode.CURSOR); } - // NEW: choose retrieval mode - public Iterator queryStream(final String language, final String query, final int batchSize, - final StreamQueryRequest.RetrievalMode mode) { - checkDatabaseIsOpen(); - stats.queries.incrementAndGet(); - - StreamQueryRequest request = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(query).setCredentials(buildCredentials()) - .setBatchSize(batchSize > 0 ? batchSize : 100).setRetrievalMode(mode) // <--- NEW - .build(); - - final BlockingClientCall responseIterator = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) - .streamQuery(request); - - return new Iterator() { - private Iterator currentBatch = Collections.emptyIterator(); - - @Override - public boolean hasNext() { - if (currentBatch.hasNext()) - return true; - - try { - while (responseIterator.hasNext()) { - QueryResult result = responseIterator.read(); - - // Defensive: skip any empty batches the server might send - if (result.getRecordsCount() == 0) { - if (result.getIsLastBatch()) - return false; - continue; - } + public Iterator queryStream(final String language, final String query, RemoteGrpcConfig config, final int batchSize) { + return queryStream(language, query, config, batchSize, StreamQueryRequest.RetrievalMode.CURSOR); + } - currentBatch = result.getRecordsList().iterator(); - return true; - } - } - catch (InterruptedException e) { - throw new RuntimeException("Interrupted while waiting for stream", e); - } - catch (StatusException e) { - handleGrpcException(e); - } + public Iterator queryStream(final String language, final String query, final int batchSize, final StreamQueryRequest.RetrievalMode mode) { + return queryStream(language, query, getDefaultRemoteGrpcConfig(), batchSize, mode); + } - return false; - } + // NEW: choose retrieval mode - @Override - public Record next() { - if (!hasNext()) - throw new NoSuchElementException(); - return grpcRecordToDBRecord(currentBatch.next()); - } - }; + public Iterator queryStream(final String language, final String query, final RemoteGrpcConfig config, final int batchSize, final StreamQueryRequest.RetrievalMode mode) { + + return queryStream(language, query, config, Map.of(), batchSize, mode); } // PARAMETERIZED variant with retrieval mode - public Iterator queryStream(final String language, final String query, final Map params, final int batchSize, - final StreamQueryRequest.RetrievalMode mode) { + public Iterator queryStream(final String language, final String query, final RemoteGrpcConfig config, final Map params, final int batchSize, final StreamQueryRequest.RetrievalMode mode) { + checkDatabaseIsOpen(); stats.queries.incrementAndGet(); @@ -726,10 +720,17 @@ public Record next() { // Keep the old signature working (defaults to CURSOR) public Iterator queryStream(final String language, final String query, final Map params, final int batchSize) { - return queryStream(language, query, params, batchSize, StreamQueryRequest.RetrievalMode.CURSOR); + + return queryStream(language, query, getDefaultRemoteGrpcConfig(), params, batchSize, StreamQueryRequest.RetrievalMode.CURSOR); } + public Iterator queryStream(final String language, final String query, final RemoteGrpcConfig config, final Map params, final int batchSize) { + + return queryStream(language, query, config, params, batchSize, StreamQueryRequest.RetrievalMode.CURSOR); + } + public static final class QueryBatch { + private final List records; private final int totalInBatch; private final long runningTotal; diff --git a/grpc/src/main/proto/arcadedb-server.proto b/grpc/src/main/proto/arcadedb-server.proto index 7cee7d312d..958e5d3c51 100644 --- a/grpc/src/main/proto/arcadedb-server.proto +++ b/grpc/src/main/proto/arcadedb-server.proto @@ -87,6 +87,20 @@ message GrpcDecimal { int32 scale = 2; // number of decimal places } +message ProjectionSettings { + bool include_projections = 1; + + // Reuse the same enum (either nested under ExecuteQueryRequest or a top-level enum) + enum ProjectionEncoding { + PROJECTION_AS_LINK = 0; + PROJECTION_AS_MAP = 1; + PROJECTION_AS_JSON = 2; + } + ProjectionEncoding projection_encoding = 2; + + optional google.protobuf.Int32Value soft_limit_bytes = 3; // <-- this is the one protoc complained about; use 3 +} + // ----------------------------------------------------------------------------- // Query (streaming) API // ----------------------------------------------------------------------------- @@ -111,6 +125,8 @@ message StreamQueryRequest { string language = 7; // e.g., "sql" (default if empty) TransactionContext transaction = 8; // optional + + ProjectionSettings projectionSettings = 9; } message QueryResult { @@ -225,6 +241,8 @@ message ExecuteCommandRequest { bool return_rows = 7; // include rows in response int32 max_rows = 8; // cap returned rows (server may still count all) + + ProjectionSettings projectionSettings = 9; } message ExecuteCommandResponse { @@ -251,17 +269,7 @@ message ExecuteQueryRequest { int32 limit = 6; int32 timeout_ms = 7; - // NEW - bool include_projections = 8; // default: false (RID only in results) - enum ProjectionEncoding { - PROJECTION_AS_LINK = 0; // RID only (GrpcValue.link_value) - PROJECTION_AS_MAP = 1; // Fully typed GrpcValue map/embedded tree - PROJECTION_AS_JSON = 2; // UTF-8 JSON bytes (GrpcValue.bytes_value, logical_type="json") - } - ProjectionEncoding projection_encoding = 9; // default: PROJECTION_AS_LINK - - // (optional, helps avoid giant messages) - google.protobuf.Int32Value projection_soft_limit_bytes = 10;// e.g., 32768; server may downshift encoding if exceeded + ProjectionSettings projectionSettings = 8; } message ExecuteQueryResponse { diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java index 866b2bf3ac..91dd19d79c 100644 --- a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java @@ -28,6 +28,7 @@ import com.arcadedb.server.ArcadeDBServer; import com.arcadedb.server.grpc.InsertOptions.ConflictMode; import com.arcadedb.server.grpc.InsertOptions.TransactionMode; +import com.arcadedb.server.grpc.ProjectionSettings.ProjectionEncoding; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonPrimitive; @@ -598,13 +599,25 @@ public void executeQuery(ExecuteQueryRequest request, StreamObserver responseObserver) { + ProjectionConfig projectionConfig = getProjectionConfigFromRequest(request); + final ServerCallStreamObserver scso = (ServerCallStreamObserver) responseObserver; final java.util.concurrent.atomic.AtomicBoolean cancelled = new java.util.concurrent.atomic.AtomicBoolean(false); @@ -802,10 +817,10 @@ public void streamQuery(StreamQueryRequest request, StreamObserver // --- Dispatch on mode (helpers do NOT manage transactions) --- switch (request.getRetrievalMode()) { - case MATERIALIZE_ALL -> streamMaterialized(db, request, batchSize, scso, cancelled); - case PAGED -> streamPaged(db, request, batchSize, scso, cancelled); - case CURSOR -> streamCursor(db, request, batchSize, scso, cancelled); - default -> streamCursor(db, request, batchSize, scso, cancelled); + case MATERIALIZE_ALL -> streamMaterialized(db, request, batchSize, scso, cancelled, projectionConfig); + case PAGED -> streamPaged(db, request, batchSize, scso, cancelled, projectionConfig); + case CURSOR -> streamCursor(db, request, batchSize, scso, cancelled, projectionConfig); + default -> streamCursor(db, request, batchSize, scso, cancelled, projectionConfig); } // If the client cancelled mid-stream, choose rollback unless caller explicitly @@ -870,58 +885,102 @@ else if (beganHere) { * Mode 1 (existing behavior-ish): run once and iterate results, batching as we * go. */ - private void streamCursor(Database db, StreamQueryRequest req, int batchSize, ServerCallStreamObserver scso, - AtomicBoolean cancelled) { + private void streamCursor(Database db, StreamQueryRequest request, int batchSize, ServerCallStreamObserver scso, + AtomicBoolean cancelled, ProjectionConfig projectionConfig) { long running = 0L; QueryResult.Builder batch = QueryResult.newBuilder(); int inBatch = 0; - try (ResultSet rs = db.query("sql", req.getQuery(), convertParameters(req.getParametersMap()))) { + try (ResultSet rs = db.query("sql", request.getQuery(), convertParameters(request.getParametersMap()))) { + while (rs.hasNext()) { + if (cancelled.get()) return; waitUntilReady(scso, cancelled); Result r = rs.next(); - if (!r.isElement()) - continue; - com.arcadedb.database.Record rec = r.getElement().get(); - batch.addRecords(convertToGrpcRecord(rec, db)); - inBatch++; - running++; + if (r.isElement()) { + + com.arcadedb.database.Record rec = r.getElement().get(); + + batch.addRecords(convertToGrpcRecord(rec, db)); + + inBatch++; + running++; - if (inBatch >= batchSize) { - safeOnNext(scso, cancelled, - batch.setTotalRecordsInBatch(inBatch).setRunningTotalEmitted(running).setIsLastBatch(false).build()); - batch = QueryResult.newBuilder(); - inBatch = 0; + if (inBatch >= batchSize) { + safeOnNext(scso, cancelled, + batch.setTotalRecordsInBatch(inBatch).setRunningTotalEmitted(running).setIsLastBatch(false).build()); + batch = QueryResult.newBuilder(); + inBatch = 0; + } } - } + else { + + GrpcRecord.Builder rb = GrpcRecord.newBuilder(); + + for (String p : r.getPropertyNames()) { + rb.putProperties(p, convertPropToGrpcValue(p, r, projectionConfig)); // overload below + } - if (!cancelled.get() && inBatch > 0) { - safeOnNext(scso, cancelled, batch.setTotalRecordsInBatch(inBatch).setRunningTotalEmitted(running).setIsLastBatch(true).build()); + batch.addRecords(rb.build()); + + inBatch++; + running++; + + if (inBatch >= batchSize) { + + safeOnNext(scso, cancelled, + batch.setTotalRecordsInBatch(inBatch).setRunningTotalEmitted(running).setIsLastBatch(false).build()); + + batch = QueryResult.newBuilder(); + inBatch = 0; + } + } } } + + if (!cancelled.get() && inBatch > 0) { + safeOnNext(scso, cancelled, batch.setTotalRecordsInBatch(inBatch).setRunningTotalEmitted(running).setIsLastBatch(true).build()); + } } - /** Mode 2: materialize everything first (simple, but can be memory-heavy). */ - private void streamMaterialized(Database db, StreamQueryRequest req, int batchSize, ServerCallStreamObserver scso, - AtomicBoolean cancelled) { + /** + * Mode 2: materialize everything first (simple, but can be memory-heavy). + * + * @param projectionConfig + */ + private void streamMaterialized(Database db, StreamQueryRequest request, int batchSize, ServerCallStreamObserver scso, + AtomicBoolean cancelled, ProjectionConfig projectionConfig) { final java.util.List all = new java.util.ArrayList<>(); - try (ResultSet rs = db.query("sql", req.getQuery(), convertParameters(req.getParametersMap()))) { + try (ResultSet rs = db.query("sql", request.getQuery(), convertParameters(request.getParametersMap()))) { while (rs.hasNext()) { if (cancelled.get()) return; + Result r = rs.next(); - if (!r.isElement()) - continue; - all.add(convertToGrpcRecord(r.getElement().get(), db)); + + if (r.isElement()) { + + all.add(convertToGrpcRecord(r.getElement().get(), db)); + } + else { + + GrpcRecord.Builder rb = GrpcRecord.newBuilder(); + + for (String p : r.getPropertyNames()) { + rb.putProperties(p, convertPropToGrpcValue(p, r, projectionConfig)); // overload below + } + + all.add(rb.build()); + } } } @@ -942,11 +1001,15 @@ private void streamMaterialized(Database db, StreamQueryRequest req, int batchSi } } - /** Mode 3: only fetch one page’s worth of rows per emission via LIMIT/SKIP. */ - private void streamPaged(Database db, StreamQueryRequest req, int batchSize, ServerCallStreamObserver scso, - AtomicBoolean cancelled) { + /** + * Mode 3: only fetch one page’s worth of rows per emission via LIMIT/SKIP. + * + * @param projectionConfig + */ + private void streamPaged(Database db, StreamQueryRequest request, int batchSize, ServerCallStreamObserver scso, + AtomicBoolean cancelled, ProjectionConfig projectionConfig) { - final String pagedSql = wrapWithSkipLimit(req.getQuery()); // see helper below + final String pagedSql = wrapWithSkipLimit(request.getQuery()); // see helper below int offset = 0; long running = 0L; @@ -955,7 +1018,7 @@ private void streamPaged(Database db, StreamQueryRequest req, int batchSize, Ser return; waitUntilReady(scso, cancelled); - java.util.Map params = new java.util.HashMap<>(convertParameters(req.getParametersMap())); + java.util.Map params = new java.util.HashMap<>(convertParameters(request.getParametersMap())); params.put("_skip", offset); params.put("_limit", batchSize); @@ -966,11 +1029,25 @@ private void streamPaged(Database db, StreamQueryRequest req, int batchSize, Ser while (rs.hasNext()) { if (cancelled.get()) return; + Result r = rs.next(); - if (!r.isElement()) - continue; - b.addRecords(convertToGrpcRecord(r.getElement().get(), db)); - count++; + + if (r.isElement()) { + + b.addRecords(convertToGrpcRecord(r.getElement().get(), db)); + count++; + } + else { + + GrpcRecord.Builder rb = GrpcRecord.newBuilder(); + + for (String p : r.getPropertyNames()) { + rb.putProperties(p, convertPropToGrpcValue(p, r, projectionConfig)); // overload below + } + + b.addRecords(rb.build()); + count++; + } } } @@ -988,6 +1065,30 @@ private void streamPaged(Database db, StreamQueryRequest req, int batchSize, Ser } } + private ProjectionConfig getProjectionConfigFromRequest(StreamQueryRequest request) { + + ProjectionConfig projectionConfig = null; + + if (request.hasProjectionSettings()) { + + var projectionSettings = request.getProjectionSettings(); + + final boolean includeProjections = projectionSettings.getIncludeProjections(); + final var projMode = projectionSettings.getProjectionEncoding(); + + int softLimit = projectionSettings.hasSoftLimitBytes() ? projectionSettings.getSoftLimitBytes().getValue() : 0; // choose your + // default + // or 0 = unlimited + projectionConfig = new ProjectionConfig(includeProjections, projMode, softLimit); + } + else { + + projectionConfig = new ProjectionConfig(true, ProjectionEncoding.PROJECTION_AS_JSON, 0); + } + + return projectionConfig; + } + /** Wrap arbitrary SQL so we can safely inject LIMIT/SKIP outside. */ private String wrapWithSkipLimit(String originalSql) { @@ -1797,10 +1898,10 @@ private GrpcValue toGrpcValue(Object o, ProjectionConfig pc) { } // In a PROJECTION row: obey encoding mode - ExecuteQueryRequest.ProjectionEncoding enc = pc.enc; + ProjectionEncoding enc = pc.enc; // 3.1 LINK mode - if (enc == ExecuteQueryRequest.ProjectionEncoding.PROJECTION_AS_LINK) { + if (enc == ProjectionEncoding.PROJECTION_AS_LINK) { if (doc.getIdentity() != null && doc.getIdentity().isValid()) { if (logger.isDebugEnabled()) { logger.debug("GRPC-ENC [toGrpcValue] PROJECTION -> LINK rid={}", doc.getIdentity()); @@ -1812,11 +1913,11 @@ private GrpcValue toGrpcValue(Object o, ProjectionConfig pc) { if (logger.isDebugEnabled()) { logger.debug("GRPC-ENC [toGrpcValue] PROJECTION LINK fallback -> MAP (no rid)"); } - enc = ExecuteQueryRequest.ProjectionEncoding.PROJECTION_AS_MAP; + enc = ProjectionEncoding.PROJECTION_AS_MAP; } // 3.2 MAP mode - if (enc == ExecuteQueryRequest.ProjectionEncoding.PROJECTION_AS_MAP) { + if (enc == ProjectionEncoding.PROJECTION_AS_MAP) { if (logger.isDebugEnabled()) { logger.debug("GRPC-ENC [toGrpcValue] PROJECTION -> MAP rid={} type={}", (doc.getIdentity() != null ? doc.getIdentity() : "null"), (doc.getType() != null ? doc.getTypeName() : "null")); @@ -1863,7 +1964,7 @@ private GrpcValue toGrpcValue(Object o, ProjectionConfig pc) { } // 3.3 JSON mode - if (enc == ExecuteQueryRequest.ProjectionEncoding.PROJECTION_AS_JSON) { + if (enc == ProjectionEncoding.PROJECTION_AS_JSON) { if (logger.isDebugEnabled()) { logger.debug("GRPC-ENC [toGrpcValue] PROJECTION -> JSON rid={} type={}", @@ -1882,7 +1983,6 @@ private GrpcValue toGrpcValue(Object o, ProjectionConfig pc) { for (String k : doc.getPropertyNames()) m.put(k, doc.get(k)); - Object sanitized = sanitizeForJson(m); byte[] jsonBytes = GSON.toJson(sanitized).getBytes(java.nio.charset.StandardCharsets.UTF_8); @@ -2825,16 +2925,16 @@ private Object sanitizeForJson(Object x) { static final class ProjectionConfig { final boolean include; - final ExecuteQueryRequest.ProjectionEncoding enc; + final ProjectionEncoding enc; final int softLimitBytes; // 0 = no soft limit // Running counter for MAP/JSON builds: final java.util.concurrent.atomic.AtomicInteger used = new java.util.concurrent.atomic.AtomicInteger(0); volatile boolean truncated = false; - ProjectionConfig(boolean include, ExecuteQueryRequest.ProjectionEncoding enc, int softLimitBytes) { + ProjectionConfig(boolean include, ProjectionEncoding enc, int softLimitBytes) { this.include = include; - this.enc = enc == null ? ExecuteQueryRequest.ProjectionEncoding.PROJECTION_AS_LINK : enc; + this.enc = enc == null ? ProjectionEncoding.PROJECTION_AS_LINK : enc; this.softLimitBytes = Math.max(softLimitBytes, 0); } From 0d9fc0a4328f0955947925327fa0abadc3583733 Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Sun, 7 Sep 2025 20:46:25 -0700 Subject: [PATCH 31/35] Checkpoint Commit --- .../remote/grpc/utils/ProtoUtils.java | 2 + .../server/grpc/ArcadeDbGrpcService.java | 246 +++++++----------- 2 files changed, 97 insertions(+), 151 deletions(-) diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/utils/ProtoUtils.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/utils/ProtoUtils.java index 0ff069ab44..5118caa59c 100644 --- a/grpc-client/src/main/java/com/arcadedb/remote/grpc/utils/ProtoUtils.java +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/utils/ProtoUtils.java @@ -310,6 +310,8 @@ public static Object fromGrpcValue(GrpcValue v) { if ("json".equalsIgnoreCase(v.getLogicalType())) { String json = v.getBytesValue().toStringUtf8(); + //System.out.println("JSON: " + json); + @SuppressWarnings("unchecked") java.util.Map map = GSON.fromJson(json, java.util.Map.class); diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java index 91dd19d79c..3c9865f709 100644 --- a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java @@ -25,6 +25,7 @@ import com.arcadedb.schema.EdgeType; import com.arcadedb.schema.Schema; import com.arcadedb.schema.VertexType; +import com.arcadedb.serializer.JsonSerializer; import com.arcadedb.server.ArcadeDBServer; import com.arcadedb.server.grpc.InsertOptions.ConflictMode; import com.arcadedb.server.grpc.InsertOptions.TransactionMode; @@ -46,17 +47,21 @@ public class ArcadeDbGrpcService extends ArcadeDbServiceGrpc.ArcadeDbServiceImpl private static final Logger logger = LoggerFactory.getLogger(ArcadeDbGrpcService.class); - private static final java.util.Base64.Encoder B64 = java.util.Base64.getEncoder(); - - private static final com.google.gson.Gson GSON = new com.google.gson.GsonBuilder().disableHtmlEscaping().serializeNulls() - .registerTypeHierarchyAdapter(java.nio.ByteBuffer.class, - (com.google.gson.JsonSerializer) (src, t, - ctx) -> new com.google.gson.JsonPrimitive(B64.encodeToString(toBytes(src)))) - // Optional: make byte[] consistent too - .registerTypeHierarchyAdapter(byte[].class, - (com.google.gson.JsonSerializer) (src, t, ctx) -> new com.google.gson.JsonPrimitive(B64.encodeToString(src))) - - .create(); + + // Pick serializer once + private static final JsonSerializer FAST = + JsonSerializer.createJsonSerializer() + .setIncludeVertexEdges(false) + .setUseVertexEdgeSize(true) + .setUseCollectionSizeForEdges(true) + .setUseCollectionSize(false); + + private static final JsonSerializer SAFE = + JsonSerializer.createJsonSerializer() + .setIncludeVertexEdges(false) + .setUseVertexEdgeSize(true) + .setUseCollectionSizeForEdges(false) + .setUseCollectionSize(false); // Transaction management private final Map activeTransactions = new ConcurrentHashMap<>(); @@ -619,6 +624,9 @@ public void executeQuery(ExecuteQueryRequest request, StreamObserver= batchSize) { - + safeOnNext(scso, cancelled, batch.setTotalRecordsInBatch(inBatch).setRunningTotalEmitted(running).setIsLastBatch(false).build()); - + batch = QueryResult.newBuilder(); inBatch = 0; } @@ -968,17 +976,17 @@ private void streamMaterialized(Database db, StreamQueryRequest request, int bat Result r = rs.next(); if (r.isElement()) { - + all.add(convertToGrpcRecord(r.getElement().get(), db)); } else { - + GrpcRecord.Builder rb = GrpcRecord.newBuilder(); - + for (String p : r.getPropertyNames()) { rb.putProperties(p, convertPropToGrpcValue(p, r, projectionConfig)); // overload below } - + all.add(rb.build()); } } @@ -1029,22 +1037,22 @@ private void streamPaged(Database db, StreamQueryRequest request, int batchSize, while (rs.hasNext()) { if (cancelled.get()) return; - + Result r = rs.next(); - + if (r.isElement()) { - + b.addRecords(convertToGrpcRecord(r.getElement().get(), db)); count++; } else { - + GrpcRecord.Builder rb = GrpcRecord.newBuilder(); - + for (String p : r.getPropertyNames()) { rb.putProperties(p, convertPropToGrpcValue(p, r, projectionConfig)); // overload below } - + b.addRecords(rb.build()); count++; } @@ -1965,51 +1973,71 @@ private GrpcValue toGrpcValue(Object o, ProjectionConfig pc) { // 3.3 JSON mode if (enc == ProjectionEncoding.PROJECTION_AS_JSON) { - if (logger.isDebugEnabled()) { logger.debug("GRPC-ENC [toGrpcValue] PROJECTION -> JSON rid={} type={}", (doc.getIdentity() != null ? doc.getIdentity() : "null"), (doc.getType() != null ? doc.getTypeName() : "null")); } - // Build a simple Map and serialize via your existing Gson - java.util.Map m = new java.util.LinkedHashMap<>(); - - if (doc.getIdentity() != null && doc.getIdentity().isValid()) - m.put("@rid", doc.getIdentity().toString()); - - if (doc.getType() != null) - m.put("@type", doc.getTypeName()); + try { + + // Let ArcadeDB serialize the document/result properly + + boolean hasEmptyCollection = false; + for (String k : doc.getPropertyNames()) { + Object v = doc.get(k); + if (v instanceof java.util.Collection c && c.isEmpty()) { + hasEmptyCollection = true; break; + } + } - for (String k : doc.getPropertyNames()) - m.put(k, doc.get(k)); + var json = (hasEmptyCollection ? SAFE : FAST).serializeDocument(doc); - Object sanitized = sanitizeForJson(m); - byte[] jsonBytes = GSON.toJson(sanitized).getBytes(java.nio.charset.StandardCharsets.UTF_8); + byte[] jsonBytes = json.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8); - if (pc.softLimitBytes > 0 && jsonBytes.length > pc.softLimitBytes) { + // Soft limit handling + if (pc.softLimitBytes > 0 && jsonBytes.length > pc.softLimitBytes) { + pc.truncated = true; + if (logger.isDebugEnabled()) { + logger.debug("GRPC-ENC [toGrpcValue] PROJECTION JSON soft-limit hit; size={} limit={}", jsonBytes.length, + pc.softLimitBytes); + } + // Prefer a RID fallback if we have one + if (doc.getIdentity() != null && doc.getIdentity().isValid()) { + return GrpcValue.newBuilder().setLinkValue(GrpcLink.newBuilder().setRid(doc.getIdentity().toString()).build()) + .setLogicalType("rid").build(); + } + else { + jsonBytes = "{\"__truncated\":true}".getBytes(java.nio.charset.StandardCharsets.UTF_8); + } + } + else { + pc.used.addAndGet(jsonBytes.length); + } - pc.truncated = true; + return GrpcValue.newBuilder().setBytesValue(com.google.protobuf.ByteString.copyFrom(jsonBytes)).setLogicalType("json") + .build(); + } + catch (Throwable t) { + // Safe fallback: send as MAP (typed GrpcValue tree) + logger.warn("GRPC-ENC [toGrpcValue] PROJECTION_AS_JSON failed for rid={} type={}, falling back to MAP; err={}", + (doc.getIdentity() != null ? doc.getIdentity() : "null"), (doc.getType() != null ? doc.getTypeName() : "null"), + t.toString()); - if (logger.isDebugEnabled()) { - logger.debug("GRPC-ENC [toGrpcValue] PROJECTION JSON soft-limit hit; size={} limit={}", jsonBytes.length, - pc.softLimitBytes); - } + GrpcMap.Builder mb = GrpcMap.newBuilder(); - // Choose your policy: truncate, or fall back to LINK if possible if (doc.getIdentity() != null && doc.getIdentity().isValid()) { - return GrpcValue.newBuilder().setLinkValue(GrpcLink.newBuilder().setRid(doc.getIdentity().toString()).build()) - .setLogicalType("rid").build(); + mb.putEntries("@rid", GrpcValue.newBuilder() + .setLinkValue(GrpcLink.newBuilder().setRid(doc.getIdentity().toString()).build()).setLogicalType("rid").build()); } - else { - // Send a tiny marker instead of giant JSON - jsonBytes = "{\"__truncated\":true}".getBytes(java.nio.charset.StandardCharsets.UTF_8); + if (doc.getType() != null) { + mb.putEntries("@type", GrpcValue.newBuilder().setStringValue(doc.getTypeName()).build()); + } + for (String k : doc.getPropertyNames()) { + mb.putEntries(k, toGrpcValue(doc.get(k))); } - } - else { - pc.used.addAndGet(jsonBytes.length); - } - return GrpcValue.newBuilder().setBytesValue(com.google.protobuf.ByteString.copyFrom(jsonBytes)).setLogicalType("json").build(); + return GrpcValue.newBuilder().setMapValue(mb.build()).build(); + } } // Shouldn’t get here, but fall back @@ -2831,98 +2859,6 @@ private GrpcValue gsonToGrpc(JsonElement n) { return b.setStringValue(n.toString()).build(); } - private static byte[] toBytes(java.nio.ByteBuffer buf) { - java.nio.ByteBuffer dup = buf.asReadOnlyBuffer(); - byte[] out = new byte[dup.remaining()]; - dup.get(out); - return out; - } - - /** - * Deep-scrub arbitrary objects into a JSON-safe graph - * (Map/List/primitive/String). - */ - private Object sanitizeForJson(Object x) { - if (x == null) - return null; - - // Primitives & simple types - if (x instanceof String || x instanceof Number || x instanceof Boolean) - return x; - - // Common binary holders -> base64 string - if (x instanceof byte[] arr) - return B64.encodeToString(arr); - if (x instanceof java.nio.ByteBuffer bb) - return B64.encodeToString(toBytes(bb)); - - // Arcade IDs/links/dates - if (x instanceof com.arcadedb.database.RID rid) - return rid.toString(); - if (x instanceof com.arcadedb.database.Identifiable id) - return id.getIdentity() != null ? id.getIdentity().toString() : null; - if (x instanceof java.util.Date d) - return java.time.Instant.ofEpochMilli(d.getTime()).toString(); - - // Arcade Result -> Map projection - if (x instanceof com.arcadedb.query.sql.executor.Result res) { - var m = new java.util.LinkedHashMap(); - try { - for (String k : res.getPropertyNames()) { - m.put(k, sanitizeForJson(res.getProperty(k))); - } - } - catch (Throwable ignore) { - } - return m; - } - - // Embedded / Document - if (x instanceof com.arcadedb.database.Document doc) { - var m = new java.util.LinkedHashMap(); - if (doc.getIdentity() != null && doc.getIdentity().isValid()) - m.put("@rid", doc.getIdentity().toString()); - if (doc.getType() != null) - m.put("@type", doc.getTypeName()); - for (String k : doc.getPropertyNames()) { - m.put(k, sanitizeForJson(doc.get(k))); - } - return m; - } - - // Collections/arrays - if (x instanceof java.util.Map in) { - var m = new java.util.LinkedHashMap(); - in.forEach((k, v) -> m.put(String.valueOf(k), sanitizeForJson(v))); - return m; - } - if (x instanceof java.util.Collection c) { - var out = new java.util.ArrayList<>(); - for (Object e : c) - out.add(sanitizeForJson(e)); - return out; - } - if (x.getClass().isArray()) { - int n = java.lang.reflect.Array.getLength(x); - var out = new java.util.ArrayList<>(n); - for (int i = 0; i < n; i++) - out.add(sanitizeForJson(java.lang.reflect.Array.get(x, i))); - return out; - } - - // FINAL GUARD: any JDK “handle-ish” or unknown classes -> describe as String - String pkg = x.getClass().getPackageName(); - if (pkg.startsWith("java.io") || pkg.startsWith("java.nio") || pkg.startsWith("sun.") || pkg.startsWith("jdk.")) { - if (logger.isDebugEnabled()) { - logger.debug("GRPC-ENC [toGrpcValue] JSON scrubbed {} via toString()", x.getClass().getName()); - } - return String.valueOf(x); - } - - // Default safe fallback - return String.valueOf(x); - } - static final class ProjectionConfig { final boolean include; final ProjectionEncoding enc; @@ -2947,5 +2883,13 @@ boolean wouldExceed(int add) { } return after > softLimitBytes; } + + public boolean isInclude() { + return include; + } + + public ProjectionEncoding getEnc() { + return enc; + } } } \ No newline at end of file From 2455ef70d3252ec82b99ecbde5a80a34187edeb9 Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Mon, 8 Sep 2025 07:53:55 -0700 Subject: [PATCH 32/35] Checkpoint Commit --- .../remote/grpc/RemoteGrpcDatabase.java | 1379 +++++++++++------ .../server/grpc/ArcadeDbGrpcService.java | 1 + 2 files changed, 934 insertions(+), 446 deletions(-) diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java index 61c06cc099..eb15f8f684 100644 --- a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java @@ -7,8 +7,13 @@ import java.util.List; import java.util.Map; import java.util.NoSuchElementException; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; + +import javax.annotation.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -79,7 +84,6 @@ import io.grpc.CallCredentials; import io.grpc.Metadata; -import io.grpc.Status; import io.grpc.StatusException; import io.grpc.StatusRuntimeException; import io.grpc.stub.BlockingClientCall; @@ -110,19 +114,19 @@ public class RemoteGrpcDatabase extends RemoteDatabase { private final String userPassword; private String databaseName; private RemoteTransactionExplicitLock explicitLock; - + protected RemoteGrpcServer remoteGrpcServer; - public RemoteGrpcDatabase(final RemoteGrpcServer remoteGrpcServer, final String server, final int grpcPort, final int httpPort, final String databaseName, final String userName, - final String userPassword) { + public RemoteGrpcDatabase(final RemoteGrpcServer remoteGrpcServer, final String server, final int grpcPort, final int httpPort, + final String databaseName, final String userName, final String userPassword) { this(remoteGrpcServer, server, grpcPort, httpPort, databaseName, userName, userPassword, new ContextConfiguration()); } - public RemoteGrpcDatabase(final RemoteGrpcServer remoteGrpcServer, final String host, final int grpcPort, final int httpPort, final String databaseName, final String userName, - final String userPassword, final ContextConfiguration configuration) { + public RemoteGrpcDatabase(final RemoteGrpcServer remoteGrpcServer, final String host, final int grpcPort, final int httpPort, + final String databaseName, final String userName, final String userPassword, final ContextConfiguration configuration) { super(host, httpPort, databaseName, userName, userPassword, configuration); - + this.remoteGrpcServer = remoteGrpcServer; this.userName = userName; @@ -187,7 +191,7 @@ public RemoteSchema getSchema() { @Override public void close() { - + if (transactionId != null) { rollback(); } @@ -203,18 +207,37 @@ public void begin(final Database.TRANSACTION_ISOLATION_LEVEL isolationLevel) { BeginTransactionRequest request = BeginTransactionRequest.newBuilder().setDatabase(getName()).setCredentials(buildCredentials()) .setIsolation(mapIsolationLevel(isolationLevel)).build(); - try { - BeginTransactionResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).beginTransaction(request); - transactionId = response.getTransactionId(); - // Store transaction ID in parent class session management - setSessionId(transactionId); - } - catch (StatusRuntimeException e) { - throw new TransactionException("Error on transaction begin", e); - } - catch (StatusException e) { - throw new TransactionException("Error on transaction begin", e); - } + callUnaryVoid("BeginTransaction", () -> { + + try { + BeginTransactionResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .beginTransaction(request); + transactionId = response.getTransactionId(); + // Store transaction ID in parent class session management + setSessionId(transactionId); + } + catch (StatusRuntimeException e) { + throw new TransactionException("Error on transaction begin", e); + } + catch (StatusException e) { + throw new TransactionException("Error on transaction begin", e); + } + + debugTx = new TxDebug(this.getName(), /* label set by TM */ null); + logTx("BEGIN(local)", null); + try { + checkCrossThreadUse("before BeginTransaction"); + // send BeginTransaction RPC ... + debugTx.beginRpcSent = true; + debugTx.rpcSeq.incrementAndGet(); + logTx("BEGIN sent", "BeginTransaction"); + } + catch (RuntimeException e) { + logTx("BEGIN failed", "BeginTransaction"); + debugTx = null; + throw e; + } + }); } @Override @@ -225,57 +248,97 @@ public void commit() { if (transactionId == null) throw new TransactionException("Transaction not begun"); - CommitTransactionRequest request = CommitTransactionRequest.newBuilder() - .setTransaction(TransactionContext.newBuilder().setTransactionId(transactionId).setDatabase(getName()).build()) - .setCredentials(buildCredentials()).build(); + logTx("COMMIT(local)", null); + checkCrossThreadUse("before CommitTransaction"); - try { - CommitTransactionResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).commitTransaction(request); - if (!response.getSuccess()) { - throw new TransactionException("Failed to commit transaction: " + response.getMessage()); + callUnaryVoid("CommitTransaction", () -> { + + try { + + CommitTransactionRequest request = CommitTransactionRequest.newBuilder() + .setTransaction(TransactionContext.newBuilder().setTransactionId(transactionId).setDatabase(getName()).build()) + .setCredentials(buildCredentials()).build(); + + try { + CommitTransactionResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .commitTransaction(request); + if (!response.getSuccess()) { + throw new TransactionException("Failed to commit transaction: " + response.getMessage()); + } + } + catch (StatusRuntimeException e) { + handleGrpcException(e); + } + catch (StatusException e) { + handleGrpcException(e); + } + finally { + transactionId = null; + setSessionId(null); + } + + if (debugTx != null) { + debugTx.committed = true; + debugTx.rpcSeq.incrementAndGet(); + } + logTx("COMMIT sent", "CommitTransaction"); } - } - catch (StatusRuntimeException e) { - handleGrpcException(e); - } - catch (StatusException e) { - handleGrpcException(e); - } - finally { - transactionId = null; - setSessionId(null); - } + finally { + debugTx = null; + } + + }); } @Override public void rollback() { + checkDatabaseIsOpen(); stats.txRollbacks.incrementAndGet(); if (transactionId == null) throw new TransactionException("Transaction not begun"); - RollbackTransactionRequest request = RollbackTransactionRequest.newBuilder() - .setTransaction(TransactionContext.newBuilder().setTransactionId(transactionId).setDatabase(getName()).build()) - .setCredentials(buildCredentials()).build(); + logTx("ROLLBACK(local)", null); + checkCrossThreadUse("before RollbackTransaction"); - try { - RollbackTransactionResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) - .rollbackTransaction(request); - if (!response.getSuccess()) { - throw new TransactionException("Failed to rollback transaction: " + response.getMessage()); + callUnaryVoid("RollbackTransaction", () -> { + + try { + + RollbackTransactionRequest request = RollbackTransactionRequest.newBuilder() + .setTransaction(TransactionContext.newBuilder().setTransactionId(transactionId).setDatabase(getName()).build()) + .setCredentials(buildCredentials()).build(); + + try { + RollbackTransactionResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .rollbackTransaction(request); + if (!response.getSuccess()) { + throw new TransactionException("Failed to rollback transaction: " + response.getMessage()); + } + } + catch (StatusRuntimeException e) { + throw new TransactionException("Error on transaction rollback", e); + } + catch (StatusException e) { + throw new TransactionException("Error on transaction rollback", e); + } + finally { + transactionId = null; + setSessionId(null); + } + + if (debugTx != null) { + debugTx.rolledBack = true; + debugTx.rpcSeq.incrementAndGet(); + } + logTx("ROLLBACK sent", "RollbackTransaction"); } - } - catch (StatusRuntimeException e) { - throw new TransactionException("Error on transaction rollback", e); - } - catch (StatusException e) { - throw new TransactionException("Error on transaction rollback", e); - } - finally { - transactionId = null; - setSessionId(null); - } + finally { + debugTx = null; + } + + }); } @Override @@ -286,20 +349,51 @@ public void deleteRecord(final Record record) { if (record.getIdentity() == null) throw new IllegalArgumentException("Cannot delete a non persistent record"); - DeleteRecordRequest request = DeleteRecordRequest.newBuilder().setDatabase(getName()).setRid(record.getIdentity().toString()) + final DeleteRecordRequest req = DeleteRecordRequest.newBuilder().setDatabase(getName()).setRid(record.getIdentity().toString()) .setCredentials(buildCredentials()).build(); try { - DeleteRecordResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).deleteRecord(request); - if (!response.getSuccess()) { - throw new DatabaseOperationException("Failed to delete record: " + response.getMessage()); + if (logger.isDebugEnabled()) { + logger.debug("CLIENT deleteRecord: db={}, tx={}, rid={}", getName(), (transactionId != null), record.getIdentity()); + } + + final DeleteRecordResponse resp = callUnary("DeleteRecord", + () -> blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).deleteRecord(req)); + + // Prefer the proto's 'deleted' flag (your other overload uses it) + if (!resp.getDeleted()) { + throw new DatabaseOperationException( + "Failed to delete record: " + (resp.getMessage().isEmpty() ? "unknown error" : resp.getMessage())); } + } - catch (StatusRuntimeException e) { - handleGrpcException(e); + catch (io.grpc.StatusRuntimeException | io.grpc.StatusException e) { + handleGrpcException(e); // rethrows mapped domain exception + throw new IllegalStateException("unreachable"); } - catch (StatusException e) { - handleGrpcException(e); + } + + public boolean deleteRecord(final String rid, final long timeoutMs) { + checkDatabaseIsOpen(); + stats.deleteRecord.incrementAndGet(); + + final DeleteRecordRequest req = DeleteRecordRequest.newBuilder().setDatabase(getName()).setRid(rid).setCredentials(buildCredentials()) + .build(); + + try { + if (logger.isDebugEnabled()) { + logger.debug("CLIENT deleteRecord: db={}, tx={}, rid={}, timeoutMs={}", getName(), (transactionId != null), rid, timeoutMs); + } + + final DeleteRecordResponse res = callUnary("DeleteRecord", + () -> blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).deleteRecord(req)); + + return res.getDeleted(); + + } + catch (io.grpc.StatusRuntimeException | io.grpc.StatusException e) { + handleGrpcException(e); // rethrows mapped domain exception + throw new IllegalStateException("unreachable"); } } @@ -315,21 +409,10 @@ public Iterator iterateType(final String typeName, final boolean polymor public Iterator iterateBucket(final String bucketName) { return streamQuery("select from bucket:`" + bucketName + "`"); } - -// @Override -// public ResultSet command(final String language, final String command) { -// checkDatabaseIsOpen(); -// stats.commands.incrementAndGet(); -// -// final Map params = new HashMap(); -// -// return commandInternal(language, command, params); -// } -// @Override public ResultSet command(final String language, final String command, final ContextConfiguration configuration, final Object... args) { - + checkDatabaseIsOpen(); stats.commands.incrementAndGet(); @@ -343,7 +426,7 @@ public ResultSet command(final String language, final String command, final Cont final Map params) { checkDatabaseIsOpen(); stats.commands.incrementAndGet(); - + return commandInternal(language, command, params); } @@ -353,10 +436,10 @@ public ResultSet command(final String language, final String command, final Obje stats.commands.incrementAndGet(); final Map params = mapArgs(args); - + return commandInternal(language, command, params); } - + @Override public ResultSet command(final String language, final String command, final Map params) { checkDatabaseIsOpen(); @@ -366,7 +449,7 @@ public ResultSet command(final String language, final String command, final Map< } private ResultSet commandInternal(final String language, final String command, final Map params) { - + ExecuteCommandRequest.Builder requestBuilder = ExecuteCommandRequest.newBuilder().setDatabase(getName()).setCommand(command) .setLanguage(language).setCredentials(buildCredentials()); @@ -377,19 +460,19 @@ private ResultSet commandInternal(final String language, final String command, f if (params != null && !params.isEmpty()) { requestBuilder.putAllParameters(convertParamsToGrpcValue(params)); } - + boolean returnRows = true; - + requestBuilder.setReturnRows(returnRows); - + try { if (logger.isDebugEnabled()) logger.debug("CLIENT executeCommand: db={}, tx={}, cmdLen={}, params={}", getName(), (transactionId != null), requestBuilder.getCommand().length(), requestBuilder.getParametersCount()); - ExecuteCommandResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) - .executeCommand(requestBuilder.build()); + final ExecuteCommandResponse response = callUnary("ExecuteCommand", + () -> blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).executeCommand(requestBuilder.build())); if (logger.isDebugEnabled()) logger.debug("CLIENT executeCommand: success = {}", response.getSuccess()); @@ -399,24 +482,24 @@ private ResultSet commandInternal(final String language, final String command, f } ResultSet resultSet; - - if (returnRows) { + + if (returnRows) { resultSet = createGrpcResultSet(response); } else { - + resultSet = new InternalResultSet(); if (response.getAffectedRecords() > 0) { - + Map result = new HashMap<>(); result.put("affected", response.getAffectedRecords()); result.put("executionTime", response.getExecutionTimeMs()); - - ((InternalResultSet)resultSet).add(new ResultInternal(result)); + + ((InternalResultSet) resultSet).add(new ResultInternal(result)); } } - + return resultSet; } catch (StatusRuntimeException e) { @@ -428,7 +511,7 @@ private ResultSet commandInternal(final String language, final String command, f return new InternalResultSet(); } } - + @Override public ResultSet query(final String language, final String query, final Object... args) { checkDatabaseIsOpen(); @@ -437,29 +520,27 @@ public ResultSet query(final String language, final String query, final Object.. final Map params = mapArgs(args); RemoteGrpcConfig remoteGrpcConfig = new RemoteGrpcConfig(true, ProjectionEncoding.PROJECTION_AS_JSON, 0); - + return query(language, query, remoteGrpcConfig, params); } - @Override public ResultSet query(final String language, final String query, final Map params) { - + RemoteGrpcConfig remoteGrpcConfig = new RemoteGrpcConfig(true, ProjectionEncoding.PROJECTION_AS_JSON, 0); - + return query(language, query, remoteGrpcConfig, params); } - public ResultSet query(final String language, final String query, RemoteGrpcConfig remoteGrpcConfig, final Object... args) { - + final Map params = mapArgs(args); return query(language, query, remoteGrpcConfig, params); } - + public ResultSet query(final String language, final String query, RemoteGrpcConfig remoteGrpcConfig, final Map params) { - + checkDatabaseIsOpen(); stats.queries.incrementAndGet(); @@ -473,23 +554,25 @@ public ResultSet query(final String language, final String query, RemoteGrpcConf if (params != null && !params.isEmpty()) { requestBuilder.putAllParameters(convertParamsToGrpcValue(params)); } - - ProjectionSettings projectionSettings = ProjectionSettings.newBuilder() - .setIncludeProjections(remoteGrpcConfig.isIncludeProjections()) - .setProjectionEncoding(remoteGrpcConfig.getProjectionEncoding()) - .setSoftLimitBytes(Int32Value.newBuilder().setValue(remoteGrpcConfig.getSoftLimitBytes()).build()) - .build(); + + ProjectionSettings projectionSettings = ProjectionSettings.newBuilder().setIncludeProjections(remoteGrpcConfig.isIncludeProjections()) + .setProjectionEncoding(remoteGrpcConfig.getProjectionEncoding()) + .setSoftLimitBytes(Int32Value.newBuilder().setValue(remoteGrpcConfig.getSoftLimitBytes()).build()).build(); requestBuilder.setProjectionSettings(projectionSettings); try { - if (logger.isDebugEnabled()) + if (logger.isDebugEnabled()) { logger.debug("CLIENT executeQuery: db={}, tx={}, queryLen={}, params={}", getName(), (transactionId != null), requestBuilder.getQuery().length(), requestBuilder.getParametersCount()); + } + + final ExecuteQueryRequest req = requestBuilder.build(); - ExecuteQueryResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) - .executeQuery(requestBuilder.build()); + final ExecuteQueryResponse response = callUnary("ExecuteQuery", + () -> blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) // or getTimeout(MODE_QUERY) + .executeQuery(req)); if (logger.isDebugEnabled()) { int _r = 0; @@ -499,17 +582,12 @@ public ResultSet query(final String language, final String query, RemoteGrpcConf } return createGrpcResultSet(response); } - catch (StatusRuntimeException e) { - handleGrpcException(e); - return new InternalResultSet(); - } - catch (StatusException e) { + catch (io.grpc.StatusException | io.grpc.StatusRuntimeException e) { handleGrpcException(e); return new InternalResultSet(); } } - - + public ExecuteCommandResponse execSql(String db, String sql, Map params, long timeoutMs) { return executeCommand(db, "sql", sql, params, /* returnRows */ false, /* maxRows */ 0, txBeginCommit(), timeoutMs); } @@ -527,14 +605,15 @@ public ExecuteCommandResponse executeCommand(String language, String command, Ma if (tx != null) reqB.setTransaction(tx); - // credentials: if your stub builds creds implicitly, set here if required reqB.setCredentials(buildCredentials()); try { - return blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).executeCommand(reqB.build()); + return callUnary("ExecuteCommand", + () -> blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).executeCommand(reqB.build())); } - catch (StatusException e) { - throw new RuntimeException("Failed to execute command: " + e.getMessage(), e); + catch (StatusException | StatusRuntimeException e) { + // handleGrpcException already called in callUnary, this is unreachable + throw new IllegalStateException("unreachable"); } } @@ -547,14 +626,15 @@ public ExecuteCommandResponse executeCommand(String database, String language, S if (tx != null) reqB.setTransaction(tx); - // credentials: if your stub builds creds implicitly, set here if required reqB.setCredentials(buildCredentials()); try { - return blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).executeCommand(reqB.build()); + return callUnary("ExecuteCommand", + () -> blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).executeCommand(reqB.build())); } - catch (StatusException e) { - throw new RuntimeException("Failed to execute command: " + e.getMessage(), e); + catch (StatusException | StatusRuntimeException e) { + // handleGrpcException already called in callUnary, this is unreachable + throw new IllegalStateException("unreachable"); } } @@ -628,20 +708,20 @@ protected RID saveRecord(final MutableDocument record) { } private RemoteGrpcConfig getDefaultRemoteGrpcConfig() { - - return new RemoteGrpcConfig(true, ProjectionEncoding.PROJECTION_AS_JSON, 0); + + return new RemoteGrpcConfig(true, ProjectionEncoding.PROJECTION_AS_JSON, 0); } - + // Convenience: default batch size stays 100, default mode = CURSOR public Iterator queryStream(final String language, final String query) { - return queryStream(language, query, getDefaultRemoteGrpcConfig(), /* batchSize */100, StreamQueryRequest.RetrievalMode.CURSOR); + return queryStream(language, query, getDefaultRemoteGrpcConfig(), /* batchSize */100, StreamQueryRequest.RetrievalMode.CURSOR); } public Iterator queryStream(final String language, final String query, final RemoteGrpcConfig config) { return queryStream(language, query, config, /* batchSize */100, StreamQueryRequest.RetrievalMode.CURSOR); } - + public Iterator queryStream(final String language, final String query, final int batchSize) { return queryStream(language, query, getDefaultRemoteGrpcConfig(), batchSize, StreamQueryRequest.RetrievalMode.CURSOR); } @@ -650,20 +730,23 @@ public Iterator queryStream(final String language, final String query, R return queryStream(language, query, config, batchSize, StreamQueryRequest.RetrievalMode.CURSOR); } - public Iterator queryStream(final String language, final String query, final int batchSize, final StreamQueryRequest.RetrievalMode mode) { - return queryStream(language, query, getDefaultRemoteGrpcConfig(), batchSize, mode); + public Iterator queryStream(final String language, final String query, final int batchSize, + final StreamQueryRequest.RetrievalMode mode) { + return queryStream(language, query, getDefaultRemoteGrpcConfig(), batchSize, mode); } // NEW: choose retrieval mode - public Iterator queryStream(final String language, final String query, final RemoteGrpcConfig config, final int batchSize, final StreamQueryRequest.RetrievalMode mode) { - - return queryStream(language, query, config, Map.of(), batchSize, mode); + public Iterator queryStream(final String language, final String query, final RemoteGrpcConfig config, final int batchSize, + final StreamQueryRequest.RetrievalMode mode) { + + return queryStream(language, query, config, Map.of(), batchSize, mode); } // PARAMETERIZED variant with retrieval mode - public Iterator queryStream(final String language, final String query, final RemoteGrpcConfig config, final Map params, final int batchSize, final StreamQueryRequest.RetrievalMode mode) { - + public Iterator queryStream(final String language, final String query, final RemoteGrpcConfig config, + final Map params, final int batchSize, final StreamQueryRequest.RetrievalMode mode) { + checkDatabaseIsOpen(); stats.queries.incrementAndGet(); @@ -674,8 +757,9 @@ public Iterator queryStream(final String language, final String query, f b.putAllParameters(convertParamsToGrpcValue(params)); } - final BlockingClientCall responseIterator = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) - .streamQuery(b.build()); + // Open the stream with deadline via helper + final BlockingClientCall responseIterator = callServerStreaming("StreamQuery", + () -> blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).streamQuery(b.build())); return new Iterator() { private Iterator currentBatch = Collections.emptyIterator(); @@ -685,27 +769,34 @@ public boolean hasNext() { if (currentBatch.hasNext()) return true; - try { + if (debugTx != null) { + checkCrossThreadUse("streamQuery.hasNext"); + } + try { while (responseIterator.hasNext()) { - - QueryResult result = responseIterator.read(); + final QueryResult result = responseIterator.read(); if (result.getRecordsCount() == 0) { if (result.getIsLastBatch()) return false; - continue; + continue; // empty non-terminal batch } currentBatch = result.getRecordsList().iterator(); return true; } - - return false; + return false; // stream exhausted + } + catch (io.grpc.StatusRuntimeException | io.grpc.StatusException e) { + handleGrpcException(e); // rethrows mapped runtime exception + throw new IllegalStateException("unreachable"); + } + catch (RuntimeException re) { + throw re; } catch (Exception e) { - - throw new RuntimeException("Interrupted while waiting for stream", e); + throw new RuntimeException("Stream failed", e); } } @@ -713,6 +804,9 @@ public boolean hasNext() { public Record next() { if (!hasNext()) throw new NoSuchElementException(); + if (debugTx != null) { + checkCrossThreadUse("streamQuery.next"); + } return grpcRecordToDBRecord(currentBatch.next()); } }; @@ -720,17 +814,18 @@ public Record next() { // Keep the old signature working (defaults to CURSOR) public Iterator queryStream(final String language, final String query, final Map params, final int batchSize) { - + return queryStream(language, query, getDefaultRemoteGrpcConfig(), params, batchSize, StreamQueryRequest.RetrievalMode.CURSOR); } - public Iterator queryStream(final String language, final String query, final RemoteGrpcConfig config, final Map params, final int batchSize) { - + public Iterator queryStream(final String language, final String query, final RemoteGrpcConfig config, + final Map params, final int batchSize) { + return queryStream(language, query, config, params, batchSize, StreamQueryRequest.RetrievalMode.CURSOR); } - + public static final class QueryBatch { - + private final List records; private final int totalInBatch; private final long runningTotal; @@ -765,15 +860,15 @@ public Iterator queryStreamBatches(final String language, final Stri checkDatabaseIsOpen(); stats.queries.incrementAndGet(); - StreamQueryRequest.Builder b = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(query).setCredentials(buildCredentials()) - .setBatchSize(batchSize > 0 ? batchSize : 100).setRetrievalMode(mode); + final StreamQueryRequest.Builder b = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(query) + .setCredentials(buildCredentials()).setBatchSize(batchSize > 0 ? batchSize : 100).setRetrievalMode(mode); if (params != null && !params.isEmpty()) { b.putAllParameters(convertParamsToGrpcValue(params)); } - final BlockingClientCall responseIterator = blockingStub.withWaitForReady() // optional, improves robustness - .withDeadlineAfter(/* e.g. */ 10, TimeUnit.MINUTES).streamQuery(b.build()); + final BlockingClientCall responseIterator = callServerStreaming("StreamQuery", + () -> blockingStub.withWaitForReady().withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).streamQuery(b.build())); return new Iterator() { private QueryBatch nextBatch = null; @@ -786,41 +881,50 @@ public boolean hasNext() { if (drained) return false; - try { + if (debugTx != null) { + checkCrossThreadUse("streamQueryBatches.hasNext"); + } + try { while (responseIterator.hasNext()) { + final QueryResult qr = responseIterator.read(); - QueryResult qr = responseIterator.read(); - - int n = qr.getTotalRecordsInBatch(); // server-populated - // Guard: some servers could omit this; fallback to list size. - if (n == 0) - n = qr.getRecordsCount(); + int totalInBatch = qr.getTotalRecordsInBatch(); + if (totalInBatch == 0) + totalInBatch = qr.getRecordsCount(); if (qr.getRecordsCount() == 0 && !qr.getIsLastBatch()) { - // skip empty non-terminal batch - continue; + continue; // empty non-terminal batch } - List converted = new ArrayList<>(qr.getRecordsCount()); + final List converted = new ArrayList<>(qr.getRecordsCount()); for (GrpcRecord gr : qr.getRecordsList()) { converted.add(grpcRecordToDBRecord(gr)); } - nextBatch = new QueryBatch(converted, n, qr.getRunningTotalEmitted(), qr.getIsLastBatch()); + nextBatch = new QueryBatch(converted, totalInBatch, qr.getRunningTotalEmitted(), qr.getIsLastBatch()); - if (qr.getIsLastBatch()) { - drained = true; // no more after this (even if server sent empty terminal) - } + if (qr.getIsLastBatch()) + drained = true; return true; } drained = true; return false; } + catch (io.grpc.StatusRuntimeException | io.grpc.StatusException e) { + handleGrpcException(e); // rethrows mapped exception + throw new IllegalStateException("unreachable"); + } + catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Stream interrupted", ie); + } + catch (RuntimeException re) { + throw re; + } catch (Exception e) { - - throw new RuntimeException("Interrupted while waiting for stream", e); + throw new RuntimeException("Stream failed", e); } } @@ -828,133 +932,186 @@ public boolean hasNext() { public QueryBatch next() { if (!hasNext()) throw new NoSuchElementException(); - QueryBatch out = nextBatch; + final QueryBatch out = nextBatch; nextBatch = null; return out; } }; } - public Iterator queryStream(String database, String sql, Map params, int batchSize, - StreamQueryRequest.RetrievalMode mode, TransactionContext tx, long timeoutMs) { - - var reqB = StreamQueryRequest.newBuilder().setDatabase(database).setQuery(sql).putAllParameters(convertParamsToGrpcValue(params)) - .setCredentials(buildCredentials()).setBatchSize(batchSize > 0 ? batchSize : 100).setRetrievalMode(mode); + public Iterator queryStream(final String database, final String sql, final Map params, final int batchSize, + final StreamQueryRequest.RetrievalMode mode, final TransactionContext tx, final long timeoutMs) { + final StreamQueryRequest.Builder reqB = StreamQueryRequest.newBuilder().setDatabase(database).setQuery(sql) + .putAllParameters(convertParamsToGrpcValue(params)).setCredentials(buildCredentials()) + .setBatchSize(batchSize > 0 ? batchSize : 100).setRetrievalMode(mode); if (tx != null) reqB.setTransaction(tx); - final BlockingClientCall it = blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).streamQuery(reqB.build()); - - return new Iterator<>() { + final BlockingClientCall it = callServerStreaming("StreamQuery", + () -> blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).streamQuery(reqB.build())); + return new Iterator() { private Iterator curr = Collections.emptyIterator(); + @Override public boolean hasNext() { - if (curr.hasNext()) return true; - try { - - if (it.hasNext()) { + if (debugTx != null) { + checkCrossThreadUse("streamQuery.hasNext"); + } - curr = it.read().getRecordsList().iterator(); - return hasNext(); + try { + while (it.hasNext()) { + final QueryResult qr = it.read(); + curr = qr.getRecordsList().iterator(); + if (curr.hasNext()) + return true; + // else loop to fetch next batch (handles empty batches) } + return false; } - catch (InterruptedException e) { - throw new RuntimeException("Interrupted while waiting for stream", e); + catch (io.grpc.StatusRuntimeException | io.grpc.StatusException e) { + handleGrpcException(e); + throw new IllegalStateException("unreachable"); } - catch (StatusException e) { - throw new RuntimeException("Stream query failed: " + e.getStatus().getDescription()); + catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Stream interrupted", ie); + } + catch (RuntimeException re) { + throw re; + } + catch (Exception e) { + throw new RuntimeException("Stream failed", e); } - - return false; } + @Override public GrpcRecord next() { if (!hasNext()) throw new NoSuchElementException(); + if (debugTx != null) { + checkCrossThreadUse("streamQuery.next"); + } return curr.next(); } }; } - public Iterator queryStreamBatches(String passLabel, String sql, Map params, int batchSize, - StreamQueryRequest.RetrievalMode mode, TransactionContext tx, long timeoutMs) { - - var reqB = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(sql).putAllParameters(convertParamsToGrpcValue(params)) - .setCredentials(buildCredentials()).setBatchSize(batchSize > 0 ? batchSize : 100).setRetrievalMode(mode); + public Iterator queryStreamBatches(final String passLabel, final String sql, final Map params, + final int batchSize, final StreamQueryRequest.RetrievalMode mode, final TransactionContext tx, final long timeoutMs) { + final StreamQueryRequest.Builder reqB = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(sql) + .putAllParameters(convertParamsToGrpcValue(params)).setCredentials(buildCredentials()) + .setBatchSize(batchSize > 0 ? batchSize : 100).setRetrievalMode(mode); if (tx != null) reqB.setTransaction(tx); - final BlockingClientCall respIter = blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS) - .streamQuery(reqB.build()); - - return new Iterator<>() { + final BlockingClientCall respIter = callServerStreaming("StreamQuery[" + passLabel + "]", + () -> blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).streamQuery(reqB.build())); + return new Iterator() { + @Override public boolean hasNext() { - + if (debugTx != null) { + checkCrossThreadUse("streamQueryBatches(" + passLabel + ").hasNext"); + } try { return respIter.hasNext(); } - catch (InterruptedException e) { - throw new RuntimeException(e); + catch (io.grpc.StatusRuntimeException | io.grpc.StatusException e) { + handleGrpcException(e); + throw new IllegalStateException("unreachable"); } - catch (StatusException e) { - throw new RuntimeException(e); + catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Stream interrupted", ie); } } + @Override public QueryBatch next() { - - QueryResult qr; - + if (debugTx != null) { + checkCrossThreadUse("streamQueryBatches(" + passLabel + ").next"); + } try { - qr = respIter.read(); - List converted = new ArrayList<>(qr.getRecordsCount()); + final QueryResult qr = respIter.read(); + + int totalInBatch = qr.getTotalRecordsInBatch(); + if (totalInBatch == 0) + totalInBatch = qr.getRecordsCount(); + final List converted = new ArrayList<>(qr.getRecordsCount()); for (GrpcRecord gr : qr.getRecordsList()) { converted.add(grpcRecordToDBRecord(gr)); } - return new QueryBatch(converted, qr.getTotalRecordsInBatch(), // int totalInBatch - qr.getRunningTotalEmitted(), // long runningTotal - qr.getIsLastBatch() // boolean lastBatch - ); + return new QueryBatch(converted, totalInBatch, qr.getRunningTotalEmitted(), qr.getIsLastBatch()); } - catch (InterruptedException e) { - throw new RuntimeException(e); + catch (io.grpc.StatusRuntimeException | io.grpc.StatusException e) { + handleGrpcException(e); + throw new IllegalStateException("unreachable"); } - catch (StatusException e) { - throw new RuntimeException(e); + catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Stream interrupted", ie); } } }; } - public String createVertex(String cls, Map props, long timeoutMs) { - // Build the nested proto GrpcRecord payload - GrpcRecord recMsg = GrpcRecord.newBuilder().putAllProperties(convertParamsToGrpcValue(props)).build(); + public String createRecord(final String cls, final Map props, final long timeoutMs) { + checkDatabaseIsOpen(); + + if (cls == null || cls.isBlank()) + throw new IllegalArgumentException("cls must be non-empty"); + if (props == null) + throw new IllegalArgumentException("props must be non-null"); - // Build the Create request - CreateRecordRequest req = CreateRecordRequest.newBuilder().setDatabase(getName()).setType(cls).setRecord(recMsg) // <<<< NESTED RECORD - // (not top-level - // properties) + // Build payload + final GrpcRecord recMsg = GrpcRecord.newBuilder().putAllProperties(convertParamsToGrpcValue(props)).build(); + + final CreateRecordRequest req = CreateRecordRequest.newBuilder().setDatabase(getName()).setType(cls).setRecord(recMsg) .setCredentials(buildCredentials()).build(); - // Call RPC - CreateRecordResponse res; + try { + if (logger.isDebugEnabled()) { + logger.debug("CLIENT createRecord: db={}, txOpen={}, type={}, propCount={}, timeoutMs={}", getName(), (transactionId != null), + cls, props.size(), timeoutMs); + } + + final CreateRecordResponse res = callUnary("CreateRecord", + () -> blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).createRecord(req)); + + return res.getRid(); // e.g. "#12:0" + + } + catch (io.grpc.StatusRuntimeException | io.grpc.StatusException e) { + handleGrpcException(e); // rethrows mapped domain exception + throw new IllegalStateException("unreachable"); + } + } + + public String createRecordTx(final String cls, final Map props, final long timeoutMs) { + checkDatabaseIsOpen(); + + final GrpcRecord recMsg = GrpcRecord.newBuilder().putAllProperties(convertParamsToGrpcValue(props)).build(); + + final CreateRecordRequest req = CreateRecordRequest.newBuilder().setDatabase(getName()).setType(cls).setRecord(recMsg) + .setTransaction(TransactionContext.newBuilder().setBegin(true).setCommit(true)).setCredentials(buildCredentials()).build(); try { - res = blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).createRecord(req); - // Response carries new RID string - return res.getRid(); // e.g., "#12:0" + final CreateRecordResponse res = callUnary("CreateRecord", + () -> blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).createRecord(req)); + return res.getRid(); + } - catch (StatusException e) { - throw new RuntimeException("Failed to create vertex", e); + catch (io.grpc.StatusRuntimeException | io.grpc.StatusException e) { + handleGrpcException(e); + throw new IllegalStateException("unreachable"); } } @@ -969,51 +1126,64 @@ public boolean updateRecord(String rid, Record dbRecord, long timeoutMs) { return updateRecordFull(rid, record, timeoutMs); } - private boolean updateRecord(String rid, PropertiesUpdate partial, long timeoutMs) { - // Build the Update request using setPartial for partial update - UpdateRecordRequest req = UpdateRecordRequest.newBuilder().setDatabase(getName()).setRid(rid).setPartial(partial) - .setTransaction(TransactionContext.newBuilder().setBegin(true).setCommit(true)).setCredentials(buildCredentials()).build(); + private boolean updateRecord(final String rid, final PropertiesUpdate partial, final long timeoutMs) { + checkDatabaseIsOpen(); - // Call RPC - UpdateRecordResponse res; + if (rid == null || rid.isBlank()) + throw new IllegalArgumentException("rid must be non-empty"); + if (partial == null) + throw new IllegalArgumentException("partial must be non-null"); + + final UpdateRecordRequest req = UpdateRecordRequest.newBuilder().setDatabase(getName()).setRid(rid).setPartial(partial) + // Per-call tx: begin+commit. If you have an outer tx, pass it instead. + .setTransaction(TransactionContext.newBuilder().setBegin(true).setCommit(true)).setCredentials(buildCredentials()).build(); try { - res = blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).updateRecord(req); - // Choose the flag your proto defines. Most builds expose getSuccess(). + if (logger.isDebugEnabled()) { + logger.debug("CLIENT updateRecord(partial): db={}, txOpen={}, rid={}, timeoutMs={}", getName(), (transactionId != null), rid, + timeoutMs); + } + + final UpdateRecordResponse res = callUnary("UpdateRecord", + () -> blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).updateRecord(req)); + + // Most builds expose getSuccess(); if your proto has getUpdated(), swap here. return res.getSuccess(); + } - catch (StatusException e) { - throw new RuntimeException("Failed to update record", e); + catch (io.grpc.StatusRuntimeException | io.grpc.StatusException e) { + handleGrpcException(e); // rethrows mapped domain exception + throw new IllegalStateException("unreachable"); } } - private boolean updateRecordFull(String rid, GrpcRecord record, long timeoutMs) { - // Build the Update request using setRecord for full replacement - UpdateRecordRequest req = UpdateRecordRequest.newBuilder().setDatabase(getName()).setRid(rid).setRecord(record) - .setTransaction(TransactionContext.newBuilder().setBegin(true).setCommit(true)).setCredentials(buildCredentials()).build(); + private boolean updateRecordFull(final String rid, final GrpcRecord record, final long timeoutMs) { + checkDatabaseIsOpen(); - // Call RPC - UpdateRecordResponse res; + if (rid == null || rid.isBlank()) + throw new IllegalArgumentException("rid must be non-empty"); + if (record == null) + throw new IllegalArgumentException("record must be non-null"); + + final UpdateRecordRequest req = UpdateRecordRequest.newBuilder().setDatabase(getName()).setRid(rid).setRecord(record) + // Per-call tx: begin+commit. If you have an outer tx, pass it instead. + .setTransaction(TransactionContext.newBuilder().setBegin(true).setCommit(true)).setCredentials(buildCredentials()).build(); try { - res = blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).updateRecord(req); - return res.getSuccess(); - } - catch (StatusException e) { - throw new RuntimeException("Failed to update record", e); - } - } + if (logger.isDebugEnabled()) { + logger.debug("CLIENT updateRecord(full): db={}, txOpen={}, rid={}, timeoutMs={}", getName(), (transactionId != null), rid, + timeoutMs); + } - public boolean deleteRecord(String rid, long timeoutMs) { - var req = DeleteRecordRequest.newBuilder().setDatabase(getName()).setRid(rid).setCredentials(buildCredentials()).build(); - DeleteRecordResponse res; + final UpdateRecordResponse res = callUnary("UpdateRecord", + () -> blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).updateRecord(req)); + + return res.getSuccess(); - try { - res = blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).deleteRecord(req); - return res.getDeleted(); } - catch (StatusException e) { - throw new RuntimeException("Failed to delete record", e); + catch (io.grpc.StatusRuntimeException | io.grpc.StatusException e) { + handleGrpcException(e); // rethrows mapped domain exception + throw new IllegalStateException("unreachable"); } } @@ -1064,24 +1234,29 @@ public Record lookupByRID(final RID rid, final boolean loadContent) { if (rid == null) throw new IllegalArgumentException("Record is null"); - LookupByRidRequest request = LookupByRidRequest.newBuilder().setDatabase(getName()).setRid(rid.toString()) + final LookupByRidRequest req = LookupByRidRequest.newBuilder().setDatabase(getName()).setRid(rid.toString()) .setCredentials(buildCredentials()).build(); try { - LookupByRidResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).lookupByRid(request); + if (logger.isDebugEnabled()) { + logger.debug("CLIENT lookupByRID: db={}, txOpen={}, rid={}, loadContent={}, timeoutMs={}", getName(), (transactionId != null), + rid, loadContent, getTimeout()); + } - if (!response.getFound()) { + final LookupByRidResponse resp = callUnary("LookupByRid", + () -> blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).lookupByRid(req)); + + if (!resp.getFound()) throw new RecordNotFoundException("Record " + rid + " not found", rid); - } - return grpcRecordToDBRecord(response.getRecord()); - } - catch (StatusRuntimeException e) { - handleGrpcException(e); - return null; + + // Note: loadContent is currently a no-op for gRPC; response already carries the + // record. + return grpcRecordToDBRecord(resp.getRecord()); + } - catch (StatusException e) { - handleGrpcException(e); - return null; + catch (io.grpc.StatusException | io.grpc.StatusRuntimeException e) { + handleGrpcException(e); // maps & rethrows proper domain exception + throw new IllegalStateException("unreachable"); } } @@ -1098,63 +1273,82 @@ public boolean existsRecord(RID rid) { } } + // Convenience overload that accepts domain rows (convert first) + public InsertSummary insertBulkAsListOfMaps(final InsertOptions options, final List> rows, final long timeoutMs) { + + List protoRows = rows.stream().map(this::toProtoRecordFromMap) // your converter + .collect(java.util.stream.Collectors.toList()); + + return insertBulk(options, protoRows, timeoutMs); + } + public InsertSummary insertBulk(final InsertOptions options, final List protoRows, final long timeoutMs) { // Ensure options carry DB + credentials as the server expects - InsertOptions.Builder ob = options.toBuilder(); + final InsertOptions.Builder ob = options.toBuilder(); - if (options.getDatabase() == null || options.getDatabase().isEmpty()) { - ob.setDatabase(getName()); // your wrapper's DB name + final String dbInOpts = options.getDatabase(); + if (dbInOpts == null || dbInOpts.isEmpty()) { + ob.setDatabase(getName()); } - boolean missingCreds = !options.hasCredentials() || options.getCredentials().getUsername().isEmpty(); + final boolean missingCreds = !options.hasCredentials() || options.getCredentials().getUsername().isEmpty(); if (missingCreds) { - ob.setCredentials(buildCredentials()); // your existing helper used by queries + ob.setCredentials(buildCredentials()); } - InsertOptions newOptions = ob.build(); + final InsertOptions newOptions = ob.build(); - BulkInsertRequest req = BulkInsertRequest.newBuilder().setOptions(newOptions).addAllRows(protoRows).build(); + final BulkInsertRequest req = BulkInsertRequest.newBuilder().setOptions(newOptions).addAllRows(protoRows).build(); try { - if (logger.isDebugEnabled()) - logger.debug("CLIENT insertBulk: rows={}, timeoutMs={}", req.getRowsCount(), timeoutMs); - return blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).bulkInsert(req); + if (logger.isDebugEnabled()) { + logger.debug("CLIENT insertBulk: rows={}, timeoutMs={}, tx={}", req.getRowsCount(), timeoutMs, (transactionId != null)); + } + + // use callUnary so tx cross-thread checks + rpcSeq happen in one place + return callUnary("BulkInsert", () -> blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).bulkInsert(req)); + } - catch (StatusException e) { - throw new RuntimeException("insertBulk() -> failed: " + e.getStatus()); + catch (io.grpc.StatusRuntimeException | io.grpc.StatusException e) { + handleGrpcException(e); // maps to your domain exceptions and rethrows + throw new IllegalStateException("unreachable"); // keep compiler happy } } - // Convenience overload that accepts domain rows (convert first) - public InsertSummary insertBulkAsListOfMaps(final InsertOptions options, final List> rows, final long timeoutMs) { + // Convenience overload + public InsertSummary ingestStreamAsListOfMaps(final InsertOptions options, final List> rows, final int chunkSize, + final long timeoutMs) throws InterruptedException { - List protoRows = rows.stream().map(this::toProtoRecordFromMap) // your converter - .collect(java.util.stream.Collectors.toList()); + List protoRows = rows.stream().map(this::toProtoRecordFromMap).collect(java.util.stream.Collectors.toList()); - return insertBulk(options, protoRows, timeoutMs); + return ingestStream(options, protoRows, chunkSize, timeoutMs); } public InsertSummary ingestStream(final InsertOptions options, final List protoRows, final int chunkSize, final long timeoutMs) throws InterruptedException { - final java.util.concurrent.CountDownLatch done = new java.util.concurrent.CountDownLatch(1); - final java.util.concurrent.atomic.AtomicReference summaryRef = new java.util.concurrent.atomic.AtomicReference<>(); + checkDatabaseIsOpen(); - // Ensure options carry DB + credentials as the server expects - InsertOptions.Builder ob = options.toBuilder(); + if (protoRows == null || protoRows.isEmpty()) + return InsertSummary.newBuilder().setReceived(0).build(); - if (options.getDatabase() == null || options.getDatabase().isEmpty()) { - ob.setDatabase(getName()); // your wrapper's DB name - } + if (chunkSize <= 0) + throw new IllegalArgumentException("chunkSize must be > 0"); - boolean missingCreds = !options.hasCredentials() || options.getCredentials().getUsername().isEmpty(); + // Ensure options carry DB + credentials + final InsertOptions.Builder ob = options.toBuilder(); + if (options.getDatabase() == null || options.getDatabase().isEmpty()) + ob.setDatabase(getName()); + if (!options.hasCredentials() || options.getCredentials().getUsername().isEmpty()) + ob.setCredentials(buildCredentials()); + final InsertOptions effOptions = ob.build(); - if (missingCreds) { - ob.setCredentials(buildCredentials()); // your existing helper used by queries - } + final CountDownLatch done = new CountDownLatch(1); + final AtomicReference summaryRef = new AtomicReference<>(); + final AtomicReference errorRef = new AtomicReference<>(); - StreamObserver resp = new StreamObserver<>() { + final StreamObserver resp = new StreamObserver<>() { @Override public void onNext(InsertSummary value) { summaryRef.set(value); @@ -1162,6 +1356,7 @@ public void onNext(InsertSummary value) { @Override public void onError(Throwable t) { + errorRef.set(t); done.countDown(); } @@ -1171,72 +1366,133 @@ public void onCompleted() { } }; - // Use the write service async stub, per-call deadline - var stub = this.asyncStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS); + if (logger.isDebugEnabled()) { + logger.debug("CLIENT ingestStream: db={}, rows={}, chunkSize={}, timeoutMs={}", getName(), protoRows.size(), chunkSize, timeoutMs); + } - final StreamObserver req = stub.insertStream(resp); + // Open the client stream via wrapper (adds deadline, tx checks, unified error + // mapping) + final StreamObserver req = callAsyncDuplex("InsertStream", timeoutMs, + (stub, responseObserver) -> stub.insertStream(responseObserver), wrapObserver("InsertStream", resp)); - // Stream chunks final String sessionId = "sess-" + System.nanoTime(); - long seq = 1; + int sent = 0; + + try { + for (int i = 0; i < protoRows.size(); i += chunkSize) { + final int end = Math.min(i + chunkSize, protoRows.size()); - for (int i = 0; i < protoRows.size(); i += chunkSize) { - final int end = Math.min(i + chunkSize, protoRows.size()); + // (Sending options on every chunk is safe; if you prefer first-chunk-only, gate + // on seq == 1.) + final InsertChunk chunk = InsertChunk.newBuilder().setSessionId(sessionId).setOptions(effOptions).setChunkSeq(seq++) + .addAllRows(protoRows.subList(i, end)).build(); + + req.onNext(chunk); + sent += (end - i); + } + req.onCompleted(); + } + catch (RuntimeException sendErr) { + // Client-side failure during onNext/onCompleted: propagate + throw sendErr; + } - final InsertChunk chunk = InsertChunk.newBuilder().setSessionId(sessionId).setOptions(ob.build()).setChunkSeq(seq++) - .addAllRows(protoRows.subList(i, end)).build(); + // Wait for server final summary or error (tiny grace after deadline) + final boolean finished = done.await(Math.max(1, timeoutMs) + 1_000, TimeUnit.MILLISECONDS); + if (!finished) { + throw new TimeoutException("ingestStream timed out waiting for server completion"); + } - req.onNext(chunk); + // If wrapObserver mapped an error, rethrow it directly + final Throwable streamErr = errorRef.get(); + if (streamErr != null) { + if (streamErr instanceof RuntimeException re) + throw re; + throw new RemoteException("gRPC stream failed: " + streamErr.getMessage(), streamErr); } - req.onCompleted(); + InsertSummary s = summaryRef.get(); + if (s == null) { + // Fallback if server completed without emitting a final summary + s = InsertSummary.newBuilder().setReceived(sent).build(); + } - done.await(timeoutMs + 5_000, TimeUnit.MILLISECONDS); + if (logger.isDebugEnabled()) { + logger.debug("CLIENT ingestStream: completed; received={}", s.getReceived()); + } - final InsertSummary s = summaryRef.get(); - return (s != null) ? s : InsertSummary.newBuilder().setReceived(protoRows.size()).build(); + return s; } - // Convenience overload - public InsertSummary ingestStreamAsListOfMaps(final InsertOptions options, final List> rows, final int chunkSize, - final long timeoutMs) throws InterruptedException { + /** + * Pushes domain {@code com.arcadedb.database.Record} rows via + * InsertBidirectional with per-batch ACKs. + */ + public InsertSummary ingestBidi(final List rows, final InsertOptions opts, final int chunkSize, + final int maxInflight, final long timeoutMs) throws InterruptedException { - List protoRows = rows.stream().map(this::toProtoRecordFromMap).collect(java.util.stream.Collectors.toList()); + return ingestBidiCore(rows, opts, chunkSize, maxInflight, timeoutMs, + (Object o) -> toProtoRecordFromDbRecord((com.arcadedb.database.Record) o)); + } - return ingestStream(options, protoRows, chunkSize, timeoutMs); + public InsertSummary ingestBidi(final List rows, final InsertOptions opts, final int chunkSize, + final int maxInflight) throws InterruptedException { + + return ingestBidiCore(rows, opts, chunkSize, maxInflight, /* timeoutMs */ 5 * 60_000L, this::toProtoRecordFromDbRecord); + } + + /** + * Pushes map-shaped rows (property map per row) via InsertBidirectional with + * per-batch ACKs. + */ + public InsertSummary ingestBidi(final InsertOptions options, final List> rows, final int chunkSize, + final int maxInflight) throws InterruptedException { + + return ingestBidiCore(rows, options, chunkSize, maxInflight, /* timeoutMs */ 5 * 60_000L, this::toProtoRecordFromMap); + } + + public InsertSummary ingestBidi(final InsertOptions options, final List> rows, final int chunkSize, + final int maxInflight, final long timeoutMs) throws InterruptedException { + + return ingestBidiCore(rows, options, chunkSize, maxInflight, timeoutMs, this::toProtoRecordFromMap); } /** * Core implementation of InsertBidirectional ingest */ + private InsertSummary ingestBidiCore(final List rows, final InsertOptions options, final int chunkSize, final int maxInflight, final long timeoutMs, final java.util.function.Function mapper) throws InterruptedException { - // 1) Ensure options carry DB + credentials the server expects - final InsertOptions.Builder ob = options.toBuilder(); + // Fast-path & guards + if (rows == null || rows.isEmpty()) + return InsertSummary.newBuilder().setReceived(0).build(); + if (chunkSize <= 0) + throw new IllegalArgumentException("chunkSize must be > 0"); + if (maxInflight <= 0) + throw new IllegalArgumentException("maxInflight must be > 0"); - if (options.getDatabase() == null || options.getDatabase().isEmpty()) { + // Options with DB + creds + final InsertOptions.Builder ob = options.toBuilder(); + if (options.getDatabase() == null || options.getDatabase().isEmpty()) ob.setDatabase(getName()); - } - - final boolean missingCreds = !options.hasCredentials() || options.getCredentials().getUsername().isEmpty(); - - if (missingCreds) { + if (!options.hasCredentials() || options.getCredentials().getUsername().isEmpty()) ob.setCredentials(buildCredentials()); - } - final InsertOptions effectiveOpts = ob.build(); - // 2) Pre-map rows → proto once (outside the observer) + // Pre-map rows → proto final List protoRows = rows.stream().map(mapper).collect(java.util.stream.Collectors.toList()); - if (logger.isDebugEnabled()) + if (logger.isDebugEnabled()) { logger.debug("CLIENT ingestBidi start: rows={}, chunkSize={}, maxInflight={}, timeoutMs={}", protoRows.size(), chunkSize, maxInflight, timeoutMs); -// 3) Streaming state + } + + // --- streaming state final String sessionId = "sess-" + System.nanoTime(); final java.util.concurrent.CountDownLatch done = new java.util.concurrent.CountDownLatch(1); + final java.util.concurrent.atomic.AtomicReference errRef = new java.util.concurrent.atomic.AtomicReference<>(); final java.util.concurrent.atomic.AtomicLong seq = new java.util.concurrent.atomic.AtomicLong(1); final java.util.concurrent.atomic.AtomicInteger cursor = new java.util.concurrent.atomic.AtomicInteger(0); final java.util.concurrent.atomic.AtomicInteger sent = new java.util.concurrent.atomic.AtomicInteger(0); @@ -1244,7 +1500,6 @@ private InsertSummary ingestBidiCore(final List rows, final InsertOptions final java.util.concurrent.atomic.AtomicReference committed = new java.util.concurrent.atomic.AtomicReference<>(); final List acks = java.util.Collections.synchronizedList(new ArrayList<>()); - // small holder to access req inside helpers final java.util.concurrent.atomic.AtomicReference> observerRef = new java.util.concurrent.atomic.AtomicReference<>(); final java.util.concurrent.atomic.AtomicBoolean commitSent = new java.util.concurrent.atomic.AtomicBoolean(false); @@ -1253,17 +1508,17 @@ private InsertSummary ingestBidiCore(final List rows, final InsertOptions t.setDaemon(true); return t; }); - final long ackGraceMillis = Math.min(Math.max(timeoutMs / 10, 1_000L), 10_000L); // e.g., 1s..10s window + final long ackGraceMillis = Math.min(Math.max(timeoutMs / 10, 1_000L), 10_000L); final Object timerLock = new Object(); final java.util.concurrent.atomic.AtomicReference> ackGraceFuture = new java.util.concurrent.atomic.AtomicReference<>(); - // helper: schedule COMMIT if still waiting final Runnable sendCommitIfNeeded = () -> { if (commitSent.compareAndSet(false, true)) { try { - if (observerRef.get() != null) { - observerRef.get().onNext(InsertRequest.newBuilder().setCommit(Commit.newBuilder().setSessionId(sessionId)).build()); - observerRef.get().onCompleted(); + final ClientCallStreamObserver r = observerRef.get(); + if (r != null) { + r.onNext(InsertRequest.newBuilder().setCommit(Commit.newBuilder().setSessionId(sessionId)).build()); + r.onCompleted(); } } catch (Throwable ignore) { @@ -1271,13 +1526,11 @@ private InsertSummary ingestBidiCore(final List rows, final InsertOptions } }; - // helper: (re)arm the grace timer final Runnable armAckGraceTimer = () -> { synchronized (timerLock) { var prev = ackGraceFuture.getAndSet(null); if (prev != null) prev.cancel(false); - // only arm if we've sent all chunks but acks are still pending if (cursor.get() >= protoRows.size() && acked.get() < sent.get() && !commitSent.get()) { var fut = scheduler.schedule(sendCommitIfNeeded, ackGraceMillis, TimeUnit.MILLISECONDS); ackGraceFuture.set(fut); @@ -1285,7 +1538,6 @@ private InsertSummary ingestBidiCore(final List rows, final InsertOptions } }; - // helper: cancel timer (e.g., once all ACKed or after COMMIT observed) final Runnable cancelAckGraceTimer = () -> { synchronized (timerLock) { var prev = ackGraceFuture.getAndSet(null); @@ -1294,15 +1546,15 @@ private InsertSummary ingestBidiCore(final List rows, final InsertOptions } }; - final ClientResponseObserver observer = new ClientResponseObserver<>() { - + // --- response observer (keeps backpressure via beforeStart) + final ClientResponseObserver respObserver = new ClientResponseObserver<>() { ClientCallStreamObserver req; volatile boolean started = false; @Override public void beforeStart(ClientCallStreamObserver r) { this.req = r; - observerRef.set(r); // <-- make req visible to helper + observerRef.set(r); r.disableAutoInboundFlowControl(); r.setOnReadyHandler(this::drain); } @@ -1313,11 +1565,8 @@ private void drain() { if (!started) { req.onNext(InsertRequest.newBuilder().setStart(Start.newBuilder().setOptions(effectiveOpts)).build()); - started = true; - - // Now that the call is started, it's safe to pull the first response - req.request(1); + req.request(1); // pull first server response } while (req.isReady() && (sent.get() - acked.get()) < maxInflight) { @@ -1336,17 +1585,11 @@ private void drain() { sent.incrementAndGet(); } - // If all chunks sent: if (cursor.get() >= protoRows.size()) { - if (acked.get() >= sent.get()) { - // all acked → commit immediately + if (acked.get() >= sent.get()) sendCommitIfNeeded.run(); - } - else { - // not all acked → (re)arm grace timer; if last ACK never arrives, timer will - // COMMIT + else armAckGraceTimer.run(); - } } } @@ -1358,7 +1601,7 @@ public void onNext(InsertResponse v) { case BATCH_ACK -> { acks.add(v.getBatchAck()); acked.incrementAndGet(); - // Each ACK may free capacity: push more & manage timer + // free capacity & manage timers drain(); if (cursor.get() >= protoRows.size()) { if (acked.get() >= sent.get()) { @@ -1374,20 +1617,26 @@ public void onNext(InsertResponse v) { committed.set(v.getCommitted().getSummary()); cancelAckGraceTimer.run(); } - case MSG_NOT_SET -> { - /* ignore */ - } case ERROR -> { - /* TBD */ + // surface as error; caller will map/throw after await + errRef.set(new StatusRuntimeException(io.grpc.Status.INTERNAL.withDescription(v.getError().getMessage()))); + cancelAckGraceTimer.run(); + try { + req.cancel("server ERROR", null); + } + catch (Throwable ignore) { + } } + case MSG_NOT_SET -> { + /* ignore */ } } - - req.request(1); // continue pulling responses + req.request(1); // keep pulling } @Override public void onError(Throwable t) { cancelAckGraceTimer.run(); + errRef.set(t); done.countDown(); } @@ -1397,26 +1646,51 @@ public void onCompleted() { done.countDown(); } }; - // 4) Kick off the bidi call with deadline + + // --- open bidi via wrapper (deadline, tx checks, unified logging) + @SuppressWarnings("unused") + final StreamObserver _req = callAsyncDuplex("InsertBidirectional", timeoutMs, + (stub, responseObs) -> stub.insertBidirectional(responseObs), wrapClientResponseObserver("InsertBidirectional", respObserver) // preserves + // beforeStart + ); try { - if (logger.isDebugEnabled()) - logger.debug("CLIENT ingestBidi opening stream: timeoutMs={}", timeoutMs); - asyncStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).insertBidirectional(observer); - @SuppressWarnings("unused") - boolean finished = done.await(timeoutMs + 5_000, TimeUnit.MILLISECONDS); - if (logger.isDebugEnabled()) + // wait for completion (tiny grace after deadline) + final boolean finished = done.await(Math.max(1, timeoutMs) + 1_000, TimeUnit.MILLISECONDS); + if (!finished) { + final ClientCallStreamObserver r = observerRef.get(); + if (r != null) { + try { + r.cancel("client timeout waiting for completion", null); + } + catch (Throwable ignore) { + } + } + throw new TimeoutException("ingestBidirectional timed out waiting for server completion"); + } + + final Throwable err = errRef.get(); + if (err != null) { + // wrapClientResponseObserver already runs through handleGrpcException(); + // if it was a non-gRPC Throwable, surface it uniformly here. + if (err instanceof RuntimeException re) + throw re; + throw new RemoteException("gRPC bidi stream failed: " + err.getMessage(), err); + } + + if (logger.isDebugEnabled()) { try { - logger.debug("CLIENT ingestBidi finished: finished={}, sent={}, acked={}", finished, sent.get(), acked.get()); + logger.debug("CLIENT ingestBidi finished: sent={}, acked={}", sent.get(), acked.get()); } catch (Throwable ignore) { } + } } finally { scheduler.shutdownNow(); } - // 6) Prefer COMMITTED summary if present; otherwise aggregate ACKs + // Prefer COMMITTED summary; else aggregate ACKs final InsertSummary finalSummary = committed.get(); if (finalSummary != null) return finalSummary; @@ -1429,39 +1703,6 @@ public void onCompleted() { return InsertSummary.newBuilder().setReceived(protoRows.size()).setInserted(ins).setUpdated(upd).setIgnored(ign).setFailed(fail).build(); } - /** - * Pushes domain {@code com.arcadedb.database.Record} rows via - * InsertBidirectional with per-batch ACKs. - */ - public InsertSummary ingestBidi(final List rows, final InsertOptions opts, final int chunkSize, - final int maxInflight, final long timeoutMs) throws InterruptedException { - - return ingestBidiCore(rows, opts, chunkSize, maxInflight, timeoutMs, - (Object o) -> toProtoRecordFromDbRecord((com.arcadedb.database.Record) o)); - } - - public InsertSummary ingestBidi(final List rows, final InsertOptions opts, final int chunkSize, - final int maxInflight) throws InterruptedException { - - return ingestBidiCore(rows, opts, chunkSize, maxInflight, /* timeoutMs */ 5 * 60_000L, this::toProtoRecordFromDbRecord); - } - - /** - * Pushes map-shaped rows (property map per row) via InsertBidirectional with - * per-batch ACKs. - */ - public InsertSummary ingestBidi(final InsertOptions options, final List> rows, final int chunkSize, - final int maxInflight) throws InterruptedException { - - return ingestBidiCore(rows, options, chunkSize, maxInflight, /* timeoutMs */ 5 * 60_000L, this::toProtoRecordFromMap); - } - - public InsertSummary ingestBidi(final InsertOptions options, final List> rows, final int chunkSize, - final int maxInflight, final long timeoutMs) throws InterruptedException { - - return ingestBidiCore(rows, options, chunkSize, maxInflight, timeoutMs, this::toProtoRecordFromMap); - } - // Map -> GrpcRecord private GrpcRecord toProtoRecordFromMap(Map row) { GrpcRecord.Builder b = GrpcRecord.newBuilder(); @@ -1523,8 +1764,8 @@ private Iterator streamQuery(final String query) { StreamQueryRequest request = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(query).setCredentials(buildCredentials()) .setBatchSize(100).build(); - final BlockingClientCall responseIterator = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) - .streamQuery(request); + final BlockingClientCall responseIterator = callServerStreaming("StreamQuery", + () -> blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).streamQuery(request)); return new Iterator() { private Iterator currentBatch = Collections.emptyIterator(); @@ -1575,14 +1816,14 @@ private ResultSet createGrpcResultSet(ExecuteQueryResponse response) { return resultSet; } - private ResultSet createGrpcResultSet(ExecuteCommandResponse response) { - InternalResultSet resultSet = new InternalResultSet(); + private ResultSet createGrpcResultSet(ExecuteCommandResponse response) { + InternalResultSet resultSet = new InternalResultSet(); for (GrpcRecord record : response.getRecordsList()) { resultSet.add(grpcRecordToResult(record)); } return resultSet; } - + private Result grpcRecordToResult(GrpcRecord grpcRecord) { Record record = grpcRecordToDBRecord(grpcRecord); @@ -1707,43 +1948,26 @@ private Object grpcValueToObject(GrpcValue grpcValue) { return out; } - private void handleGrpcException(StatusRuntimeException e) { - Status status = e.getStatus(); - switch (status.getCode()) { - case NOT_FOUND: - throw new RecordNotFoundException(status.getDescription(), null); - case ALREADY_EXISTS: - throw new DuplicatedKeyException("", "", null); - case ABORTED: - throw new ConcurrentModificationException(status.getDescription()); - case DEADLINE_EXCEEDED: - throw new TimeoutException(status.getDescription()); - case PERMISSION_DENIED: - throw new SecurityException(status.getDescription()); - case UNAVAILABLE: - throw new NeedRetryException(status.getDescription()); - default: - throw new RemoteException("gRPC error: " + status.getDescription(), e); - } - } + private void handleGrpcException(Throwable e) { + // Works for StatusException, StatusRuntimeException, and anything else + io.grpc.Status status = io.grpc.Status.fromThrowable(e); + String msg = status.getDescription() != null ? status.getDescription() : status.getCode().name(); - private void handleGrpcException(StatusException e) { - Status status = e.getStatus(); switch (status.getCode()) { case NOT_FOUND: - throw new RecordNotFoundException(status.getDescription(), null); + throw new RecordNotFoundException(msg, null); case ALREADY_EXISTS: throw new DuplicatedKeyException("", "", null); case ABORTED: - throw new ConcurrentModificationException(status.getDescription()); + throw new ConcurrentModificationException(msg); case DEADLINE_EXCEEDED: - throw new TimeoutException(status.getDescription()); + throw new TimeoutException(msg); case PERMISSION_DENIED: - throw new SecurityException(status.getDescription()); + throw new SecurityException(msg); case UNAVAILABLE: - throw new NeedRetryException(status.getDescription()); + throw new NeedRetryException(msg); default: - throw new RemoteException("gRPC error: " + status.getDescription(), e); + throw new RemoteException("gRPC error: " + msg, e); } } @@ -1753,7 +1977,7 @@ public RemoteGrpcTransactionExplicitLock(RemoteGrpcDatabase database) { super(database); } } - + // --- Debug helpers (client) --- private static String summarize(Object o) { if (o == null) @@ -1802,5 +2026,268 @@ private static String summarize(GrpcRecord r) { String ty = r.getType(); int props = r.getPropertiesCount(); return "GrpcRecord{rid=" + rid + ", type=" + ty + ", props=" + props + "}"; - } + } + + // RemoteGrpcDatabase.java + + // ---- fields ---- + private volatile TxDebug debugTx; + + private static final class TxDebug { + final long id = System.nanoTime(); // local correlation id + final Thread ownerThread = Thread.currentThread(); + final String dbName; + volatile String txLabel; // optional + final Exception beginSite = new Exception("begin site"); // capture stack + final java.util.concurrent.atomic.AtomicLong rpcSeq = new java.util.concurrent.atomic.AtomicLong(); + volatile boolean beginRpcSent, committed, rolledBack; + + @SuppressWarnings("unused") + TxDebug(String db, String label) { + this.dbName = db; + this.txLabel = label; + } + } + + // one place to handle JDK 17/21 differences + private static String tidName(Thread t) { + try { + // Java 19+: threadId() + long tid = (long) Thread.class.getMethod("threadId").invoke(t); + return tid + ":" + t.getName(); + } + catch (ReflectiveOperationException ignore) { + return legacyTidName(t); + } + } + + private static String legacyTidName(Thread t) { + return t.threadId() + ":" + t.getName(); + } + + // Optional knobs you can toggle from the TM for a single run + private volatile boolean txDebugEnabled = true; + + public void enableTxDebug(boolean on) { + this.txDebugEnabled = on; + } + + public boolean isLocalTxActive() { + return debugTx != null; + } + + public @Nullable String currentLocalTxId() { + return (debugTx != null ? Long.toString(debugTx.id) : null); + } + + public void setCurrentTxLabel(String label) { + if (debugTx != null) + debugTx.txLabel = label; + } + + private void logTx(String phase, String rpcOp) { + if (debugTx == null || !logger.isDebugEnabled()) + return; + TxDebug d = debugTx; + logger.debug("TXDBG {} db={} tx#{} label={} owner={} now={} rpcOp={} rpcSeq={} beginSent={} committed={} rolledBack={}", phase, d.dbName, + d.id, d.txLabel, tidName(d.ownerThread), tidName(Thread.currentThread()), rpcOp, d.rpcSeq.get(), d.beginRpcSent, d.committed, + d.rolledBack); + } + + private void checkCrossThreadUse(String where) { + TxDebug d = debugTx; + if (d == null) + return; + Thread now = Thread.currentThread(); + if (now != d.ownerThread) { + logger.warn("TXDBG CROSS-THREAD {} db={} tx#{} owner={} now={} label={} (begin site follows)", where, d.dbName, d.id, + tidName(d.ownerThread), tidName(now), d.txLabel, d.beginSite); + } + } + + @FunctionalInterface + private interface Rpc { + T run() throws io.grpc.StatusException; // V2 throws this; v1 lambdas compile fine too + } + + private Resp callUnary(String opName, Rpc rpc) throws io.grpc.StatusException { + if (debugTx != null) { + checkCrossThreadUse("RPC " + opName); + logTx("RPC(local)", opName); + } + try { + return rpc.run(); + } + finally { + if (debugTx != null) { + debugTx.rpcSeq.incrementAndGet(); + } + } + } + + // unary that returns void + private void callUnaryVoid(String opName, Runnable rpc) { + if (debugTx != null) { + checkCrossThreadUse("RPC " + opName); + logTx("RPC(unary)", opName); + } + rpc.run(); + if (debugTx != null) { + debugTx.rpcSeq.incrementAndGet(); + } + } + + // Helper for server-streaming RPCs (mirrors callUnary) + private BlockingClientCall callServerStreaming(String opName, Supplier> rpc) { + if (debugTx != null) { + checkCrossThreadUse("RPC " + opName); + logTx("RPC(open-stream)", opName); + } + try { + final BlockingClientCall it = rpc.get(); + if (debugTx != null) { + debugTx.rpcSeq.incrementAndGet(); + } + return it; + } + catch (io.grpc.StatusRuntimeException e) { + handleGrpcException(e); // rethrows mapped runtime exception + throw new IllegalStateException("unreachable"); + } + } + + // For async "client-streaming" and "bidirectional" calls that RETURN a request + // StreamObserver + private StreamObserver callAsyncDuplex(String opName, long timeoutMs, + java.util.function.BiFunction, StreamObserver> starter, + StreamObserver responseObserver) { + if (debugTx != null) { + checkCrossThreadUse("STREAM " + opName); + logTx("STREAM(local)", opName); + } + final var stub = asyncStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS); + StreamObserver reqObs = starter.apply(stub, wrapObserver(opName, responseObserver)); + if (debugTx != null) { + debugTx.rpcSeq.incrementAndGet(); + } + return reqObs; + } + + // For async "server-streaming" calls that take (request, responseObserver) and + // return void + private void callAsyncServerStreaming(String opName, long timeoutMs, Req request, + java.util.function.BiConsumer> invoker, + StreamObserver responseObserver) { + if (debugTx != null) { + checkCrossThreadUse("STREAM " + opName); + logTx("STREAM(local)", opName); + } + final var stub = asyncStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS); + invoker.accept(stub, wrapObserver(opName, responseObserver)); + if (debugTx != null) { + debugTx.rpcSeq.incrementAndGet(); + } + } + + // Wrap a plain StreamObserver to translate gRPC Status into your domain + // exceptions + private StreamObserver wrapObserver(String opName, StreamObserver delegate) { + return new StreamObserver<>() { + @Override + public void onNext(T value) { + delegate.onNext(value); + } + + @Override + public void onError(Throwable t) { + try { + handleGrpcException(t); + } + catch (RuntimeException mapped) { + delegate.onError(mapped); + return; + } + delegate.onError(t); + } + + @Override + public void onCompleted() { + delegate.onCompleted(); + } + }; + } + + // Same idea, but preserves ClientResponseObserver features (beforeStart, flow + // control) + private ClientResponseObserver wrapObserver(String opName, ClientResponseObserver delegate) { + return new ClientResponseObserver<>() { + @Override + public void beforeStart(io.grpc.stub.ClientCallStreamObserver r) { + // pass through; your delegate may set onReady handler, request(n), etc. + delegate.beforeStart(r); + } + + @Override + public void onNext(Resp value) { + delegate.onNext(value); + } + + @Override + public void onError(Throwable t) { + try { + handleGrpcException(t); + } + catch (RuntimeException mapped) { + delegate.onError(mapped); + return; + } + delegate.onError(t); + } + + @Override + public void onCompleted() { + delegate.onCompleted(); + } + }; + } + + private ClientResponseObserver wrapClientResponseObserver(String opName, + ClientResponseObserver delegate) { + return new ClientResponseObserver<>() { + @Override + public void beforeStart(ClientCallStreamObserver requestStream) { + // Preserve delegate’s backpressure hooks + delegate.beforeStart(requestStream); + } + + @Override + public void onNext(RespT value) { + delegate.onNext(value); + } + + @Override + public void onError(Throwable t) { + // Normalize gRPC errors through your handler, then pass the mapped exception + try { + if (t instanceof io.grpc.StatusRuntimeException sre) { + handleGrpcException(sre); // throws + } + else if (t instanceof io.grpc.StatusException se) { + handleGrpcException(se); // throws + } + // Non-gRPC error: forward as-is + delegate.onError(t); + } + catch (RuntimeException mapped) { + // forward the mapped exception to delegate + delegate.onError(mapped); + } + } + + @Override + public void onCompleted() { + delegate.onCompleted(); + } + }; + } } \ No newline at end of file diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java index 3c9865f709..a94596ea2a 100644 --- a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java @@ -754,6 +754,7 @@ public void beginTransaction(BeginTransactionRequest request, StreamObserver responseObserver) { try { Database database = activeTransactions.remove(request.getTransaction().getTransactionId()); + if (database == null) { throw new IllegalArgumentException("Invalid transaction ID"); } From efd1ac8e4bd6024ff474054cec29099f06b6ed04 Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Mon, 8 Sep 2025 08:17:30 -0700 Subject: [PATCH 33/35] Checkpoint Commit --- grpc/src/main/proto/arcadedb-server.proto | 8 +- .../server/grpc/ArcadeDbGrpcService.java | 116 ++++++++++-------- 2 files changed, 68 insertions(+), 56 deletions(-) diff --git a/grpc/src/main/proto/arcadedb-server.proto b/grpc/src/main/proto/arcadedb-server.proto index 958e5d3c51..45954eac83 100644 --- a/grpc/src/main/proto/arcadedb-server.proto +++ b/grpc/src/main/proto/arcadedb-server.proto @@ -191,8 +191,9 @@ message CommitTransactionRequest { message CommitTransactionResponse { bool success = 1; - string message = 2; - int64 timestamp = 3; + bool committed = 2; + string message = 3; + int64 timestamp = 4; } message RollbackTransactionRequest { @@ -202,7 +203,8 @@ message RollbackTransactionRequest { message RollbackTransactionResponse { bool success = 1; - string message = 2; + bool rolled_back = 2; + string message = 3; } // ----------------------------------------------------------------------------- diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java index a94596ea2a..d845e6b1a7 100644 --- a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java @@ -5,6 +5,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; +import javax.annotation.Nullable; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,21 +49,12 @@ public class ArcadeDbGrpcService extends ArcadeDbServiceGrpc.ArcadeDbServiceImpl private static final Logger logger = LoggerFactory.getLogger(ArcadeDbGrpcService.class); - // Pick serializer once - private static final JsonSerializer FAST = - JsonSerializer.createJsonSerializer() - .setIncludeVertexEdges(false) - .setUseVertexEdgeSize(true) - .setUseCollectionSizeForEdges(true) - .setUseCollectionSize(false); - - private static final JsonSerializer SAFE = - JsonSerializer.createJsonSerializer() - .setIncludeVertexEdges(false) - .setUseVertexEdgeSize(true) - .setUseCollectionSizeForEdges(false) - .setUseCollectionSize(false); + private static final JsonSerializer FAST = JsonSerializer.createJsonSerializer().setIncludeVertexEdges(false).setUseVertexEdgeSize(true) + .setUseCollectionSizeForEdges(true).setUseCollectionSize(false); + + private static final JsonSerializer SAFE = JsonSerializer.createJsonSerializer().setIncludeVertexEdges(false).setUseVertexEdgeSize(true) + .setUseCollectionSizeForEdges(false).setUseCollectionSize(false); // Transaction management private final Map activeTransactions = new ConcurrentHashMap<>(); @@ -225,18 +218,23 @@ public void executeCommand(ExecuteCommandRequest req, StreamObserver commit > begin-only⇒commit if (hasTx) { + if (tx.getRollback()) { + + logger.debug("executeCommand(): after - rolling back db={} tid={}", db.getName(), tx.getTransactionId()); db.rollback(); } else if (tx.getCommit()) { + logger.debug("executeCommand(): after - committing [tx.getCommit() == true] db={} tid={}", db.getName(), tx.getTransactionId()); db.commit(); } else if (beganHere) { // Began but no explicit commit/rollback flag — default to commit (HTTP parity) + logger.debug("executeCommand(): after - committing [beganHere == true] db={} tid={}", db.getName(), tx.getTransactionId()); db.commit(); } } @@ -751,51 +749,62 @@ public void beginTransaction(BeginTransactionRequest request, StreamObserver responseObserver) { - try { - Database database = activeTransactions.remove(request.getTransaction().getTransactionId()); - - if (database == null) { - throw new IllegalArgumentException("Invalid transaction ID"); - } + public void commitTransaction(CommitTransactionRequest req, StreamObserver rsp) { + + final String txId = req.getTransaction().getTransactionId(); - // Commit transaction - database.commit(); + if (txId == null || txId.isBlank()) { + rsp.onError(Status.INVALID_ARGUMENT.withDescription("Missing transaction id").asException()); + return; + } - CommitTransactionResponse response = CommitTransactionResponse.newBuilder().setSuccess(true) - .setMessage("Transaction committed successfully").setTimestamp(System.currentTimeMillis()).build(); + // remove atomically to avoid double commit races + final Database db = activeTransactions.remove(txId); - responseObserver.onNext(response); - responseObserver.onCompleted(); + if (db == null) { + // Idempotent no-op: OK but not committed because nothing was open. + rsp.onNext(CommitTransactionResponse.newBuilder().setCommitted(false) + .setMessage("No active transaction for id=" + txId + " (already committed/rolled back?)").build()); + rsp.onCompleted(); + return; + } + try { + db.commit(); + rsp.onNext(CommitTransactionResponse.newBuilder().setCommitted(true).build()); + rsp.onCompleted(); } - catch (Exception e) { - logger.error("Error committing transaction: {}", e.getMessage(), e); - responseObserver.onError(Status.INTERNAL.withDescription("Failed to commit transaction: " + e.getMessage()).asException()); + catch (Throwable t) { + // Put nothing back in the map. The tx is unusable now. + rsp.onError(Status.ABORTED.withDescription("Commit failed: " + t.getMessage()).asException()); } } @Override - public void rollbackTransaction(RollbackTransactionRequest request, StreamObserver responseObserver) { - try { - Database database = activeTransactions.remove(request.getTransaction().getTransactionId()); - if (database == null) { - throw new IllegalArgumentException("Invalid transaction ID"); - } + public void rollbackTransaction(RollbackTransactionRequest req, StreamObserver rsp) { - // Rollback transaction - database.rollback(); + final String txId = req.getTransaction().getTransactionId(); + if (txId == null || txId.isBlank()) { + rsp.onError(Status.INVALID_ARGUMENT.withDescription("Missing transaction id").asException()); + return; + } - RollbackTransactionResponse response = RollbackTransactionResponse.newBuilder().setSuccess(true) - .setMessage("Transaction rolled back successfully").build(); + final Database db = activeTransactions.remove(txId); - responseObserver.onNext(response); - responseObserver.onCompleted(); + if (db == null) { + rsp.onNext(RollbackTransactionResponse.newBuilder().setRolledBack(false) + .setMessage("No active transaction for id=" + txId + " (already committed/rolled back?)").build()); + rsp.onCompleted(); + return; + } + try { + db.rollback(); + rsp.onNext(RollbackTransactionResponse.newBuilder().setRolledBack(true).build()); + rsp.onCompleted(); } - catch (Exception e) { - logger.error("Error rolling back transaction: {}", e.getMessage(), e); - responseObserver.onError(Status.INTERNAL.withDescription("Failed to rollback transaction: " + e.getMessage()).asException()); + catch (Throwable t) { + rsp.onError(Status.ABORTED.withDescription("Rollback failed: " + t.getMessage()).asException()); } } @@ -1980,15 +1989,16 @@ private GrpcValue toGrpcValue(Object o, ProjectionConfig pc) { } try { - + // Let ArcadeDB serialize the document/result properly - + boolean hasEmptyCollection = false; for (String k : doc.getPropertyNames()) { - Object v = doc.get(k); - if (v instanceof java.util.Collection c && c.isEmpty()) { - hasEmptyCollection = true; break; - } + Object v = doc.get(k); + if (v instanceof java.util.Collection c && c.isEmpty()) { + hasEmptyCollection = true; + break; + } } var json = (hasEmptyCollection ? SAFE : FAST).serializeDocument(doc); @@ -2210,7 +2220,7 @@ void closeQuietly() { } // Helper methods - + private Database getDatabase(String databaseName, DatabaseCredentials credentials) { // Validate credentials From aa764ffb879b9b87bacfe7b55a6ef004e3203add Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Mon, 8 Sep 2025 10:06:24 -0700 Subject: [PATCH 34/35] Checkpoint Commit --- .../remote/grpc/RemoteGrpcDatabase.java | 17 ++- .../server/grpc/ArcadeDbGrpcService.java | 110 ++++++++++++++---- 2 files changed, 106 insertions(+), 21 deletions(-) diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java index eb15f8f684..ecd7c8c296 100644 --- a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java @@ -242,11 +242,13 @@ public void begin(final Database.TRANSACTION_ISOLATION_LEVEL isolationLevel) { @Override public void commit() { + checkDatabaseIsOpen(); stats.txCommits.incrementAndGet(); - if (transactionId == null) + if (transactionId == null) { throw new TransactionException("Transaction not begun"); + } logTx("COMMIT(local)", null); checkCrossThreadUse("before CommitTransaction"); @@ -260,27 +262,36 @@ public void commit() { .setCredentials(buildCredentials()).build(); try { + CommitTransactionResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) .commitTransaction(request); + + logger.debug("[After commit] Success: {} Committed: {}", response.getSuccess(), response.getCommitted()); + if (!response.getSuccess()) { throw new TransactionException("Failed to commit transaction: " + response.getMessage()); } } catch (StatusRuntimeException e) { + handleGrpcException(e); } catch (StatusException e) { + handleGrpcException(e); } finally { + transactionId = null; setSessionId(null); } if (debugTx != null) { + debugTx.committed = true; debugTx.rpcSeq.incrementAndGet(); } + logTx("COMMIT sent", "CommitTransaction"); } finally { @@ -311,8 +322,12 @@ public void rollback() { .setCredentials(buildCredentials()).build(); try { + RollbackTransactionResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) .rollbackTransaction(request); + + logger.debug("[After rollback] Success: {} Committed: {}", response.getSuccess(), response.getRolledBack()); + if (!response.getSuccess()) { throw new TransactionException("Failed to rollback transaction: " + response.getMessage()); } diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java index d845e6b1a7..66ad20d64b 100644 --- a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java @@ -229,7 +229,8 @@ public void executeCommand(ExecuteCommandRequest req, StreamObserver @rid = {}", dbRecord.asDocument().get("@rid")); + logger.debug("executeQuery(): dbRecord -> @rid = {}", dbRecord.getIdentity().toString()); GrpcRecord grpcRecord = convertToGrpcRecord(dbRecord, database); @@ -725,85 +726,154 @@ public void executeQuery(ExecuteQueryRequest request, StreamObserver responseObserver) { + final String reqDb = request.getDatabase(); + final String user = (request.hasCredentials() ? request.getCredentials().getUsername() : null); + + if (logger.isDebugEnabled()) { + logger.debug("beginTransaction(): received request db={} user={} activeTxCount(before)={}", reqDb, (user != null ? user : ""), + activeTransactions.size()); + } + try { - Database database = getDatabase(request.getDatabase(), request.getCredentials()); + final Database database = getDatabase(reqDb, request.getCredentials()); + + if (logger.isDebugEnabled()) { + logger.debug("beginTransaction(): resolved database instance dbName={} class={} hash={}", + (database != null ? database.getName() : ""), (database != null ? database.getClass().getSimpleName() : ""), + (database != null ? System.identityHashCode(database) : 0)); + logger.debug("beginTransaction(): calling database.begin()"); + } // Begin transaction database.begin(); - // Generate transaction ID - String transactionId = generateTransactionId(); + // Generate transaction ID and register + final String transactionId = generateTransactionId(); activeTransactions.put(transactionId, database); - BeginTransactionResponse response = BeginTransactionResponse.newBuilder().setTransactionId(transactionId) + if (logger.isDebugEnabled()) { + logger.debug("beginTransaction(): started txId={} for db={} activeTxCount(after)={}", transactionId, + (database != null ? database.getName() : ""), activeTransactions.size()); + } + + final BeginTransactionResponse response = BeginTransactionResponse.newBuilder().setTransactionId(transactionId) .setTimestamp(System.currentTimeMillis()).build(); responseObserver.onNext(response); responseObserver.onCompleted(); - } - catch (Exception e) { - logger.error("Error beginning transaction: {}", e.getMessage(), e); - responseObserver.onError(Status.INTERNAL.withDescription("Failed to begin transaction: " + e.getMessage()).asException()); + catch (Throwable t) { + if (logger.isDebugEnabled()) { + logger.debug("beginTransaction(): FAILED db={} user={} err={}", reqDb, (user != null ? user : ""), t.toString(), t); + } + logger.error("Error beginning transaction: {}", t.getMessage(), t); + responseObserver.onError(Status.INTERNAL.withDescription("Failed to begin transaction: " + t.getMessage()).asException()); } } @Override public void commitTransaction(CommitTransactionRequest req, StreamObserver rsp) { - final String txId = req.getTransaction().getTransactionId(); + if (logger.isDebugEnabled()) { + logger.debug("commitTransaction(): received request txId={}", txId); + } + if (txId == null || txId.isBlank()) { + if (logger.isDebugEnabled()) { + logger.debug("commitTransaction(): missing/blank txId"); + } rsp.onError(Status.INVALID_ARGUMENT.withDescription("Missing transaction id").asException()); return; } - // remove atomically to avoid double commit races + // remove atomically to avoid double-commit races final Database db = activeTransactions.remove(txId); + if (logger.isDebugEnabled()) { + logger.debug("commitTransaction(): removed txId={}, presentPreviously={}, remainingActiveTx={}", txId, db != null, + activeTransactions.size()); + } + if (db == null) { - // Idempotent no-op: OK but not committed because nothing was open. - rsp.onNext(CommitTransactionResponse.newBuilder().setCommitted(false) + // Idempotent no-op + if (logger.isDebugEnabled()) { + logger.debug("commitTransaction(): no active tx for id={}, responding committed=false", txId); + } + rsp.onNext(CommitTransactionResponse.newBuilder().setSuccess(true).setCommitted(false) .setMessage("No active transaction for id=" + txId + " (already committed/rolled back?)").build()); rsp.onCompleted(); return; } try { + if (logger.isDebugEnabled()) { + logger.debug("commitTransaction(): committing txId={} on db={}", txId, db.getName()); + } db.commit(); - rsp.onNext(CommitTransactionResponse.newBuilder().setCommitted(true).build()); + if (logger.isDebugEnabled()) { + logger.debug("commitTransaction(): commit OK txId={}", txId); + } + rsp.onNext(CommitTransactionResponse.newBuilder().setSuccess(true).setCommitted(true).build()); rsp.onCompleted(); } catch (Throwable t) { - // Put nothing back in the map. The tx is unusable now. + if (logger.isDebugEnabled()) { + logger.debug("commitTransaction(): commit FAILED txId={} err={}", txId, t.toString(), t); + } + // tx is unusable; do not reinsert into the map rsp.onError(Status.ABORTED.withDescription("Commit failed: " + t.getMessage()).asException()); } } @Override public void rollbackTransaction(RollbackTransactionRequest req, StreamObserver rsp) { - final String txId = req.getTransaction().getTransactionId(); + + if (logger.isDebugEnabled()) { + logger.debug("rollbackTransaction(): received request txId={}", txId); + } + if (txId == null || txId.isBlank()) { + if (logger.isDebugEnabled()) { + logger.debug("rollbackTransaction(): missing/blank txId"); + } rsp.onError(Status.INVALID_ARGUMENT.withDescription("Missing transaction id").asException()); return; } final Database db = activeTransactions.remove(txId); + if (logger.isDebugEnabled()) { + logger.debug("rollbackTransaction(): removed txId={}, presentPreviously={}, remainingActiveTx={}", txId, db != null, + activeTransactions.size()); + } + if (db == null) { - rsp.onNext(RollbackTransactionResponse.newBuilder().setRolledBack(false) + if (logger.isDebugEnabled()) { + logger.debug("rollbackTransaction(): no active tx for id={}, responding rolledBack=false", txId); + } + rsp.onNext(RollbackTransactionResponse.newBuilder().setSuccess(true).setRolledBack(false) .setMessage("No active transaction for id=" + txId + " (already committed/rolled back?)").build()); rsp.onCompleted(); return; } try { + if (logger.isDebugEnabled()) { + logger.debug("rollbackTransaction(): rolling back txId={} on db={}", txId, db.getName()); + } db.rollback(); - rsp.onNext(RollbackTransactionResponse.newBuilder().setRolledBack(true).build()); + if (logger.isDebugEnabled()) { + logger.debug("rollbackTransaction(): rollback OK txId={}", txId); + } + rsp.onNext(RollbackTransactionResponse.newBuilder().setSuccess(true).setRolledBack(true).build()); rsp.onCompleted(); } catch (Throwable t) { + if (logger.isDebugEnabled()) { + logger.debug("rollbackTransaction(): rollback FAILED txId={} err={}", txId, t.toString(), t); + } rsp.onError(Status.ABORTED.withDescription("Rollback failed: " + t.getMessage()).asException()); } } @@ -2220,7 +2290,7 @@ void closeQuietly() { } // Helper methods - + private Database getDatabase(String databaseName, DatabaseCredentials credentials) { // Validate credentials From 33e55f535c7fff2ee3de8b9f751ea6c0aba78f41 Mon Sep 17 00:00:00 2001 From: Oleg Cohen Date: Mon, 8 Sep 2025 18:00:51 -0700 Subject: [PATCH 35/35] Checkpoint Commit --- .../remote/grpc/RemoteGrpcDatabase.java | 451 ++++++++++++------ 1 file changed, 299 insertions(+), 152 deletions(-) diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java index ecd7c8c296..feeaa1f0de 100644 --- a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java @@ -10,6 +10,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; @@ -242,7 +243,7 @@ public void begin(final Database.TRANSACTION_ISOLATION_LEVEL isolationLevel) { @Override public void commit() { - + checkDatabaseIsOpen(); stats.txCommits.incrementAndGet(); @@ -262,32 +263,32 @@ public void commit() { .setCredentials(buildCredentials()).build(); try { - + CommitTransactionResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) .commitTransaction(request); logger.debug("[After commit] Success: {} Committed: {}", response.getSuccess(), response.getCommitted()); - + if (!response.getSuccess()) { throw new TransactionException("Failed to commit transaction: " + response.getMessage()); } } catch (StatusRuntimeException e) { - + handleGrpcException(e); } catch (StatusException e) { - + handleGrpcException(e); } finally { - + transactionId = null; setSessionId(null); } if (debugTx != null) { - + debugTx.committed = true; debugTx.rpcSeq.incrementAndGet(); } @@ -322,10 +323,10 @@ public void rollback() { .setCredentials(buildCredentials()).build(); try { - + RollbackTransactionResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) .rollbackTransaction(request); - + logger.debug("[After rollback] Success: {} Committed: {}", response.getSuccess(), response.getRolledBack()); if (!response.getSuccess()) { @@ -412,19 +413,6 @@ public boolean deleteRecord(final String rid, final long timeoutMs) { } } - @Override - public Iterator iterateType(final String typeName, final boolean polymorphic) { - String query = "select from `" + typeName + "`"; - if (!polymorphic) - query += " where @type = '" + typeName + "'"; - return streamQuery(query); - } - - @Override - public Iterator iterateBucket(final String bucketName) { - return streamQuery("select from bucket:`" + bucketName + "`"); - } - @Override public ResultSet command(final String language, final String command, final ContextConfiguration configuration, final Object... args) { @@ -729,149 +717,283 @@ private RemoteGrpcConfig getDefaultRemoteGrpcConfig() { // Convenience: default batch size stays 100, default mode = CURSOR - public Iterator queryStream(final String language, final String query) { + public ResultSet queryStream(final String language, final String query) { return queryStream(language, query, getDefaultRemoteGrpcConfig(), /* batchSize */100, StreamQueryRequest.RetrievalMode.CURSOR); } - public Iterator queryStream(final String language, final String query, final RemoteGrpcConfig config) { + public ResultSet queryStream(final String language, final String query, final RemoteGrpcConfig config) { return queryStream(language, query, config, /* batchSize */100, StreamQueryRequest.RetrievalMode.CURSOR); } - public Iterator queryStream(final String language, final String query, final int batchSize) { + public ResultSet queryStream(final String language, final String query, final int batchSize) { return queryStream(language, query, getDefaultRemoteGrpcConfig(), batchSize, StreamQueryRequest.RetrievalMode.CURSOR); } - public Iterator queryStream(final String language, final String query, RemoteGrpcConfig config, final int batchSize) { + public ResultSet queryStream(final String language, final String query, RemoteGrpcConfig config, final int batchSize) { return queryStream(language, query, config, batchSize, StreamQueryRequest.RetrievalMode.CURSOR); } - public Iterator queryStream(final String language, final String query, final int batchSize, - final StreamQueryRequest.RetrievalMode mode) { + public ResultSet queryStream(final String language, final String query, final int batchSize, final StreamQueryRequest.RetrievalMode mode) { return queryStream(language, query, getDefaultRemoteGrpcConfig(), batchSize, mode); } // NEW: choose retrieval mode - public Iterator queryStream(final String language, final String query, final RemoteGrpcConfig config, final int batchSize, + public ResultSet queryStream(final String language, final String query, final RemoteGrpcConfig config, final int batchSize, final StreamQueryRequest.RetrievalMode mode) { return queryStream(language, query, config, Map.of(), batchSize, mode); } - // PARAMETERIZED variant with retrieval mode - public Iterator queryStream(final String language, final String query, final RemoteGrpcConfig config, - final Map params, final int batchSize, final StreamQueryRequest.RetrievalMode mode) { + /** + * Stream query results as a ResultSet that fetches data lazily in batches. This + * provides consistency with non-streaming query methods and supports non-Record + * results like projections and aggregations. + */ + public ResultSet queryStream(final String language, final String query, final RemoteGrpcConfig config, final Map params, + final int batchSize, final StreamQueryRequest.RetrievalMode mode) { checkDatabaseIsOpen(); stats.queries.incrementAndGet(); StreamQueryRequest.Builder b = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(query).setCredentials(buildCredentials()) - .setBatchSize(batchSize > 0 ? batchSize : 100).setRetrievalMode(mode); // <--- NEW + .setBatchSize(batchSize > 0 ? batchSize : 100).setRetrievalMode(mode); if (params != null && !params.isEmpty()) { b.putAllParameters(convertParamsToGrpcValue(params)); } - // Open the stream with deadline via helper final BlockingClientCall responseIterator = callServerStreaming("StreamQuery", () -> blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).streamQuery(b.build())); - return new Iterator() { - private Iterator currentBatch = Collections.emptyIterator(); + // Return a streaming ResultSet implementation + return new StreamingResultSet(responseIterator, this); + } - @Override - public boolean hasNext() { - if (currentBatch.hasNext()) - return true; + // Keep the old signature working (defaults to CURSOR) + public ResultSet queryStream(final String language, final String query, final Map params, final int batchSize) { - if (debugTx != null) { - checkCrossThreadUse("streamQuery.hasNext"); - } + return queryStream(language, query, getDefaultRemoteGrpcConfig(), params, batchSize, StreamQueryRequest.RetrievalMode.CURSOR); + } - try { - while (responseIterator.hasNext()) { - final QueryResult result = responseIterator.read(); + public ResultSet queryStream(final String language, final String query, final RemoteGrpcConfig config, final Map params, + final int batchSize) { - if (result.getRecordsCount() == 0) { - if (result.getIsLastBatch()) - return false; - continue; // empty non-terminal batch + return queryStream(language, query, config, params, batchSize, StreamQueryRequest.RetrievalMode.CURSOR); + } + + /** + * A ResultSet implementation that lazily fetches results from a gRPC stream. + * Supports both Record results and projection/aggregation results. + */ + private static class StreamingResultSet implements ResultSet { + private static final Logger logger = LoggerFactory.getLogger(StreamingResultSet.class); + + private final BlockingClientCall stream; + private final RemoteGrpcDatabase db; + protected Iterator currentBatch = Collections.emptyIterator(); + private boolean streamExhausted = false; + private Result nextResult = null; + private final AtomicLong totalProcessed = new AtomicLong(0); + + StreamingResultSet(BlockingClientCall stream, RemoteGrpcDatabase db) { + this.stream = stream; + this.db = db; + } + + @Override + public boolean hasNext() { + if (nextResult != null) { + return true; + } + + // Try to get next from current batch + if (currentBatch.hasNext()) { + nextResult = currentBatch.next(); + return true; + } + + // Current batch exhausted, try to fetch next batch + if (streamExhausted) { + return false; + } + + if (db.debugTx != null) { + db.checkCrossThreadUse("streamQuery.hasNext"); + } + + try { + while (stream.hasNext()) { + final QueryResult queryResult = stream.read(); + + if (logger.isDebugEnabled()) { + logger.debug("Received batch with {} records, isLastBatch={}", queryResult.getRecordsCount(), + queryResult.getIsLastBatch()); + } + + if (queryResult.getRecordsCount() == 0) { + if (queryResult.getIsLastBatch()) { + streamExhausted = true; + return false; } + continue; // empty non-terminal batch + } - currentBatch = result.getRecordsList().iterator(); + // Convert GrpcRecords to Results + currentBatch = convertBatchToResults(queryResult); + + if (currentBatch.hasNext()) { + nextResult = currentBatch.next(); return true; } - return false; // stream exhausted - } - catch (io.grpc.StatusRuntimeException | io.grpc.StatusException e) { - handleGrpcException(e); // rethrows mapped runtime exception - throw new IllegalStateException("unreachable"); - } - catch (RuntimeException re) { - throw re; - } - catch (Exception e) { - throw new RuntimeException("Stream failed", e); } + + streamExhausted = true; + return false; + + } + catch (io.grpc.StatusRuntimeException | io.grpc.StatusException e) { + db.handleGrpcException(e); + throw new IllegalStateException("unreachable"); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Stream interrupted", e); } + catch (RuntimeException re) { + throw re; + } + catch (Exception e) { + throw new RuntimeException("Stream failed", e); + } + } - @Override - public Record next() { - if (!hasNext()) - throw new NoSuchElementException(); - if (debugTx != null) { - checkCrossThreadUse("streamQuery.next"); + /** + * Convert a QueryResult batch to an Iterator of Result objects + */ + protected Iterator convertBatchToResults(QueryResult queryResult) { + List results = new ArrayList<>(queryResult.getRecordsCount()); + + for (GrpcRecord grpcRecord : queryResult.getRecordsList()) { + // Use the existing grpcRecordToResult method from RemoteGrpcDatabase + Result result = db.grpcRecordToResult(grpcRecord); + results.add(result); + } + + return results.iterator(); + } + + @Override + public Result next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + if (db.debugTx != null) { + db.checkCrossThreadUse("streamQuery.next"); + } + + Result result = nextResult; + nextResult = null; + totalProcessed.incrementAndGet(); + return result; + } + + @Override + public void close() { + + try { + // Drain any remaining results + while (stream.hasNext()) { + stream.read(); } - return grpcRecordToDBRecord(currentBatch.next()); } - }; + catch (Exception e) { + logger.debug("Exception while draining stream during close", e); + } + + // BlockingClientCall doesn't implement AutoCloseable + // No need to cast or check instanceof + } } - // Keep the old signature working (defaults to CURSOR) - public Iterator queryStream(final String language, final String query, final Map params, final int batchSize) { + /** + * Enhanced streaming with batch-aware ResultSet for better memory control and + * performance monitoring. + */ + public ResultSet queryStreamBatched(final String language, final String query, final Map params, final int batchSize, + final StreamQueryRequest.RetrievalMode mode) { - return queryStream(language, query, getDefaultRemoteGrpcConfig(), params, batchSize, StreamQueryRequest.RetrievalMode.CURSOR); - } + checkDatabaseIsOpen(); + stats.queries.incrementAndGet(); - public Iterator queryStream(final String language, final String query, final RemoteGrpcConfig config, - final Map params, final int batchSize) { + final StreamQueryRequest.Builder b = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(query) + .setCredentials(buildCredentials()).setBatchSize(batchSize > 0 ? batchSize : 100).setRetrievalMode(mode); - return queryStream(language, query, config, params, batchSize, StreamQueryRequest.RetrievalMode.CURSOR); + if (params != null && !params.isEmpty()) { + b.putAllParameters(convertParamsToGrpcValue(params)); + } + + final BlockingClientCall responseIterator = callServerStreaming("StreamQuery", + () -> blockingStub.withWaitForReady().withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).streamQuery(b.build())); + + return new BatchedStreamingResultSet(responseIterator, this); } - public static final class QueryBatch { + /** + * A ResultSet that exposes batch boundaries for advanced use cases while + * maintaining the standard ResultSet interface. + */ + private static class BatchedStreamingResultSet extends StreamingResultSet { - private final List records; - private final int totalInBatch; - private final long runningTotal; - private final boolean lastBatch; + private int currentBatchSize = 0; + private boolean isLastBatch = false; + private long runningTotal = 0; - public QueryBatch(List list, int totalInBatch, long runningTotal, boolean lastBatch) { - this.records = list; - this.totalInBatch = totalInBatch; - this.runningTotal = runningTotal; - this.lastBatch = lastBatch; + BatchedStreamingResultSet(BlockingClientCall stream, RemoteGrpcDatabase db) { + super(stream, db); } - public List records() { - return records; - } + @Override + public boolean hasNext() { + boolean hasMore = super.hasNext(); - public int totalInBatch() { - return totalInBatch; + // Update batch info when we fetch a new batch + if (hasMore && currentBatch != null) { + // Track batch metadata here if needed + } + + return hasMore; } - public long runningTotal() { + @Override + protected Iterator convertBatchToResults(QueryResult queryResult) { + // Capture batch metadata + this.currentBatchSize = queryResult.getRecordsCount(); + this.isLastBatch = queryResult.getIsLastBatch(); + this.runningTotal = queryResult.getRunningTotalEmitted(); + + // Call parent implementation + return super.convertBatchToResults(queryResult); + } + + // Additional methods for batch-aware processing + public long getRunningTotal() { return runningTotal; } + public int getCurrentBatchSize() { + return currentBatchSize; + } + public boolean isLastBatch() { - return lastBatch; + return isLastBatch; } } - public Iterator queryStreamBatches(final String language, final String query, final Map params, + public Iterator queryStreamBatchesIterator(final String language, final String query, final Map params, final int batchSize, final StreamQueryRequest.RetrievalMode mode) { + checkDatabaseIsOpen(); stats.queries.incrementAndGet(); @@ -901,26 +1023,32 @@ public boolean hasNext() { } try { + while (responseIterator.hasNext()) { final QueryResult qr = responseIterator.read(); int totalInBatch = qr.getTotalRecordsInBatch(); - if (totalInBatch == 0) + if (totalInBatch == 0) { totalInBatch = qr.getRecordsCount(); + } if (qr.getRecordsCount() == 0 && !qr.getIsLastBatch()) { continue; // empty non-terminal batch } - final List converted = new ArrayList<>(qr.getRecordsCount()); + // Convert GrpcRecords to Results (not Records) + final List convertedResults = new ArrayList<>(qr.getRecordsCount()); for (GrpcRecord gr : qr.getRecordsList()) { - converted.add(grpcRecordToDBRecord(gr)); + Result result = grpcRecordToResult(gr); + convertedResults.add(result); } - nextBatch = new QueryBatch(converted, totalInBatch, qr.getRunningTotalEmitted(), qr.getIsLastBatch()); + // Create QueryBatch with Results instead of Records + nextBatch = new QueryBatch(convertedResults, totalInBatch, qr.getRunningTotalEmitted(), qr.getIsLastBatch()); - if (qr.getIsLastBatch()) + if (qr.getIsLastBatch()) { drained = true; + } return true; } @@ -928,7 +1056,7 @@ public boolean hasNext() { return false; } catch (io.grpc.StatusRuntimeException | io.grpc.StatusException e) { - handleGrpcException(e); // rethrows mapped exception + handleGrpcException(e); throw new IllegalStateException("unreachable"); } catch (InterruptedException ie) { @@ -954,6 +1082,49 @@ public QueryBatch next() { }; } + public static final class QueryBatch { + private final List results; // Changed from List + private final int totalInBatch; + private final long runningTotal; + private final boolean lastBatch; + + public QueryBatch(List results, int totalInBatch, long runningTotal, boolean lastBatch) { + this.results = results; + this.totalInBatch = totalInBatch; + this.runningTotal = runningTotal; + this.lastBatch = lastBatch; + } + + public List results() { + return results; + } + + // Backward compatibility: provide records() method that extracts Records from + // Results + @Deprecated + public List records() { + List records = new ArrayList<>(results.size()); + for (Result result : results) { + if (result.isElement()) { + result.getRecord().ifPresent(records::add); + } + } + return records; + } + + public int totalInBatch() { + return totalInBatch; + } + + public long runningTotal() { + return runningTotal; + } + + public boolean isLastBatch() { + return lastBatch; + } + } + public Iterator queryStream(final String database, final String sql, final Map params, final int batchSize, final StreamQueryRequest.RetrievalMode mode, final TransactionContext tx, final long timeoutMs) { final StreamQueryRequest.Builder reqB = StreamQueryRequest.newBuilder().setDatabase(database).setQuery(sql) @@ -1016,68 +1187,44 @@ public GrpcRecord next() { }; } - public Iterator queryStreamBatches(final String passLabel, final String sql, final Map params, - final int batchSize, final StreamQueryRequest.RetrievalMode mode, final TransactionContext tx, final long timeoutMs) { - final StreamQueryRequest.Builder reqB = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(sql) - .putAllParameters(convertParamsToGrpcValue(params)).setCredentials(buildCredentials()) - .setBatchSize(batchSize > 0 ? batchSize : 100).setRetrievalMode(mode); - - if (tx != null) - reqB.setTransaction(tx); + @Deprecated + public Iterator queryStreamAsRecordIterator(final String language, final String query, final Map params, + final int batchSize) { + ResultSet rs = queryStream(language, query, params, batchSize); - final BlockingClientCall respIter = callServerStreaming("StreamQuery[" + passLabel + "]", - () -> blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).streamQuery(reqB.build())); - - return new Iterator() { + // Convert ResultSet to Iterator + return new Iterator() { @Override public boolean hasNext() { - if (debugTx != null) { - checkCrossThreadUse("streamQueryBatches(" + passLabel + ").hasNext"); - } - try { - return respIter.hasNext(); - } - catch (io.grpc.StatusRuntimeException | io.grpc.StatusException e) { - handleGrpcException(e); - throw new IllegalStateException("unreachable"); - } - catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw new RuntimeException("Stream interrupted", ie); - } + return rs.hasNext(); } @Override - public QueryBatch next() { - if (debugTx != null) { - checkCrossThreadUse("streamQueryBatches(" + passLabel + ").next"); - } - try { - final QueryResult qr = respIter.read(); - - int totalInBatch = qr.getTotalRecordsInBatch(); - if (totalInBatch == 0) - totalInBatch = qr.getRecordsCount(); - - final List converted = new ArrayList<>(qr.getRecordsCount()); - for (GrpcRecord gr : qr.getRecordsList()) { - converted.add(grpcRecordToDBRecord(gr)); - } - - return new QueryBatch(converted, totalInBatch, qr.getRunningTotalEmitted(), qr.getIsLastBatch()); - } - catch (io.grpc.StatusRuntimeException | io.grpc.StatusException e) { - handleGrpcException(e); - throw new IllegalStateException("unreachable"); - } - catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw new RuntimeException("Stream interrupted", ie); + public Record next() { + Result result = rs.next(); + // Try to get Record from Result + if (result.isElement()) { + return result.getRecord().orElseThrow(() -> new IllegalStateException("Result claims to be element but has no Record")); } + // For non-Record results, throw or skip + throw new IllegalStateException("Result is not a Record: " + result); } }; } + @Override + public Iterator iterateType(final String typeName, final boolean polymorphic) { + String query = "select from `" + typeName + "`"; + if (!polymorphic) + query += " where @type = '" + typeName + "'"; + return streamQuery(query); + } + + @Override + public Iterator iterateBucket(final String bucketName) { + return streamQuery("select from bucket:`" + bucketName + "`"); + } + public String createRecord(final String cls, final Map props, final long timeoutMs) { checkDatabaseIsOpen();