diff --git a/.github/workflows/studio-security-audit.yml b/.github/workflows/studio-security-audit.yml index c0428a025a..8c34311c1d 100644 --- a/.github/workflows/studio-security-audit.yml +++ b/.github/workflows/studio-security-audit.yml @@ -146,7 +146,7 @@ jobs: uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: "Dependency Review" - uses: actions/dependency-review-action@595b5aeba73380359d98a5e087f648dbb0edce1b # v4.7.3 + uses: actions/dependency-review-action@bc41886e18ea39df68b1b1245f4184881938e050 # v4.7.2 with: fail-on-severity: moderate allow-dependencies-licenses: "MIT, Apache-2.0, BSD-2-Clause, BSD-3-Clause, ISC, 0BSD" diff --git a/e2e-js/package-lock.json b/e2e-js/package-lock.json index ca0a6a7a77..026a23e709 100644 --- a/e2e-js/package-lock.json +++ b/e2e-js/package-lock.json @@ -57,22 +57,22 @@ } }, "node_modules/@babel/core": { - "version": "7.28.3", - "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.28.3.tgz", - "integrity": "sha512-yDBHV9kQNcr2/sUr9jghVyz9C3Y5G2zUM2H2lo+9mKv4sFgbA8s8Z9t8D1jiTkGoO/NoIfKMyKWr4s6CN23ZwQ==", + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.28.0.tgz", + "integrity": "sha512-UlLAnTPrFdNGoFtbSXwcGFQBtQZJCNjaN6hQNP3UPvuNXT1i82N26KL3dZeIpNalWywr9IuQuncaAfUaS1g6sQ==", "dev": true, "license": "MIT", "dependencies": { "@ampproject/remapping": "^2.2.0", "@babel/code-frame": "^7.27.1", - "@babel/generator": "^7.28.3", + "@babel/generator": "^7.28.0", "@babel/helper-compilation-targets": "^7.27.2", - "@babel/helper-module-transforms": "^7.28.3", - "@babel/helpers": "^7.28.3", - "@babel/parser": "^7.28.3", + "@babel/helper-module-transforms": "^7.27.3", + "@babel/helpers": "^7.27.6", + "@babel/parser": "^7.28.0", "@babel/template": "^7.27.2", - "@babel/traverse": "^7.28.3", - "@babel/types": "^7.28.2", + "@babel/traverse": "^7.28.0", + "@babel/types": "^7.28.0", "convert-source-map": "^2.0.0", "debug": "^4.1.0", "gensync": "^1.0.0-beta.2", @@ -88,14 +88,14 @@ } }, "node_modules/@babel/generator": { - "version": "7.28.3", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.28.3.tgz", - "integrity": "sha512-3lSpxGgvnmZznmBkCRnVREPUFJv2wrv9iAoFDvADJc0ypmdOxdUtcLeBgBJ6zE0PMeTKnxeQzyk0xTBq4Ep7zw==", + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.28.0.tgz", + "integrity": "sha512-lJjzvrbEeWrhB4P3QBsH7tey117PjLZnDbLiQEKjQ/fNJTjuq4HSqgFA+UNSwZT8D7dxxbnuSBMsa1lrWzKlQg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/parser": "^7.28.3", - "@babel/types": "^7.28.2", + "@babel/parser": "^7.28.0", + "@babel/types": "^7.28.0", "@jridgewell/gen-mapping": "^0.3.12", "@jridgewell/trace-mapping": "^0.3.28", "jsesc": "^3.0.2" @@ -146,15 +146,15 @@ } }, "node_modules/@babel/helper-module-transforms": { - "version": "7.28.3", - "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.28.3.tgz", - "integrity": "sha512-gytXUbs8k2sXS9PnQptz5o0QnpLL51SwASIORY6XaBKF88nsOT0Zw9szLqlSGQDP/4TljBAD5y98p2U1fqkdsw==", + "version": "7.27.3", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.27.3.tgz", + "integrity": "sha512-dSOvYwvyLsWBeIRyOeHXp5vPj5l1I011r52FM1+r1jCERv+aFXYk4whgQccYEGYxK2H3ZAIA8nuPkQ0HaUo3qg==", "dev": true, "license": "MIT", "dependencies": { "@babel/helper-module-imports": "^7.27.1", "@babel/helper-validator-identifier": "^7.27.1", - "@babel/traverse": "^7.28.3" + "@babel/traverse": "^7.27.3" }, "engines": { "node": ">=6.9.0" @@ -204,9 +204,9 @@ } }, "node_modules/@babel/helpers": { - "version": "7.28.3", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.28.3.tgz", - "integrity": "sha512-PTNtvUQihsAsDHMOP5pfobP8C6CM4JWXmP8DrEIt46c3r2bf87Ua1zoqevsMo9g+tWDwgWrFP5EIxuBx5RudAw==", + "version": "7.28.2", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.28.2.tgz", + "integrity": "sha512-/V9771t+EgXz62aCcyofnQhGM8DQACbRhvzKFsXKC9QM+5MadF8ZmIm0crDMaz3+o0h0zXfJnd4EhbYbxsrcFw==", "dev": true, "license": "MIT", "dependencies": { @@ -218,13 +218,13 @@ } }, "node_modules/@babel/parser": { - "version": "7.28.3", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.28.3.tgz", - "integrity": "sha512-7+Ey1mAgYqFAx2h0RuoxcQT5+MlG3GTV0TQrgr7/ZliKsm/MNDxVVutlWaziMq7wJNAz8MTqz55XLpWvva6StA==", + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.28.0.tgz", + "integrity": "sha512-jVZGvOxOuNSsuQuLRTh13nU0AogFlw32w/MT+LV6D3sP5WdbW61E77RnkbaO2dUvmPAYrBDJXGn5gGS6tH4j8g==", "dev": true, "license": "MIT", "dependencies": { - "@babel/types": "^7.28.2" + "@babel/types": "^7.28.0" }, "bin": { "parser": "bin/babel-parser.js" @@ -488,18 +488,18 @@ } }, "node_modules/@babel/traverse": { - "version": "7.28.3", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.28.3.tgz", - "integrity": "sha512-7w4kZYHneL3A6NP2nxzHvT3HCZ7puDZZjFMqDpBPECub79sTtSO5CGXDkKrTQq8ksAwfD/XI2MRFX23njdDaIQ==", + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.28.0.tgz", + "integrity": "sha512-mGe7UK5wWyh0bKRfupsUchrQGqvDbZDbKJw+kcRGSmdHVYrv+ltd0pnpDTVpiTqnaBru9iEvA8pz8W46v0Amwg==", "dev": true, "license": "MIT", "dependencies": { "@babel/code-frame": "^7.27.1", - "@babel/generator": "^7.28.3", + "@babel/generator": "^7.28.0", "@babel/helper-globals": "^7.28.0", - "@babel/parser": "^7.28.3", + "@babel/parser": "^7.28.0", "@babel/template": "^7.27.2", - "@babel/types": "^7.28.2", + "@babel/types": "^7.28.0", "debug": "^4.3.1" }, "engines": { @@ -534,21 +534,21 @@ "license": "MIT" }, "node_modules/@emnapi/core": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.5.0.tgz", - "integrity": "sha512-sbP8GzB1WDzacS8fgNPpHlp6C9VZe+SJP3F90W9rLemaQj2PzIuTEl1qDOYQf58YIpyjViI24y9aPWCjEzY2cg==", + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.4.5.tgz", + "integrity": "sha512-XsLw1dEOpkSX/WucdqUhPWP7hDxSvZiY+fsUC14h+FtQ2Ifni4znbBt8punRX+Uj2JG/uDb8nEHVKvrVlvdZ5Q==", "dev": true, "license": "MIT", "optional": true, "dependencies": { - "@emnapi/wasi-threads": "1.1.0", + "@emnapi/wasi-threads": "1.0.4", "tslib": "^2.4.0" } }, "node_modules/@emnapi/runtime": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.5.0.tgz", - "integrity": "sha512-97/BJ3iXHww3djw6hYIfErCZFee7qCtrneuLa20UXFCOTCfBM2cvQHjWJ2EG0s0MtdNwInarqCTz35i4wWXHsQ==", + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.4.5.tgz", + "integrity": "sha512-++LApOtY0pEEz1zrd9vy1/zXVaVJJ/EbAF3u0fXIzPJEDtnITsBGbbK0EkM72amhl/R5b+5xx0Y/QhcVOpuulg==", "dev": true, "license": "MIT", "optional": true, @@ -557,9 +557,9 @@ } }, "node_modules/@emnapi/wasi-threads": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.1.0.tgz", - "integrity": "sha512-WI0DdZ8xFSbgMjR1sFsKABJ/C5OnRrjT06JXbZKexJGrDuPTzZdDYfFlsgcCXCyf+suG5QU2e/y1Wo2V/OapLQ==", + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.0.4.tgz", + "integrity": "sha512-PJR+bOmMOPH8AtcTGAyYNiuJ3/Fcoj2XN/gBEWzDIKh254XO+mM9XoXHk5GNEhodxeMznbg7BlRojVbKN+gC6g==", "dev": true, "license": "MIT", "optional": true, @@ -729,16 +729,16 @@ } }, "node_modules/@jest/console": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/@jest/console/-/console-30.1.1.tgz", - "integrity": "sha512-f7TGqR1k4GtN5pyFrKmq+ZVndesiwLU33yDpJIGMS9aW+j6hKjue7ljeAdznBsH9kAnxUWe2Y+Y3fLV/FJt3gA==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/@jest/console/-/console-30.0.5.tgz", + "integrity": "sha512-xY6b0XiL0Nav3ReresUarwl2oIz1gTnxGbGpho9/rbUWsLH0f1OD/VT84xs8c7VmH7MChnLb0pag6PhZhAdDiA==", "dev": true, "license": "MIT", "dependencies": { "@jest/types": "30.0.5", "@types/node": "*", "chalk": "^4.1.2", - "jest-message-util": "30.1.0", + "jest-message-util": "30.0.5", "jest-util": "30.0.5", "slash": "^3.0.0" }, @@ -747,17 +747,17 @@ } }, "node_modules/@jest/core": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/@jest/core/-/core-30.1.1.tgz", - "integrity": "sha512-3ncU9peZ3D2VdgRkdZtUceTrDgX5yiDRwAFjtxNfU22IiZrpVWlv/FogzDLYSJQptQGfFo3PcHK86a2oG6WUGg==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/@jest/core/-/core-30.0.5.tgz", + "integrity": "sha512-fKD0OulvRsXF1hmaFgHhVJzczWzA1RXMMo9LTPuFXo9q/alDbME3JIyWYqovWsUBWSoBcsHaGPSLF9rz4l9Qeg==", "dev": true, "license": "MIT", "dependencies": { - "@jest/console": "30.1.1", + "@jest/console": "30.0.5", "@jest/pattern": "30.0.1", - "@jest/reporters": "30.1.1", - "@jest/test-result": "30.1.1", - "@jest/transform": "30.1.1", + "@jest/reporters": "30.0.5", + "@jest/test-result": "30.0.5", + "@jest/transform": "30.0.5", "@jest/types": "30.0.5", "@types/node": "*", "ansi-escapes": "^4.3.2", @@ -766,18 +766,18 @@ "exit-x": "^0.2.2", "graceful-fs": "^4.2.11", "jest-changed-files": "30.0.5", - "jest-config": "30.1.1", - "jest-haste-map": "30.1.0", - "jest-message-util": "30.1.0", + "jest-config": "30.0.5", + "jest-haste-map": "30.0.5", + "jest-message-util": "30.0.5", "jest-regex-util": "30.0.1", - "jest-resolve": "30.1.0", - "jest-resolve-dependencies": "30.1.1", - "jest-runner": "30.1.1", - "jest-runtime": "30.1.1", - "jest-snapshot": "30.1.1", + "jest-resolve": "30.0.5", + "jest-resolve-dependencies": "30.0.5", + "jest-runner": "30.0.5", + "jest-runtime": "30.0.5", + "jest-snapshot": "30.0.5", "jest-util": "30.0.5", - "jest-validate": "30.1.0", - "jest-watcher": "30.1.1", + "jest-validate": "30.0.5", + "jest-watcher": "30.0.5", "micromatch": "^4.0.8", "pretty-format": "30.0.5", "slash": "^3.0.0" @@ -805,13 +805,13 @@ } }, "node_modules/@jest/environment": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-30.1.1.tgz", - "integrity": "sha512-yWHbU+3j7ehQE+NRpnxRvHvpUhoohIjMePBbIr8lfe0cWVb0WeTf80DNux1GPJa18CDHiIU5DtksGUfxcDE+Rw==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-30.0.5.tgz", + "integrity": "sha512-aRX7WoaWx1oaOkDQvCWImVQ8XNtdv5sEWgk4gxR6NXb7WBUnL5sRak4WRzIQRZ1VTWPvV4VI4mgGjNL9TeKMYA==", "dev": true, "license": "MIT", "dependencies": { - "@jest/fake-timers": "30.1.1", + "@jest/fake-timers": "30.0.5", "@jest/types": "30.0.5", "@types/node": "*", "jest-mock": "30.0.5" @@ -821,43 +821,43 @@ } }, "node_modules/@jest/expect": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-30.1.1.tgz", - "integrity": "sha512-3vHIHsF+qd3D8FU2c7U5l3rg1fhDwAYcGyHyZAi94YIlTwcJ+boNhRyJf373cl4wxbOX+0Q7dF40RTrTFTSuig==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-30.0.5.tgz", + "integrity": "sha512-6udac8KKrtTtC+AXZ2iUN/R7dp7Ydry+Fo6FPFnDG54wjVMnb6vW/XNlf7Xj8UDjAE3aAVAsR4KFyKk3TCXmTA==", "dev": true, "license": "MIT", "dependencies": { - "expect": "30.1.1", - "jest-snapshot": "30.1.1" + "expect": "30.0.5", + "jest-snapshot": "30.0.5" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/expect-utils": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-30.1.1.tgz", - "integrity": "sha512-5YUHr27fpJ64dnvtu+tt11ewATynrHkGYD+uSFgRr8V2eFJis/vEXgToyLwccIwqBihVfz9jwio+Zr1ab1Zihw==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-30.0.5.tgz", + "integrity": "sha512-F3lmTT7CXWYywoVUGTCmom0vXq3HTTkaZyTAzIy+bXSBizB7o5qzlC9VCtq0arOa8GqmNsbg/cE9C6HLn7Szew==", "dev": true, "license": "MIT", "dependencies": { - "@jest/get-type": "30.1.0" + "@jest/get-type": "30.0.1" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/@jest/fake-timers": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-30.1.1.tgz", - "integrity": "sha512-fK/25dNgBNYPw3eLi2CRs57g1H04qBAFNMsUY3IRzkfx/m4THe0E1zF+yGQBOMKKc2XQVdc9EYbJ4hEm7/2UtA==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-30.0.5.tgz", + "integrity": "sha512-ZO5DHfNV+kgEAeP3gK3XlpJLL4U3Sz6ebl/n68Uwt64qFFs5bv4bfEEjyRGK5uM0C90ewooNgFuKMdkbEoMEXw==", "dev": true, "license": "MIT", "dependencies": { "@jest/types": "30.0.5", "@sinonjs/fake-timers": "^13.0.0", "@types/node": "*", - "jest-message-util": "30.1.0", + "jest-message-util": "30.0.5", "jest-mock": "30.0.5", "jest-util": "30.0.5" }, @@ -866,9 +866,9 @@ } }, "node_modules/@jest/get-type": { - "version": "30.1.0", - "resolved": "https://registry.npmjs.org/@jest/get-type/-/get-type-30.1.0.tgz", - "integrity": "sha512-eMbZE2hUnx1WV0pmURZY9XoXPkUYjpc55mb0CrhtdWLtzMQPFvu/rZkTLZFTsdaVQa+Tr4eWAteqcUzoawq/uA==", + "version": "30.0.1", + "resolved": "https://registry.npmjs.org/@jest/get-type/-/get-type-30.0.1.tgz", + "integrity": "sha512-AyYdemXCptSRFirI5EPazNxyPwAL0jXt3zceFjaj8NFiKP9pOi0bfXonf6qkf82z2t3QWPeLCWWw4stPBzctLw==", "dev": true, "license": "MIT", "engines": { @@ -876,14 +876,14 @@ } }, "node_modules/@jest/globals": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-30.1.1.tgz", - "integrity": "sha512-NNUUkHT2TU/xztZl6r1UXvJL+zvCwmZsQDmK69fVHHcB9fBtlu3FInnzOve/ZoyKnWY8JXWJNT+Lkmu1+ubXUA==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-30.0.5.tgz", + "integrity": "sha512-7oEJT19WW4oe6HR7oLRvHxwlJk2gev0U9px3ufs8sX9PoD1Eza68KF0/tlN7X0dq/WVsBScXQGgCldA1V9Y/jA==", "dev": true, "license": "MIT", "dependencies": { - "@jest/environment": "30.1.1", - "@jest/expect": "30.1.1", + "@jest/environment": "30.0.5", + "@jest/expect": "30.0.5", "@jest/types": "30.0.5", "jest-mock": "30.0.5" }, @@ -906,16 +906,16 @@ } }, "node_modules/@jest/reporters": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-30.1.1.tgz", - "integrity": "sha512-Hb2Bq80kahOC6Sv2waEaH1rEU6VdFcM6WHaRBWQF9tf30+nJHxhl/Upbgo9+25f0mOgbphxvbwSMjSgy9gW/FA==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-30.0.5.tgz", + "integrity": "sha512-mafft7VBX4jzED1FwGC1o/9QUM2xebzavImZMeqnsklgcyxBto8mV4HzNSzUrryJ+8R9MFOM3HgYuDradWR+4g==", "dev": true, "license": "MIT", "dependencies": { "@bcoe/v8-coverage": "^0.2.3", - "@jest/console": "30.1.1", - "@jest/test-result": "30.1.1", - "@jest/transform": "30.1.1", + "@jest/console": "30.0.5", + "@jest/test-result": "30.0.5", + "@jest/transform": "30.0.5", "@jest/types": "30.0.5", "@jridgewell/trace-mapping": "^0.3.25", "@types/node": "*", @@ -929,9 +929,9 @@ "istanbul-lib-report": "^3.0.0", "istanbul-lib-source-maps": "^5.0.0", "istanbul-reports": "^3.1.3", - "jest-message-util": "30.1.0", + "jest-message-util": "30.0.5", "jest-util": "30.0.5", - "jest-worker": "30.1.0", + "jest-worker": "30.0.5", "slash": "^3.0.0", "string-length": "^4.0.2", "v8-to-istanbul": "^9.0.1" @@ -962,9 +962,9 @@ } }, "node_modules/@jest/snapshot-utils": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/@jest/snapshot-utils/-/snapshot-utils-30.1.1.tgz", - "integrity": "sha512-TkVBc9wuN22TT8hESRFmjjg/xIMu7z0J3UDYtIRydzCqlLPTB7jK1DDBKdnTUZ4zL3z3rnPpzV6rL1Uzh87sXg==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/@jest/snapshot-utils/-/snapshot-utils-30.0.5.tgz", + "integrity": "sha512-XcCQ5qWHLvi29UUrowgDFvV4t7ETxX91CbDczMnoqXPOIcZOxyNdSjm6kV5XMc8+HkxfRegU/MUmnTbJRzGrUQ==", "dev": true, "license": "MIT", "dependencies": { @@ -993,13 +993,13 @@ } }, "node_modules/@jest/test-result": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-30.1.1.tgz", - "integrity": "sha512-bMdj7fNu8iZuBPSnbVir5ezvWmVo4jrw7xDE+A33Yb3ENCoiJK9XgOLgal+rJ9XSKjsL7aPUMIo87zhN7I5o2w==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-30.0.5.tgz", + "integrity": "sha512-wPyztnK0gbDMQAJZ43tdMro+qblDHH1Ru/ylzUo21TBKqt88ZqnKKK2m30LKmLLoKtR2lxdpCC/P3g1vfKcawQ==", "dev": true, "license": "MIT", "dependencies": { - "@jest/console": "30.1.1", + "@jest/console": "30.0.5", "@jest/types": "30.0.5", "@types/istanbul-lib-coverage": "^2.0.6", "collect-v8-coverage": "^1.0.2" @@ -1009,15 +1009,15 @@ } }, "node_modules/@jest/test-sequencer": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-30.1.1.tgz", - "integrity": "sha512-yruRdLXSA3HYD/MTNykgJ6VYEacNcXDFRMqKVAwlYegmxICUiT/B++CNuhJnYJzKYks61iYnjVsMwbUqmmAYJg==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-30.0.5.tgz", + "integrity": "sha512-Aea/G1egWoIIozmDD7PBXUOxkekXl7ueGzrsGGi1SbeKgQqCYCIf+wfbflEbf2LiPxL8j2JZGLyrzZagjvW4YQ==", "dev": true, "license": "MIT", "dependencies": { - "@jest/test-result": "30.1.1", + "@jest/test-result": "30.0.5", "graceful-fs": "^4.2.11", - "jest-haste-map": "30.1.0", + "jest-haste-map": "30.0.5", "slash": "^3.0.0" }, "engines": { @@ -1025,9 +1025,9 @@ } }, "node_modules/@jest/transform": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-30.1.1.tgz", - "integrity": "sha512-PHIA2AbAASBfk6evkNifvmx9lkOSkmvaQoO6VSpuL8+kQqDMHeDoJ7RU3YP1wWAMD7AyQn9UL5iheuFYCC4lqQ==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-30.0.5.tgz", + "integrity": "sha512-Vk8amLQCmuZyy6GbBht1Jfo9RSdBtg7Lks+B0PecnjI8J+PCLQPGh7uI8Q/2wwpW2gLdiAfiHNsmekKlywULqg==", "dev": true, "license": "MIT", "dependencies": { @@ -1039,7 +1039,7 @@ "convert-source-map": "^2.0.0", "fast-json-stable-stringify": "^2.1.0", "graceful-fs": "^4.2.11", - "jest-haste-map": "30.1.0", + "jest-haste-map": "30.0.5", "jest-regex-util": "30.0.1", "jest-util": "30.0.5", "micromatch": "^4.0.8", @@ -1071,9 +1071,9 @@ } }, "node_modules/@jridgewell/gen-mapping": { - "version": "0.3.13", - "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", - "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "version": "0.3.12", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.12.tgz", + "integrity": "sha512-OuLGC46TjB5BbN1dH8JULVVZY4WTdkF7tV9Ys6wLL1rubZnCMstOhNHueU5bLCrnRuDhKPDM4g6sw4Bel5Gzqg==", "dev": true, "license": "MIT", "dependencies": { @@ -1092,16 +1092,16 @@ } }, "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.5", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", - "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.4.tgz", + "integrity": "sha512-VT2+G1VQs/9oz078bLrYbecdZKs912zQlkelYpuf+SXF+QvZDYJlbx/LSx+meSAwdDFnF8FVXW92AVjjkVmgFw==", "dev": true, "license": "MIT" }, "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.30", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.30.tgz", - "integrity": "sha512-GQ7Nw5G2lTu/BtHTKfXhKHok2WGetd4XYcVKGx00SjAk8GMwgJM3zr6zORiPGuOE+/vkc90KtTosSSvaCjKb2Q==", + "version": "0.3.29", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.29.tgz", + "integrity": "sha512-uw6guiW/gcAGPDhLmd77/6lW8QLeiV5RUTsAX46Db6oLhGaVj4lhnPwb184s1bkc8kdVg/+h988dro8GRDpmYQ==", "dev": true, "license": "MIT", "dependencies": { @@ -1221,9 +1221,9 @@ "dev": true }, "node_modules/@sinclair/typebox": { - "version": "0.34.41", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.34.41.tgz", - "integrity": "sha512-6gS8pZzSXdyRHTIqoqSVknxolr1kzfy4/CeDnrzsVz8TTIWUbOBr6gnzOmTYJ3eXQNh4IYHIGi5aIL7sOZ2G/g==", + "version": "0.34.38", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.34.38.tgz", + "integrity": "sha512-HpkxMmc2XmZKhvaKIZZThlHmx1L0I/V1hWK1NubtlFnr6ZqdiOpV72TKudZUNQjZNsyDBay72qFEhEvb+bcwcA==", "dev": true, "license": "MIT" }, @@ -1294,13 +1294,13 @@ } }, "node_modules/@types/babel__traverse": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.28.0.tgz", - "integrity": "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==", + "version": "7.20.7", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.20.7.tgz", + "integrity": "sha512-dkO5fhS7+/oos4ciWxyEyjWe48zmG6wbCheo/G2ZnHx4fs3EU6YC6UM8rk56gAjNJ9P3MTH2jo5jb92/K6wbng==", "dev": true, "license": "MIT", "dependencies": { - "@babel/types": "^7.28.2" + "@babel/types": "^7.20.7" } }, "node_modules/@types/docker-modem": { @@ -1844,13 +1844,13 @@ "license": "Apache-2.0" }, "node_modules/babel-jest": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-30.1.1.tgz", - "integrity": "sha512-1bZfC/V03qBCzASvZpNFhx3Ouj6LgOd4KFJm4br/fYOS+tSSvVCE61QmcAVbMTwq/GoB7KN4pzGMoyr9cMxSvQ==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-30.0.5.tgz", + "integrity": "sha512-mRijnKimhGDMsizTvBTWotwNpzrkHr+VvZUQBof2AufXKB8NXrL1W69TG20EvOz7aevx6FTJIaBuBkYxS8zolg==", "dev": true, "license": "MIT", "dependencies": { - "@jest/transform": "30.1.1", + "@jest/transform": "30.0.5", "@types/babel__core": "^7.20.5", "babel-plugin-istanbul": "^7.0.0", "babel-preset-jest": "30.0.1", @@ -1898,9 +1898,9 @@ } }, "node_modules/babel-preset-current-node-syntax": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/babel-preset-current-node-syntax/-/babel-preset-current-node-syntax-1.2.0.tgz", - "integrity": "sha512-E/VlAEzRrsLEb2+dv8yp3bo4scof3l9nR4lrld+Iy5NyVqgVYUJnDAmunkhPMisRI32Qc4iRiz425d8vM++2fg==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/babel-preset-current-node-syntax/-/babel-preset-current-node-syntax-1.1.0.tgz", + "integrity": "sha512-ldYss8SbBlWva1bs28q78Ju5Zq1F+8BrqBZZ0VFhLBvhh6lCpC2o3gDJi/5DRLs9FgYZCnmPYIVFU4lRXCkyUw==", "dev": true, "license": "MIT", "dependencies": { @@ -1921,7 +1921,7 @@ "@babel/plugin-syntax-top-level-await": "^7.14.5" }, "peerDependencies": { - "@babel/core": "^7.0.0 || ^8.0.0-0" + "@babel/core": "^7.0.0" } }, "node_modules/babel-preset-jest": { @@ -2125,9 +2125,9 @@ } }, "node_modules/browserslist": { - "version": "4.25.4", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.25.4.tgz", - "integrity": "sha512-4jYpcjabC606xJ3kw2QwGEZKX0Aw7sgQdZCvIK9dhVSPh76BKo+C+btT1RRofH7B+8iNpEbgGNVWiLki5q93yg==", + "version": "4.25.1", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.25.1.tgz", + "integrity": "sha512-KGj0KoOMXLpSNkkEI6Z6mShmQy0bc1I+T7K9N81k4WWMrfz+6fQ6es80B/YLAeRoKvjYE1YSHHOW1qe9xIVzHw==", "dev": true, "funding": [ { @@ -2145,8 +2145,8 @@ ], "license": "MIT", "dependencies": { - "caniuse-lite": "^1.0.30001737", - "electron-to-chromium": "^1.5.211", + "caniuse-lite": "^1.0.30001726", + "electron-to-chromium": "^1.5.173", "node-releases": "^2.0.19", "update-browserslist-db": "^1.1.3" }, @@ -2250,9 +2250,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001739", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001739.tgz", - "integrity": "sha512-y+j60d6ulelrNSwpPyrHdl+9mJnQzHBr08xm48Qno0nSk4h3Qojh+ziv2qE6rXf4k3tadF4o1J/1tAbVm1NtnA==", + "version": "1.0.30001727", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001727.tgz", + "integrity": "sha512-pB68nIHmbN6L/4C6MH1DokyR3bYqFwjaSs/sWDHGj4CTcFtQUQMuJftVwWkXq7mNWOybD3KhUv3oWHoGxgP14Q==", "dev": true, "funding": [ { @@ -2649,9 +2649,9 @@ "license": "MIT" }, "node_modules/electron-to-chromium": { - "version": "1.5.211", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.211.tgz", - "integrity": "sha512-IGBvimJkotaLzFnwIVgW9/UD/AOJ2tByUmeOrtqBfACSbAw5b1G0XpvdaieKyc7ULmbwXVx+4e4Be8pOPBrYkw==", + "version": "1.5.191", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.191.tgz", + "integrity": "sha512-xcwe9ELcuxYLUFqZZxL19Z6HVKcvNkIwhbHUz7L3us6u12yR+7uY89dSl570f/IqNthx8dAw3tojG7i4Ni4tDA==", "dev": true, "license": "ISC" }, @@ -2783,16 +2783,16 @@ } }, "node_modules/expect": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/expect/-/expect-30.1.1.tgz", - "integrity": "sha512-OKe7cdic4qbfWd/CcgwJvvCrNX2KWfuMZee9AfJHL1gTYmvqjBjZG1a2NwfhspBzxzlXwsN75WWpKTYfsJpBxg==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/expect/-/expect-30.0.5.tgz", + "integrity": "sha512-P0te2pt+hHI5qLJkIR+iMvS+lYUZml8rKKsohVHAGY+uClp9XVbdyYNJOIjSRpHVp8s8YqxJCiHUkSYZGr8rtQ==", "dev": true, "license": "MIT", "dependencies": { - "@jest/expect-utils": "30.1.1", - "@jest/get-type": "30.1.0", - "jest-matcher-utils": "30.1.1", - "jest-message-util": "30.1.0", + "@jest/expect-utils": "30.0.5", + "@jest/get-type": "30.0.1", + "jest-matcher-utils": "30.0.5", + "jest-message-util": "30.0.5", "jest-mock": "30.0.5", "jest-util": "30.0.5" }, @@ -3225,9 +3225,9 @@ } }, "node_modules/istanbul-reports": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", - "integrity": "sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==", + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.1.7.tgz", + "integrity": "sha512-BewmUXImeuRk2YY0PVbxgKAysvhRPUQE0h5QRM++nVWyubKGV0l8qQ5op8+B2DOmwSe63Jivj0BjkPQVf8fP5g==", "dev": true, "license": "BSD-3-Clause", "dependencies": { @@ -3255,16 +3255,16 @@ } }, "node_modules/jest": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/jest/-/jest-30.1.1.tgz", - "integrity": "sha512-yC3JvpP/ZcAZX5rYCtXO/g9k6VTCQz0VFE2v1FpxytWzUqfDtu0XL/pwnNvptzYItvGwomh1ehomRNMOyhCJKw==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest/-/jest-30.0.5.tgz", + "integrity": "sha512-y2mfcJywuTUkvLm2Lp1/pFX8kTgMO5yyQGq/Sk/n2mN7XWYp4JsCZ/QXW34M8YScgk8bPZlREH04f6blPnoHnQ==", "dev": true, "license": "MIT", "dependencies": { - "@jest/core": "30.1.1", + "@jest/core": "30.0.5", "@jest/types": "30.0.5", "import-local": "^3.2.0", - "jest-cli": "30.1.1" + "jest-cli": "30.0.5" }, "bin": { "jest": "bin/jest.js" @@ -3297,26 +3297,26 @@ } }, "node_modules/jest-circus": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/jest-circus/-/jest-circus-30.1.1.tgz", - "integrity": "sha512-M3Vd4x5wD7eSJspuTvRF55AkOOBndRxgW3gqQBDlFvbH3X+ASdi8jc+EqXEeAFd/UHulVYIlC4XKJABOhLw6UA==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-circus/-/jest-circus-30.0.5.tgz", + "integrity": "sha512-h/sjXEs4GS+NFFfqBDYT7y5Msfxh04EwWLhQi0F8kuWpe+J/7tICSlswU8qvBqumR3kFgHbfu7vU6qruWWBPug==", "dev": true, "license": "MIT", "dependencies": { - "@jest/environment": "30.1.1", - "@jest/expect": "30.1.1", - "@jest/test-result": "30.1.1", + "@jest/environment": "30.0.5", + "@jest/expect": "30.0.5", + "@jest/test-result": "30.0.5", "@jest/types": "30.0.5", "@types/node": "*", "chalk": "^4.1.2", "co": "^4.6.0", "dedent": "^1.6.0", "is-generator-fn": "^2.1.0", - "jest-each": "30.1.0", - "jest-matcher-utils": "30.1.1", - "jest-message-util": "30.1.0", - "jest-runtime": "30.1.1", - "jest-snapshot": "30.1.1", + "jest-each": "30.0.5", + "jest-matcher-utils": "30.0.5", + "jest-message-util": "30.0.5", + "jest-runtime": "30.0.5", + "jest-snapshot": "30.0.5", "jest-util": "30.0.5", "p-limit": "^3.1.0", "pretty-format": "30.0.5", @@ -3329,21 +3329,21 @@ } }, "node_modules/jest-cli": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-30.1.1.tgz", - "integrity": "sha512-xm9llxuh5OoI5KZaYzlMhklryHBwg9LZy/gEaaMlXlxb+cZekGNzukU0iblbDo3XOBuN6N0CgK4ykgNRYSEb6g==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-30.0.5.tgz", + "integrity": "sha512-Sa45PGMkBZzF94HMrlX4kUyPOwUpdZasaliKN3mifvDmkhLYqLLg8HQTzn6gq7vJGahFYMQjXgyJWfYImKZzOw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/core": "30.1.1", - "@jest/test-result": "30.1.1", + "@jest/core": "30.0.5", + "@jest/test-result": "30.0.5", "@jest/types": "30.0.5", "chalk": "^4.1.2", "exit-x": "^0.2.2", "import-local": "^3.2.0", - "jest-config": "30.1.1", + "jest-config": "30.0.5", "jest-util": "30.0.5", - "jest-validate": "30.1.0", + "jest-validate": "30.0.5", "yargs": "^17.7.2" }, "bin": { @@ -3362,31 +3362,31 @@ } }, "node_modules/jest-config": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/jest-config/-/jest-config-30.1.1.tgz", - "integrity": "sha512-xuPGUGDw+9fPPnGmddnLnHS/mhKUiJOW7K65vErYmglEPKq65NKwSRchkQ7iv6gqjs2l+YNEsAtbsplxozdOWg==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-config/-/jest-config-30.0.5.tgz", + "integrity": "sha512-aIVh+JNOOpzUgzUnPn5FLtyVnqc3TQHVMupYtyeURSb//iLColiMIR8TxCIDKyx9ZgjKnXGucuW68hCxgbrwmA==", "dev": true, "license": "MIT", "dependencies": { "@babel/core": "^7.27.4", - "@jest/get-type": "30.1.0", + "@jest/get-type": "30.0.1", "@jest/pattern": "30.0.1", - "@jest/test-sequencer": "30.1.1", + "@jest/test-sequencer": "30.0.5", "@jest/types": "30.0.5", - "babel-jest": "30.1.1", + "babel-jest": "30.0.5", "chalk": "^4.1.2", "ci-info": "^4.2.0", "deepmerge": "^4.3.1", "glob": "^10.3.10", "graceful-fs": "^4.2.11", - "jest-circus": "30.1.1", + "jest-circus": "30.0.5", "jest-docblock": "30.0.1", - "jest-environment-node": "30.1.1", + "jest-environment-node": "30.0.5", "jest-regex-util": "30.0.1", - "jest-resolve": "30.1.0", - "jest-runner": "30.1.1", + "jest-resolve": "30.0.5", + "jest-runner": "30.0.5", "jest-util": "30.0.5", - "jest-validate": "30.1.0", + "jest-validate": "30.0.5", "micromatch": "^4.0.8", "parse-json": "^5.2.0", "pretty-format": "30.0.5", @@ -3414,14 +3414,14 @@ } }, "node_modules/jest-diff": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.1.1.tgz", - "integrity": "sha512-LUU2Gx8EhYxpdzTR6BmjL1ifgOAQJQELTHOiPv9KITaKjZvJ9Jmgigx01tuZ49id37LorpGc9dPBPlXTboXScw==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-30.0.5.tgz", + "integrity": "sha512-1UIqE9PoEKaHcIKvq2vbibrCog4Y8G0zmOxgQUVEiTqwR5hJVMCoDsN1vFvI5JvwD37hjueZ1C4l2FyGnfpE0A==", "dev": true, "license": "MIT", "dependencies": { "@jest/diff-sequences": "30.0.1", - "@jest/get-type": "30.1.0", + "@jest/get-type": "30.0.1", "chalk": "^4.1.2", "pretty-format": "30.0.5" }, @@ -3443,13 +3443,13 @@ } }, "node_modules/jest-each": { - "version": "30.1.0", - "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-30.1.0.tgz", - "integrity": "sha512-A+9FKzxPluqogNahpCv04UJvcZ9B3HamqpDNWNKDjtxVRYB8xbZLFuCr8JAJFpNp83CA0anGQFlpQna9Me+/tQ==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-30.0.5.tgz", + "integrity": "sha512-dKjRsx1uZ96TVyejD3/aAWcNKy6ajMaN531CwWIsrazIqIoXI9TnnpPlkrEYku/8rkS3dh2rbH+kMOyiEIv0xQ==", "dev": true, "license": "MIT", "dependencies": { - "@jest/get-type": "30.1.0", + "@jest/get-type": "30.0.1", "@jest/types": "30.0.5", "chalk": "^4.1.2", "jest-util": "30.0.5", @@ -3460,28 +3460,28 @@ } }, "node_modules/jest-environment-node": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-30.1.1.tgz", - "integrity": "sha512-IaMoaA6saxnJimqCppUDqKck+LKM0Jg+OxyMUIvs1yGd2neiC22o8zXo90k04+tO+49OmgMR4jTgM5e4B0S62Q==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-30.0.5.tgz", + "integrity": "sha512-ppYizXdLMSvciGsRsMEnv/5EFpvOdXBaXRBzFUDPWrsfmog4kYrOGWXarLllz6AXan6ZAA/kYokgDWuos1IKDA==", "dev": true, "license": "MIT", "dependencies": { - "@jest/environment": "30.1.1", - "@jest/fake-timers": "30.1.1", + "@jest/environment": "30.0.5", + "@jest/fake-timers": "30.0.5", "@jest/types": "30.0.5", "@types/node": "*", "jest-mock": "30.0.5", "jest-util": "30.0.5", - "jest-validate": "30.1.0" + "jest-validate": "30.0.5" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-haste-map": { - "version": "30.1.0", - "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-30.1.0.tgz", - "integrity": "sha512-JLeM84kNjpRkggcGpQLsV7B8W4LNUWz7oDNVnY1Vjj22b5/fAb3kk3htiD+4Na8bmJmjJR7rBtS2Rmq/NEcADg==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-30.0.5.tgz", + "integrity": "sha512-dkmlWNlsTSR0nH3nRfW5BKbqHefLZv0/6LCccG0xFCTWcJu8TuEwG+5Cm75iBfjVoockmO6J35o5gxtFSn5xeg==", "dev": true, "license": "MIT", "dependencies": { @@ -3492,7 +3492,7 @@ "graceful-fs": "^4.2.11", "jest-regex-util": "30.0.1", "jest-util": "30.0.5", - "jest-worker": "30.1.0", + "jest-worker": "30.0.5", "micromatch": "^4.0.8", "walker": "^1.0.8" }, @@ -3520,13 +3520,13 @@ } }, "node_modules/jest-leak-detector": { - "version": "30.1.0", - "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-30.1.0.tgz", - "integrity": "sha512-AoFvJzwxK+4KohH60vRuHaqXfWmeBATFZpzpmzNmYTtmRMiyGPVhkXpBqxUQunw+dQB48bDf4NpUs6ivVbRv1g==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-30.0.5.tgz", + "integrity": "sha512-3Uxr5uP8jmHMcsOtYMRB/zf1gXN3yUIc+iPorhNETG54gErFIiUhLvyY/OggYpSMOEYqsmRxmuU4ZOoX5jpRFg==", "dev": true, "license": "MIT", "dependencies": { - "@jest/get-type": "30.1.0", + "@jest/get-type": "30.0.1", "pretty-format": "30.0.5" }, "engines": { @@ -3534,15 +3534,15 @@ } }, "node_modules/jest-matcher-utils": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-30.1.1.tgz", - "integrity": "sha512-SuH2QVemK48BNTqReti6FtjsMPFsSOD/ZzRxU1TttR7RiRsRSe78d03bb4Cx6D4bQC/80Q8U4VnaaAH9FlbZ9w==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-30.0.5.tgz", + "integrity": "sha512-uQgGWt7GOrRLP1P7IwNWwK1WAQbq+m//ZY0yXygyfWp0rJlksMSLQAA4wYQC3b6wl3zfnchyTx+k3HZ5aPtCbQ==", "dev": true, "license": "MIT", "dependencies": { - "@jest/get-type": "30.1.0", + "@jest/get-type": "30.0.1", "chalk": "^4.1.2", - "jest-diff": "30.1.1", + "jest-diff": "30.0.5", "pretty-format": "30.0.5" }, "engines": { @@ -3550,9 +3550,9 @@ } }, "node_modules/jest-message-util": { - "version": "30.1.0", - "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.1.0.tgz", - "integrity": "sha512-HizKDGG98cYkWmaLUHChq4iN+oCENohQLb7Z5guBPumYs+/etonmNFlg1Ps6yN9LTPyZn+M+b/9BbnHx3WTMDg==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-30.0.5.tgz", + "integrity": "sha512-NAiDOhsK3V7RU0Aa/HnrQo+E4JlbarbmI3q6Pi4KcxicdtjV82gcIUrejOtczChtVQR4kddu1E1EJlW6EN9IyA==", "dev": true, "license": "MIT", "dependencies": { @@ -3614,18 +3614,18 @@ } }, "node_modules/jest-resolve": { - "version": "30.1.0", - "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-30.1.0.tgz", - "integrity": "sha512-hASe7D/wRtZw8Cm607NrlF7fi3HWC5wmA5jCVc2QjQAB2pTwP9eVZILGEi6OeSLNUtE1zb04sXRowsdh5CUjwA==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-30.0.5.tgz", + "integrity": "sha512-d+DjBQ1tIhdz91B79mywH5yYu76bZuE96sSbxj8MkjWVx5WNdt1deEFRONVL4UkKLSrAbMkdhb24XN691yDRHg==", "dev": true, "license": "MIT", "dependencies": { "chalk": "^4.1.2", "graceful-fs": "^4.2.11", - "jest-haste-map": "30.1.0", + "jest-haste-map": "30.0.5", "jest-pnp-resolver": "^1.2.3", "jest-util": "30.0.5", - "jest-validate": "30.1.0", + "jest-validate": "30.0.5", "slash": "^3.0.0", "unrs-resolver": "^1.7.11" }, @@ -3634,30 +3634,30 @@ } }, "node_modules/jest-resolve-dependencies": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-30.1.1.tgz", - "integrity": "sha512-tRtaaoH8Ws1Gn1o/9pedt19dvVgr81WwdmvJSP9Ow3amOUOP2nN9j94u5jC9XlIfa2Q1FQKIWWQwL4ajqsjCGQ==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-30.0.5.tgz", + "integrity": "sha512-/xMvBR4MpwkrHW4ikZIWRttBBRZgWK4d6xt3xW1iRDSKt4tXzYkMkyPfBnSCgv96cpkrctfXs6gexeqMYqdEpw==", "dev": true, "license": "MIT", "dependencies": { "jest-regex-util": "30.0.1", - "jest-snapshot": "30.1.1" + "jest-snapshot": "30.0.5" }, "engines": { "node": "^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0" } }, "node_modules/jest-runner": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/jest-runner/-/jest-runner-30.1.1.tgz", - "integrity": "sha512-ATe6372SOfJvCRExtCAr06I4rGujwFdKg44b6i7/aOgFnULwjxzugJ0Y4AnG+jeSeQi8dU7R6oqLGmsxRUbErQ==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-runner/-/jest-runner-30.0.5.tgz", + "integrity": "sha512-JcCOucZmgp+YuGgLAXHNy7ualBx4wYSgJVWrYMRBnb79j9PD0Jxh0EHvR5Cx/r0Ce+ZBC4hCdz2AzFFLl9hCiw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/console": "30.1.1", - "@jest/environment": "30.1.1", - "@jest/test-result": "30.1.1", - "@jest/transform": "30.1.1", + "@jest/console": "30.0.5", + "@jest/environment": "30.0.5", + "@jest/test-result": "30.0.5", + "@jest/transform": "30.0.5", "@jest/types": "30.0.5", "@types/node": "*", "chalk": "^4.1.2", @@ -3665,15 +3665,15 @@ "exit-x": "^0.2.2", "graceful-fs": "^4.2.11", "jest-docblock": "30.0.1", - "jest-environment-node": "30.1.1", - "jest-haste-map": "30.1.0", - "jest-leak-detector": "30.1.0", - "jest-message-util": "30.1.0", - "jest-resolve": "30.1.0", - "jest-runtime": "30.1.1", + "jest-environment-node": "30.0.5", + "jest-haste-map": "30.0.5", + "jest-leak-detector": "30.0.5", + "jest-message-util": "30.0.5", + "jest-resolve": "30.0.5", + "jest-runtime": "30.0.5", "jest-util": "30.0.5", - "jest-watcher": "30.1.1", - "jest-worker": "30.1.0", + "jest-watcher": "30.0.5", + "jest-worker": "30.0.5", "p-limit": "^3.1.0", "source-map-support": "0.5.13" }, @@ -3682,18 +3682,18 @@ } }, "node_modules/jest-runtime": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-30.1.1.tgz", - "integrity": "sha512-7sOyR0Oekw4OesQqqBHuYJRB52QtXiq0NNgLRzVogiMSxKCMiliUd6RrXHCnG5f12Age/ggidCBiQftzcA9XKw==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-30.0.5.tgz", + "integrity": "sha512-7oySNDkqpe4xpX5PPiJTe5vEa+Ak/NnNz2bGYZrA1ftG3RL3EFlHaUkA1Cjx+R8IhK0Vg43RML5mJedGTPNz3A==", "dev": true, "license": "MIT", "dependencies": { - "@jest/environment": "30.1.1", - "@jest/fake-timers": "30.1.1", - "@jest/globals": "30.1.1", + "@jest/environment": "30.0.5", + "@jest/fake-timers": "30.0.5", + "@jest/globals": "30.0.5", "@jest/source-map": "30.0.1", - "@jest/test-result": "30.1.1", - "@jest/transform": "30.1.1", + "@jest/test-result": "30.0.5", + "@jest/transform": "30.0.5", "@jest/types": "30.0.5", "@types/node": "*", "chalk": "^4.1.2", @@ -3701,12 +3701,12 @@ "collect-v8-coverage": "^1.0.2", "glob": "^10.3.10", "graceful-fs": "^4.2.11", - "jest-haste-map": "30.1.0", - "jest-message-util": "30.1.0", + "jest-haste-map": "30.0.5", + "jest-message-util": "30.0.5", "jest-mock": "30.0.5", "jest-regex-util": "30.0.1", - "jest-resolve": "30.1.0", - "jest-snapshot": "30.1.1", + "jest-resolve": "30.0.5", + "jest-snapshot": "30.0.5", "jest-util": "30.0.5", "slash": "^3.0.0", "strip-bom": "^4.0.0" @@ -3716,9 +3716,9 @@ } }, "node_modules/jest-snapshot": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-30.1.1.tgz", - "integrity": "sha512-7/iBEzoJqEt2TjkQY+mPLHP8cbPhLReZVkkxjTMzIzoTC4cZufg7HzKo/n9cIkXKj2LG0x3mmBHsZto+7TOmFg==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-30.0.5.tgz", + "integrity": "sha512-T00dWU/Ek3LqTp4+DcW6PraVxjk28WY5Ua/s+3zUKSERZSNyxTqhDXCWKG5p2HAJ+crVQ3WJ2P9YVHpj1tkW+g==", "dev": true, "license": "MIT", "dependencies": { @@ -3727,18 +3727,18 @@ "@babel/plugin-syntax-jsx": "^7.27.1", "@babel/plugin-syntax-typescript": "^7.27.1", "@babel/types": "^7.27.3", - "@jest/expect-utils": "30.1.1", - "@jest/get-type": "30.1.0", - "@jest/snapshot-utils": "30.1.1", - "@jest/transform": "30.1.1", + "@jest/expect-utils": "30.0.5", + "@jest/get-type": "30.0.1", + "@jest/snapshot-utils": "30.0.5", + "@jest/transform": "30.0.5", "@jest/types": "30.0.5", "babel-preset-current-node-syntax": "^1.1.0", "chalk": "^4.1.2", - "expect": "30.1.1", + "expect": "30.0.5", "graceful-fs": "^4.2.11", - "jest-diff": "30.1.1", - "jest-matcher-utils": "30.1.1", - "jest-message-util": "30.1.0", + "jest-diff": "30.0.5", + "jest-matcher-utils": "30.0.5", + "jest-message-util": "30.0.5", "jest-util": "30.0.5", "pretty-format": "30.0.5", "semver": "^7.7.2", @@ -3793,13 +3793,13 @@ } }, "node_modules/jest-validate": { - "version": "30.1.0", - "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-30.1.0.tgz", - "integrity": "sha512-7P3ZlCFW/vhfQ8pE7zW6Oi4EzvuB4sgR72Q1INfW9m0FGo0GADYlPwIkf4CyPq7wq85g+kPMtPOHNAdWHeBOaA==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-30.0.5.tgz", + "integrity": "sha512-ouTm6VFHaS2boyl+k4u+Qip4TSH7Uld5tyD8psQ8abGgt2uYYB8VwVfAHWHjHc0NWmGGbwO5h0sCPOGHHevefw==", "dev": true, "license": "MIT", "dependencies": { - "@jest/get-type": "30.1.0", + "@jest/get-type": "30.0.1", "@jest/types": "30.0.5", "camelcase": "^6.3.0", "chalk": "^4.1.2", @@ -3824,13 +3824,13 @@ } }, "node_modules/jest-watcher": { - "version": "30.1.1", - "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-30.1.1.tgz", - "integrity": "sha512-CrAQ73LlaS6KGQQw6NBi71g7qvP7scy+4+2c0jKX6+CWaYg85lZiig5nQQVTsS5a5sffNPL3uxXnaE9d7v9eQg==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-30.0.5.tgz", + "integrity": "sha512-z9slj/0vOwBDBjN3L4z4ZYaA+pG56d6p3kTUhFRYGvXbXMWhXmb/FIxREZCD06DYUwDKKnj2T80+Pb71CQ0KEg==", "dev": true, "license": "MIT", "dependencies": { - "@jest/test-result": "30.1.1", + "@jest/test-result": "30.0.5", "@jest/types": "30.0.5", "@types/node": "*", "ansi-escapes": "^4.3.2", @@ -3844,9 +3844,9 @@ } }, "node_modules/jest-worker": { - "version": "30.1.0", - "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-30.1.0.tgz", - "integrity": "sha512-uvWcSjlwAAgIu133Tt77A05H7RIk3Ho8tZL50bQM2AkvLdluw9NG48lRCl3Dt+MOH719n/0nnb5YxUwcuJiKRA==", + "version": "30.0.5", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-30.0.5.tgz", + "integrity": "sha512-ojRXsWzEP16NdUuBw/4H/zkZdHOa7MMYCk4E430l+8fELeLg/mqmMlRhjL7UNZvQrDmnovWZV4DxX03fZF48fQ==", "dev": true, "license": "MIT", "dependencies": { @@ -4166,9 +4166,9 @@ "optional": true }, "node_modules/napi-postinstall": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/napi-postinstall/-/napi-postinstall-0.3.3.tgz", - "integrity": "sha512-uTp172LLXSxuSYHv/kou+f6KW3SMppU9ivthaVTXian9sOt3XM/zHYHpRZiLgQoxeWfYUnslNWQHF1+G71xcow==", + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/napi-postinstall/-/napi-postinstall-0.3.2.tgz", + "integrity": "sha512-tWVJxJHmBWLy69PvO96TZMZDrzmw5KeiZBz3RHmiM2XZ9grBJ2WgMAFVVg25nqp3ZjTFUs2Ftw1JhscL3Teliw==", "dev": true, "license": "MIT", "bin": { diff --git a/grpc-client/docs/Readme.txt b/grpc-client/docs/Readme.txt new file mode 100644 index 0000000000..9ea64f2f92 --- /dev/null +++ b/grpc-client/docs/Readme.txt @@ -0,0 +1,47 @@ + +PWD: /Users/ocohen/git/Verdance/ArcadeDB-GRPC/arcadedb-25.8.1-SNAPSHOT + + +./bin/server.sh -Darcadedb.server.rootPassword=root1234 -Darcadedb.server.name=Arcade_GRPC_Test -Darcadedb.dumpConfigAtStartup=true -Darcadedb.server.mode=development -Darcadedb.server.rootPath=../var/arcadedb -Darcadedb.server.plugins=GRPC:com.arcadedb.server.grpc.GrpcServerPlugin -Xms512M -Xmx4096M -XX:InitialRAMPercentage=50.0 -XX:MaxRAMPercentage=75.0 -Darcadedb.server.httpIncomingPort=2489 -Darcadedb.grpc.enabled=true -Darcadedb.grpc.port=50059 -Darcadedb.grpc.mode=standard -Darcadedb.grpc.reflection.enabled=true -Darcadedb.grpc.health.enabled=true + + +Ports: + +HTTP: 2489 +GTPC: 50059 + +root +root1234 + +Logging: + +d exec -it arcadedb1-vulcan sh + + +vi /home/arcadedb/config/arcadedb-log.properties + + +handlers = java.util.logging.ConsoleHandler, java.util.logging.FileHandler + +.level = INFO +com.arcadedb.level = INFO +com.arcadedb.server.grpc.level = FINE + +java.util.logging.ConsoleHandler.level = INFO +java.util.logging.ConsoleHandler.formatter = com.arcadedb.utility.AnsiLogFormatter + +java.util.logging.FileHandler.level = FINE +java.util.logging.FileHandler.pattern=./log/arcadedb.log +java.util.logging.FileHandler.formatter = com.arcadedb.log.LogFormatter +java.util.logging.FileHandler.limit=100000000 +java.util.logging.FileHandler.count=10 + + + + +tail -f /home/arcadedb/log/arcadedb.log.0 + + + + +d exec -it arcadedb1-vulcan sh \ No newline at end of file diff --git a/grpc-client/pom.xml b/grpc-client/pom.xml new file mode 100644 index 0000000000..f95c91a141 --- /dev/null +++ b/grpc-client/pom.xml @@ -0,0 +1,150 @@ + + + + 4.0.0 + + + com.arcadedb + arcadedb-parent + 25.8.1-SNAPSHOT + ../pom.xml + + + arcadedb-grpc-client + jar + ArcadeDB gRPC Client + + + 5.15.2 + + + + + + com.arcadedb + arcadedb-engine + ${project.parent.version} + compile + + + + com.arcadedb + arcadedb-network + ${project.parent.version} + compile + + + + com.arcadedb + arcadedb-grpc + ${project.parent.version} + grpc-interface + + + * + * + + + + + + + io.grpc + grpc-netty-shaded + ${grpc.version} + + + io.grpc + grpc-protobuf + ${grpc.version} + + + io.grpc + grpc-stub + ${grpc.version} + + + io.grpc + grpc-services + ${grpc.version} + + + + + com.google.protobuf + protobuf-java + ${protobuf-java.version} + + + com.google.protobuf + protobuf-java-util + ${protobuf-java.version} + + + + + com.google.api.grpc + proto-google-common-protos + ${com.google.api.grpc.version} + + + + + javax.annotation + javax.annotation-api + ${javax.annotation-api.version} + + + + + io.grpc + grpc-testing + ${grpc.version} + test + + + + org.slf4j + slf4j-simple + 2.0.17 + provided + + + + com.fasterxml.jackson.core + jackson-databind + 2.19.2 + provided + + + + + + + + + + + + + + diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcConfig.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcConfig.java new file mode 100644 index 0000000000..f055e10a1e --- /dev/null +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcConfig.java @@ -0,0 +1,41 @@ +package com.arcadedb.remote.grpc; + +import com.arcadedb.server.grpc.ProjectionSettings.ProjectionEncoding; + +public class RemoteGrpcConfig { + + private boolean includeProjections; + private ProjectionEncoding projectionEncoding; + private int softLimitBytes; + + public RemoteGrpcConfig(boolean includeProjections, ProjectionEncoding projectionEncoding, int softLimitBytes) { + this.includeProjections = includeProjections; + this.projectionEncoding = projectionEncoding; + this.softLimitBytes = softLimitBytes; + } + + public boolean isIncludeProjections() { + return includeProjections; + } + + public void setIncludeProjections(boolean includeProjections) { + this.includeProjections = includeProjections; + } + + public ProjectionEncoding getProjectionEncoding() { + return projectionEncoding; + } + + public void setProjectionEncoding(ProjectionEncoding projectionEncoding) { + this.projectionEncoding = projectionEncoding; + } + + public int getSoftLimitBytes() { + return softLimitBytes; + } + + public void setSoftLimitBytes(int softLimitBytes) { + this.softLimitBytes = softLimitBytes; + } + +} diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java new file mode 100644 index 0000000000..feeaa1f0de --- /dev/null +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabase.java @@ -0,0 +1,2455 @@ +package com.arcadedb.remote.grpc; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; + +import javax.annotation.Nullable; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.arcadedb.ContextConfiguration; +import com.arcadedb.database.Database; +import com.arcadedb.database.MutableDocument; +import com.arcadedb.database.RID; +import com.arcadedb.database.Record; +import com.arcadedb.exception.ConcurrentModificationException; +import com.arcadedb.exception.DatabaseOperationException; +import com.arcadedb.exception.DuplicatedKeyException; +import com.arcadedb.exception.NeedRetryException; +import com.arcadedb.exception.RecordNotFoundException; +import com.arcadedb.exception.TimeoutException; +import com.arcadedb.exception.TransactionException; +import com.arcadedb.query.sql.executor.InternalResultSet; +import com.arcadedb.query.sql.executor.Result; +import com.arcadedb.query.sql.executor.ResultInternal; +import com.arcadedb.query.sql.executor.ResultSet; +import com.arcadedb.remote.RemoteDatabase; +import com.arcadedb.remote.RemoteException; +import com.arcadedb.remote.RemoteImmutableDocument; +import com.arcadedb.remote.RemoteImmutableEdge; +import com.arcadedb.remote.RemoteImmutableVertex; +import com.arcadedb.remote.RemoteSchema; +import com.arcadedb.remote.RemoteTransactionExplicitLock; +import com.arcadedb.remote.grpc.utils.ProtoUtils; +import com.arcadedb.server.grpc.ArcadeDbServiceGrpc; +import com.arcadedb.server.grpc.BatchAck; +import com.arcadedb.server.grpc.BeginTransactionRequest; +import com.arcadedb.server.grpc.BeginTransactionResponse; +import com.arcadedb.server.grpc.BulkInsertRequest; +import com.arcadedb.server.grpc.Commit; +import com.arcadedb.server.grpc.CommitTransactionRequest; +import com.arcadedb.server.grpc.CommitTransactionResponse; +import com.arcadedb.server.grpc.CreateRecordRequest; +import com.arcadedb.server.grpc.CreateRecordResponse; +import com.arcadedb.server.grpc.DatabaseCredentials; +import com.arcadedb.server.grpc.DeleteRecordRequest; +import com.arcadedb.server.grpc.DeleteRecordResponse; +import com.arcadedb.server.grpc.ExecuteCommandRequest; +import com.arcadedb.server.grpc.ExecuteCommandResponse; +import com.arcadedb.server.grpc.ExecuteQueryRequest; +import com.arcadedb.server.grpc.ExecuteQueryResponse; +import com.arcadedb.server.grpc.GrpcRecord; +import com.arcadedb.server.grpc.GrpcValue; +import com.arcadedb.server.grpc.InsertChunk; +import com.arcadedb.server.grpc.InsertOptions; +import com.arcadedb.server.grpc.InsertRequest; +import com.arcadedb.server.grpc.InsertResponse; +import com.arcadedb.server.grpc.InsertSummary; +import com.arcadedb.server.grpc.LookupByRidRequest; +import com.arcadedb.server.grpc.LookupByRidResponse; +import com.arcadedb.server.grpc.ProjectionSettings; +import com.arcadedb.server.grpc.ProjectionSettings.ProjectionEncoding; +import com.arcadedb.server.grpc.PropertiesUpdate; +import com.arcadedb.server.grpc.QueryResult; +import com.arcadedb.server.grpc.RollbackTransactionRequest; +import com.arcadedb.server.grpc.RollbackTransactionResponse; +import com.arcadedb.server.grpc.Start; +import com.arcadedb.server.grpc.StreamQueryRequest; +import com.arcadedb.server.grpc.TransactionContext; +import com.arcadedb.server.grpc.TransactionIsolation; +import com.arcadedb.server.grpc.UpdateRecordRequest; +import com.arcadedb.server.grpc.UpdateRecordResponse; +import com.google.protobuf.Int32Value; + +import io.grpc.CallCredentials; +import io.grpc.Metadata; +import io.grpc.StatusException; +import io.grpc.StatusRuntimeException; +import io.grpc.stub.BlockingClientCall; +import io.grpc.stub.ClientCallStreamObserver; +import io.grpc.stub.ClientResponseObserver; +import io.grpc.stub.StreamObserver; + +/** + * Remote Database implementation using gRPC protocol instead of HTTP. Extends + * RemoteDatabase to maintain compatibility while overriding network operations. + * It's not thread safe. For multi-thread usage create one instance of + * RemoteGrpcDatabase per thread. + * + * @author Oleg Cohen (oleg.cohen@gmail.com) + */ +public class RemoteGrpcDatabase extends RemoteDatabase { + + private static final Logger logger = LoggerFactory.getLogger(RemoteGrpcDatabase.class); + + private final ArcadeDbServiceGrpc.ArcadeDbServiceBlockingV2Stub blockingStub; + private final ArcadeDbServiceGrpc.ArcadeDbServiceStub asyncStub; + + private String transactionId; + + private RemoteSchema schema; + + private final String userName; + private final String userPassword; + private String databaseName; + private RemoteTransactionExplicitLock explicitLock; + + protected RemoteGrpcServer remoteGrpcServer; + + public RemoteGrpcDatabase(final RemoteGrpcServer remoteGrpcServer, final String server, final int grpcPort, final int httpPort, + final String databaseName, final String userName, final String userPassword) { + this(remoteGrpcServer, server, grpcPort, httpPort, databaseName, userName, userPassword, new ContextConfiguration()); + } + + public RemoteGrpcDatabase(final RemoteGrpcServer remoteGrpcServer, final String host, final int grpcPort, final int httpPort, + final String databaseName, final String userName, final String userPassword, final ContextConfiguration configuration) { + + super(host, httpPort, databaseName, userName, userPassword, configuration); + + this.remoteGrpcServer = remoteGrpcServer; + + this.userName = userName; + this.userPassword = userPassword; + + this.databaseName = databaseName; + + this.blockingStub = createBlockingStub(); + + this.asyncStub = createAsyncStub(); + + this.schema = new RemoteSchema(this); + } + + /** + * Creates call credentials for authentication + */ + protected CallCredentials createCallCredentials(String userName, String userPassword) { + return new CallCredentials() { + @Override + public void applyRequestMetadata(RequestInfo requestInfo, Executor appExecutor, MetadataApplier applier) { + Metadata headers = new Metadata(); + headers.put(Metadata.Key.of("username", Metadata.ASCII_STRING_MARSHALLER), userName); + headers.put(Metadata.Key.of("password", Metadata.ASCII_STRING_MARSHALLER), userPassword); + headers.put(Metadata.Key.of("x-arcade-user", Metadata.ASCII_STRING_MARSHALLER), userName); + headers.put(Metadata.Key.of("x-arcade-password", Metadata.ASCII_STRING_MARSHALLER), userPassword); + headers.put(Metadata.Key.of("x-arcade-database", Metadata.ASCII_STRING_MARSHALLER), databaseName); + applier.apply(headers); + } + + @Override + public void thisUsesUnstableApi() { + // Required by the interface + } + }; + } + + protected CallCredentials createCredentials() { + return createCallCredentials(userName, userPassword); + } + + /** + * Override this method to customize blocking stub creation + */ + protected ArcadeDbServiceGrpc.ArcadeDbServiceBlockingV2Stub createBlockingStub() { + return this.remoteGrpcServer.newBlockingStub(getTimeout()); + } + + protected ArcadeDbServiceGrpc.ArcadeDbServiceStub createAsyncStub() { + return this.remoteGrpcServer.newAsyncStub(getTimeout()); + } + + @Override + public String getDatabasePath() { + return "grpc://" + this.remoteGrpcServer.channel().authority() + "/" + getName(); + } + + @Override + public RemoteSchema getSchema() { + return schema; + } + + @Override + public void close() { + + if (transactionId != null) { + rollback(); + } + super.close(); + } + + @Override + public void begin(final Database.TRANSACTION_ISOLATION_LEVEL isolationLevel) { + checkDatabaseIsOpen(); + if (transactionId != null) + throw new TransactionException("Transaction already begun"); + + BeginTransactionRequest request = BeginTransactionRequest.newBuilder().setDatabase(getName()).setCredentials(buildCredentials()) + .setIsolation(mapIsolationLevel(isolationLevel)).build(); + + callUnaryVoid("BeginTransaction", () -> { + + try { + BeginTransactionResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .beginTransaction(request); + transactionId = response.getTransactionId(); + // Store transaction ID in parent class session management + setSessionId(transactionId); + } + catch (StatusRuntimeException e) { + throw new TransactionException("Error on transaction begin", e); + } + catch (StatusException e) { + throw new TransactionException("Error on transaction begin", e); + } + + debugTx = new TxDebug(this.getName(), /* label set by TM */ null); + logTx("BEGIN(local)", null); + try { + checkCrossThreadUse("before BeginTransaction"); + // send BeginTransaction RPC ... + debugTx.beginRpcSent = true; + debugTx.rpcSeq.incrementAndGet(); + logTx("BEGIN sent", "BeginTransaction"); + } + catch (RuntimeException e) { + logTx("BEGIN failed", "BeginTransaction"); + debugTx = null; + throw e; + } + }); + } + + @Override + public void commit() { + + checkDatabaseIsOpen(); + stats.txCommits.incrementAndGet(); + + if (transactionId == null) { + throw new TransactionException("Transaction not begun"); + } + + logTx("COMMIT(local)", null); + checkCrossThreadUse("before CommitTransaction"); + + callUnaryVoid("CommitTransaction", () -> { + + try { + + CommitTransactionRequest request = CommitTransactionRequest.newBuilder() + .setTransaction(TransactionContext.newBuilder().setTransactionId(transactionId).setDatabase(getName()).build()) + .setCredentials(buildCredentials()).build(); + + try { + + CommitTransactionResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .commitTransaction(request); + + logger.debug("[After commit] Success: {} Committed: {}", response.getSuccess(), response.getCommitted()); + + if (!response.getSuccess()) { + throw new TransactionException("Failed to commit transaction: " + response.getMessage()); + } + } + catch (StatusRuntimeException e) { + + handleGrpcException(e); + } + catch (StatusException e) { + + handleGrpcException(e); + } + finally { + + transactionId = null; + setSessionId(null); + } + + if (debugTx != null) { + + debugTx.committed = true; + debugTx.rpcSeq.incrementAndGet(); + } + + logTx("COMMIT sent", "CommitTransaction"); + } + finally { + debugTx = null; + } + + }); + } + + @Override + public void rollback() { + + checkDatabaseIsOpen(); + stats.txRollbacks.incrementAndGet(); + + if (transactionId == null) + throw new TransactionException("Transaction not begun"); + + logTx("ROLLBACK(local)", null); + checkCrossThreadUse("before RollbackTransaction"); + + callUnaryVoid("RollbackTransaction", () -> { + + try { + + RollbackTransactionRequest request = RollbackTransactionRequest.newBuilder() + .setTransaction(TransactionContext.newBuilder().setTransactionId(transactionId).setDatabase(getName()).build()) + .setCredentials(buildCredentials()).build(); + + try { + + RollbackTransactionResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) + .rollbackTransaction(request); + + logger.debug("[After rollback] Success: {} Committed: {}", response.getSuccess(), response.getRolledBack()); + + if (!response.getSuccess()) { + throw new TransactionException("Failed to rollback transaction: " + response.getMessage()); + } + } + catch (StatusRuntimeException e) { + throw new TransactionException("Error on transaction rollback", e); + } + catch (StatusException e) { + throw new TransactionException("Error on transaction rollback", e); + } + finally { + transactionId = null; + setSessionId(null); + } + + if (debugTx != null) { + debugTx.rolledBack = true; + debugTx.rpcSeq.incrementAndGet(); + } + logTx("ROLLBACK sent", "RollbackTransaction"); + } + finally { + debugTx = null; + } + + }); + } + + @Override + public void deleteRecord(final Record record) { + checkDatabaseIsOpen(); + stats.deleteRecord.incrementAndGet(); + + if (record.getIdentity() == null) + throw new IllegalArgumentException("Cannot delete a non persistent record"); + + final DeleteRecordRequest req = DeleteRecordRequest.newBuilder().setDatabase(getName()).setRid(record.getIdentity().toString()) + .setCredentials(buildCredentials()).build(); + + try { + if (logger.isDebugEnabled()) { + logger.debug("CLIENT deleteRecord: db={}, tx={}, rid={}", getName(), (transactionId != null), record.getIdentity()); + } + + final DeleteRecordResponse resp = callUnary("DeleteRecord", + () -> blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).deleteRecord(req)); + + // Prefer the proto's 'deleted' flag (your other overload uses it) + if (!resp.getDeleted()) { + throw new DatabaseOperationException( + "Failed to delete record: " + (resp.getMessage().isEmpty() ? "unknown error" : resp.getMessage())); + } + + } + catch (io.grpc.StatusRuntimeException | io.grpc.StatusException e) { + handleGrpcException(e); // rethrows mapped domain exception + throw new IllegalStateException("unreachable"); + } + } + + public boolean deleteRecord(final String rid, final long timeoutMs) { + checkDatabaseIsOpen(); + stats.deleteRecord.incrementAndGet(); + + final DeleteRecordRequest req = DeleteRecordRequest.newBuilder().setDatabase(getName()).setRid(rid).setCredentials(buildCredentials()) + .build(); + + try { + if (logger.isDebugEnabled()) { + logger.debug("CLIENT deleteRecord: db={}, tx={}, rid={}, timeoutMs={}", getName(), (transactionId != null), rid, timeoutMs); + } + + final DeleteRecordResponse res = callUnary("DeleteRecord", + () -> blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).deleteRecord(req)); + + return res.getDeleted(); + + } + catch (io.grpc.StatusRuntimeException | io.grpc.StatusException e) { + handleGrpcException(e); // rethrows mapped domain exception + throw new IllegalStateException("unreachable"); + } + } + + @Override + public ResultSet command(final String language, final String command, final ContextConfiguration configuration, final Object... args) { + + checkDatabaseIsOpen(); + stats.commands.incrementAndGet(); + + final Map params = mapArgs(args); + + return commandInternal(language, command, params); + } + + @Override + public ResultSet command(final String language, final String command, final ContextConfiguration configuration, + final Map params) { + checkDatabaseIsOpen(); + stats.commands.incrementAndGet(); + + return commandInternal(language, command, params); + } + + @Override + public ResultSet command(final String language, final String command, final Object... args) { + checkDatabaseIsOpen(); + stats.commands.incrementAndGet(); + + final Map params = mapArgs(args); + + return commandInternal(language, command, params); + } + + @Override + public ResultSet command(final String language, final String command, final Map params) { + checkDatabaseIsOpen(); + stats.commands.incrementAndGet(); + + return commandInternal(language, command, params); + } + + private ResultSet commandInternal(final String language, final String command, final Map params) { + + ExecuteCommandRequest.Builder requestBuilder = ExecuteCommandRequest.newBuilder().setDatabase(getName()).setCommand(command) + .setLanguage(language).setCredentials(buildCredentials()); + + if (transactionId != null) { + requestBuilder.setTransaction(TransactionContext.newBuilder().setTransactionId(transactionId).setDatabase(getName()).build()); + } + + if (params != null && !params.isEmpty()) { + requestBuilder.putAllParameters(convertParamsToGrpcValue(params)); + } + + boolean returnRows = true; + + requestBuilder.setReturnRows(returnRows); + + try { + + if (logger.isDebugEnabled()) + logger.debug("CLIENT executeCommand: db={}, tx={}, cmdLen={}, params={}", getName(), (transactionId != null), + requestBuilder.getCommand().length(), requestBuilder.getParametersCount()); + + final ExecuteCommandResponse response = callUnary("ExecuteCommand", + () -> blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).executeCommand(requestBuilder.build())); + + if (logger.isDebugEnabled()) + logger.debug("CLIENT executeCommand: success = {}", response.getSuccess()); + + if (!response.getSuccess()) { + throw new DatabaseOperationException("Failed to execute command: " + response.getMessage()); + } + + ResultSet resultSet; + + if (returnRows) { + resultSet = createGrpcResultSet(response); + } + else { + + resultSet = new InternalResultSet(); + + if (response.getAffectedRecords() > 0) { + + Map result = new HashMap<>(); + result.put("affected", response.getAffectedRecords()); + result.put("executionTime", response.getExecutionTimeMs()); + + ((InternalResultSet) resultSet).add(new ResultInternal(result)); + } + } + + return resultSet; + } + catch (StatusRuntimeException e) { + handleGrpcException(e); + return new InternalResultSet(); + } + catch (StatusException e) { + handleGrpcException(e); + return new InternalResultSet(); + } + } + + @Override + public ResultSet query(final String language, final String query, final Object... args) { + checkDatabaseIsOpen(); + stats.queries.incrementAndGet(); + + final Map params = mapArgs(args); + + RemoteGrpcConfig remoteGrpcConfig = new RemoteGrpcConfig(true, ProjectionEncoding.PROJECTION_AS_JSON, 0); + + return query(language, query, remoteGrpcConfig, params); + } + + @Override + public ResultSet query(final String language, final String query, final Map params) { + + RemoteGrpcConfig remoteGrpcConfig = new RemoteGrpcConfig(true, ProjectionEncoding.PROJECTION_AS_JSON, 0); + + return query(language, query, remoteGrpcConfig, params); + } + + public ResultSet query(final String language, final String query, RemoteGrpcConfig remoteGrpcConfig, final Object... args) { + + final Map params = mapArgs(args); + + return query(language, query, remoteGrpcConfig, params); + } + + public ResultSet query(final String language, final String query, RemoteGrpcConfig remoteGrpcConfig, final Map params) { + + checkDatabaseIsOpen(); + stats.queries.incrementAndGet(); + + ExecuteQueryRequest.Builder requestBuilder = ExecuteQueryRequest.newBuilder().setDatabase(getName()).setQuery(query) + .setCredentials(buildCredentials()); + + if (transactionId != null) { + requestBuilder.setTransaction(TransactionContext.newBuilder().setTransactionId(transactionId).setDatabase(getName()).build()); + } + + if (params != null && !params.isEmpty()) { + requestBuilder.putAllParameters(convertParamsToGrpcValue(params)); + } + + ProjectionSettings projectionSettings = ProjectionSettings.newBuilder().setIncludeProjections(remoteGrpcConfig.isIncludeProjections()) + .setProjectionEncoding(remoteGrpcConfig.getProjectionEncoding()) + .setSoftLimitBytes(Int32Value.newBuilder().setValue(remoteGrpcConfig.getSoftLimitBytes()).build()).build(); + + requestBuilder.setProjectionSettings(projectionSettings); + + try { + + if (logger.isDebugEnabled()) { + logger.debug("CLIENT executeQuery: db={}, tx={}, queryLen={}, params={}", getName(), (transactionId != null), + requestBuilder.getQuery().length(), requestBuilder.getParametersCount()); + } + + final ExecuteQueryRequest req = requestBuilder.build(); + + final ExecuteQueryResponse response = callUnary("ExecuteQuery", + () -> blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS) // or getTimeout(MODE_QUERY) + .executeQuery(req)); + + if (logger.isDebugEnabled()) { + int _r = 0; + for (var qr : response.getResultsList()) + _r += qr.getRecordsCount(); + logger.debug("CLIENT executeQuery: results={}", _r); + } + return createGrpcResultSet(response); + } + catch (io.grpc.StatusException | io.grpc.StatusRuntimeException e) { + handleGrpcException(e); + return new InternalResultSet(); + } + } + + public ExecuteCommandResponse execSql(String db, String sql, Map params, long timeoutMs) { + return executeCommand(db, "sql", sql, params, /* returnRows */ false, /* maxRows */ 0, txBeginCommit(), timeoutMs); + } + + public ExecuteCommandResponse execSql(String sql, Map params, long timeoutMs) { + return executeCommand(databaseName, "sql", sql, params, /* returnRows */ false, /* maxRows */ 0, txBeginCommit(), timeoutMs); + } + + public ExecuteCommandResponse executeCommand(String language, String command, Map params, boolean returnRows, int maxRows, + TransactionContext tx, long timeoutMs) { + + var reqB = ExecuteCommandRequest.newBuilder().setDatabase(databaseName).setCommand(command) + .putAllParameters(convertParamsToGrpcValue(params)).setLanguage(langOrDefault(language)).setReturnRows(returnRows) + .setMaxRows(maxRows > 0 ? maxRows : 0); + + if (tx != null) + reqB.setTransaction(tx); + reqB.setCredentials(buildCredentials()); + + try { + return callUnary("ExecuteCommand", + () -> blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).executeCommand(reqB.build())); + } + catch (StatusException | StatusRuntimeException e) { + // handleGrpcException already called in callUnary, this is unreachable + throw new IllegalStateException("unreachable"); + } + } + + public ExecuteCommandResponse executeCommand(String database, String language, String command, Map params, + boolean returnRows, int maxRows, TransactionContext tx, long timeoutMs) { + + var reqB = ExecuteCommandRequest.newBuilder().setDatabase(database).setCommand(command) + .putAllParameters(convertParamsToGrpcValue(params)).setLanguage(langOrDefault(language)).setReturnRows(returnRows) + .setMaxRows(maxRows > 0 ? maxRows : 0); + + if (tx != null) + reqB.setTransaction(tx); + reqB.setCredentials(buildCredentials()); + + try { + return callUnary("ExecuteCommand", + () -> blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).executeCommand(reqB.build())); + } + catch (StatusException | StatusRuntimeException e) { + // handleGrpcException already called in callUnary, this is unreachable + throw new IllegalStateException("unreachable"); + } + } + + @Override + protected RID saveRecord(final MutableDocument record) { + stats.createRecord.incrementAndGet(); + + final RID rid = record.getIdentity(); + + if (rid != null) { + // -------- UPDATE (partial) -------- + PropertiesUpdate partial = PropertiesUpdate.newBuilder().putAllProperties(convertParamsToGrpcValue(record.toMap(false))).build(); + + UpdateRecordRequest request = UpdateRecordRequest.newBuilder().setDatabase(getName()).setRid(rid.toString()).setPartial(partial) + .setDatabase(databaseName).setCredentials(buildCredentials()).build(); + + try { + + @SuppressWarnings("unused") + UpdateRecordResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).updateRecord(request); + + // If your proto has flags, you can check response.getSuccess()/getUpdated() + // Otherwise, treat non-exception as success. + return rid; + } + catch (StatusRuntimeException e) { + handleGrpcException(e); + return null; + } + catch (StatusException e) { + handleGrpcException(e); + return null; + } + } + else { + // -------- CREATE -------- + GrpcRecord recMsg = GrpcRecord.newBuilder().putAllProperties(convertParamsToGrpcValue(record.toMap(false))).build(); + + CreateRecordRequest request = CreateRecordRequest.newBuilder().setDatabase(getName()).setType(record.getTypeName()).setRecord(recMsg) // nested + // GrpcRecord + // payload + .setCredentials(buildCredentials()).build(); + + try { + CreateRecordResponse response = blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).createRecord(request); + + // Proto returns the newly created RID as a string + final String ridStr = response.getRid(); + if (ridStr == null || ridStr.isEmpty()) { + throw new DatabaseOperationException("Failed to create record (empty RID)"); + } + + // Construct a RID from the returned string + try { + return new RID(ridStr); + } + catch (NoSuchMethodError | IllegalArgumentException ex) { + // Fallback for older APIs expecting (Database, String) + return new RID(this, ridStr); + } + } + catch (StatusRuntimeException e) { + handleGrpcException(e); + return null; + } + catch (StatusException e) { + handleGrpcException(e); + return null; + } + } + } + + private RemoteGrpcConfig getDefaultRemoteGrpcConfig() { + + return new RemoteGrpcConfig(true, ProjectionEncoding.PROJECTION_AS_JSON, 0); + } + + // Convenience: default batch size stays 100, default mode = CURSOR + + public ResultSet queryStream(final String language, final String query) { + return queryStream(language, query, getDefaultRemoteGrpcConfig(), /* batchSize */100, StreamQueryRequest.RetrievalMode.CURSOR); + } + + public ResultSet queryStream(final String language, final String query, final RemoteGrpcConfig config) { + return queryStream(language, query, config, /* batchSize */100, StreamQueryRequest.RetrievalMode.CURSOR); + } + + public ResultSet queryStream(final String language, final String query, final int batchSize) { + return queryStream(language, query, getDefaultRemoteGrpcConfig(), batchSize, StreamQueryRequest.RetrievalMode.CURSOR); + } + + public ResultSet queryStream(final String language, final String query, RemoteGrpcConfig config, final int batchSize) { + return queryStream(language, query, config, batchSize, StreamQueryRequest.RetrievalMode.CURSOR); + } + + public ResultSet queryStream(final String language, final String query, final int batchSize, final StreamQueryRequest.RetrievalMode mode) { + return queryStream(language, query, getDefaultRemoteGrpcConfig(), batchSize, mode); + } + + // NEW: choose retrieval mode + + public ResultSet queryStream(final String language, final String query, final RemoteGrpcConfig config, final int batchSize, + final StreamQueryRequest.RetrievalMode mode) { + + return queryStream(language, query, config, Map.of(), batchSize, mode); + } + + /** + * Stream query results as a ResultSet that fetches data lazily in batches. This + * provides consistency with non-streaming query methods and supports non-Record + * results like projections and aggregations. + */ + public ResultSet queryStream(final String language, final String query, final RemoteGrpcConfig config, final Map params, + final int batchSize, final StreamQueryRequest.RetrievalMode mode) { + + checkDatabaseIsOpen(); + stats.queries.incrementAndGet(); + + StreamQueryRequest.Builder b = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(query).setCredentials(buildCredentials()) + .setBatchSize(batchSize > 0 ? batchSize : 100).setRetrievalMode(mode); + + if (params != null && !params.isEmpty()) { + b.putAllParameters(convertParamsToGrpcValue(params)); + } + + final BlockingClientCall responseIterator = callServerStreaming("StreamQuery", + () -> blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).streamQuery(b.build())); + + // Return a streaming ResultSet implementation + return new StreamingResultSet(responseIterator, this); + } + + // Keep the old signature working (defaults to CURSOR) + public ResultSet queryStream(final String language, final String query, final Map params, final int batchSize) { + + return queryStream(language, query, getDefaultRemoteGrpcConfig(), params, batchSize, StreamQueryRequest.RetrievalMode.CURSOR); + } + + public ResultSet queryStream(final String language, final String query, final RemoteGrpcConfig config, final Map params, + final int batchSize) { + + return queryStream(language, query, config, params, batchSize, StreamQueryRequest.RetrievalMode.CURSOR); + } + + /** + * A ResultSet implementation that lazily fetches results from a gRPC stream. + * Supports both Record results and projection/aggregation results. + */ + private static class StreamingResultSet implements ResultSet { + private static final Logger logger = LoggerFactory.getLogger(StreamingResultSet.class); + + private final BlockingClientCall stream; + private final RemoteGrpcDatabase db; + protected Iterator currentBatch = Collections.emptyIterator(); + private boolean streamExhausted = false; + private Result nextResult = null; + private final AtomicLong totalProcessed = new AtomicLong(0); + + StreamingResultSet(BlockingClientCall stream, RemoteGrpcDatabase db) { + this.stream = stream; + this.db = db; + } + + @Override + public boolean hasNext() { + if (nextResult != null) { + return true; + } + + // Try to get next from current batch + if (currentBatch.hasNext()) { + nextResult = currentBatch.next(); + return true; + } + + // Current batch exhausted, try to fetch next batch + if (streamExhausted) { + return false; + } + + if (db.debugTx != null) { + db.checkCrossThreadUse("streamQuery.hasNext"); + } + + try { + while (stream.hasNext()) { + final QueryResult queryResult = stream.read(); + + if (logger.isDebugEnabled()) { + logger.debug("Received batch with {} records, isLastBatch={}", queryResult.getRecordsCount(), + queryResult.getIsLastBatch()); + } + + if (queryResult.getRecordsCount() == 0) { + if (queryResult.getIsLastBatch()) { + streamExhausted = true; + return false; + } + continue; // empty non-terminal batch + } + + // Convert GrpcRecords to Results + currentBatch = convertBatchToResults(queryResult); + + if (currentBatch.hasNext()) { + nextResult = currentBatch.next(); + return true; + } + } + + streamExhausted = true; + return false; + + } + catch (io.grpc.StatusRuntimeException | io.grpc.StatusException e) { + db.handleGrpcException(e); + throw new IllegalStateException("unreachable"); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Stream interrupted", e); + } + catch (RuntimeException re) { + throw re; + } + catch (Exception e) { + throw new RuntimeException("Stream failed", e); + } + } + + /** + * Convert a QueryResult batch to an Iterator of Result objects + */ + protected Iterator convertBatchToResults(QueryResult queryResult) { + List results = new ArrayList<>(queryResult.getRecordsCount()); + + for (GrpcRecord grpcRecord : queryResult.getRecordsList()) { + // Use the existing grpcRecordToResult method from RemoteGrpcDatabase + Result result = db.grpcRecordToResult(grpcRecord); + results.add(result); + } + + return results.iterator(); + } + + @Override + public Result next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + + if (db.debugTx != null) { + db.checkCrossThreadUse("streamQuery.next"); + } + + Result result = nextResult; + nextResult = null; + totalProcessed.incrementAndGet(); + return result; + } + + @Override + public void close() { + + try { + // Drain any remaining results + while (stream.hasNext()) { + stream.read(); + } + } + catch (Exception e) { + logger.debug("Exception while draining stream during close", e); + } + + // BlockingClientCall doesn't implement AutoCloseable + // No need to cast or check instanceof + } + } + + /** + * Enhanced streaming with batch-aware ResultSet for better memory control and + * performance monitoring. + */ + public ResultSet queryStreamBatched(final String language, final String query, final Map params, final int batchSize, + final StreamQueryRequest.RetrievalMode mode) { + + checkDatabaseIsOpen(); + stats.queries.incrementAndGet(); + + final StreamQueryRequest.Builder b = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(query) + .setCredentials(buildCredentials()).setBatchSize(batchSize > 0 ? batchSize : 100).setRetrievalMode(mode); + + if (params != null && !params.isEmpty()) { + b.putAllParameters(convertParamsToGrpcValue(params)); + } + + final BlockingClientCall responseIterator = callServerStreaming("StreamQuery", + () -> blockingStub.withWaitForReady().withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).streamQuery(b.build())); + + return new BatchedStreamingResultSet(responseIterator, this); + } + + /** + * A ResultSet that exposes batch boundaries for advanced use cases while + * maintaining the standard ResultSet interface. + */ + private static class BatchedStreamingResultSet extends StreamingResultSet { + + private int currentBatchSize = 0; + private boolean isLastBatch = false; + private long runningTotal = 0; + + BatchedStreamingResultSet(BlockingClientCall stream, RemoteGrpcDatabase db) { + super(stream, db); + } + + @Override + public boolean hasNext() { + boolean hasMore = super.hasNext(); + + // Update batch info when we fetch a new batch + if (hasMore && currentBatch != null) { + // Track batch metadata here if needed + } + + return hasMore; + } + + @Override + protected Iterator convertBatchToResults(QueryResult queryResult) { + // Capture batch metadata + this.currentBatchSize = queryResult.getRecordsCount(); + this.isLastBatch = queryResult.getIsLastBatch(); + this.runningTotal = queryResult.getRunningTotalEmitted(); + + // Call parent implementation + return super.convertBatchToResults(queryResult); + } + + // Additional methods for batch-aware processing + public long getRunningTotal() { + return runningTotal; + } + + public int getCurrentBatchSize() { + return currentBatchSize; + } + + public boolean isLastBatch() { + return isLastBatch; + } + } + + public Iterator queryStreamBatchesIterator(final String language, final String query, final Map params, + final int batchSize, final StreamQueryRequest.RetrievalMode mode) { + + checkDatabaseIsOpen(); + stats.queries.incrementAndGet(); + + final StreamQueryRequest.Builder b = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(query) + .setCredentials(buildCredentials()).setBatchSize(batchSize > 0 ? batchSize : 100).setRetrievalMode(mode); + + if (params != null && !params.isEmpty()) { + b.putAllParameters(convertParamsToGrpcValue(params)); + } + + final BlockingClientCall responseIterator = callServerStreaming("StreamQuery", + () -> blockingStub.withWaitForReady().withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).streamQuery(b.build())); + + return new Iterator() { + private QueryBatch nextBatch = null; + private boolean drained = false; + + @Override + public boolean hasNext() { + if (nextBatch != null) + return true; + if (drained) + return false; + + if (debugTx != null) { + checkCrossThreadUse("streamQueryBatches.hasNext"); + } + + try { + + while (responseIterator.hasNext()) { + final QueryResult qr = responseIterator.read(); + + int totalInBatch = qr.getTotalRecordsInBatch(); + if (totalInBatch == 0) { + totalInBatch = qr.getRecordsCount(); + } + + if (qr.getRecordsCount() == 0 && !qr.getIsLastBatch()) { + continue; // empty non-terminal batch + } + + // Convert GrpcRecords to Results (not Records) + final List convertedResults = new ArrayList<>(qr.getRecordsCount()); + for (GrpcRecord gr : qr.getRecordsList()) { + Result result = grpcRecordToResult(gr); + convertedResults.add(result); + } + + // Create QueryBatch with Results instead of Records + nextBatch = new QueryBatch(convertedResults, totalInBatch, qr.getRunningTotalEmitted(), qr.getIsLastBatch()); + + if (qr.getIsLastBatch()) { + drained = true; + } + return true; + } + + drained = true; + return false; + } + catch (io.grpc.StatusRuntimeException | io.grpc.StatusException e) { + handleGrpcException(e); + throw new IllegalStateException("unreachable"); + } + catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Stream interrupted", ie); + } + catch (RuntimeException re) { + throw re; + } + catch (Exception e) { + throw new RuntimeException("Stream failed", e); + } + } + + @Override + public QueryBatch next() { + if (!hasNext()) + throw new NoSuchElementException(); + final QueryBatch out = nextBatch; + nextBatch = null; + return out; + } + }; + } + + public static final class QueryBatch { + private final List results; // Changed from List + private final int totalInBatch; + private final long runningTotal; + private final boolean lastBatch; + + public QueryBatch(List results, int totalInBatch, long runningTotal, boolean lastBatch) { + this.results = results; + this.totalInBatch = totalInBatch; + this.runningTotal = runningTotal; + this.lastBatch = lastBatch; + } + + public List results() { + return results; + } + + // Backward compatibility: provide records() method that extracts Records from + // Results + @Deprecated + public List records() { + List records = new ArrayList<>(results.size()); + for (Result result : results) { + if (result.isElement()) { + result.getRecord().ifPresent(records::add); + } + } + return records; + } + + public int totalInBatch() { + return totalInBatch; + } + + public long runningTotal() { + return runningTotal; + } + + public boolean isLastBatch() { + return lastBatch; + } + } + + public Iterator queryStream(final String database, final String sql, final Map params, final int batchSize, + final StreamQueryRequest.RetrievalMode mode, final TransactionContext tx, final long timeoutMs) { + final StreamQueryRequest.Builder reqB = StreamQueryRequest.newBuilder().setDatabase(database).setQuery(sql) + .putAllParameters(convertParamsToGrpcValue(params)).setCredentials(buildCredentials()) + .setBatchSize(batchSize > 0 ? batchSize : 100).setRetrievalMode(mode); + + if (tx != null) + reqB.setTransaction(tx); + + final BlockingClientCall it = callServerStreaming("StreamQuery", + () -> blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).streamQuery(reqB.build())); + + return new Iterator() { + private Iterator curr = Collections.emptyIterator(); + + @Override + public boolean hasNext() { + if (curr.hasNext()) + return true; + + if (debugTx != null) { + checkCrossThreadUse("streamQuery.hasNext"); + } + + try { + while (it.hasNext()) { + final QueryResult qr = it.read(); + curr = qr.getRecordsList().iterator(); + if (curr.hasNext()) + return true; + // else loop to fetch next batch (handles empty batches) + } + return false; + } + catch (io.grpc.StatusRuntimeException | io.grpc.StatusException e) { + handleGrpcException(e); + throw new IllegalStateException("unreachable"); + } + catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Stream interrupted", ie); + } + catch (RuntimeException re) { + throw re; + } + catch (Exception e) { + throw new RuntimeException("Stream failed", e); + } + } + + @Override + public GrpcRecord next() { + if (!hasNext()) + throw new NoSuchElementException(); + if (debugTx != null) { + checkCrossThreadUse("streamQuery.next"); + } + return curr.next(); + } + }; + } + + @Deprecated + public Iterator queryStreamAsRecordIterator(final String language, final String query, final Map params, + final int batchSize) { + ResultSet rs = queryStream(language, query, params, batchSize); + + // Convert ResultSet to Iterator + return new Iterator() { + @Override + public boolean hasNext() { + return rs.hasNext(); + } + + @Override + public Record next() { + Result result = rs.next(); + // Try to get Record from Result + if (result.isElement()) { + return result.getRecord().orElseThrow(() -> new IllegalStateException("Result claims to be element but has no Record")); + } + // For non-Record results, throw or skip + throw new IllegalStateException("Result is not a Record: " + result); + } + }; + } + + @Override + public Iterator iterateType(final String typeName, final boolean polymorphic) { + String query = "select from `" + typeName + "`"; + if (!polymorphic) + query += " where @type = '" + typeName + "'"; + return streamQuery(query); + } + + @Override + public Iterator iterateBucket(final String bucketName) { + return streamQuery("select from bucket:`" + bucketName + "`"); + } + + public String createRecord(final String cls, final Map props, final long timeoutMs) { + checkDatabaseIsOpen(); + + if (cls == null || cls.isBlank()) + throw new IllegalArgumentException("cls must be non-empty"); + if (props == null) + throw new IllegalArgumentException("props must be non-null"); + + // Build payload + final GrpcRecord recMsg = GrpcRecord.newBuilder().putAllProperties(convertParamsToGrpcValue(props)).build(); + + final CreateRecordRequest req = CreateRecordRequest.newBuilder().setDatabase(getName()).setType(cls).setRecord(recMsg) + .setCredentials(buildCredentials()).build(); + + try { + if (logger.isDebugEnabled()) { + logger.debug("CLIENT createRecord: db={}, txOpen={}, type={}, propCount={}, timeoutMs={}", getName(), (transactionId != null), + cls, props.size(), timeoutMs); + } + + final CreateRecordResponse res = callUnary("CreateRecord", + () -> blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).createRecord(req)); + + return res.getRid(); // e.g. "#12:0" + + } + catch (io.grpc.StatusRuntimeException | io.grpc.StatusException e) { + handleGrpcException(e); // rethrows mapped domain exception + throw new IllegalStateException("unreachable"); + } + } + + public String createRecordTx(final String cls, final Map props, final long timeoutMs) { + checkDatabaseIsOpen(); + + final GrpcRecord recMsg = GrpcRecord.newBuilder().putAllProperties(convertParamsToGrpcValue(props)).build(); + + final CreateRecordRequest req = CreateRecordRequest.newBuilder().setDatabase(getName()).setType(cls).setRecord(recMsg) + .setTransaction(TransactionContext.newBuilder().setBegin(true).setCommit(true)).setCredentials(buildCredentials()).build(); + + try { + final CreateRecordResponse res = callUnary("CreateRecord", + () -> blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).createRecord(req)); + return res.getRid(); + + } + catch (io.grpc.StatusRuntimeException | io.grpc.StatusException e) { + handleGrpcException(e); + throw new IllegalStateException("unreachable"); + } + } + + public boolean updateRecord(String rid, Map props, long timeoutMs) { + PropertiesUpdate partial = PropertiesUpdate.newBuilder().putAllProperties(convertParamsToGrpcValue(props)).build(); + + return updateRecord(rid, partial, timeoutMs); + } + + public boolean updateRecord(String rid, Record dbRecord, long timeoutMs) { + GrpcRecord record = ProtoUtils.toProtoRecord(dbRecord); + return updateRecordFull(rid, record, timeoutMs); + } + + private boolean updateRecord(final String rid, final PropertiesUpdate partial, final long timeoutMs) { + checkDatabaseIsOpen(); + + if (rid == null || rid.isBlank()) + throw new IllegalArgumentException("rid must be non-empty"); + if (partial == null) + throw new IllegalArgumentException("partial must be non-null"); + + final UpdateRecordRequest req = UpdateRecordRequest.newBuilder().setDatabase(getName()).setRid(rid).setPartial(partial) + // Per-call tx: begin+commit. If you have an outer tx, pass it instead. + .setTransaction(TransactionContext.newBuilder().setBegin(true).setCommit(true)).setCredentials(buildCredentials()).build(); + + try { + if (logger.isDebugEnabled()) { + logger.debug("CLIENT updateRecord(partial): db={}, txOpen={}, rid={}, timeoutMs={}", getName(), (transactionId != null), rid, + timeoutMs); + } + + final UpdateRecordResponse res = callUnary("UpdateRecord", + () -> blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).updateRecord(req)); + + // Most builds expose getSuccess(); if your proto has getUpdated(), swap here. + return res.getSuccess(); + + } + catch (io.grpc.StatusRuntimeException | io.grpc.StatusException e) { + handleGrpcException(e); // rethrows mapped domain exception + throw new IllegalStateException("unreachable"); + } + } + + private boolean updateRecordFull(final String rid, final GrpcRecord record, final long timeoutMs) { + checkDatabaseIsOpen(); + + if (rid == null || rid.isBlank()) + throw new IllegalArgumentException("rid must be non-empty"); + if (record == null) + throw new IllegalArgumentException("record must be non-null"); + + final UpdateRecordRequest req = UpdateRecordRequest.newBuilder().setDatabase(getName()).setRid(rid).setRecord(record) + // Per-call tx: begin+commit. If you have an outer tx, pass it instead. + .setTransaction(TransactionContext.newBuilder().setBegin(true).setCommit(true)).setCredentials(buildCredentials()).build(); + + try { + if (logger.isDebugEnabled()) { + logger.debug("CLIENT updateRecord(full): db={}, txOpen={}, rid={}, timeoutMs={}", getName(), (transactionId != null), rid, + timeoutMs); + } + + final UpdateRecordResponse res = callUnary("UpdateRecord", + () -> blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).updateRecord(req)); + + return res.getSuccess(); + + } + catch (io.grpc.StatusRuntimeException | io.grpc.StatusException e) { + handleGrpcException(e); // rethrows mapped domain exception + throw new IllegalStateException("unreachable"); + } + } + + @Override + public RemoteTransactionExplicitLock acquireLock() { + // Need gRPC-specific implementation + if (explicitLock == null) + explicitLock = new RemoteGrpcTransactionExplicitLock(this); + + return explicitLock; + } + + @Override + public long countBucket(final String bucketName) { + checkDatabaseIsOpen(); + stats.countBucket.incrementAndGet(); + ResultSet result = query("sql", "select count(*) as count from bucket:" + bucketName); + if (result.hasNext()) { + Number count = result.next().getProperty("count"); + return count != null ? count.longValue() : 0; + } + return 0; + } + + @Override + public long countType(final String typeName, final boolean polymorphic) { + checkDatabaseIsOpen(); + stats.countType.incrementAndGet(); + final String appendix = polymorphic ? "" : " where @type = '" + typeName + "'"; + ResultSet result = query("sql", "select count(*) as count from " + typeName + appendix); + if (result.hasNext()) { + Number count = result.next().getProperty("count"); + return count != null ? count.longValue() : 0; + } + return 0; + } + + @Override + public Record lookupByRID(final RID rid) { + return lookupByRID(rid, true); + } + + @Override + public Record lookupByRID(final RID rid, final boolean loadContent) { + checkDatabaseIsOpen(); + stats.readRecord.incrementAndGet(); + + if (rid == null) + throw new IllegalArgumentException("Record is null"); + + final LookupByRidRequest req = LookupByRidRequest.newBuilder().setDatabase(getName()).setRid(rid.toString()) + .setCredentials(buildCredentials()).build(); + + try { + if (logger.isDebugEnabled()) { + logger.debug("CLIENT lookupByRID: db={}, txOpen={}, rid={}, loadContent={}, timeoutMs={}", getName(), (transactionId != null), + rid, loadContent, getTimeout()); + } + + final LookupByRidResponse resp = callUnary("LookupByRid", + () -> blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).lookupByRid(req)); + + if (!resp.getFound()) + throw new RecordNotFoundException("Record " + rid + " not found", rid); + + // Note: loadContent is currently a no-op for gRPC; response already carries the + // record. + return grpcRecordToDBRecord(resp.getRecord()); + + } + catch (io.grpc.StatusException | io.grpc.StatusRuntimeException e) { + handleGrpcException(e); // maps & rethrows proper domain exception + throw new IllegalStateException("unreachable"); + } + } + + @Override + public boolean existsRecord(RID rid) { + stats.existsRecord.incrementAndGet(); + if (rid == null) + throw new IllegalArgumentException("Record is null"); + try { + return lookupByRID(rid, false) != null; + } + catch (RecordNotFoundException e) { + return false; + } + } + + // Convenience overload that accepts domain rows (convert first) + public InsertSummary insertBulkAsListOfMaps(final InsertOptions options, final List> rows, final long timeoutMs) { + + List protoRows = rows.stream().map(this::toProtoRecordFromMap) // your converter + .collect(java.util.stream.Collectors.toList()); + + return insertBulk(options, protoRows, timeoutMs); + } + + public InsertSummary insertBulk(final InsertOptions options, final List protoRows, final long timeoutMs) { + + // Ensure options carry DB + credentials as the server expects + final InsertOptions.Builder ob = options.toBuilder(); + + final String dbInOpts = options.getDatabase(); + if (dbInOpts == null || dbInOpts.isEmpty()) { + ob.setDatabase(getName()); + } + + final boolean missingCreds = !options.hasCredentials() || options.getCredentials().getUsername().isEmpty(); + if (missingCreds) { + ob.setCredentials(buildCredentials()); + } + + final InsertOptions newOptions = ob.build(); + + final BulkInsertRequest req = BulkInsertRequest.newBuilder().setOptions(newOptions).addAllRows(protoRows).build(); + + try { + if (logger.isDebugEnabled()) { + logger.debug("CLIENT insertBulk: rows={}, timeoutMs={}, tx={}", req.getRowsCount(), timeoutMs, (transactionId != null)); + } + + // use callUnary so tx cross-thread checks + rpcSeq happen in one place + return callUnary("BulkInsert", () -> blockingStub.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS).bulkInsert(req)); + + } + catch (io.grpc.StatusRuntimeException | io.grpc.StatusException e) { + handleGrpcException(e); // maps to your domain exceptions and rethrows + throw new IllegalStateException("unreachable"); // keep compiler happy + } + } + + // Convenience overload + public InsertSummary ingestStreamAsListOfMaps(final InsertOptions options, final List> rows, final int chunkSize, + final long timeoutMs) throws InterruptedException { + + List protoRows = rows.stream().map(this::toProtoRecordFromMap).collect(java.util.stream.Collectors.toList()); + + return ingestStream(options, protoRows, chunkSize, timeoutMs); + } + + public InsertSummary ingestStream(final InsertOptions options, final List protoRows, final int chunkSize, final long timeoutMs) + throws InterruptedException { + + checkDatabaseIsOpen(); + + if (protoRows == null || protoRows.isEmpty()) + return InsertSummary.newBuilder().setReceived(0).build(); + + if (chunkSize <= 0) + throw new IllegalArgumentException("chunkSize must be > 0"); + + // Ensure options carry DB + credentials + final InsertOptions.Builder ob = options.toBuilder(); + if (options.getDatabase() == null || options.getDatabase().isEmpty()) + ob.setDatabase(getName()); + if (!options.hasCredentials() || options.getCredentials().getUsername().isEmpty()) + ob.setCredentials(buildCredentials()); + final InsertOptions effOptions = ob.build(); + + final CountDownLatch done = new CountDownLatch(1); + final AtomicReference summaryRef = new AtomicReference<>(); + final AtomicReference errorRef = new AtomicReference<>(); + + final StreamObserver resp = new StreamObserver<>() { + @Override + public void onNext(InsertSummary value) { + summaryRef.set(value); + } + + @Override + public void onError(Throwable t) { + errorRef.set(t); + done.countDown(); + } + + @Override + public void onCompleted() { + done.countDown(); + } + }; + + if (logger.isDebugEnabled()) { + logger.debug("CLIENT ingestStream: db={}, rows={}, chunkSize={}, timeoutMs={}", getName(), protoRows.size(), chunkSize, timeoutMs); + } + + // Open the client stream via wrapper (adds deadline, tx checks, unified error + // mapping) + final StreamObserver req = callAsyncDuplex("InsertStream", timeoutMs, + (stub, responseObserver) -> stub.insertStream(responseObserver), wrapObserver("InsertStream", resp)); + + final String sessionId = "sess-" + System.nanoTime(); + long seq = 1; + int sent = 0; + + try { + for (int i = 0; i < protoRows.size(); i += chunkSize) { + final int end = Math.min(i + chunkSize, protoRows.size()); + + // (Sending options on every chunk is safe; if you prefer first-chunk-only, gate + // on seq == 1.) + final InsertChunk chunk = InsertChunk.newBuilder().setSessionId(sessionId).setOptions(effOptions).setChunkSeq(seq++) + .addAllRows(protoRows.subList(i, end)).build(); + + req.onNext(chunk); + sent += (end - i); + } + req.onCompleted(); + } + catch (RuntimeException sendErr) { + // Client-side failure during onNext/onCompleted: propagate + throw sendErr; + } + + // Wait for server final summary or error (tiny grace after deadline) + final boolean finished = done.await(Math.max(1, timeoutMs) + 1_000, TimeUnit.MILLISECONDS); + if (!finished) { + throw new TimeoutException("ingestStream timed out waiting for server completion"); + } + + // If wrapObserver mapped an error, rethrow it directly + final Throwable streamErr = errorRef.get(); + if (streamErr != null) { + if (streamErr instanceof RuntimeException re) + throw re; + throw new RemoteException("gRPC stream failed: " + streamErr.getMessage(), streamErr); + } + + InsertSummary s = summaryRef.get(); + if (s == null) { + // Fallback if server completed without emitting a final summary + s = InsertSummary.newBuilder().setReceived(sent).build(); + } + + if (logger.isDebugEnabled()) { + logger.debug("CLIENT ingestStream: completed; received={}", s.getReceived()); + } + + return s; + } + + /** + * Pushes domain {@code com.arcadedb.database.Record} rows via + * InsertBidirectional with per-batch ACKs. + */ + public InsertSummary ingestBidi(final List rows, final InsertOptions opts, final int chunkSize, + final int maxInflight, final long timeoutMs) throws InterruptedException { + + return ingestBidiCore(rows, opts, chunkSize, maxInflight, timeoutMs, + (Object o) -> toProtoRecordFromDbRecord((com.arcadedb.database.Record) o)); + } + + public InsertSummary ingestBidi(final List rows, final InsertOptions opts, final int chunkSize, + final int maxInflight) throws InterruptedException { + + return ingestBidiCore(rows, opts, chunkSize, maxInflight, /* timeoutMs */ 5 * 60_000L, this::toProtoRecordFromDbRecord); + } + + /** + * Pushes map-shaped rows (property map per row) via InsertBidirectional with + * per-batch ACKs. + */ + public InsertSummary ingestBidi(final InsertOptions options, final List> rows, final int chunkSize, + final int maxInflight) throws InterruptedException { + + return ingestBidiCore(rows, options, chunkSize, maxInflight, /* timeoutMs */ 5 * 60_000L, this::toProtoRecordFromMap); + } + + public InsertSummary ingestBidi(final InsertOptions options, final List> rows, final int chunkSize, + final int maxInflight, final long timeoutMs) throws InterruptedException { + + return ingestBidiCore(rows, options, chunkSize, maxInflight, timeoutMs, this::toProtoRecordFromMap); + } + + /** + * Core implementation of InsertBidirectional ingest + */ + + private InsertSummary ingestBidiCore(final List rows, final InsertOptions options, final int chunkSize, final int maxInflight, + final long timeoutMs, final java.util.function.Function mapper) throws InterruptedException { + + // Fast-path & guards + if (rows == null || rows.isEmpty()) + return InsertSummary.newBuilder().setReceived(0).build(); + if (chunkSize <= 0) + throw new IllegalArgumentException("chunkSize must be > 0"); + if (maxInflight <= 0) + throw new IllegalArgumentException("maxInflight must be > 0"); + + // Options with DB + creds + final InsertOptions.Builder ob = options.toBuilder(); + if (options.getDatabase() == null || options.getDatabase().isEmpty()) + ob.setDatabase(getName()); + if (!options.hasCredentials() || options.getCredentials().getUsername().isEmpty()) + ob.setCredentials(buildCredentials()); + final InsertOptions effectiveOpts = ob.build(); + + // Pre-map rows → proto + final List protoRows = rows.stream().map(mapper).collect(java.util.stream.Collectors.toList()); + + if (logger.isDebugEnabled()) { + logger.debug("CLIENT ingestBidi start: rows={}, chunkSize={}, maxInflight={}, timeoutMs={}", protoRows.size(), chunkSize, + maxInflight, timeoutMs); + } + + // --- streaming state + final String sessionId = "sess-" + System.nanoTime(); + final java.util.concurrent.CountDownLatch done = new java.util.concurrent.CountDownLatch(1); + final java.util.concurrent.atomic.AtomicReference errRef = new java.util.concurrent.atomic.AtomicReference<>(); + final java.util.concurrent.atomic.AtomicLong seq = new java.util.concurrent.atomic.AtomicLong(1); + final java.util.concurrent.atomic.AtomicInteger cursor = new java.util.concurrent.atomic.AtomicInteger(0); + final java.util.concurrent.atomic.AtomicInteger sent = new java.util.concurrent.atomic.AtomicInteger(0); + final java.util.concurrent.atomic.AtomicInteger acked = new java.util.concurrent.atomic.AtomicInteger(0); + final java.util.concurrent.atomic.AtomicReference committed = new java.util.concurrent.atomic.AtomicReference<>(); + final List acks = java.util.Collections.synchronizedList(new ArrayList<>()); + + final java.util.concurrent.atomic.AtomicReference> observerRef = new java.util.concurrent.atomic.AtomicReference<>(); + + final java.util.concurrent.atomic.AtomicBoolean commitSent = new java.util.concurrent.atomic.AtomicBoolean(false); + final java.util.concurrent.ScheduledExecutorService scheduler = java.util.concurrent.Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "grpc-ack-grace-timer"); + t.setDaemon(true); + return t; + }); + final long ackGraceMillis = Math.min(Math.max(timeoutMs / 10, 1_000L), 10_000L); + final Object timerLock = new Object(); + final java.util.concurrent.atomic.AtomicReference> ackGraceFuture = new java.util.concurrent.atomic.AtomicReference<>(); + + final Runnable sendCommitIfNeeded = () -> { + if (commitSent.compareAndSet(false, true)) { + try { + final ClientCallStreamObserver r = observerRef.get(); + if (r != null) { + r.onNext(InsertRequest.newBuilder().setCommit(Commit.newBuilder().setSessionId(sessionId)).build()); + r.onCompleted(); + } + } + catch (Throwable ignore) { + /* best effort */ } + } + }; + + final Runnable armAckGraceTimer = () -> { + synchronized (timerLock) { + var prev = ackGraceFuture.getAndSet(null); + if (prev != null) + prev.cancel(false); + if (cursor.get() >= protoRows.size() && acked.get() < sent.get() && !commitSent.get()) { + var fut = scheduler.schedule(sendCommitIfNeeded, ackGraceMillis, TimeUnit.MILLISECONDS); + ackGraceFuture.set(fut); + } + } + }; + + final Runnable cancelAckGraceTimer = () -> { + synchronized (timerLock) { + var prev = ackGraceFuture.getAndSet(null); + if (prev != null) + prev.cancel(false); + } + }; + + // --- response observer (keeps backpressure via beforeStart) + final ClientResponseObserver respObserver = new ClientResponseObserver<>() { + ClientCallStreamObserver req; + volatile boolean started = false; + + @Override + public void beforeStart(ClientCallStreamObserver r) { + this.req = r; + observerRef.set(r); + r.disableAutoInboundFlowControl(); + r.setOnReadyHandler(this::drain); + } + + private void drain() { + if (!req.isReady()) + return; + + if (!started) { + req.onNext(InsertRequest.newBuilder().setStart(Start.newBuilder().setOptions(effectiveOpts)).build()); + started = true; + req.request(1); // pull first server response + } + + while (req.isReady() && (sent.get() - acked.get()) < maxInflight) { + final int start = cursor.get(); + if (start >= protoRows.size()) + break; + + final int end = Math.min(start + chunkSize, protoRows.size()); + final var slice = protoRows.subList(start, end); + + final var chunk = InsertChunk.newBuilder().setSessionId(sessionId).setChunkSeq(seq.getAndIncrement()).addAllRows(slice) + .build(); + + req.onNext(InsertRequest.newBuilder().setChunk(chunk).build()); + cursor.set(end); + sent.incrementAndGet(); + } + + if (cursor.get() >= protoRows.size()) { + if (acked.get() >= sent.get()) + sendCommitIfNeeded.run(); + else + armAckGraceTimer.run(); + } + } + + @Override + public void onNext(InsertResponse v) { + switch (v.getMsgCase()) { + case STARTED -> { + /* ok */ } + case BATCH_ACK -> { + acks.add(v.getBatchAck()); + acked.incrementAndGet(); + // free capacity & manage timers + drain(); + if (cursor.get() >= protoRows.size()) { + if (acked.get() >= sent.get()) { + cancelAckGraceTimer.run(); + sendCommitIfNeeded.run(); + } + else { + armAckGraceTimer.run(); + } + } + } + case COMMITTED -> { + committed.set(v.getCommitted().getSummary()); + cancelAckGraceTimer.run(); + } + case ERROR -> { + // surface as error; caller will map/throw after await + errRef.set(new StatusRuntimeException(io.grpc.Status.INTERNAL.withDescription(v.getError().getMessage()))); + cancelAckGraceTimer.run(); + try { + req.cancel("server ERROR", null); + } + catch (Throwable ignore) { + } + } + case MSG_NOT_SET -> { + /* ignore */ } + } + req.request(1); // keep pulling + } + + @Override + public void onError(Throwable t) { + cancelAckGraceTimer.run(); + errRef.set(t); + done.countDown(); + } + + @Override + public void onCompleted() { + cancelAckGraceTimer.run(); + done.countDown(); + } + }; + + // --- open bidi via wrapper (deadline, tx checks, unified logging) + @SuppressWarnings("unused") + final StreamObserver _req = callAsyncDuplex("InsertBidirectional", timeoutMs, + (stub, responseObs) -> stub.insertBidirectional(responseObs), wrapClientResponseObserver("InsertBidirectional", respObserver) // preserves + // beforeStart + ); + + try { + // wait for completion (tiny grace after deadline) + final boolean finished = done.await(Math.max(1, timeoutMs) + 1_000, TimeUnit.MILLISECONDS); + if (!finished) { + final ClientCallStreamObserver r = observerRef.get(); + if (r != null) { + try { + r.cancel("client timeout waiting for completion", null); + } + catch (Throwable ignore) { + } + } + throw new TimeoutException("ingestBidirectional timed out waiting for server completion"); + } + + final Throwable err = errRef.get(); + if (err != null) { + // wrapClientResponseObserver already runs through handleGrpcException(); + // if it was a non-gRPC Throwable, surface it uniformly here. + if (err instanceof RuntimeException re) + throw re; + throw new RemoteException("gRPC bidi stream failed: " + err.getMessage(), err); + } + + if (logger.isDebugEnabled()) { + try { + logger.debug("CLIENT ingestBidi finished: sent={}, acked={}", sent.get(), acked.get()); + } + catch (Throwable ignore) { + } + } + } + finally { + scheduler.shutdownNow(); + } + + // Prefer COMMITTED summary; else aggregate ACKs + final InsertSummary finalSummary = committed.get(); + if (finalSummary != null) + return finalSummary; + + final long ins = acks.stream().mapToLong(BatchAck::getInserted).sum(); + final long upd = acks.stream().mapToLong(BatchAck::getUpdated).sum(); + final long ign = acks.stream().mapToLong(BatchAck::getIgnored).sum(); + final long fail = acks.stream().mapToLong(BatchAck::getFailed).sum(); + + return InsertSummary.newBuilder().setReceived(protoRows.size()).setInserted(ins).setUpdated(upd).setIgnored(ign).setFailed(fail).build(); + } + + // Map -> GrpcRecord + private GrpcRecord toProtoRecordFromMap(Map row) { + GrpcRecord.Builder b = GrpcRecord.newBuilder(); + row.forEach((k, v) -> b.putProperties(k, objectToGrpcValue(v))); + GrpcRecord rec = b.build(); + if (logger.isDebugEnabled()) + logger.debug("CLIENT toProtoRecordFromMap: {}", summarize(rec)); + return rec; + } + + // Domain Record (storage) -> GrpcRecord + private GrpcRecord toProtoRecordFromDbRecord(com.arcadedb.database.Record rec) { + // Use ProtoUtils for proper conversion + GrpcRecord out = ProtoUtils.toProtoRecord(rec); + if (logger.isDebugEnabled()) + logger.debug("CLIENT toProtoRecordFromDbRecord: {}", summarize(out)); + return out; + } + + // Convert Java object -> GrpcValue (extend as needed) + private GrpcValue objectToGrpcValue(Object v) { + return ProtoUtils.toGrpcValue(v); + } + + public DatabaseCredentials buildCredentials() { + return DatabaseCredentials.newBuilder().setUsername(getUserName()).setPassword(getUserPassword()).build(); + } + + private TransactionIsolation mapIsolationLevel(Database.TRANSACTION_ISOLATION_LEVEL level) { + switch (level) { +// case READ_UNCOMMITTED: +// return TransactionIsolation.READ_UNCOMMITTED; + case READ_COMMITTED: + return TransactionIsolation.READ_COMMITTED; + case REPEATABLE_READ: + return TransactionIsolation.REPEATABLE_READ; +// case SERIALIZABLE: +// return TransactionIsolation.SERIALIZABLE; + default: + return TransactionIsolation.READ_COMMITTED; + } + } + + // ---- TX helpers ------------------------------------------------------------- + private static TransactionContext txBeginCommit() { + return TransactionContext.newBuilder().setBegin(true).setCommit(true).build(); + } + + private static TransactionContext txNone() { + return TransactionContext.getDefaultInstance(); + } + + // Optional: language defaulting (server defaults to "sql" too) + private static String langOrDefault(String language) { + return (language == null || language.isEmpty()) ? "sql" : language; + } + + private Iterator streamQuery(final String query) { + StreamQueryRequest request = StreamQueryRequest.newBuilder().setDatabase(getName()).setQuery(query).setCredentials(buildCredentials()) + .setBatchSize(100).build(); + + final BlockingClientCall responseIterator = callServerStreaming("StreamQuery", + () -> blockingStub.withDeadlineAfter(getTimeout(), TimeUnit.MILLISECONDS).streamQuery(request)); + + return new Iterator() { + private Iterator currentBatch = Collections.emptyIterator(); + + @Override + public boolean hasNext() { + + if (currentBatch.hasNext()) { + return true; + } + + try { + if (responseIterator.hasNext()) { + + QueryResult result; + result = responseIterator.read(); + currentBatch = result.getRecordsList().iterator(); + return currentBatch.hasNext(); + } + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + catch (StatusException e) { + throw new RuntimeException(e); + } + + return false; + } + + @Override + public Record next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return grpcRecordToDBRecord(currentBatch.next()); + } + }; + } + + private ResultSet createGrpcResultSet(ExecuteQueryResponse response) { + InternalResultSet resultSet = new InternalResultSet(); + for (QueryResult queryResult : response.getResultsList()) { + for (GrpcRecord record : queryResult.getRecordsList()) { + resultSet.add(grpcRecordToResult(record)); + } + } + return resultSet; + } + + private ResultSet createGrpcResultSet(ExecuteCommandResponse response) { + InternalResultSet resultSet = new InternalResultSet(); + for (GrpcRecord record : response.getRecordsList()) { + resultSet.add(grpcRecordToResult(record)); + } + return resultSet; + } + + private Result grpcRecordToResult(GrpcRecord grpcRecord) { + + Record record = grpcRecordToDBRecord(grpcRecord); + + if (record == null) { + + Map properties = new HashMap<>(); + grpcRecord.getPropertiesMap().forEach((k, v) -> properties.put(k, grpcValueToObject(v))); + return new ResultInternal(properties); + } + + return new ResultInternal(record); + } + + private Record grpcRecordToDBRecord(GrpcRecord grpcRecord) { + + Map map = new HashMap<>(); + + // Convert properties + grpcRecord.getPropertiesMap().forEach((k, v) -> map.put(k, grpcValueToObject(v))); + + // Add metadata + map.put("@rid", grpcRecord.getRid()); + map.put("@type", grpcRecord.getType()); + + GrpcValue catFromGrpcRecord = grpcRecord.getPropertiesMap().get("@cat"); + + String cat = null; + + if (catFromGrpcRecord != null) { + + cat = catFromGrpcRecord.getStringValue(); + } + else { + + cat = mapRecordType(grpcRecord); + } + + if (cat != null) { + + map.put("@cat", cat); + } + + if (cat == null) { + + return null; + } + + switch (cat) { + case "d": + + return new RemoteImmutableDocument(this, map); + + case "v": + + return new RemoteImmutableVertex(this, map); + + case "e": + + return new RemoteImmutableEdge(this, map); + + default: + return null; + } + } + + private String mapRecordType(GrpcRecord grpcRecord) { + + // Determine record category from type name + String typeName = grpcRecord.getType(); + + // Check schema to determine actual type + try { + + if (typeName != null && !typeName.isBlank() && getSchema().existsType(typeName)) { + + Object type = getSchema().getType(typeName); + + if (type instanceof com.arcadedb.schema.VertexType) { + + return "v"; + } + else if (type instanceof com.arcadedb.schema.EdgeType) { + + return "e"; + } + else if (type instanceof com.arcadedb.schema.DocumentType) { + + return "d"; + } + else { + + return null; + } + } + else { + + return null; + } + } + catch (Exception e) { + + throw new RuntimeException(e); + } + } + + private Map convertParamsToGrpcValue(Map params) { + Map grpcParams = new HashMap<>(); + + for (Map.Entry entry : params.entrySet()) { + GrpcValue value = objectToGrpcValue(entry.getValue()); + grpcParams.put(entry.getKey(), value); + } + + return grpcParams; + } + + private Object grpcValueToObject(GrpcValue grpcValue) { + Object out = ProtoUtils.fromGrpcValue(grpcValue); + if (logger.isDebugEnabled()) + logger.debug("CLIENT decode grpcValueToObject: {} -> {}", summarize(grpcValue), summarize(out)); + return out; + } + + private void handleGrpcException(Throwable e) { + // Works for StatusException, StatusRuntimeException, and anything else + io.grpc.Status status = io.grpc.Status.fromThrowable(e); + String msg = status.getDescription() != null ? status.getDescription() : status.getCode().name(); + + switch (status.getCode()) { + case NOT_FOUND: + throw new RecordNotFoundException(msg, null); + case ALREADY_EXISTS: + throw new DuplicatedKeyException("", "", null); + case ABORTED: + throw new ConcurrentModificationException(msg); + case DEADLINE_EXCEEDED: + throw new TimeoutException(msg); + case PERMISSION_DENIED: + throw new SecurityException(msg); + case UNAVAILABLE: + throw new NeedRetryException(msg); + default: + throw new RemoteException("gRPC error: " + msg, e); + } + } + + // Add the missing RemoteGrpcTransactionExplicitLock class reference + private static class RemoteGrpcTransactionExplicitLock extends RemoteTransactionExplicitLock { + public RemoteGrpcTransactionExplicitLock(RemoteGrpcDatabase database) { + super(database); + } + } + + // --- Debug helpers (client) --- + private static String summarize(Object o) { + if (o == null) + return "null"; + try { + if (o instanceof CharSequence s) + return "String(" + s.length() + ")"; + if (o instanceof byte[] b) + return "bytes[" + b.length + "]"; + if (o instanceof java.util.Collection c) + return o.getClass().getSimpleName() + "[size=" + c.size() + "]"; + if (o instanceof java.util.Map m) + return o.getClass().getSimpleName() + "[size=" + m.size() + "]"; + return o.getClass().getSimpleName(); + } + catch (Exception e) { + return o.getClass().getSimpleName(); + } + } + + private static String summarize(GrpcValue v) { + if (v == null) + return "GrpcValue(null)"; + return switch (v.getKindCase()) { + case BOOL_VALUE -> "BOOL"; + case INT32_VALUE -> "INT32"; + case INT64_VALUE -> "INT64"; + case FLOAT_VALUE -> "FLOAT"; + case DOUBLE_VALUE -> "DOUBLE"; + case STRING_VALUE -> "STRING(" + v.getStringValue().length() + ")"; + case BYTES_VALUE -> "BYTES[" + v.getBytesValue().size() + "]"; + case TIMESTAMP_VALUE -> "TIMESTAMP"; + case LIST_VALUE -> "LIST[" + v.getListValue().getValuesCount() + "]"; + case MAP_VALUE -> "MAP[" + v.getMapValue().getEntriesCount() + "]"; + case EMBEDDED_VALUE -> "EMBEDDED"; + case LINK_VALUE -> "LINK(" + v.getLinkValue().getRid() + ")"; + case DECIMAL_VALUE -> "DECIMAL(scale=" + v.getDecimalValue().getScale() + ")"; + case KIND_NOT_SET -> "KIND_NOT_SET"; + }; + } + + private static String summarize(GrpcRecord r) { + if (r == null) + return "GrpcRecord(null)"; + String rid = r.getRid(); + String ty = r.getType(); + int props = r.getPropertiesCount(); + return "GrpcRecord{rid=" + rid + ", type=" + ty + ", props=" + props + "}"; + } + + // RemoteGrpcDatabase.java + + // ---- fields ---- + private volatile TxDebug debugTx; + + private static final class TxDebug { + final long id = System.nanoTime(); // local correlation id + final Thread ownerThread = Thread.currentThread(); + final String dbName; + volatile String txLabel; // optional + final Exception beginSite = new Exception("begin site"); // capture stack + final java.util.concurrent.atomic.AtomicLong rpcSeq = new java.util.concurrent.atomic.AtomicLong(); + volatile boolean beginRpcSent, committed, rolledBack; + + @SuppressWarnings("unused") + TxDebug(String db, String label) { + this.dbName = db; + this.txLabel = label; + } + } + + // one place to handle JDK 17/21 differences + private static String tidName(Thread t) { + try { + // Java 19+: threadId() + long tid = (long) Thread.class.getMethod("threadId").invoke(t); + return tid + ":" + t.getName(); + } + catch (ReflectiveOperationException ignore) { + return legacyTidName(t); + } + } + + private static String legacyTidName(Thread t) { + return t.threadId() + ":" + t.getName(); + } + + // Optional knobs you can toggle from the TM for a single run + private volatile boolean txDebugEnabled = true; + + public void enableTxDebug(boolean on) { + this.txDebugEnabled = on; + } + + public boolean isLocalTxActive() { + return debugTx != null; + } + + public @Nullable String currentLocalTxId() { + return (debugTx != null ? Long.toString(debugTx.id) : null); + } + + public void setCurrentTxLabel(String label) { + if (debugTx != null) + debugTx.txLabel = label; + } + + private void logTx(String phase, String rpcOp) { + if (debugTx == null || !logger.isDebugEnabled()) + return; + TxDebug d = debugTx; + logger.debug("TXDBG {} db={} tx#{} label={} owner={} now={} rpcOp={} rpcSeq={} beginSent={} committed={} rolledBack={}", phase, d.dbName, + d.id, d.txLabel, tidName(d.ownerThread), tidName(Thread.currentThread()), rpcOp, d.rpcSeq.get(), d.beginRpcSent, d.committed, + d.rolledBack); + } + + private void checkCrossThreadUse(String where) { + TxDebug d = debugTx; + if (d == null) + return; + Thread now = Thread.currentThread(); + if (now != d.ownerThread) { + logger.warn("TXDBG CROSS-THREAD {} db={} tx#{} owner={} now={} label={} (begin site follows)", where, d.dbName, d.id, + tidName(d.ownerThread), tidName(now), d.txLabel, d.beginSite); + } + } + + @FunctionalInterface + private interface Rpc { + T run() throws io.grpc.StatusException; // V2 throws this; v1 lambdas compile fine too + } + + private Resp callUnary(String opName, Rpc rpc) throws io.grpc.StatusException { + if (debugTx != null) { + checkCrossThreadUse("RPC " + opName); + logTx("RPC(local)", opName); + } + try { + return rpc.run(); + } + finally { + if (debugTx != null) { + debugTx.rpcSeq.incrementAndGet(); + } + } + } + + // unary that returns void + private void callUnaryVoid(String opName, Runnable rpc) { + if (debugTx != null) { + checkCrossThreadUse("RPC " + opName); + logTx("RPC(unary)", opName); + } + rpc.run(); + if (debugTx != null) { + debugTx.rpcSeq.incrementAndGet(); + } + } + + // Helper for server-streaming RPCs (mirrors callUnary) + private BlockingClientCall callServerStreaming(String opName, Supplier> rpc) { + if (debugTx != null) { + checkCrossThreadUse("RPC " + opName); + logTx("RPC(open-stream)", opName); + } + try { + final BlockingClientCall it = rpc.get(); + if (debugTx != null) { + debugTx.rpcSeq.incrementAndGet(); + } + return it; + } + catch (io.grpc.StatusRuntimeException e) { + handleGrpcException(e); // rethrows mapped runtime exception + throw new IllegalStateException("unreachable"); + } + } + + // For async "client-streaming" and "bidirectional" calls that RETURN a request + // StreamObserver + private StreamObserver callAsyncDuplex(String opName, long timeoutMs, + java.util.function.BiFunction, StreamObserver> starter, + StreamObserver responseObserver) { + if (debugTx != null) { + checkCrossThreadUse("STREAM " + opName); + logTx("STREAM(local)", opName); + } + final var stub = asyncStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS); + StreamObserver reqObs = starter.apply(stub, wrapObserver(opName, responseObserver)); + if (debugTx != null) { + debugTx.rpcSeq.incrementAndGet(); + } + return reqObs; + } + + // For async "server-streaming" calls that take (request, responseObserver) and + // return void + private void callAsyncServerStreaming(String opName, long timeoutMs, Req request, + java.util.function.BiConsumer> invoker, + StreamObserver responseObserver) { + if (debugTx != null) { + checkCrossThreadUse("STREAM " + opName); + logTx("STREAM(local)", opName); + } + final var stub = asyncStub.withDeadlineAfter(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS); + invoker.accept(stub, wrapObserver(opName, responseObserver)); + if (debugTx != null) { + debugTx.rpcSeq.incrementAndGet(); + } + } + + // Wrap a plain StreamObserver to translate gRPC Status into your domain + // exceptions + private StreamObserver wrapObserver(String opName, StreamObserver delegate) { + return new StreamObserver<>() { + @Override + public void onNext(T value) { + delegate.onNext(value); + } + + @Override + public void onError(Throwable t) { + try { + handleGrpcException(t); + } + catch (RuntimeException mapped) { + delegate.onError(mapped); + return; + } + delegate.onError(t); + } + + @Override + public void onCompleted() { + delegate.onCompleted(); + } + }; + } + + // Same idea, but preserves ClientResponseObserver features (beforeStart, flow + // control) + private ClientResponseObserver wrapObserver(String opName, ClientResponseObserver delegate) { + return new ClientResponseObserver<>() { + @Override + public void beforeStart(io.grpc.stub.ClientCallStreamObserver r) { + // pass through; your delegate may set onReady handler, request(n), etc. + delegate.beforeStart(r); + } + + @Override + public void onNext(Resp value) { + delegate.onNext(value); + } + + @Override + public void onError(Throwable t) { + try { + handleGrpcException(t); + } + catch (RuntimeException mapped) { + delegate.onError(mapped); + return; + } + delegate.onError(t); + } + + @Override + public void onCompleted() { + delegate.onCompleted(); + } + }; + } + + private ClientResponseObserver wrapClientResponseObserver(String opName, + ClientResponseObserver delegate) { + return new ClientResponseObserver<>() { + @Override + public void beforeStart(ClientCallStreamObserver requestStream) { + // Preserve delegate’s backpressure hooks + delegate.beforeStart(requestStream); + } + + @Override + public void onNext(RespT value) { + delegate.onNext(value); + } + + @Override + public void onError(Throwable t) { + // Normalize gRPC errors through your handler, then pass the mapped exception + try { + if (t instanceof io.grpc.StatusRuntimeException sre) { + handleGrpcException(sre); // throws + } + else if (t instanceof io.grpc.StatusException se) { + handleGrpcException(se); // throws + } + // Non-gRPC error: forward as-is + delegate.onError(t); + } + catch (RuntimeException mapped) { + // forward the mapped exception to delegate + delegate.onError(mapped); + } + } + + @Override + public void onCompleted() { + delegate.onCompleted(); + } + }; + } +} \ No newline at end of file diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseWithCompression.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseWithCompression.java new file mode 100644 index 0000000000..e985407250 --- /dev/null +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseWithCompression.java @@ -0,0 +1,14 @@ +package com.arcadedb.remote.grpc; + +import com.arcadedb.ContextConfiguration; + +/** + * Extended RemoteGrpcDatabase with compression support + */ +public class RemoteGrpcDatabaseWithCompression extends RemoteGrpcDatabase { + + public RemoteGrpcDatabaseWithCompression(RemoteGrpcServer remoteGrpcServer, String host, int grpcPort, int httpPort, String databaseName, String userName, + String userPassword, ContextConfiguration configuration) { + super(remoteGrpcServer, host, grpcPort, httpPort, databaseName, userName, userPassword, configuration); + } +} diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcServer.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcServer.java new file mode 100644 index 0000000000..74b8d9689b --- /dev/null +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcServer.java @@ -0,0 +1,289 @@ +package com.arcadedb.remote.grpc; + +import java.util.List; +import java.util.Objects; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; + +import javax.annotation.PreDestroy; + +import com.arcadedb.server.grpc.ArcadeDbAdminServiceGrpc; +import com.arcadedb.server.grpc.ArcadeDbServiceGrpc; +import com.arcadedb.server.grpc.CreateDatabaseRequest; +import com.arcadedb.server.grpc.DatabaseCredentials; +import com.arcadedb.server.grpc.DropDatabaseRequest; +import com.arcadedb.server.grpc.ListDatabasesRequest; +import com.arcadedb.server.grpc.ListDatabasesResponse; + +import io.grpc.CallCredentials; +import io.grpc.Channel; +import io.grpc.ClientInterceptor; +import io.grpc.ClientInterceptors; +import io.grpc.CompressorRegistry; +import io.grpc.DecompressorRegistry; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.Metadata; +import io.grpc.StatusException; +import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder; +import io.grpc.netty.shaded.io.netty.channel.EventLoopGroup; +import io.grpc.netty.shaded.io.netty.channel.nio.NioEventLoopGroup; +import io.grpc.netty.shaded.io.netty.channel.socket.nio.NioSocketChannel; +import io.grpc.stub.AbstractStub; + +/** + * Minimal server-scope gRPC wrapper (HTTP RemoteServer equivalent), implemented + * ONLY with RPCs present in your current proto. + * + * Features: - listDatabases() - existsDatabase(name) via listDatabases() - + * createDatabase(name, type) - createDatabaseIfMissing(name, type) - + * dropDatabase(name, force?) // 'force' only if defined in your proto; + * otherwise ignored + * + * Add more methods later when you extend the proto (ping, serverInfo, user + * mgmt, etc.). + */ +public class RemoteGrpcServer implements AutoCloseable { + + private final String host; + private final int port; + private final String userName; + private final String userPassword; + + private final long defaultTimeoutMs; + + private final List interceptors; + private final boolean plaintext; + + private ManagedChannel channel; + + private ArcadeDbAdminServiceGrpc.ArcadeDbAdminServiceBlockingV2Stub adminServiceBlockingV2Stub; + + public RemoteGrpcServer(final String host, final int port, final String user, final String pass, boolean plaintext, List interceptors) { + this(host, port, user, pass, plaintext, interceptors, 30_000); + } + + public RemoteGrpcServer(final String host, final int port, final String user, final String pass, boolean plaintext, List interceptors, final long defaultTimeoutMs) { + + this.host = Objects.requireNonNull(host, "host"); + + this.port = port; + + this.plaintext = plaintext; + this.interceptors = interceptors == null ? List.of() : List.copyOf(interceptors); + + this.userName = Objects.requireNonNull(user, "user"); + + this.userPassword = Objects.requireNonNull(pass, "pass"); + + this.defaultTimeoutMs = defaultTimeoutMs > 0 ? defaultTimeoutMs : 30_000; + } + + + public synchronized void start() { + + if (channel != null) return; + +// NettyChannelBuilder b = NettyChannelBuilder.forAddress(host, port) +// .maxInboundMessageSize(100 * 1024 * 1024) // 100MB max message size +// .keepAliveTime(30, TimeUnit.SECONDS) // Keep-alive configuration +// .keepAliveTimeout(10, TimeUnit.SECONDS) +// .keepAliveWithoutCalls(true) +// .decompressorRegistry(DecompressorRegistry.getDefaultInstance()) +// .compressorRegistry(CompressorRegistry.getDefaultInstance()); + + + EventLoopGroup elg = new NioEventLoopGroup(); + + NettyChannelBuilder chBuilder = NettyChannelBuilder.forAddress(host, port) + .eventLoopGroup(elg) // share across clients + .channelType(NioSocketChannel.class) + .maxInboundMessageSize(150 * 1024 * 1024) + .maxInboundMetadataSize(32 * 1024 * 1024) + .keepAliveTime(30, TimeUnit.SECONDS) + .keepAliveTimeout(10, TimeUnit.SECONDS) + .keepAliveWithoutCalls(true) + .decompressorRegistry(DecompressorRegistry.getDefaultInstance()) + .compressorRegistry(CompressorRegistry.getDefaultInstance()) + .flowControlWindow(8 * 1024 * 1024); + + // .proxyDetector(myProxyDetector) + + if (plaintext) chBuilder.usePlaintext(); + + channel = chBuilder.build(); + } + + /** Returns a Channel (wrapped with interceptors if provided). */ + public Channel channel() { + + if (channel == null) { + start(); + } + + return interceptors.isEmpty() ? channel : ClientInterceptors.intercept(channel, interceptors); + } + + public ArcadeDbServiceGrpc.ArcadeDbServiceBlockingV2Stub newBlockingStub(int timeout) { + + return ArcadeDbServiceGrpc.newBlockingV2Stub(channel()) + .withCallCredentials(createCredentials()) + .withDeadlineAfter(timeout, TimeUnit.MILLISECONDS) + .withCompression("gzip"); + } + + public ArcadeDbServiceGrpc.ArcadeDbServiceStub newAsyncStub(int timeout) { + + return ArcadeDbServiceGrpc.newStub(channel()) + .withCallCredentials(createCredentials()) + .withDeadlineAfter(timeout, TimeUnit.MILLISECONDS) + .withCompression("gzip"); + } + + private ArcadeDbAdminServiceGrpc.ArcadeDbAdminServiceBlockingV2Stub adminServiceBlockingV2Stub() { + + if (this.adminServiceBlockingV2Stub == null) { + + this.adminServiceBlockingV2Stub = ArcadeDbAdminServiceGrpc.newBlockingV2Stub(channel()) + .withCallCredentials(createCallCredentials(userName, userPassword)); + } + + return this.adminServiceBlockingV2Stub; + } + + @PreDestroy + @Override + public void close() { + if (channel == null) + return; + try { + channel.shutdown(); + if (!channel.awaitTermination(5, TimeUnit.SECONDS)) { + channel.shutdownNow(); + channel.awaitTermination(2, TimeUnit.SECONDS); + } + } + catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + channel.shutdownNow(); + } + finally { + // log.info("gRPC channel to {}:{} closed.", host, port); + } + } + + private > S withDeadline(S stub, long timeoutMs) { + long t = (timeoutMs > 0) ? timeoutMs : defaultTimeoutMs; + return stub.withDeadlineAfter(t, TimeUnit.MILLISECONDS); + } + + public DatabaseCredentials buildCredentials() { + return DatabaseCredentials.newBuilder().setUsername(userName == null ? "" : userName) + .setPassword(userPassword == null ? "" : userPassword).build(); + } + + /** Returns the list of database names from the server. */ + public List listDatabases() { + + ListDatabasesResponse resp = null; + + try { + + resp = withDeadline(adminServiceBlockingV2Stub(), defaultTimeoutMs) + .listDatabases(ListDatabasesRequest.newBuilder().setCredentials(buildCredentials()).build()); + return resp.getDatabasesList(); + } + catch (StatusException e) { + + throw new RuntimeException("Failed to list databases: " + e.getMessage(), e); + } + } + + /** Checks existence by listing (no ExistsDatabaseRequest needed). */ + public boolean existsDatabase(final String database) { + return listDatabases().stream().anyMatch(n -> n.equalsIgnoreCase(database)); + } + + /** Creates a database with type "graph" or "document". */ + public void createDatabase(final String database) { + + try { + withDeadline(adminServiceBlockingV2Stub(), defaultTimeoutMs) + .createDatabase(CreateDatabaseRequest.newBuilder().setName(database).setCredentials(buildCredentials()).build()); + } + catch (StatusException e) { + throw new RuntimeException("Failed to create database: " + e.getMessage(), e); + } + } + + /** No-op if already present; creates otherwise. */ + public void createDatabaseIfMissing(final String database) { + if (!existsDatabase(database)) { + createDatabase(database); + } + } + + /** Drops a database. If your proto supports 'force', add it here. */ + public void dropDatabase(final String database) { + + try { + withDeadline(adminServiceBlockingV2Stub(), defaultTimeoutMs) + .dropDatabase(DropDatabaseRequest.newBuilder().setName(database).setCredentials(buildCredentials()).build()); + } + catch (StatusException e) { + throw new RuntimeException("Failed to drop database: " + e.getMessage(), e); + } + } + + public String endpoint() { + return host + ":" + port; + } + + @Override + public String toString() { + return "RemoteGrpcServer{endpoint=" + endpoint() + ", userName='" + userName + "'}"; + } + + /** + * Creates call credentials for authentication + */ + protected CallCredentials createCallCredentials(String userName, String userPassword) { + return new CallCredentials() { + @Override + public void applyRequestMetadata(RequestInfo requestInfo, Executor appExecutor, MetadataApplier applier) { + Metadata headers = new Metadata(); + headers.put(Metadata.Key.of("username", Metadata.ASCII_STRING_MARSHALLER), userName); + headers.put(Metadata.Key.of("password", Metadata.ASCII_STRING_MARSHALLER), userPassword); + headers.put(Metadata.Key.of("x-arcade-user", Metadata.ASCII_STRING_MARSHALLER), userName); + headers.put(Metadata.Key.of("x-arcade-password", Metadata.ASCII_STRING_MARSHALLER), userPassword); + applier.apply(headers); + } + + @Override + public void thisUsesUnstableApi() { + // Required by the interface + } + }; + } + + protected CallCredentials createCredentials() { + return new CallCredentials() { + @Override + public void applyRequestMetadata(RequestInfo requestInfo, Executor appExecutor, MetadataApplier applier) { + Metadata headers = new Metadata(); + headers.put(Metadata.Key.of("username", Metadata.ASCII_STRING_MARSHALLER), userName); + headers.put(Metadata.Key.of("password", Metadata.ASCII_STRING_MARSHALLER), userPassword); + headers.put(Metadata.Key.of("x-arcade-user", Metadata.ASCII_STRING_MARSHALLER), userName); + headers.put(Metadata.Key.of("x-arcade-password", Metadata.ASCII_STRING_MARSHALLER), userPassword); + applier.apply(headers); + } + + // x-arcade-user: root" -H "x-arcade-password: oY9uU2uJ8nD8iY7t" -H + // "x-arcade-database: local_shakeiq_curonix_poc-app" + @Override + public void thisUsesUnstableApi() { + // Required by the interface + } + }; + } +} \ No newline at end of file diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcTransactionExplicitLock.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcTransactionExplicitLock.java new file mode 100644 index 0000000000..ecea2ac893 --- /dev/null +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/RemoteGrpcTransactionExplicitLock.java @@ -0,0 +1,74 @@ +/* + * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com) + * SPDX-License-Identifier: Apache-2.0 + */ +/* Generated By:JJTree: Do not edit this line. OAlterClassStatement.java Version 4.3 */ +/* JavaCCOptions:MULTI=true,NODE_USES_PARSER=false,VISITOR=true,TRACK_TOKENS=true,NODE_PREFIX=O,NODE_EXTENDS=,NODE_FACTORY=,SUPPORT_USERTYPE_VISIBILITY_PUBLIC=true */ +package com.arcadedb.remote.grpc; + +import com.arcadedb.database.TransactionExplicitLock; +import com.arcadedb.remote.RemoteTransactionExplicitLock; + +import java.util.*; + +/** + * Explicit lock on a transaction to lock buckets and types in pessimistic way. + * This avoids the retry mechanism of default implicit locking. + * + * @author Luca Garulli (l.garulli@arcadedata.com) + */ +public class RemoteGrpcTransactionExplicitLock extends RemoteTransactionExplicitLock implements TransactionExplicitLock { + private final RemoteGrpcDatabase database; + private final Set lockedTypeNames = new HashSet<>(); + private final Set lockedBucketNames = new HashSet<>(); + + public RemoteGrpcTransactionExplicitLock(final RemoteGrpcDatabase database) { + super(database); + this.database = database; + } + + @Override + public RemoteGrpcTransactionExplicitLock bucket(final String bucketName) { + lockedBucketNames.add(bucketName); + return this; + } + + @Override + public RemoteGrpcTransactionExplicitLock type(final String typeName) { + lockedTypeNames.add(typeName); + return this; + } + + @Override + public void lock() { + final StringBuilder command = new StringBuilder(); + command.append("LOCK"); + if (!lockedTypeNames.isEmpty()) { + command.append(" TYPE "); + command.append(String.join(", ", lockedTypeNames)); + } + if (!lockedBucketNames.isEmpty()) { + command.append(" BUCKET "); + command.append(String.join(", ", lockedBucketNames)); + } + + lockedBucketNames.clear(); + lockedTypeNames.clear(); + + database.command("sql", command.toString()); + } +} diff --git a/grpc-client/src/main/java/com/arcadedb/remote/grpc/utils/ProtoUtils.java b/grpc-client/src/main/java/com/arcadedb/remote/grpc/utils/ProtoUtils.java new file mode 100644 index 0000000000..5118caa59c --- /dev/null +++ b/grpc-client/src/main/java/com/arcadedb/remote/grpc/utils/ProtoUtils.java @@ -0,0 +1,358 @@ +package com.arcadedb.remote.grpc.utils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.Map; + +import com.arcadedb.database.Document; +import com.arcadedb.database.RID; +import com.arcadedb.database.Record; +import com.arcadedb.graph.Edge; +import com.arcadedb.graph.Vertex; +import com.arcadedb.server.grpc.GrpcRecord; +import com.arcadedb.server.grpc.GrpcValue; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.ToNumberPolicy; + +public class ProtoUtils { + + private static final Logger logger = LoggerFactory.getLogger(ProtoUtils.class); + + private static final Gson GSON = new GsonBuilder().disableHtmlEscaping().serializeNulls(). + setObjectToNumberStrategy(ToNumberPolicy.LONG_OR_DOUBLE).create(); + + // Logging config via system properties (optional) + private static final int LOG_STR_PREVIEW = Integer.getInteger("arcadedb.grpc.log.preview", 96); + private static final boolean LOG_SHOW_STR_AT_DEBUG = Boolean.getBoolean("arcadedb.grpc.log.showString"); + + // Optional per-thread context for logs + private static final ThreadLocal LOG_CTX = new ThreadLocal<>(); + + public static AutoCloseable pushLogContext(String ctx) { + final String prev = LOG_CTX.get(); + LOG_CTX.set(ctx); + return () -> LOG_CTX.set(prev); + } + + private static String ctx() { + String c = LOG_CTX.get(); + return (c == null || c.isEmpty()) ? "" : " " + c; + } + + private static String summarizeJava(Object o) { + if (o == null) + return "null"; + try { + if (o instanceof CharSequence s) { + if (logger.isTraceEnabled() || LOG_SHOW_STR_AT_DEBUG) { + String txt = s.toString(); + return "String(" + s.length() + ")=\"" + (txt.length() > LOG_STR_PREVIEW ? txt.substring(0, LOG_STR_PREVIEW) + "…" : txt) + + "\""; + } + return "String(" + s.length() + ")"; + } + if (o instanceof byte[] b) + return "bytes[" + b.length + "]"; + if (o instanceof java.util.Collection c) + return o.getClass().getSimpleName() + "[size=" + c.size() + "]"; + if (o instanceof java.util.Map m) + return o.getClass().getSimpleName() + "[size=" + m.size() + "]"; + return o.getClass().getSimpleName() + "(" + String.valueOf(o) + ")"; + } + catch (Exception e) { + return o.getClass().getSimpleName(); + } + } + + private static String summarizeGrpc(GrpcValue v) { + if (v == null) + return "GrpcValue(null)"; + switch (v.getKindCase()) { + case BOOL_VALUE: + return "BOOL(" + v.getBoolValue() + ")"; + case INT32_VALUE: + return "INT32(" + v.getInt32Value() + ")"; + case INT64_VALUE: + return "INT64(" + v.getInt64Value() + ")"; + case FLOAT_VALUE: + return "FLOAT(" + v.getFloatValue() + ")"; + case DOUBLE_VALUE: + return "DOUBLE(" + v.getDoubleValue() + ")"; + case STRING_VALUE: { + String s = v.getStringValue(); + if (logger.isTraceEnabled() || LOG_SHOW_STR_AT_DEBUG) { + String txt = s; + return "STRING(" + s.length() + ")=\"" + (txt.length() > LOG_STR_PREVIEW ? txt.substring(0, LOG_STR_PREVIEW) + "…" : txt) + "\""; + } + return "STRING(" + s.length() + ")"; + } + case BYTES_VALUE: + return "BYTES[" + v.getBytesValue().size() + "]"; + case TIMESTAMP_VALUE: + return "TIMESTAMP(" + v.getTimestampValue().getSeconds() + "." + v.getTimestampValue().getNanos() + ")"; + case LIST_VALUE: + return "LIST[size=" + v.getListValue().getValuesCount() + "]"; + case MAP_VALUE: + return "MAP[size=" + v.getMapValue().getEntriesCount() + "]"; + case EMBEDDED_VALUE: + return "EMBEDDED[type=" + v.getEmbeddedValue().getType() + ", size=" + v.getEmbeddedValue().getFieldsCount() + "]"; + case LINK_VALUE: + return "LINK(" + v.getLinkValue().getRid() + ")"; + case DECIMAL_VALUE: + return "DECIMAL(unscaled=" + v.getDecimalValue().getUnscaled() + ", scale=" + v.getDecimalValue().getScale() + ")"; + case KIND_NOT_SET: + default: + return "GrpcValue(KIND_NOT_SET)"; + } + } + + private static GrpcValue dbgEnc(String where, Object in, GrpcValue out) { + if (logger.isDebugEnabled()) + logger.debug("CLIENT GRPC-ENC [{}]{} in={} -> out={}", where, ctx(), summarizeJava(in), summarizeGrpc(out)); + return out; + } + + private static Object dbgDec(String where, GrpcValue in, Object out) { + if (logger.isDebugEnabled()) + logger.debug("CLIENT GRPC-DEC [{}]{} in={} -> out={}", where, ctx(), summarizeGrpc(in), summarizeJava(out)); + return out; + } + + /** + * Converts an ArcadeDB Record into a gRPC GrpcRecord message. + * + * @param rec ArcadeDB record (Document, Vertex, or Edge) + * @return Proto GrpcRecord message + */ + public static GrpcRecord toProtoRecord(Record rec) { + + if (rec == null) { + throw new IllegalArgumentException("Record cannot be null"); + } + + GrpcRecord.Builder builder = GrpcRecord.newBuilder(); + + // Set RID if available + if (rec.getIdentity() != null) { + builder.setRid(rec.getIdentity().toString()); + } + + // Determine logical type/class name + String typeName = null; + + if (rec instanceof Vertex) { + Vertex vertex = rec.asVertex(); + if (vertex != null && vertex.getType() != null) { + typeName = vertex.getType().getName(); + } + } + else if (rec instanceof Edge) { + Edge edge = rec.asEdge(); + if (edge != null && edge.getType() != null) { + typeName = edge.getType().getName(); + } + } + else if (rec instanceof Document) { + Document doc = rec.asDocument(); + if (doc != null && doc.getType() != null) { + typeName = doc.getType().getName(); + } + } + + if (typeName != null && !typeName.isEmpty()) { + builder.setType(typeName); + } + + // Convert properties + if (rec instanceof Document) { + Document doc = (Document) rec; + for (String propName : doc.getPropertyNames()) { + Object value = doc.get(propName); + + System.out.print("toProtoRecord: " + propName + ": " + value); + + builder.putProperties(propName, toGrpcValue(value)); + } + } + + return builder.build(); + } + + /** + * Converts a Map into a gRPC GrpcRecord message. + * + * @param map Map with properties + * @param rid Optional RID + * @param type Optional type name + * @return Proto GrpcRecord message + */ + public static GrpcRecord mapToProtoRecord(Map map, String rid, String type) { + GrpcRecord.Builder builder = GrpcRecord.newBuilder(); + + if (rid != null && !rid.isEmpty()) { + builder.setRid(rid); + } + + if (type != null && !type.isEmpty()) { + builder.setType(type); + } + + for (Map.Entry entry : map.entrySet()) { + builder.putProperties(entry.getKey(), toGrpcValue(entry.getValue())); + } + + return builder.build(); + } + + /** + * Converts a Java Object into a GrpcValue for GrpcRecord.properties map. + * + * Supports primitives, strings, numbers, booleans, lists, maps, dates, and + * RIDs. + * + * @param value Object value + * @return GrpcValue + */ + + public static GrpcValue toGrpcValue(Object value) { + GrpcValue.Builder b = GrpcValue.newBuilder(); + if (value == null) + return dbgEnc("toGrpcValue", value, b.build()); + + if (value instanceof Boolean v) + return dbgEnc("toGrpcValue", value, b.setBoolValue(v).build()); + if (value instanceof Integer v) + return dbgEnc("toGrpcValue", value, b.setInt32Value(v).build()); + if (value instanceof Long v) + return dbgEnc("toGrpcValue", value, b.setInt64Value(v).build()); + if (value instanceof Float v) + return dbgEnc("toGrpcValue", value, b.setFloatValue(v).build()); + if (value instanceof Double v) + return dbgEnc("toGrpcValue", value, b.setDoubleValue(v).build()); + if (value instanceof CharSequence v) + return dbgEnc("toGrpcValue", value, b.setStringValue(v.toString()).build()); + if (value instanceof byte[] v) + return dbgEnc("toGrpcValue", value, b.setBytesValue(com.google.protobuf.ByteString.copyFrom(v)).build()); + + if (value instanceof java.util.Date v) { + return dbgEnc("toGrpcValue", value, + GrpcValue.newBuilder().setTimestampValue(com.google.protobuf.Timestamp.newBuilder() + .setSeconds(Math.floorDiv(v.getTime(), 1000L)).setNanos((int) Math.floorMod(v.getTime(), 1000L) * 1_000_000).build()) + .setLogicalType("datetime").build()); + } + + if (value instanceof RID rid) { + return dbgEnc("toGrpcValue", value, GrpcValue.newBuilder() + .setLinkValue(com.arcadedb.server.grpc.GrpcLink.newBuilder().setRid(rid.toString()).build()).setLogicalType("rid").build()); + } + + if (value instanceof java.math.BigDecimal v) { + java.math.BigInteger unscaled = v.unscaledValue(); + if (unscaled.bitLength() <= 63) { + return dbgEnc("toGrpcValue", value, + GrpcValue.newBuilder().setDecimalValue( + com.arcadedb.server.grpc.GrpcDecimal.newBuilder().setUnscaled(unscaled.longValue()).setScale(v.scale()).build()) + .setLogicalType("decimal").build()); + } + else { + return dbgEnc("toGrpcValue", value, GrpcValue.newBuilder().setStringValue(v.toPlainString()).setLogicalType("decimal").build()); + } + } + + if (value instanceof Document edoc && (edoc.getIdentity() == null)) { + com.arcadedb.server.grpc.GrpcEmbedded.Builder eb = com.arcadedb.server.grpc.GrpcEmbedded.newBuilder(); + if (edoc.getType() != null) + eb.setType(edoc.getType().getName()); + for (String k : edoc.getPropertyNames()) { + eb.putFields(k, toGrpcValue(edoc.get(k))); + } + return dbgEnc("toGrpcValue", value, GrpcValue.newBuilder().setEmbeddedValue(eb.build()).build()); + } + + if (value instanceof Map m) { + com.arcadedb.server.grpc.GrpcMap.Builder mb = com.arcadedb.server.grpc.GrpcMap.newBuilder(); + for (var e : m.entrySet()) { + mb.putEntries(String.valueOf(e.getKey()), toGrpcValue(e.getValue())); + } + return dbgEnc("toGrpcValue", value, GrpcValue.newBuilder().setMapValue(mb.build()).build()); + } + + if (value instanceof Collection c) { + com.arcadedb.server.grpc.GrpcList.Builder lb = com.arcadedb.server.grpc.GrpcList.newBuilder(); + for (Object e : c) + lb.addValues(toGrpcValue(e)); + return dbgEnc("toGrpcValue", value, GrpcValue.newBuilder().setListValue(lb.build()).build()); + } + + return dbgEnc("toGrpcValue", value, GrpcValue.newBuilder().setStringValue(String.valueOf(value)).build()); + } + + public static Object fromGrpcValue(GrpcValue v) { + + switch (v.getKindCase()) { + case BOOL_VALUE: + return dbgDec("fromGrpcValue", v, v.getBoolValue()); + case INT32_VALUE: + return dbgDec("fromGrpcValue", v, v.getInt32Value()); + case INT64_VALUE: + return dbgDec("fromGrpcValue", v, v.getInt64Value()); + case FLOAT_VALUE: + return dbgDec("fromGrpcValue", v, v.getFloatValue()); + case DOUBLE_VALUE: + return dbgDec("fromGrpcValue", v, v.getDoubleValue()); + case STRING_VALUE: + return dbgDec("fromGrpcValue", v, v.getStringValue()); + case BYTES_VALUE: + + if ("json".equalsIgnoreCase(v.getLogicalType())) { + String json = v.getBytesValue().toStringUtf8(); + + //System.out.println("JSON: " + json); + + @SuppressWarnings("unchecked") + java.util.Map map = GSON.fromJson(json, java.util.Map.class); + + return dbgDec("fromGrpcValue", v, map); + } + return dbgDec("fromGrpcValue", v, v.getBytesValue().toByteArray()); + + case TIMESTAMP_VALUE: + return dbgDec("fromGrpcValue", v, tsToMillis(v.getTimestampValue())); // or Instant + case LIST_VALUE: { + var out = new java.util.ArrayList<>(); + for (GrpcValue e : v.getListValue().getValuesList()) + out.add(fromGrpcValue(e)); + return dbgDec("fromGrpcValue", v, out); + } + case MAP_VALUE: { + var out = new java.util.LinkedHashMap(); + v.getMapValue().getEntriesMap().forEach((k, vv) -> out.put(k, fromGrpcValue(vv))); + return dbgDec("fromGrpcValue", v, out); + } + case EMBEDDED_VALUE: { + var out = new java.util.LinkedHashMap(); + v.getEmbeddedValue().getFieldsMap().forEach((k, vv) -> out.put(k, fromGrpcValue(vv))); + // v.getEmbeddedValue().getType() is available if you want to instantiate a + // typed DTO + return dbgDec("fromGrpcValue", v, out); + } + case LINK_VALUE: + return dbgDec("fromGrpcValue", v, v.getLinkValue().getRid()); // or a Link object + case DECIMAL_VALUE: { + var d = v.getDecimalValue(); + return dbgDec("fromGrpcValue", v, new java.math.BigDecimal(java.math.BigInteger.valueOf(d.getUnscaled()), d.getScale())); + } + case KIND_NOT_SET: + return dbgDec("fromGrpcValue", v, null); + } + + return dbgDec("fromGrpcValue", v, null); + } + + static long tsToMillis(com.google.protobuf.Timestamp ts) { + return ts.getSeconds() * 1000L + ts.getNanos() / 1_000_000L; + } +} \ No newline at end of file diff --git a/grpc-client/src/test/java/com/arcadedb/remote/grpc/ArcadeDbHTTPTvsGRPCBench.java b/grpc-client/src/test/java/com/arcadedb/remote/grpc/ArcadeDbHTTPTvsGRPCBench.java new file mode 100644 index 0000000000..abbe71d7c2 --- /dev/null +++ b/grpc-client/src/test/java/com/arcadedb/remote/grpc/ArcadeDbHTTPTvsGRPCBench.java @@ -0,0 +1,322 @@ +package com.arcadedb.remote.grpc; + +import java.io.File; +import java.nio.file.Files; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import com.arcadedb.remote.RemoteDatabase; // HTTP client +import com.arcadedb.server.grpc.InsertOptions; +import com.arcadedb.server.grpc.InsertOptions.ConflictMode; +import com.arcadedb.server.grpc.InsertOptions.TransactionMode; +import com.arcadedb.server.grpc.InsertSummary; +import com.arcadedb.server.grpc.StreamQueryRequest; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class ArcadeDbHTTPTvsGRPCBench { + + // ---- Config (edit or pass via env/args) ---- + static String DB_NAME = System.getenv().getOrDefault("ARCADE_DB", "ArcadeDB"); + static String HTTP_HOST = System.getenv().getOrDefault("ARCADE_HTTP_HOST", "127.0.0.1"); + static int HTTP_PORT = Integer.parseInt(System.getenv().getOrDefault("ARCADE_HTTP_PORT", "2489")); + static String GRPC_HOST = System.getenv().getOrDefault("ARCADE_GRPC_HOST", "127.0.0.1"); + static int GRPC_PORT = Integer.parseInt(System.getenv().getOrDefault("ARCADE_GRPC_PORT", "50059")); + + static String USER = System.getenv().getOrDefault("ARCADE_USER", "root"); + static String PASS = System.getenv().getOrDefault("ARCADE_PASS", "root1234"); + + static ObjectMapper objectMapper = new ObjectMapper(); + + public static void main(String[] args) throws Exception { + + // override via args if provided: host/ports/db/user/pass + for (String a : args) { + if (a.startsWith("--db=")) + DB_NAME = a.substring(5); + else if (a.startsWith("--http=")) { + var hp = a.substring(7).split(":"); + HTTP_HOST = hp[0]; + HTTP_PORT = Integer.parseInt(hp[1]); + } + else if (a.startsWith("--grpc=")) { + var hp = a.substring(7).split(":"); + GRPC_HOST = hp[0]; + GRPC_PORT = Integer.parseInt(hp[1]); + } + else if (a.startsWith("--user=")) + USER = a.substring(7); + else if (a.startsWith("--pass=")) + PASS = a.substring(7); + } + + System.out.printf("Benchmarking DB='%s' HTTP=%s:%d gRPC=%s:%d as %s%n", DB_NAME, HTTP_HOST, HTTP_PORT, GRPC_HOST, GRPC_PORT, USER); + + RemoteGrpcServer grpcServer = new RemoteGrpcServer(GRPC_HOST, GRPC_PORT, USER, PASS, true, List.of()); + + if (grpcServer.existsDatabase(DB_NAME)) { + + System.out.printf("Database %s exists%n", DB_NAME); + } + else { + + System.out.printf("Creating database %s%n", DB_NAME); + + grpcServer.createDatabase(DB_NAME); + } + + // ---------- Open both clients ---------- + try (RemoteDatabase http = new RemoteDatabase(HTTP_HOST, HTTP_PORT, DB_NAME, USER, PASS); + RemoteGrpcDatabase grpc = new RemoteGrpcDatabase(grpcServer, GRPC_HOST, GRPC_PORT, HTTP_PORT, DB_NAME, USER, PASS)) { + + // ensure schema aligned + prepareSchemaHTTP(http); + prepareSchemaGRPC(grpc); + + // --- apples-to-apples data set (clear prior run for repeatability) --- + cleanupHTTP(http); + cleanupGRPC(grpc); + + // ---------- BULK INSERT (100 rows) ---------- + List> rows = buildFeedbackRows(100); + + String jsonContent = objectMapper.writeValueAsString(rows); + + System.out.printf("JSON:%n%s%n", jsonContent); + + time("HTTP bulkInsert(100)", () -> { + + http.begin(); + + http.command("sqlscript", "INSERT INTO UserFeedback CONTENT " + jsonContent); + + http.commit(); + }); + + time("gRPC bulkInsert(100)", () -> { + grpc.begin(); + InsertOptions opts = defaultInsertOptions(grpc, "UserFeedback", List.of("id")); // id unique -> idempotent + InsertSummary s = grpc.insertBulkAsListOfMaps(opts, rows, 120_000); + grpc.commit(); + System.out.printf(" gRPC summary: received=%d, inserted=%d, updated=%d, ignored=%d, failed=%d, errors=%d%n", s.getReceived(), + s.getInserted(), s.getUpdated(), s.getIgnored(), s.getFailed(), s.getErrorsCount()); + }); + + // ---------- Single insert ---------- + Map one = uf("UF-SINGLE", "tenantZ", "UI", "FEATURE", "Single insert via API", null, Instant.now()); + time("HTTP insert(1)", () -> { + http.begin(); + http.command("sqlscript", "INSERT INTO UserFeedback CONTENT :m", Map.of("m", one)); + http.commit(); + }); + time("gRPC insert(1)", () -> { + grpc.begin(); + InsertOptions opts = defaultInsertOptions(grpc, "UserFeedback", List.of("id")); + grpc.insertBulkAsListOfMaps(opts, List.of(one), 60_000); + grpc.commit(); + }); + + // ---------- Update ---------- + time("HTTP update by id", () -> { + http.begin(); + http.command("sqlscript", "UPDATE UserFeedback SET feedback = 'UPDATED_HTTP' WHERE id = 'UF-0001'"); + http.commit(); + }); + time("gRPC update by id", () -> { + grpc.begin(); + grpc.command("sql", "UPDATE UserFeedback SET feedback = 'UPDATED_GRPC' WHERE id = 'UF-0001'", Map.of(), false, 0, + /* tx */ null, 60_000); + grpc.commit(); + }); + + // ---------- Point read by index ---------- + time("HTTP point read (id)", () -> { + var rs = http.query("sql", "SELECT FROM UserFeedback WHERE id = 'UF-0001'"); + int count = 0; + while (rs.hasNext()) { + rs.next(); + count++; + } + System.out.println(" HTTP rows=" + count); + }); + time("gRPC point read (id)", () -> { + var it = grpc.queryStream(DB_NAME, "SELECT FROM UserFeedback WHERE id = 'UF-0001'", Map.of(), 50, + StreamQueryRequest.RetrievalMode.CURSOR, /* tx */ null, 60_000); + int count = 0; + while (it.hasNext()) { + it.next(); + count++; + } + System.out.println(" gRPC rows=" + count); + }); + + // ---------- Streaming query (3 modes) ---------- + runStreamBench(grpc, "SELECT FROM UserFeedback", 200, StreamQueryRequest.RetrievalMode.CURSOR); + runStreamBench(grpc, "SELECT FROM UserFeedback", 200, StreamQueryRequest.RetrievalMode.MATERIALIZE_ALL); + runStreamBench(grpc, "SELECT FROM UserFeedback", 200, StreamQueryRequest.RetrievalMode.PAGED); + + // ---------- Delete subset ---------- + time("HTTP delete subset", () -> { + http.begin(); + http.command("sql", "DELETE FROM UserFeedback WHERE id LIKE 'UF-00%'"); + http.commit(); + }); + time("gRPC delete subset", () -> { + grpc.begin(); + grpc.executeCommand("sql", "DELETE FROM UserFeedback WHERE id LIKE 'UF-00%'", Map.of(), false, 0, null, 60_000); + grpc.commit(); + }); + + } // try-with-resources + } + + // ------------------------------------------------------ + // Schema prep - HTTP + // ------------------------------------------------------ + private static void prepareSchemaHTTP(RemoteDatabase http) { + // Create vertex type + http.command("sql", "CREATE VERTEX TYPE UserFeedback IF NOT EXISTS"); + + // Create properties + http.command("sqlscript", "CREATE PROPERTY UserFeedback.id IF NOT EXISTS STRING"); + http.command("sqlscript", "CREATE PROPERTY UserFeedback.applicationArea IF NOT EXISTS STRING"); + http.command("sqlscript", "CREATE PROPERTY UserFeedback.empowerTenantId IF NOT EXISTS STRING"); + http.command("sqlscript", "CREATE PROPERTY UserFeedback.empowerType IF NOT EXISTS STRING"); + http.command("sqlscript", "CREATE PROPERTY UserFeedback.feedback IF NOT EXISTS STRING"); + http.command("sqlscript", "CREATE PROPERTY UserFeedback.timestamp IF NOT EXISTS DATETIME"); + http.command("sqlscript", "CREATE PROPERTY UserFeedback.image IF NOT EXISTS EMBEDDED"); + + // Create index + http.command("sqlscript", "CREATE INDEX IF NOT EXISTS ON UserFeedback (id) UNIQUE"); + } + + // ------------------------------------------------------ + // Schema prep - gRPC + // ------------------------------------------------------ + private static void prepareSchemaGRPC(RemoteGrpcDatabase grpc) { + + // Create vertex type + grpc.executeCommand("sqlscript", "CREATE VERTEX TYPE UserFeedback IF NOT EXISTS", Map.of(), false, 0, null, 60_000); + + // Create properties + grpc.executeCommand("sqlscript", "CREATE PROPERTY UserFeedback.id IF NOT EXISTS STRING", Map.of(), false, 0, null, 60_000); + grpc.executeCommand("sqlscript", "CREATE PROPERTY UserFeedback.applicationArea IF NOT EXISTS STRING", Map.of(), false, 0, null, 60_000); + grpc.executeCommand("sqlscript", "CREATE PROPERTY UserFeedback.empowerTenantId IF NOT EXISTS STRING", Map.of(), false, 0, null, 60_000); + grpc.executeCommand("sqlscript", "CREATE PROPERTY UserFeedback.empowerType IF NOT EXISTS STRING", Map.of(), false, 0, null, 60_000); + grpc.executeCommand("sqlscript", "CREATE PROPERTY UserFeedback.feedback IF NOT EXISTS STRING", Map.of(), false, 0, null, 60_000); + grpc.executeCommand("sqlscript", "CREATE PROPERTY UserFeedback.timestamp IF NOT EXISTS DATETIME", Map.of(), false, 0, null, 60_000); + grpc.executeCommand("sqlscript", "CREATE PROPERTY UserFeedback.image IF NOT EXISTS EMBEDDED", Map.of(), false, 0, null, 60_000); + + // Create index + grpc.executeCommand("sqlscript", "CREATE INDEX IF NOT EXISTS ON UserFeedback (id) UNIQUE", Map.of(), false, 0, null, 60_000); + } + + private static void cleanupHTTP(RemoteDatabase http) { + http.command("sql", "DELETE FROM UserFeedback"); + } + + private static void cleanupGRPC(RemoteGrpcDatabase grpc) { + grpc.executeCommand("sql", "DELETE FROM UserFeedback", Map.of(), false, 0, null, 60_000); + } + + // ------------------------------------------------------ + // Workloads + // ------------------------------------------------------ + + private static void runStreamBench(RemoteGrpcDatabase grpc, String sql, int batchSize, StreamQueryRequest.RetrievalMode mode) { + time("gRPC stream " + mode + " (batchSize=" + batchSize + ")", () -> { + + var it = grpc.queryStream(DB_NAME, sql, Map.of(), batchSize, mode, /* tx */ null, 120_000); + var total = new AtomicInteger(0); + while (it.hasNext()) { + it.next(); + total.incrementAndGet(); + } + + System.out.println(" rows=" + total.get()); + }); + } + + // ------------------------------------------------------ + // Data builders + // ------------------------------------------------------ + + private static List> buildFeedbackRows(int n) throws Exception { + // optional: shared embedded image payload + byte[] png = loadDefaultPNG(); + String b64 = (png != null) ? Base64.getEncoder().encodeToString(png) : null; + Map embeddedImage = (b64 != null) ? Map.of("mime", "image/png", "data", b64) : null; + + List> out = new ArrayList<>(n); + for (int i = 1; i <= n; i++) { + out.add(uf(String.format("UF-%04d", i), "tenantA", "UI", "USER_FEEDBACK", "Bulk feedback #" + i, embeddedImage, Instant.now())); + } + return out; + } + + private static Map uf(String id, String tenant, String area, String type, String feedback, Map image, + Instant ts) { + Map m = new LinkedHashMap<>(); + m.put("id", id); + m.put("empowerTenantId", tenant); + m.put("applicationArea", area); + m.put("empowerType", type); + m.put("feedback", feedback); + if (image != null) + m.put("image", image); // EMBEDDED map (server coerces to embedded doc) + m.put("timestamp", Date.from(ts)); + return m; + } + + private static byte[] loadDefaultPNG() { + try { + File f = new File("./data/Curonix/Logos/default.png"); + if (!f.exists()) + return null; + return Files.readAllBytes(f.toPath()); + } + catch (Exception ignore) { + return null; + } + } + + private static InsertOptions defaultInsertOptions(RemoteGrpcDatabase grpc, String targetClass, List keyCols) { + // Minimal options: target class + key columns for idempotency; update + // last-write-wins + return InsertOptions.newBuilder().setDatabase(DB_NAME).setTargetClass(targetClass).addAllKeyColumns(keyCols) + .setConflictMode(ConflictMode.CONFLICT_UPDATE) + .addAllUpdateColumnsOnConflict(Arrays.asList("feedback", "image", "timestamp", "applicationArea", "empowerType")) + .setTransactionMode(TransactionMode.PER_BATCH).setServerBatchSize(500) + .setCredentials(grpc.buildCredentials()) // if your server validates per-request creds + .build(); + } + + // ------------------------------------------------------ + // Timing helper + // ------------------------------------------------------ + + @FunctionalInterface + interface ThrowingRunnable { + void run() throws Exception; + } + + private static void time(String label, ThrowingRunnable r) { + long t0 = System.currentTimeMillis(); + try { + r.run(); + long ms = System.currentTimeMillis() - t0; + System.out.printf("%-35s %8d ms%n", label + ":", ms); + } + catch (Exception e) { + long ms = System.currentTimeMillis() - t0; + System.out.printf("%-35s %8d ms (ERROR: %s)%n", label + ":", ms, e.getMessage()); + throw new RuntimeException(e); + } + } +} \ No newline at end of file diff --git a/grpc-client/src/test/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseRegressionTest.java b/grpc-client/src/test/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseRegressionTest.java new file mode 100644 index 0000000000..5a6512d769 --- /dev/null +++ b/grpc-client/src/test/java/com/arcadedb/remote/grpc/RemoteGrpcDatabaseRegressionTest.java @@ -0,0 +1,285 @@ +package com.arcadedb.remote.grpc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +import com.arcadedb.query.sql.executor.Result; +import com.arcadedb.query.sql.executor.ResultSet; +import com.arcadedb.server.grpc.InsertOptions; +import com.arcadedb.server.grpc.InsertOptions.ConflictMode; +import com.arcadedb.server.grpc.InsertOptions.TransactionMode; +import com.arcadedb.server.grpc.InsertSummary; + +/** + * Regression tests that exercise ONLY the gRPC Remote database client. + * + * Requirements: - ArcadeDB server must be running with gRPC enabled. - + * Connection parameters may be provided via env vars: ARCADE_DB, + * ARCADE_GRPC_HOST, ARCADE_GRPC_PORT, ARCADE_HTTP_PORT, ARCADE_USER, + * ARCADE_PASS + * + * Defaults (match the sample bench): DB=ArcadeDB, gRPC=127.0.0.1:50059, + * HTTP=127.0.0.1:2489, user=root, pass=root1234 + * + * These tests create an isolated vertex type "RG_Feedback" and clean up their + * own data. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class RemoteGrpcDatabaseRegressionTest { + + // -------- Config (env overrides supported) -------- + + static final String DB_NAME = System.getenv().getOrDefault("ARCADE_DB", "test"); + static final String GRPC_HOST = System.getenv().getOrDefault("ARCADE_GRPC_HOST", "127.0.0.1"); + static final int GRPC_PORT = Integer.parseInt(System.getenv().getOrDefault("ARCADE_GRPC_PORT", "50051")); + static final String HTTP_HOST = System.getenv().getOrDefault("ARCADE_HTTP_HOST", "127.0.0.1"); + static final int HTTP_PORT = Integer.parseInt(System.getenv().getOrDefault("ARCADE_HTTP_PORT", "2480")); + static final String USER = System.getenv().getOrDefault("ARCADE_USER", "root"); + static final String PASS = System.getenv().getOrDefault("ARCADE_PASS", "oY9uU2uJ8nD8iY7t"); + + // Test type & props + static final String TYPE = "RG_Feedback"; + + private RemoteGrpcServer grpcServer; + private RemoteGrpcDatabase grpc; + + @BeforeAll + void ensureDatabaseExists() { + + this.grpcServer = new RemoteGrpcServer(GRPC_HOST, GRPC_PORT, USER, PASS, true, List.of()); + + + // Prefer using the gRPC admin helper if available (same package). + if (!grpcServer.existsDatabase(DB_NAME)) { + grpcServer.createDatabase(DB_NAME); + } + } + + @BeforeEach + void open() { + + grpc = new RemoteGrpcDatabase(this.grpcServer, GRPC_HOST, GRPC_PORT, HTTP_PORT, DB_NAME, USER, PASS); + + // Create isolated schema for these tests (id unique, name string, n integer) + grpc.command("sql", "CREATE VERTEX TYPE `" + TYPE + "` IF NOT EXISTS", Map.of()); + grpc.command("sql", "CREATE PROPERTY `" + TYPE + "`.id IF NOT EXISTS STRING", Map.of()); + grpc.command("sql", "CREATE PROPERTY `" + TYPE + "`.name IF NOT EXISTS STRING", Map.of()); + grpc.command("sql", "CREATE PROPERTY `" + TYPE + "`.n IF NOT EXISTS INTEGER", Map.of()); + grpc.command("sql", "CREATE INDEX IF NOT EXISTS ON " + TYPE + " (id) UNIQUE", Map.of()); + // Ensure clean slate for each test method + grpc.command("sql", "DELETE FROM `" + TYPE + "`", Map.of()); + } + + @AfterEach + void close() { + if (grpc != null) { + try { + grpc.rollback(); + } + catch (Throwable ignore) { + } + grpc.close(); + } + } + + // ---------- Helpers ---------- + + private InsertOptions defaultInsertOptions(final String targetClass, final List keyCols, final List updateCols) { + return InsertOptions.newBuilder().setDatabase(DB_NAME).setTargetClass(targetClass).addAllKeyColumns(keyCols) + .setConflictMode(ConflictMode.CONFLICT_UPDATE) // idempotent: upsert by keys + .addAllUpdateColumnsOnConflict(updateCols) // LWW on these fields + .setTransactionMode(TransactionMode.PER_BATCH).setServerBatchSize(256).setCredentials(grpc.buildCredentials()) // package-private + // in same + // package + .build(); + } + + private long countAll(String type) { + + String sql = "SELECT count(*) AS c FROM " + type; + + try (ResultSet rs = grpc.query("sql", sql, Map.of())) { + long c = 0; + while (rs.hasNext()) { + Result r = rs.next(); + // System.out.println("Count all: r = " + r); + Number n = r.getProperty("c"); + c = (n == null) ? 0 : n.longValue(); + } + return c; + } + } + + private Map row(String id, String name, int n) { + Map m = new HashMap<>(); + m.put("id", id); + m.put("name", name); + m.put("n", n); + return m; + } + + // ---------- Tests ---------- + + @Test + @DisplayName("Bulk insert via gRPC is idempotent by key and supports updates on conflict") + void bulkInsertIdempotentAndUpdate() { + // Prepare rows + List> rows = new ArrayList<>(); + rows.add(row("r1", "alpha", 1)); + rows.add(row("r2", "beta", 2)); + rows.add(row("r3", "gamma", 3)); + + grpc.begin(); + InsertOptions opts = defaultInsertOptions(TYPE, List.of("id"), Arrays.asList("name", "n")); + InsertSummary s1 = grpc.insertBulkAsListOfMaps(opts, rows, 60_000); + grpc.commit(); + + assertEquals(3, s1.getInserted(), "first insert should insert 3"); + assertEquals(0, s1.getUpdated(), "first insert should not update"); + assertEquals(3, countAll(TYPE), "row count after first insert"); + + // Re-insert with one changed record (r2) to force an update-on-conflict + rows.set(1, row("r2", "beta-UPDATED", 22)); + + grpc.begin(); + InsertSummary s2 = grpc.insertBulkAsListOfMaps(opts, rows, 60_000); + grpc.commit(); + + assertEquals(0, s2.getInserted(), "second insert should not insert new rows"); + assertTrue(s2.getUpdated() >= 1, "should update at least 1 row on conflict"); + assertEquals(3, countAll(TYPE), "row count unchanged after upsert"); + + // Verify the updated record + try (ResultSet rs = grpc.query("sql", "SELECT from `" + TYPE + "` WHERE id = :id", Map.of("id", "r2"))) { + assertTrue(rs.hasNext(), "record r2 must exist"); + Result r = rs.next(); + assertEquals("beta-UPDATED", r.getProperty("name")); + assertEquals(22, r.getProperty("n").intValue()); + } + } + + @Test + @DisplayName("Basic CRUD via SQL commands over gRPC") + void basicCrudViaCommand() { + // Create + grpc.command("sql", "INSERT INTO `" + TYPE + "` set id = :id, name = :name, n = :n", Map.of("id", "x1", "name", "one", "n", 1)); + grpc.command("sql", "INSERT INTO `" + TYPE + "` set id = :id, name = :name, n = :n", Map.of("id", "x2", "name", "two", "n", 2)); + + assertEquals(2, countAll(TYPE), "two rows inserted"); + + // Read + try (ResultSet rs = grpc.query("sql", "SELECT from `" + TYPE + "` WHERE id = :id", Map.of("id", "x1"))) { + assertTrue(rs.hasNext()); + Result r = rs.next(); + assertEquals("one", r.getProperty("name")); + } + + // Update + grpc.begin(); + grpc.command("sql", "UPDATE `" + TYPE + "` SET name = :name, n = :n WHERE id = :id", Map.of("name", "ONE!", "n", 11, "id", "x1")); + grpc.commit(); + + try (ResultSet rs = grpc.query("sql", "SELECT from `" + TYPE + "` WHERE id = :id", Map.of("id", "x1"))) { + Result r = rs.next(); + assertEquals("ONE!", r.getProperty("name")); + assertEquals(11, r.getProperty("n").intValue()); + } + + // Delete + grpc.command("sql", "DELETE FROM `" + TYPE + "` WHERE id = :id", Map.of("id", "x2")); + assertEquals(1, countAll(TYPE), "one row remains after delete"); + } + + @Test + @DisplayName("Transaction: rollback undoes changes; commit persists") + void transactionsRollbackAndCommit() { + long before = countAll(TYPE); + + // Rollback path + grpc.begin(); + grpc.command("sql", "INSERT INTO `" + TYPE + "` set id = :id, name = :name, n = :n", Map.of("id", "tx1", "name", "temp", "n", 99)); + grpc.rollback(); + assertEquals(before, countAll(TYPE), "rollback must revert insert"); + + // Commit path + grpc.begin(); + grpc.command("sql", "INSERT INTO `" + TYPE + "` set id = :id, name = :name, n = :n", Map.of("id", "tx2", "name", "persisted", "n", 100)); + grpc.commit(); + assertEquals(before + 1, countAll(TYPE), "commit must persist insert"); + } + + @Test + @DisplayName("schema:types → properties decoded as List>") + void schemaTypesPropertiesDecodedAsMaps() { + + try (ResultSet rs = grpc.query("sql", "SELECT FROM schema:types WHERE name = :name", Map.of("name", TYPE))) { + + assertTrue(rs.hasNext(), "schema:types should contain our test type"); + + Result r = rs.next(); + + Object propsObj = r.getProperty("properties"); + + assertTrue(propsObj instanceof java.util.List, "properties must be a List"); + + java.util.List props = (java.util.List) propsObj; + assertTrue(!props.isEmpty(), "properties list should not be empty"); + + Object first = props.get(0); + assertTrue(first instanceof java.util.Map, "each property is expected to be a Map"); + + java.util.Map p0 = (java.util.Map) first; + + // Spot-check expected keys + assertTrue(p0.containsKey("name"), "property map must have 'name'"); + assertTrue(p0.containsKey("type"), "property map must have 'type'"); + } + } + + @Test + @DisplayName("Embedded _auditMetadata map round-trips with Long timestamps") + void embeddedAuditMetadataRoundTrip() { + + final String recId = "audit1"; + final java.util.Map audit = new java.util.LinkedHashMap<>(); + + audit.put("createdDate", 1720225210408L); + audit.put("createdByUser", "service-account-empower-platform-admin"); + audit.put("lastModifiedDate", 1741795459718L); + audit.put("lastModifiedByUser", "service-account-empower-platform-admin"); + + grpc.command("sql", "INSERT INTO `" + TYPE + "` SET id = :id, name = :name, n = :n, _auditMetadata = :audit", + Map.of("id", recId, "name", "with-audit", "n", 1, "audit", audit)); + + try (ResultSet rs = grpc.query("sql", "SELECT FROM `" + TYPE + "` WHERE id = :id", Map.of("id", recId))) { + assertTrue(rs.hasNext(), "inserted record should be queriable"); + Result r = rs.next(); + Object auditObj = r.getProperty("_auditMetadata"); + assertTrue(auditObj instanceof java.util.Map, "_auditMetadata must be a Map"); + @SuppressWarnings("unchecked") + java.util.Map m = (java.util.Map) auditObj; + + Object cd = m.get("createdDate"); + Object md = m.get("lastModifiedDate"); + assertTrue(cd instanceof Number, "createdDate must be numeric"); + assertTrue(md instanceof Number, "lastModifiedDate must be numeric"); + assertEquals(1720225210408L, ((Number) cd).longValue(), "createdDate must be precise long"); + assertEquals(1741795459718L, ((Number) md).longValue(), "lastModifiedDate must be precise long"); + assertEquals("service-account-empower-platform-admin", m.get("createdByUser")); + assertEquals("service-account-empower-platform-admin", m.get("lastModifiedByUser")); + } + } +} diff --git a/grpc-client/src/test/resources/arcadedb-log.properties b/grpc-client/src/test/resources/arcadedb-log.properties new file mode 100644 index 0000000000..29b52b4421 --- /dev/null +++ b/grpc-client/src/test/resources/arcadedb-log.properties @@ -0,0 +1,32 @@ +# +# Copyright 2021-present Arcade Data Ltd +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Specify the handlers to create in the root logger +# (all loggers are children of the root logger) +# The following creates two handlers +handlers=java.util.logging.ConsoleHandler +# Set the default logging level for the root logger +.level=WARNING +com.arcadedb.level=WARNING +com.arcadedb.remote.grpc.level=FINE +# Set the default logging level for new ConsoleHandler instances +java.util.logging.ConsoleHandler.level=WARNING +# Set the default formatter for new ConsoleHandler instances +java.util.logging.ConsoleHandler.formatter=com.arcadedb.utility.AnsiLogFormatter diff --git a/grpc/gRPC-Readme.txt b/grpc/gRPC-Readme.txt new file mode 100644 index 0000000000..bc24c8bc07 --- /dev/null +++ b/grpc/gRPC-Readme.txt @@ -0,0 +1,422 @@ +# ArcadeDB gRPC Server Configuration Examples + +# ========================================== +# Basic Configuration (arcadedb-server.properties) +# ========================================== + +# Enable gRPC server (default: true) +grpc.enabled=true + +# Server mode: standard, xds, or both (default: standard) +grpc.mode=standard + +# Standard gRPC server port (default: 50051) +grpc.port=50051 + +# Bind host (default: 0.0.0.0) +grpc.host=0.0.0.0 + +# Max message size in MB (default: 100) +grpc.maxMessageSize=100 + +# Enable gRPC reflection for debugging with grpcurl (default: true) +grpc.reflection.enabled=true + +# Enable health checking endpoint (default: true) +grpc.health.enabled=true + +# ========================================== +# XDS Configuration (for Service Mesh) +# ========================================== + +# Enable XDS mode for Envoy/Istio integration +grpc.mode=xds +# or run both standard and XDS +# grpc.mode=both + +# XDS server port (default: 50052) +grpc.xds.port=50052 + +# ========================================== +# TLS Configuration +# ========================================== + +# Enable TLS (default: false) +grpc.tls.enabled=true + +# Path to TLS certificate +grpc.tls.cert=/path/to/server.crt + +# Path to TLS private key +grpc.tls.key=/path/to/server.key + + +arcadedb.grpc.compression.enabled=true # Enable compression support +arcadedb.grpc.compression.force=false # Force compression on all responses +arcadedb.grpc.compression.type=gzip # Compression algorithm + + + +# ========================================== +# Docker Compose Example +# ========================================== +# docker-compose.yml: +# +# version: '3.8' +# services: +# arcadedb: +# image: arcadedb/arcadedb:latest +# ports: +# - "2480:2480" # HTTP +# - "2424:2424" # Binary +# - "50051:50051" # gRPC +# environment: +# - ARCADEDB_OPTS_MEMORY=-Xms2G -Xmx2G +# - arcadedb.grpc.enabled=true +# - arcadedb.grpc.port=50051 +# volumes: +# - ./databases:/var/arcadedb/databases + +# ========================================== +# Kubernetes ConfigMap Example +# ========================================== +# apiVersion: v1 +# kind: ConfigMap +# metadata: +# name: arcadedb-config +# data: +# arcadedb.properties: | +# grpc.enabled=true +# grpc.mode=xds +# grpc.xds.port=50052 +# grpc.health.enabled=true +# grpc.reflection.enabled=false + +# ========================================== +# Istio Service Mesh Example +# ========================================== +# apiVersion: v1 +# kind: Service +# metadata: +# name: arcadedb-grpc +# spec: +# ports: +# - name: grpc-xds +# port: 50052 +# protocol: TCP +# - name: grpc +# port: 50051 +# protocol: TCP +# selector: +# app: arcadedb + + + + + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.6.0 + + + jar-with-dependencies + + + + com.arcadedb.server.grpc.GrpcStandaloneLauncher + + + + + + make-grpc-standalone + package + + single + + + arcadedb-grpc-server-${project.version} + false + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.5.1 + + + package + + shade + + + false + + + com.arcadedb.server.grpc.GrpcStandaloneLauncher + + + + + + META-INF/services/io.grpc.ServerProvider + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + true + grpc-standalone + + + + + + + +properties# Basic +grpc.enabled=true +grpc.port=50051 +grpc.host=0.0.0.0 + +# Advanced +grpc.mode=standard|xds|both +grpc.maxMessageSize=100 +grpc.reflection.enabled=true +grpc.health.enabled=true + +# Security +grpc.tls.enabled=true +grpc.tls.cert=/path/to/cert +grpc.tls.key=/path/to/key + + + + +version: '3.8' +services: + arcadedb: + image: arcadedb/arcadedb:latest + ports: + - "50051:50051" # gRPC + environment: + - arcadedb.grpc.enabled=true + +apiVersion: v1 +kind: Service +metadata: + name: arcadedb-grpc +spec: + ports: + - name: grpc-xds + port: 50052 + protocol: TCP + + + + +# Development +mvn exec:java -Dexec.mainClass="com.arcadedb.server.grpc.GrpcStandaloneLauncher" + +# Production +java -jar arcadedb-grpc-server.jar --config /etc/arcadedb/grpc.properties + + + + +# Using grpcurl +grpcurl -plaintext localhost:50051 list +grpcurl -plaintext localhost:50051 describe com.arcadedb.grpc.ArcadeDbService + +# Health check +grpcurl -plaintext localhost:50051 grpc.health.v1.Health/Check + + + +# Basic Authentication +grpcurl -H "x-arcade-user: admin" \ + -H "x-arcade-password: admin" \ + -H "x-arcade-database: mydb" \ + -plaintext localhost:50051 \ + com.arcadedb.grpc.ArcadeDbService/Ping + +# Bearer Token (for future implementation) +grpcurl -H "authorization: Bearer " \ + -plaintext localhost:50051 \ + com.arcadedb.grpc.ArcadeDbService/Ping + + +server.registerPlugin(new GrpcServerPlugin()); + +grpc.enabled=true +grpc.port=50051 +grpc.mode=standard +grpc.reflection.enabled=true +grpc.health.enabled=true + + + +# Check health +grpcurl -plaintext localhost:50051 grpc.health.v1.Health/Check + +# List services +grpcurl -plaintext localhost:50051 list + +# Test with authentication +grpcurl -H "x-arcade-user: admin" \ + -H "x-arcade-password: admin" \ + -H "x-arcade-database: mydb" \ + -plaintext localhost:50051 \ + com.arcadedb.grpc.ArcadeDbService/Ping + + + + +arcadedb.grpc.enabled=true +arcadedb.grpc.port=50051 +arcadedb.grpc.host=0.0.0.0 +arcadedb.grpc.mode=standard +arcadedb.grpc.reflection.enabled=true +arcadedb.grpc.health.enabled=true +arcadedb.grpc.maxMessageSize=100 + + + +java -Darcadedb.grpc.enabled=true \ + -Darcadedb.grpc.port=50051 \ + -Darcadedb.grpc.mode=standard \ + -jar arcadedb-server.jar + +# Check if gRPC is running +grpcurl -plaintext localhost:50051 list + +# Check health +grpcurl -plaintext localhost:50051 grpc.health.v1.Health/Check + + + + + +# Complete build with Docker images +mvn clean install -DskipTests && \ +cd package && \ +mvn package -Pdocker + +# Multi-platform build (slower but supports ARM and x64) +mvn package -Pdocker,multiplatform + +# Build and push to registry +mvn deploy -Pdocker -Ddocker.username=youruser -Ddocker.password=yourpass + +# Skip tests and build quickly +mvn package -Pdocker -DskipTests -Dmaven.test.skip=true + + + + +# Run the Docker image with gRPC enabled +docker run -d \ + --name arcadedb-test \ + -p 2480:2480 \ + -p 50051:50051 \ + -e arcadedb.grpc.enabled=true \ + arcadedata/arcadedb:latest + + +# Run the Docker image with gRPC enabled +docker run -d \ + --name arcadedb-test \ + -p 2480:2480 \ + -p 50051:50051 \ + -e JAVA_OPTS="-Darcadedb.server.rootPassword=oY9uU2uJ8nD8iY7t -Darcadedb.server.name=ArcadeDB1 -Darcadedb.dumpConfigAtStartup=true -Darcadedb.server.mode=development -Darcadedb.server.plugins=GRPC:com.arcadedb.server.grpc.GrpcServerPlugin -Darcadedb.grpc.enabled=true -Darcadedb.grpc.mode=standard -Darcadedb.grpc.port=50051" \ + -e ARCADEDB_OPTS_MEMORY="-Xms4g -Xmx8g -XX:InitialRAMPercentage=50.0 -XX:MaxRAMPercentage=75.0" \ + arcadedata/arcadedb:25.8.1-SNAPSHOT + + + +# Test gRPC connection +grpcurl -plaintext localhost:50051 list + +# Check health +grpcurl -plaintext localhost:50051 grpc.health.v1.Health/Check + +# Stop and remove container +docker stop arcadedb-test +docker rm arcadedb-test + + + + + + + +version: '3.8' + +services: + arcadedb: + image: arcadedata/arcadedb:25.8.1-SNAPSHOT + container_name: arcadedb-grpc + ports: + - "2480:2480" # HTTP API + - "2424:2424" # Binary protocol + - "50051:50051" # gRPC + - "50052:50052" # gRPC XDS (if enabled) + environment: + - ARCADEDB_OPTS_MEMORY=-Xms2G -Xmx2G + - arcadedb.grpc.enabled=true + - arcadedb.grpc.port=50051 + - arcadedb.grpc.mode=standard + - arcadedb.grpc.reflection.enabled=true + - arcadedb.grpc.health.enabled=true + volumes: + - ./databases:/var/arcadedb/databases + - ./config:/var/arcadedb/config + networks: + - arcadedb-network + + # Optional: gRPC UI for testing + grpcui: + image: fullstorydev/grpcui:latest + container_name: grpcui + ports: + - "8080:8080" + command: + - -plaintext + - -port + - "8080" + - arcadedb:50051 + depends_on: + - arcadedb + networks: + - arcadedb-network + +networks: + arcadedb-network: + driver: bridge + + + + +Docker: + + + +grpcui -plaintext -bind 0.0.0.0 -port 8080 localhost:50051 + +mvn clean package -Pdocker \ No newline at end of file diff --git a/grpc/gRPC-Support-Case.md b/grpc/gRPC-Support-Case.md new file mode 100644 index 0000000000..1d0971d938 --- /dev/null +++ b/grpc/gRPC-Support-Case.md @@ -0,0 +1,191 @@ +# Make gRPC a First-Class API in ArcadeDB + +## TL;DR + +- **Unary ops (local, HTTP/2):** gRPC ≈ JSON/HTTP/2 on localhost—expected because ArcadeDB already moved to HTTP/2 (Undertow), removing most HTTP/1.1 overhead. +- **Streaming ops:** gRPC is a step-change: + - **Time-to-first-row:** 0–3 ms (vs ~175–227 ms with batched JSON). + - **Total time (7,746 rows):** 50–74 ms (vs 175–227 ms). + - That’s roughly a **3.1×–3.6× throughput gain** plus near-instant first byte. + +**Bottom line:** gRPC streaming enables real-time, low-latency graph I/O and unlocks new product capabilities, while unary parity on localhost is normal given HTTP/2 parity. + +## What I Measured (Local, Same Machine & Query) + +| Scenario | Transport | Passes | Time to First Response | Time to All | Rows | +|----------|-----------|--------|-----------------------|-------------|------| +| Large single query (batch JSON vs streamed gRPC) | JSON/HTTP (HTTP/2) | 4 | ~175–227 ms (first byte bundled with full payload) | 175–227 ms | 7,746 | +| | gRPC streaming (batchSize 100/500/800) | 3 | 0–3 ms | 50–74 ms | 7,746 | +| Many smaller queries (unary) | JSON/HTTP | 1 | — | 544 ms | — | +| | gRPC (unary) | 1 | — | 694 ms | — | + +**Why unary parity/slight regressions locally?** With HTTP/2 (Undertow), the big wins over HTTP/1.1 (handshakes, head-of-line blocking, connection churn) are already gone. On localhost, JSON parse costs are negligible; gRPC’s binary/framing advantages tend to pay off with network latency, concurrency, or streaming. + +## Why Leadership Should Care (Business Outcomes) + +- **Differentiate with “real-time graph.”** “Milliseconds to first result” demos incredibly well. Streaming queries feel instant—a clear edge vs REST-only graph stores. +- **Lower TCO under load.** Streaming reduces server memory pressure (no giant JSON payloads), tail latencies, and egress bytes—fewer cores, less RAM, less bandwidth for the same work. +- **Mobile & edge advantage.** On lossy/WAN links, HTTP/2 + gRPC streams tolerate jitter and deliver partial results promptly—snappier UX for apps and agents. +- **Enterprise readiness.** gRPC offers typed SDKs (Java, Go, Python, Node, Rust, Swift), mTLS, and OpenTelemetry—accelerating evals, POCs, and platform integrations. +- **New revenue surfaces.** Package Real-time Subscriptions, Live Traversal Streams, and Bulk Ingest via Client-Streaming as enterprise add-ons. + +## Why Engineering Should Care (Technical Benefits) + +1. **Server/Client Streaming** + - Time-to-first-row in ms, not 100s of ms. + - Backpressure built-in (HTTP/2 flow control); avoids buffering entire result sets. + - Enables live queries / CDC / tailing (e.g., “stream new edges matching X”). + +2. **Resource Efficiency** + - Smaller on-wire frames (binary protobuf) vs verbose JSON. + - Lower GC churn (no giant JSON strings/DOMs); steadier memory profile. + +3. **Concurrency at Scale** + - Single multiplexed channel per client; avoids socket storms. + - No app-layer head-of-line blocking; better tail latencies under load. + +4. **Typed Contracts & Evolvability** + - Proto schemas → generated clients, fewer integration bugs. + - Back/forward compatibility via field numbering and optionality. + +5. **Observability & Control** + - Standard interceptors for auth, quotas, retries, deadlines, metrics, tracing. + - First-class OpenTelemetry spans per RPC for SLOs and latency budgets. + +6. **Security & Networking** + - mTLS/ALPN by default; tight per-RPC auth (JWT, service tokens). + - Works seamlessly with Envoy/Istio and gRPC-Web for browsers. + +7. **Mesh-Native Traffic Policy via gRPC xDS (Kubernetes/Istio)** + - With Istio (or any xDS control plane), gRPC can consume xDS (CDS/EDS/LDS/RDS) to get dynamic, centrally managed traffic policy—no app redeploys. + - Smart LB & resilience: locality-aware load-balancing, fast failover, retries/hedging/timeouts/circuit-breaking/outlier detection by config. + - Progressive delivery: weighted canaries, blue-green, traffic mirroring, and fault injection for chaos tests—uniformly applied to gRPC traffic. + - Security & observability: mesh-wide mTLS (SPIFFE), consistent authz, and unified metrics/traces (Envoy/Otel). + +## Product Capabilities gRPC Unlocks + +- **Live Results / Progressive UI**: Stream traversals and long paths as they’re discovered (great for GraphRAG, visual explorers, dashboards). +- **Subscription Queries / CDC**: Subscribe to patterns (“new Person→Facility edges in Region=…”) and receive server-push updates in real time. +- **Bulk Write Pipelines**: Client-streaming for batch insert/upsert; server replies with a summary (counts, errors) at stream end. +- **Chunked Binary Blocks**: Stream rows in row-block or columnar frames for faster client deserialization and vectorization. +- **Back-office & Intra-cluster RPC**: Use the same gRPC surface for internal services (analytics, ingestion, connectors) to remove REST glue code. + +## Kubernetes + Istio + gRPC xDS (Advanced Benefit) + +When ArcadeDB runs in Kubernetes with Istio (or another xDS control plane), you can apply centralized, dynamic traffic policy to gRPC without code changes. + +### Two deployment patterns +1. **Sidecar-terminated gRPC (most common)**: App uses standard gRPC; Envoy sidecars enforce policy, mTLS, and routing—no client changes. +2. **Client-side xDS (sidecar-optional)**: Enable the gRPC xDS client to pull policy directly from the mesh’s xDS server (via a bootstrap file/env). Useful for sidecarless or off-mesh clients that still need mesh policy. + +### Example Istio policy (weighted canary + retry tuning for gRPC) + +``` +apiVersion: networking.istio.io/v1beta1 +kind: DestinationRule +metadata: + name: arcadedb +spec: + host: arcadedb.svc.cluster.local + trafficPolicy: + outlierDetection: + consecutive5xx: 5 + interval: 5s + baseEjectionTime: 30s + subsets: + - name: v1 + labels: { version: v1 } + - name: v2 + labels: { version: v2 } +--- +apiVersion: networking.istio.io/v1beta1 +kind: VirtualService +metadata: + name: arcadedb +spec: + hosts: [arcadedb.svc.cluster.local] + http: + - retries: + attempts: 3 + perTryTimeout: 200ms + retryOn: > + cancelled,connect-failure,refused-stream,resource-exhausted,unavailable + route: + - destination: { host: arcadedb.svc.cluster.local, subset: v1 } + weight: 90 + - destination: { host: arcadedb.svc.cluster.local, subset: v2 } + weight: 10 +``` + +**Notes:** +- Apply retries only to idempotent operations. +- Tune timeouts for long-running traversals; consider hedging for tail-latency control. +- Enable locality-weighted LB for multi-zone clusters. +- For browser apps, expose gRPC via gRPC-Web through the Istio ingress gateway. + +## Suggested MVP (Fast Path) + +### Services +- `QueryService.ExecuteStream(Query, Params) -> stream` + - Row blocks carry N rows + schema, flow-controlled. +- `IngestService.StreamWrites(stream) -> Summary` +- `SubscriptionService.Subscribe(Query) -> stream` +- `Health` (gRPC health checking), `Auth` (JWT/mTLS). + +### Wire Format +- Protobuf frames by default; allow JSON payloads only for interop, not default. + +### Packaging +- Ship `.proto`; publish official clients for Java & Node first. +- Provide gRPC-Web via Envoy for browser apps. + +### Config +- Toggle gRPC alongside REST; shared auth; TLS on by default. + +## Benchmark Plan (Beyond Localhost) + +- **Axes**: LAN vs WAN, concurrency (1, 16, 128 clients), payloads (1K, 10K, 100K rows), TTFB vs total time, p95/p99. +- **Controls**: warm-ups, stabilized JIT, pinned CPU governor, fixed GC (G1/ZGC), channel reuse, compression on/off. +- **Server**: Undertow HTTP/2 vs Netty gRPC; identical query engine/result materialization; measure server CPU/RAM and egress bytes. + +**Expectation**: Streaming gRPC yields lower p95/p99, smaller egress, and better CPU-per-result at medium/high concurrency; unary parity remains close to HTTP/2 JSON. + +## Risks & Mitigations + +- **Browser support**: Use gRPC-Web via Envoy; keep REST for simple use cases. +- **Schema evolution**: Lock proto CI checks; reserve tags; add compatibility tests. +- **Operational complexity**: Ship Helm charts + sample Envoy config; provide OpenTelemetry defaults. +- **Client fragmentation**: Provide first-party clients and examples; keep REST for long tail. + +## Call to Action + +1. Green-light a gRPC Streaming MVP (services above). +2. Publish proto + Java/Node SDKs with examples (streaming query, subscription, bulk ingest). +3. Add docs & a “Real-Time Streaming API” page featuring the metrics below (show 0–3 ms TTFB demo and ~3× throughput). +4. Run external benchmarks and publish guidance on when to choose streaming gRPC vs REST. + +## Appendix: Raw Local Test Results + +``` +testDBPerformance(): Case 1 [Pass 1] - JSON/HTTP: Query Execution Time = 227 ms; # of results = 7746 rows +testDBPerformance(): Case 1 [Pass 2] - JSON/HTTP: Query Execution Time = 179 ms; # of results = 7746 rows +testDBPerformance(): Case 1 [Pass 3] - JSON/HTTP: Query Execution Time = 177 ms; # of results = 7746 rows +testDBPerformance(): Case 1 [Pass 4] - JSON/HTTP: Query Execution Time = 175 ms; # of results = 7746 rows +testDBPerformance(): Case 1 [Pass 1] - gRPC. Query Execution Time = 327 ms; # of results = 7746 rows +testDBPerformance(): Case 1 [Pass 2] - gRPC. Query Execution Time = 202 ms; # of results = 7746 rows +testDBPerformance(): Case 1 [Pass 3] - gRPC. Query Execution Time = 184 ms; # of results = 7746 rows +testDBPerformance(): Case 1 [Pass 4] - gRPC. Query Execution Time = 185 ms; # of results = 7746 rows + +testGRPCLargeSingleQueryWithStreaming(): [Pass 1; batchSize = 100] - gRPC. To 1st response => Query Execution Time = 3 ms +testGRPCLargeSingleQueryWithStreaming(): [Pass 1] - gRPC. To all responses => Query Execution Time = 74 ms; Total Records = 7746 +testGRPCLargeSingleQueryWithStreaming(): [Pass 2; batchSize = 500] - gRPC. To 1st response => Query Execution Time = 0 ms +testGRPCLargeSingleQueryWithStreaming(): [Pass 2] - gRPC. To all responses => Query Execution Time = 52 ms; Total Records = 7746 +testGRPCLargeSingleQueryWithStreaming(): [Pass 3; batchSize = 800] - gRPC. To 1st response => Query Execution Time = 0 ms +testGRPCLargeSingleQueryWithStreaming(): [Pass 3] - gRPC. To all responses => Query Execution Time = 50 ms; Total Records = 7746 + +testHTTPManySmallerQueries(): Entry ... +testDBPerformance(): Case 2 - JSON/HTTP: Query Execution Time = 544 ms + +testGRPCManySmallerQueries(): Entry ... +testDBPerformance(): Case 2 - gRPC: Query Execution Time = 694 ms +``` \ No newline at end of file diff --git a/grpc/gRPC-vs-HTTP-PerformanceTest.txt b/grpc/gRPC-vs-HTTP-PerformanceTest.txt new file mode 100644 index 0000000000..65ad7e0f12 --- /dev/null +++ b/grpc/gRPC-vs-HTTP-PerformanceTest.txt @@ -0,0 +1,179 @@ +QUERY: +====== + + +TRAVERSE out(), in() FROM (SELECT FROM SalesShakeInstance WHERE @rid=:rid) +WHILE $depth <= 4 +STRATEGY BREADTH_FIRST + + +MATCH {class: SalesShakeInstance, as: s, where: (@rid=:rid)} + .out(){as:v1} + .out(){as:v2} +RETURN s, collectDistinct(v1) as lvl1, collectDistinct(v2) as lvl2 + + + +SELECT + $vertices AS vertices, + $edges AS edges +LET + $root = (SELECT FROM SalesShakeInstance WHERE @rid = :rid), + $walk = (TRAVERSE out(), in(), outE(), inE() + FROM $root + WHILE $depth <= 4), -- adjust depth as needed + $vertices = (SELECT FROM $walk WHERE @type = 'd' AND class() IN ['SalesInsight','SalesObjectiveInstance','SalesShakeInstance','SalesRep','SalesShakeAssist','Hospital','Physician']), + $edges = (SELECT FROM $walk WHERE @type = 'e'); + + + +RID = #1137:5 + + +SELECT + $vertices AS vertices, + $edges AS edges +LET + $root = (SELECT FROM SalesShakeInstance WHERE @rid = #1137:5), + $walk = (TRAVERSE out(), in(), outE(), inE() + FROM $root + WHILE $depth <= 4), -- adjust depth as needed + $vertices = (SELECT FROM $walk WHERE @type = 'd' AND class() IN ['SalesInsight','SalesObjectiveInstance','SalesShakeInstance','SalesRep','SalesShakeAssist','Hospital','Physician']), + $edges = (SELECT FROM $walk WHERE @type = 'e'); + + + +The following is about 7700 records + +TRAVERSE out(), in() FROM (SELECT FROM SalesShakeInstance) WHILE $depth <= 7 STRATEGY BREADTH_FIRST + +root/oY9uU2uJ8nD8iY7t + +HTTP: +===== + + +POST + +http://localhost:2480/api/v1/query/local_shakeiq_curonix_poc-app + + +{ + "language": "sql", + "command": "TRAVERSE out(), in() FROM (SELECT FROM SalesShakeInstance) WHILE $depth <= 7 STRATEGY BREADTH_FIRST", + "parameters": {}, + "limit": 10000, + "returnExpanded": true +} + + "command": "SELECT COUNT(*) FROM (TRAVERSE out(), in() FROM (SELECT FROM SalesShakeInstance) WHILE $depth <= 7 STRATEGY BREADTH_FIRST)", + + + +gtime -f "\nElapsed time: %E (%e seconds)" curl -sS --compressed -o /dev/null -w "HTTP elapsed: %{time_total}s size: %{size_download}\n" \ + -u root:oY9uU2uJ8nD8iY7t \ + -H 'Content-Type: application/json' \ + -X POST \ + 'http://localhost:2480/api/v1/command/local_shakeiq_curonix_poc-app' \ + -d '{"language":"sql","command":"TRAVERSE out(), in() FROM (SELECT FROM SalesShakeInstance) WHILE $depth <= 7 STRATEGY BREADTH_FIRST","parameters":{},"limit":10000,"returnExpanded":true}' + + +HTTP elapsed: 0.299217s size: 6,750,599 + + +curl \ + -u root:oY9uU2uJ8nD8iY7t \ + -H 'Content-Type: application/json' \ + -X POST \ + 'http://localhost:2480/api/v1/command/local_shakeiq_curonix_poc-app' \ + -d '{"language":"sql","command":"TRAVERSE out(), in() FROM (SELECT FROM SalesShakeInstance) WHILE $depth <= 7 STRATEGY BREADTH_FIRST","parameters":{},"limit":10000,"returnExpanded":true}' + + +GRPC: +===== + + +{ + "database": "local_shakeiq_curonix_poc-app", + "query": "TRAVERSE out(), in() FROM (SELECT FROM SalesShakeInstance) WHILE $depth <= 7 STRATEGY BREADTH_FIRST", + "parameters": { }, + "credentials": { "username": "root", "password": "oY9uU2uJ8nD8iY7t" }, + "limit": 10000, + "timeoutMs": 30000 +} + +x-arcade-user: root +x-arcade-password: oY9uU2uJ8nD8iY7t +x-arcade-database: local_shakeiq_curonix_poc-app + +grpcurl -plaintext -max-msg-sz 33554432 \ + -H "grpc-accept-encoding: gzip" \ + -H "x-arcade-user: root" \ + -H "x-arcade-password: oY9uU2uJ8nD8iY7t" \ + -H "x-arcade-database: local_shakeiq_curonix_poc-app" \ + -d '{ + "database":"local_shakeiq_curonix_poc-app", + "query":"TRAVERSE out(), in() FROM (SELECT FROM SalesShakeInstance) WHILE $depth <= 7 STRATEGY BREADTH_FIRST", + "parameters":{}, + "credentials":{"username":"root","password":"oY9uU2uJ8nD8iY7t"}, + "limit":10000, + "timeout_ms":30000 + }' \ + localhost:50051 com.arcadedb.grpc.ArcadeDbService/ExecuteQuery + + + + +grpcurl -plaintext -max-msg-sz 33554432 \ + -H "grpc-accept-encoding: gzip" \ + -H "x-arcade-user: root" \ + -H "x-arcade-password: oY9uU2uJ8nD8iY7t" \ + -H "x-arcade-database: local_shakeiq_curonix_poc-app" \ + -d '{"database":"local_shakeiq_curonix_poc-app","query":"TRAVERSE out(), in() FROM (SELECT FROM SalesShakeInstance) WHILE $depth <= 7 STRATEGY BREADTH_FIRST","parameters":{},"credentials":{"username":"root","password":"oY9uU2uJ8nD8iY7t"},"limit":10000,"timeout_ms":30000}' \ + localhost:50051 com.arcadedb.grpc.ArcadeDbService/ExecuteQuery + + + +date +%s%3N; grpcurl -plaintext -max-msg-sz 33554432 \ + -H "grpc-accept-encoding: gzip" \ + -H "x-arcade-user: root" \ + -H "x-arcade-password: oY9uU2uJ8nD8iY7t" \ + -H "x-arcade-database: local_shakeiq_curonix_poc-app" \ + -d '{"database":"local_shakeiq_curonix_poc-app","query":"TRAVERSE out(), in() FROM (SELECT FROM SalesShakeInstance) WHILE $depth <= 7 STRATEGY BREADTH_FIRST","parameters":{},"credentials":{"username":"root","password":"oY9uU2uJ8nD8iY7t"},"limit":10000,"timeout_ms":30000}' \ + localhost:50051 com.arcadedb.grpc.ArcadeDbService/ExecuteQuery \ + >/dev/null; date +%s%3N + + +start=$(date +%s%3N) +grpcurl -plaintext -max-msg-sz 33554432 \ + -H "grpc-accept-encoding: gzip" \ + -H "x-arcade-user: root" \ + -H "x-arcade-password: oY9uU2uJ8nD8iY7t" \ + -H "x-arcade-database: local_shakeiq_curonix_poc-app" \ + -d '{"database":"local_shakeiq_curonix_poc-app","query":"TRAVERSE out(), in() FROM (SELECT FROM SalesShakeInstance) WHILE $depth <= 7 STRATEGY BREADTH_FIRST","parameters":{},"credentials":{"username":"root","password":"oY9uU2uJ8nD8iY7t"},"limit":10000,"timeout_ms":30000}' \ + localhost:50051 com.arcadedb.grpc.ArcadeDbService/ExecuteQuery \ + >/dev/null +end=$(date +%s%3N) +echo "Elapsed time: $((end - start)) ms" + + +cd /Users/ocohen/git/arcadedb/server/src/main/proto + +gtime -f "\nElapsed time: %E (%e seconds)" grpcurl -proto arcadedb-server.proto -plaintext -max-msg-sz 33554432 -H "grpc-accept-encoding: gzip" -H "x-arcade-user: root" -H "x-arcade-password: oY9uU2uJ8nD8iY7t" -H "x-arcade-database: local_shakeiq_curonix_poc-app" -d '{"database":"local_shakeiq_curonix_poc-app","query":"TRAVERSE out(), in() FROM (SELECT FROM SalesShakeInstance) WHILE $depth <= 7 STRATEGY BREADTH_FIRST","parameters":{},"credentials":{"username":"root","password":"oY9uU2uJ8nD8iY7t"},"limit":10000,"timeout_ms":30000}' localhost:50051 com.arcadedb.grpc.ArcadeDbService/ExecuteQuery > /dev/null + + +grpcurl -proto arcadedb-server.proto -plaintext -max-msg-sz 33554432 -H "grpc-accept-encoding: gzip" -H "x-arcade-user: root" -H "x-arcade-password: oY9uU2uJ8nD8iY7t" -H "x-arcade-database: local_shakeiq_curonix_poc-app" -d '{"database":"local_shakeiq_curonix_poc-app","query":"TRAVERSE out(), in() FROM (SELECT FROM SalesShakeInstance) WHILE $depth <= 7 STRATEGY BREADTH_FIRST","parameters":{},"credentials":{"username":"root","password":"oY9uU2uJ8nD8iY7t"},"limit":10000,"timeout_ms":30000}' localhost:50051 com.arcadedb.grpc.ArcadeDbService/ExecuteQuery | wc -c + +8,229,923 + +ghz --enable-compression --insecure \ + --proto arcadedb-server.proto \ + --call com.arcadedb.grpc.ArcadeDbService.ExecuteQuery \ + --metadata '{"grpc-accept-encoding":"gzip","x-arcade-user":"root","x-arcade-password":"oY9uU2uJ8nD8iY7t","x-arcade-database":"local_shakeiq_curonix_poc-app"}' \ + --data '{"database":"local_shakeiq_curonix_poc-app","query":"TRAVERSE out(), in() FROM (SELECT FROM SalesShakeInstance) WHILE $depth <= 7 STRATEGY BREADTH_FIRST","parameters":{},"credentials":{"username":"root","password":"oY9uU2uJ8nD8iY7t"},"limit":10000,"timeout_ms":30000}' \ + --max-send-message-size 33554432 --max-recv-message-size 33554432 \ + --concurrency 5 \ + --total 1 \ + localhost:50051 + + \ No newline at end of file diff --git a/grpc/pom.xml b/grpc/pom.xml new file mode 100644 index 0000000000..6e75f4c65e --- /dev/null +++ b/grpc/pom.xml @@ -0,0 +1,163 @@ + + 4.0.0 + + + com.arcadedb + arcadedb-parent + 25.8.1-SNAPSHOT + + + arcadedb-grpc + ArcadeDB gRPC Stubs + jar + + + + + com.arcadedb + arcadedb-engine + ${project.parent.version} + provided + + + + + io.grpc + grpc-netty-shaded + ${grpc.version} + + + io.grpc + grpc-protobuf + ${grpc.version} + + + io.grpc + grpc-stub + ${grpc.version} + + + io.grpc + grpc-services + ${grpc.version} + + + io.grpc + grpc-xds + ${grpc.version} + + + + + com.google.protobuf + protobuf-java + ${protobuf-java.version} + + + com.google.protobuf + protobuf-java-util + ${protobuf-java.version} + + + + + com.google.api.grpc + proto-google-common-protos + ${com.google.api.grpc.version} + + + + + javax.annotation + javax.annotation-api + ${javax.annotation-api.version} + + + + + io.grpc + grpc-testing + ${grpc.version} + test + + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + grpc-interface + ${project.build.directory}/classes + + com/arcadedb/server/grpc/* + + + pack-grpc-client + package + + jar + + + + default-jar + package + + jar + + + + + + + + io.github.ascopes + protobuf-maven-plugin + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.6.0 + + + add-source + generate-sources + + add-source + + + + src/generated/protobuf/java + + + + + + + + org.apache.maven.plugins + maven-clean-plugin + 3.3.1 + + + + src/generated + + + + + + + + \ No newline at end of file diff --git a/grpc/src/main/proto/arcadedb-server.proto b/grpc/src/main/proto/arcadedb-server.proto new file mode 100644 index 0000000000..45954eac83 --- /dev/null +++ b/grpc/src/main/proto/arcadedb-server.proto @@ -0,0 +1,584 @@ +syntax = "proto3"; + +package com.arcadedb.grpc; + +option java_multiple_files = true; +option java_package = "com.arcadedb.server.grpc"; +option java_outer_classname = "ArcadeDbProto"; + +import "google/protobuf/empty.proto"; +import "google/protobuf/struct.proto"; +import "google/protobuf/timestamp.proto"; +import "google/protobuf/wrappers.proto"; + +// ----------------------------------------------------------------------------- +// Common types +// ----------------------------------------------------------------------------- + +message DatabaseCredentials { + string username = 1; + string password = 2; +} + +message TransactionContext { + string transaction_id = 1; // optional correlation + string database = 2; // optional if request already has database + bool begin = 3; // start a tx if not already active + bool commit = 4; // commit at end of call/stream + bool rollback = 5; // rollback at end of call/stream + int64 timeout_ms = 6; // optional per-tx timeout + bool read_only = 7; // optional hint +} + +message RowError { + uint64 row_index = 1; string code = 2; string message = 3; string field = 4; +} + +// ----------------------------------------------------------------------------- +// GRPC Record +// ----------------------------------------------------------------------------- + +// A record (vertex/document) +message GrpcRecord { + string rid = 1; // empty for embedded docs + string type = 2; // ArcadeDB class/type name (optional for embedded) + map properties = 3; +} + +// Typed values, recursively composable +message GrpcValue { + oneof kind { + bool bool_value = 1; + int32 int32_value = 2; + int64 int64_value = 3; // epoch ms, counters, etc. + float float_value = 4; + double double_value = 5; + string string_value = 6; + bytes bytes_value = 7; // binary + google.protobuf.Timestamp timestamp_value = 8; // date/datetime + GrpcList list_value = 9; // LIST + GrpcMap map_value = 10; // schemaless MAP + GrpcEmbedded embedded_value = 11; // embedded doc + GrpcLink link_value = 12; // RID link + GrpcDecimal decimal_value = 13; // exact decimal + } + + // Optional semantic hint (non-breaking): "date", "datetime", "rid", "decimal(38,9)", etc. + string logical_type = 14; +} + +// Containers +message GrpcList { repeated GrpcValue values = 1; } +message GrpcMap { map entries = 1; } +message GrpcEmbedded{ + string type = 1; // ArcadeDB class name if present; may be empty + map fields = 2; +} + +// RID link +message GrpcLink { + string rid = 1; // e.g., "#12:0" + string type = 2; // optional target class +} + +// Exact decimal without loss +message GrpcDecimal { + sint64 unscaled = 1; // unscaled value + int32 scale = 2; // number of decimal places +} + +message ProjectionSettings { + bool include_projections = 1; + + // Reuse the same enum (either nested under ExecuteQueryRequest or a top-level enum) + enum ProjectionEncoding { + PROJECTION_AS_LINK = 0; + PROJECTION_AS_MAP = 1; + PROJECTION_AS_JSON = 2; + } + ProjectionEncoding projection_encoding = 2; + + optional google.protobuf.Int32Value soft_limit_bytes = 3; // <-- this is the one protoc complained about; use 3 +} + +// ----------------------------------------------------------------------------- +// Query (streaming) API +// ----------------------------------------------------------------------------- + +message StreamQueryRequest { + + string database = 1; + string query = 2; + + map parameters = 3; + + DatabaseCredentials credentials = 4; + + int32 batch_size = 5; + + enum RetrievalMode { + CURSOR = 0; // run once, stream as you iterate + MATERIALIZE_ALL = 1; // load all then emit in batches + PAGED = 2; // re-issue with LIMIT/SKIP per batch + } + RetrievalMode retrieval_mode = 6; + + string language = 7; // e.g., "sql" (default if empty) + TransactionContext transaction = 8; // optional + + ProjectionSettings projectionSettings = 9; +} + +message QueryResult { + repeated GrpcRecord records = 1; + int32 total_records_in_batch = 2; + int64 running_total_emitted = 3; + bool is_last_batch = 4; +} + +service ArcadeDbService { + + // Streaming queries + rpc StreamQuery (StreamQueryRequest) returns (stream QueryResult); + + // Execute arbitrary command (INSERT/UPDATE/DDL/etc.) + rpc ExecuteCommand (ExecuteCommandRequest) returns (ExecuteCommandResponse); + rpc ExecuteQuery(ExecuteQueryRequest) returns (ExecuteQueryResponse); + + // Single-record helpers + rpc CreateRecord (CreateRecordRequest) returns (CreateRecordResponse); + rpc UpdateRecord (UpdateRecordRequest) returns (UpdateRecordResponse); + rpc LookupByRid (LookupByRidRequest) returns (LookupByRidResponse); + rpc DeleteRecord(DeleteRecordRequest) returns (DeleteRecordResponse); + + // Insert APIs (unary, client-streaming, and bidi) + rpc BulkInsert (BulkInsertRequest) returns (InsertSummary); + rpc InsertStream (stream InsertChunk) returns (InsertSummary); + rpc InsertBidirectional (stream InsertRequest) returns (stream InsertResponse); + + // Transaction operations + rpc BeginTransaction(BeginTransactionRequest) returns (BeginTransactionResponse); + rpc CommitTransaction(CommitTransactionRequest) returns (CommitTransactionResponse); + rpc RollbackTransaction(RollbackTransactionRequest) returns (RollbackTransactionResponse); +} + +// ----------------------------------------------------------------------------- +// Transaction operations +// ----------------------------------------------------------------------------- + +message BeginTransactionRequest { + string database = 1; + DatabaseCredentials credentials = 2; + TransactionIsolation isolation = 3; +} + +enum TransactionIsolation { + READ_UNCOMMITTED = 0; + READ_COMMITTED = 1; + REPEATABLE_READ = 2; + SERIALIZABLE = 3; +} + +message BeginTransactionResponse { + string transaction_id = 1; + int64 timestamp = 2; +} + +message CommitTransactionRequest { + TransactionContext transaction = 1; + DatabaseCredentials credentials = 2; +} + +message CommitTransactionResponse { + bool success = 1; + bool committed = 2; + string message = 3; + int64 timestamp = 4; +} + +message RollbackTransactionRequest { + TransactionContext transaction = 1; + DatabaseCredentials credentials = 2; +} + +message RollbackTransactionResponse { + bool success = 1; + bool rolled_back = 2; + string message = 3; +} + +// ----------------------------------------------------------------------------- +// Delete Record +// ----------------------------------------------------------------------------- + +message DeleteRecordRequest { + string database = 1; + string rid = 2; + DatabaseCredentials credentials = 3; + TransactionContext transaction = 4; // optional begin/commit/rollback flags +} + +message DeleteRecordResponse { + bool success = 1; // call succeeded + bool deleted = 2; // record actually removed + string message = 3; // optional info ("not found", error text if success=false, etc.) +} + +// ----------------------------------------------------------------------------- +// Execute Command +// ----------------------------------------------------------------------------- + +message ExecuteCommandRequest { + + string database = 1; + + string command = 2; + + map parameters = 3; + + DatabaseCredentials credentials = 4; + TransactionContext transaction = 5; + + string language = 6; // "sql" if empty + + bool return_rows = 7; // include rows in response + int32 max_rows = 8; // cap returned rows (server may still count all) + + ProjectionSettings projectionSettings = 9; +} + +message ExecuteCommandResponse { + bool success = 1; + string message = 2; + + int64 affected_records = 3; // server-computed (elements + numeric scalars) + int64 execution_time_ms = 4; + + repeated GrpcRecord records = 5; // optional row payload if return_rows=true +} + +// ----------------------------------------------------------------------------- +// Execute Query +// ----------------------------------------------------------------------------- + +message ExecuteQueryRequest { + string database = 1; + string query = 2; + map parameters = 3; + + DatabaseCredentials credentials = 4; + TransactionContext transaction = 5; + int32 limit = 6; + int32 timeout_ms = 7; + + ProjectionSettings projectionSettings = 8; +} + +message ExecuteQueryResponse { + repeated QueryResult results = 1; + int64 execution_time_ms = 2; + string query_plan = 3; +} + +// ----------------------------------------------------------------------------- +// GrpcRecord CRUD helpers +// ----------------------------------------------------------------------------- + +message CreateRecordRequest { + string database = 1; + DatabaseCredentials credentials = 2; + string type = 3; // target class/type + GrpcRecord record = 4; // payload + TransactionContext transaction = 5; +} + +message CreateRecordResponse { + string rid = 1; +} + +// Helper wrapper so we can use a map with oneof +message PropertiesUpdate { + map properties = 1; +} + +message UpdateRecordRequest { + string database = 1; + DatabaseCredentials credentials = 2; + string rid = 3; + + oneof payload { + GrpcRecord record = 4; // full replacement + PropertiesUpdate partial = 5; // partial update (map wrapped) + } + + TransactionContext transaction = 6; +} + +message UpdateRecordResponse { + bool success = 1; + bool updated = 2; +} + +message LookupByRidRequest { + string database = 1; + DatabaseCredentials credentials = 2; + string rid = 3; +} +message LookupByRidResponse { + bool found = 1; + GrpcRecord record = 2; // present if found +} + +// ----------------------------------------------------------------------------- +// Insert API (shared types) +// ----------------------------------------------------------------------------- + +message InsertError { + string message = 1; + int64 row_index = 2; // index within the chunk (or -1 if not applicable) + string code = 3; // optional machine-usable code + string field = 4; +} + +message InsertSummary { + int64 received = 1; + int64 inserted = 2; + int64 updated = 3; + int64 ignored = 4; + int64 failed = 5; + repeated InsertError errors = 6; + int64 execution_time_ms = 7; + google.protobuf.Timestamp started_at = 8; + google.protobuf.Timestamp finished_at = 9; +} + +message InsertOptions { + + string target_class = 1; // required + repeated string key_columns = 2; + + enum ConflictMode { + CONFLICT_ERROR = 0; // default (fail on dup) + CONFLICT_UPDATE = 1; // upsert/update + CONFLICT_IGNORE = 2; // drop dup + CONFLICT_ABORT = 3; + } + + ConflictMode conflict_mode = 3; + + repeated string update_columns_on_conflict = 4; + + enum TransactionMode { + PER_REQUEST = 0; // one tx for entire RPC + PER_BATCH = 1; // commit per streamed chunk + PER_STREAM = 2; // commit per streamed chunk + PER_ROW = 3; // one tx for entire RPC + NONE = 4; // caller manages externally + } + + TransactionMode transaction_mode = 5; + + int32 server_batch_size = 6; // hint; 0 => server chooses + + bool validate_only = 7; // dry-run (no commit) + + string database = 8; + DatabaseCredentials credentials = 9; +} + +// ----------------------------------------------------------------------------- +// Insert: Unary Bulk +// ----------------------------------------------------------------------------- + +message BulkInsertRequest { + string database = 1; + DatabaseCredentials credentials = 2; + + InsertOptions options = 3; + repeated GrpcRecord rows = 4; + + TransactionContext transaction = 5; // optional +} + +// ----------------------------------------------------------------------------- +// Insert: Client-Streaming +// ----------------------------------------------------------------------------- + +message InsertChunk { + // REQUIRED on the first chunk; optional on subsequent chunks (server caches) + string database = 1; + DatabaseCredentials credentials = 2; + InsertOptions options = 3; + TransactionContext transaction = 4; + + // Streaming envelope + string session_id = 5; // client-generated stable ID + int64 chunk_seq = 6; // 1,2,3,... + + // Payload + repeated GrpcRecord rows = 7; + + // Optional client hint + bool last = 8; +} + +// ----------------------------------------------------------------------------- +// Insert: Bidirectional +// ----------------------------------------------------------------------------- + +message InsertRequest { + oneof msg { + Start start = 1; + InsertChunk chunk = 2; + Commit commit = 3; + } +} + +message Start { + string database = 1; + DatabaseCredentials credentials = 2; + InsertOptions options = 3; + TransactionContext transaction = 4; + string session_id = 5; // may be empty; server can echo a generated one +} + +message Commit { + string session_id = 1; + bool commit = 2; // if both false, server may default to commit + bool rollback = 3; +} + +message InsertResponse { + oneof msg { + Started started = 1; + BatchAck batch_ack = 2; + Committed committed = 3; + InsertError error = 4; // immediate error if any + } +} + +message Started { + string session_id = 1; +} + +message BatchAck { + string session_id = 1; + int64 chunk_seq = 2; + + // Tallies for this chunk only + int64 inserted = 3; + int64 updated = 4; + int64 ignored = 5; + int64 failed = 6; + + repeated InsertError errors = 7; // batch-scoped +} + +message Committed { + InsertSummary summary = 1; // full-stream totals +} + +// ----------------------------------------------------------------------------- +// Admin service +// ----------------------------------------------------------------------------- + +service ArcadeDbAdminService { + rpc Ping (PingRequest) returns (PingResponse); + rpc GetServerInfo (GetServerInfoRequest) returns (GetServerInfoResponse); + + rpc ListDatabases (ListDatabasesRequest) returns (ListDatabasesResponse); + rpc ExistsDatabase (ExistsDatabaseRequest) returns (ExistsDatabaseResponse); + rpc CreateDatabase (CreateDatabaseRequest) returns (CreateDatabaseResponse); + rpc DropDatabase (DropDatabaseRequest) returns (DropDatabaseResponse); + + rpc GetDatabaseInfo (GetDatabaseInfoRequest) returns (GetDatabaseInfoResponse); + + rpc CreateUser (CreateUserRequest) returns (CreateUserResponse); + rpc DeleteUser (DeleteUserRequest) returns (DeleteUserResponse); +} + +message PingRequest { + DatabaseCredentials credentials = 1; // optional if you allow unauth ping +} +message PingResponse { + bool ok = 1; + int64 server_time_ms = 2; +} + +message GetServerInfoRequest { + DatabaseCredentials credentials = 1; +} +message GetServerInfoResponse { + string version = 1; // e.g., "25.8.1-SNAPSHOT" + string edition = 2; // "CE"/"EE" if applicable + int64 start_time_ms = 3; + int64 uptime_ms = 4; + int32 http_port = 5; // -1 if not available + int32 grpc_port = 6; // -1 if not available + int32 binary_port = 7; // -1 if not available + int32 databases_count = 8; + map features = 9; // optional flags +} + +message ListDatabasesRequest { + DatabaseCredentials credentials = 1; +} + +message ListDatabasesResponse { + repeated string databases = 1; +} + +message ExistsDatabaseRequest { + DatabaseCredentials credentials = 1; + string name = 2; // database name +} +message ExistsDatabaseResponse { + bool exists = 1; +} + +message CreateDatabaseRequest { + DatabaseCredentials credentials = 1; + string name = 2; // database name + string type = 3; // "graph" or "document" (logical) +} +message CreateDatabaseResponse {} + +message DropDatabaseRequest { + DatabaseCredentials credentials = 1; + string name = 2; // database name +} +message DropDatabaseResponse {} + +message GetDatabaseInfoRequest { + DatabaseCredentials credentials = 1; + string name = 2; // database name +} +message GetDatabaseInfoResponse { + string database = 1; + string type = 2; // "graph"/"document" + int64 records = 3; + int32 classes = 4; + int32 indexes = 5; + int64 size_bytes = 6; // optional if you can provide + map properties = 7; // extra info +} + +message CreateUserRequest { + DatabaseCredentials credentials = 1; // admin creds + string user = 2; + string password = 3; + string role = 4; // optional, depending on your model +} +message CreateUserResponse { + bool success = 1; + string message = 2; +} + +message DeleteUserRequest { + DatabaseCredentials credentials = 1; // admin creds + string user = 2; +} +message DeleteUserResponse { + bool success = 1; + string message = 2; +} \ No newline at end of file diff --git a/grpcw/pom.xml b/grpcw/pom.xml new file mode 100644 index 0000000000..658032885e --- /dev/null +++ b/grpcw/pom.xml @@ -0,0 +1,64 @@ + + + + 4.0.0 + + + com.arcadedb + arcadedb-parent + 25.8.1-SNAPSHOT + ../pom.xml + + + arcadedb-grpcw + jar + ArcadeDB gRpcW + + + + + + + com.arcadedb + arcadedb-server + ${project.parent.version} + provided + + + com.arcadedb + arcadedb-server + ${project.parent.version} + test + test-jar + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + + diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcAdminService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcAdminService.java new file mode 100644 index 0000000000..eee4d73cca --- /dev/null +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcAdminService.java @@ -0,0 +1,524 @@ +package com.arcadedb.server.grpc; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import com.arcadedb.database.Database; +import com.arcadedb.engine.ComponentFile; +import com.arcadedb.index.Index; +import com.arcadedb.schema.DocumentType; +import com.arcadedb.schema.Schema; +import com.arcadedb.schema.VertexType; +import com.arcadedb.server.ArcadeDBServer; +import com.arcadedb.server.security.credential.CredentialsValidator; + +import io.grpc.Status; +import io.grpc.stub.StreamObserver; + +/** + * Consolidated, compile-clean admin service for the new ArcadeDbAdminService. - + * No usage of APIs that were missing in your build - Safe fallbacks for + * version/ports/start time if not exposed by server - Array .length for indexes + * - Inference of db kind (graph/document) via vertex types + */ +public class ArcadeDbGrpcAdminService extends ArcadeDbAdminServiceGrpc.ArcadeDbAdminServiceImplBase { + + // Replace with your concrete server class if needed + private final ArcadeDBServer server; + private final CredentialsValidator credentialsValidator; + + public ArcadeDbGrpcAdminService(final ArcadeDBServer server, CredentialsValidator credentialsValidator) { + + this.server = Objects.requireNonNull(server, "server"); + this.credentialsValidator = Objects.requireNonNull(credentialsValidator, "credentialsValidator"); + } + + // ------------------------------------------------------------------------------------ + // RPCs + // ------------------------------------------------------------------------------------ + + @Override + public void ping(PingRequest req, StreamObserver resp) { + try { + // If you want ping to be open, comment out the next line + authenticate(req.getCredentials()); + + PingResponse out = PingResponse.newBuilder().setOk(true).setServerTimeMs(System.currentTimeMillis()).build(); + resp.onNext(out); + resp.onCompleted(); + } + catch (SecurityException se) { + resp.onError(Status.UNAUTHENTICATED.withDescription(se.getMessage()).asException()); + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("ping: " + e.getMessage()).asException()); + } + } + + @Override + public void getServerInfo(GetServerInfoRequest req, StreamObserver resp) { + try { + authenticate(req.getCredentials()); + + // Safer fallbacks (adjust these 3 helpers to your server if you have getters) + final String version = safeServerVersion(); + final long startMs = safeServerStartMs(); + final long uptime = (startMs > 0) ? Math.max(0, System.currentTimeMillis() - startMs) : 0L; + + final int httpPort = safeHttpPort(); + final int grpcPort = safeGrpcPort(); + final int binaryPort = safeBinaryPort(); + + final List dbNames = new ArrayList<>(getDatabaseNames()); + int dbCount = dbNames.size(); + + GetServerInfoResponse out = GetServerInfoResponse.newBuilder().setVersion(version).setEdition("CE") // adjust if you expose edition + .setStartTimeMs(startMs).setUptimeMs(uptime).setHttpPort(httpPort).setGrpcPort(grpcPort).setBinaryPort(binaryPort) + .setDatabasesCount(dbCount).build(); + + resp.onNext(out); + resp.onCompleted(); + } + catch (SecurityException se) { + resp.onError(Status.UNAUTHENTICATED.withDescription(se.getMessage()).asException()); + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("getServerInfo: " + e.getMessage()).asException()); + } + } + + @Override + public void listDatabases(ListDatabasesRequest req, StreamObserver resp) { + + try { + + authenticate(req.getCredentials()); + + ArrayList names = new ArrayList<>(getDatabaseNames()); + names.sort(String.CASE_INSENSITIVE_ORDER); + + resp.onNext(ListDatabasesResponse.newBuilder().addAllDatabases(names).build()); + resp.onCompleted(); + } + catch (SecurityException se) { + resp.onError(Status.UNAUTHENTICATED.withDescription(se.getMessage()).asException()); + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("listDatabases: " + e.getMessage()).asException()); + } + } + + @Override + public void existsDatabase(ExistsDatabaseRequest req, StreamObserver resp) { + + try { + + authenticate(req.getCredentials()); + + final String name = req.getName(); // proto should define 'name' for the DB + + boolean exists = containsDatabaseIgnoreCase(name); + + resp.onNext(ExistsDatabaseResponse.newBuilder().setExists(exists).build()); + resp.onCompleted(); + } + catch (SecurityException se) { + resp.onError(Status.UNAUTHENTICATED.withDescription(se.getMessage()).asException()); + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("existsDatabase: " + e.getMessage()).asException()); + } + } + + @Override + public void createDatabase(CreateDatabaseRequest req, StreamObserver resp) { + + try { + authenticate(req.getCredentials()); + + final String name = req.getName(); // DB name in proto + final String type = req.getType(); // "graph" or "document" (logical) + + if (containsDatabaseIgnoreCase(name)) { + + resp.onNext(CreateDatabaseResponse.newBuilder().build()); + resp.onCompleted(); + return; + } + + // Physical creation (READ_WRITE is the common default) + createDatabasePhysical(name); + + // Optional: if requested 'graph', initialize default graph types + if ("graph".equalsIgnoreCase(type)) { + try (Database db = openDatabase(name)) { + Schema s = db.getSchema(); + if (!existsVertexType(s, "V")) + s.createVertexType("V"); + if (!existsEdgeType(s, "E")) + s.createEdgeType("E"); + } + } + + resp.onNext(CreateDatabaseResponse.newBuilder().build()); + resp.onCompleted(); + } + catch (SecurityException se) { + resp.onError(Status.UNAUTHENTICATED.withDescription(se.getMessage()).asException()); + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("createDatabase: " + e.getMessage()).asException()); + } + } + + @Override + public void dropDatabase(DropDatabaseRequest req, StreamObserver resp) { + + try { + + authenticate(req.getCredentials()); + + final String name = req.getName(); + + if (!containsDatabaseIgnoreCase(name)) { + resp.onNext(DropDatabaseResponse.newBuilder().build()); + resp.onCompleted(); + return; + } + + dropDatabasePhysical(name); + + resp.onNext(DropDatabaseResponse.newBuilder().build()); + resp.onCompleted(); + } + catch (SecurityException se) { + resp.onError(Status.UNAUTHENTICATED.withDescription(se.getMessage()).asException()); + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("dropDatabase: " + e.getMessage()).asException()); + } + } + + @Override + public void getDatabaseInfo(GetDatabaseInfoRequest req, StreamObserver resp) { + + try { + + authenticate(req.getCredentials()); + + final String name = req.getName(); + + if (!containsDatabaseIgnoreCase(name)) { + resp.onError(Status.NOT_FOUND.withDescription("Database not found: " + name).asException()); + return; + } + + try (Database db = openDatabase(name)) { + + Schema schema = db.getSchema(); + + // Count classes + int classes = 0; + try { + classes = schema.getTypes().size(); + } + catch (Throwable ignore) { + } + + // Count indexes (Index[] in your build) + int indexes = 0; + try { + Index[] idx = schema.getIndexes(); + indexes = (idx != null) ? idx.length : 0; + } + catch (Throwable ignore) { + } + + // Approximate record count (fast-ish; adjust to your needs) + long records = approximateRecordCount(db); + + // Infer db kind: "graph" if any vertex type exists + String type = "document"; + + try { + + var vIter = schema.getTypes().stream().filter(t -> t instanceof VertexType); + + if (vIter != null && vIter.iterator().hasNext()) + type = "graph"; + } + catch (Throwable ignore) { + } + + GetDatabaseInfoResponse out = GetDatabaseInfoResponse.newBuilder() + .setDatabase(name) + .setClasses(classes).setIndexes(indexes).setRecords(records).setType(type) + .build(); + + resp.onNext(out); + resp.onCompleted(); + } + } + catch (SecurityException se) { + resp.onError(Status.UNAUTHENTICATED.withDescription(se.getMessage()).asException()); + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("getDatabaseInfo: " + e.getMessage()).asException()); + } + } + + @Override + public void createUser(CreateUserRequest req, StreamObserver resp) { + + try { + + authenticate(req.getCredentials()); + + final String user = req.getUser(); + final String pwd = req.getPassword(); + final String role = req.getRole(); + + if (user == null || user.isBlank()) + throw new IllegalArgumentException("user is required"); + if (pwd == null || pwd.isBlank()) + throw new IllegalArgumentException("password is required"); + + // TODO: wire into your security backend here + // server.getSecurity().createUser(user, pwd, role); + + resp.onNext(CreateUserResponse.newBuilder().setSuccess(true).setMessage("OK").build()); + resp.onCompleted(); + } + catch (SecurityException se) { + resp.onError(Status.PERMISSION_DENIED.withDescription(se.getMessage()).asException()); + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("createUser: " + e.getMessage()).asException()); + } + } + + @Override + public void deleteUser(DeleteUserRequest req, StreamObserver resp) { + try { + authenticate(req.getCredentials()); + final String user = req.getUser(); + if (user == null || user.isBlank()) + throw new IllegalArgumentException("user is required"); + + // TODO: wire into your security backend here + // server.getSecurity().deleteUser(user); + + resp.onNext(DeleteUserResponse.newBuilder().setSuccess(true).setMessage("OK").build()); + resp.onCompleted(); + } + catch (SecurityException se) { + resp.onError(Status.PERMISSION_DENIED.withDescription(se.getMessage()).asException()); + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("deleteUser: " + e.getMessage()).asException()); + } + } + + // ------------------------------------------------------------------------------------ + // Helpers + // ------------------------------------------------------------------------------------ + + private void authenticate(DatabaseCredentials creds) { + + if (creds == null) + throw new SecurityException("Authentication required"); + final String user = creds.getUsername(); // matches your proto (not getUser()) + final String pass = creds.getPassword(); + + if (user == null || user.isBlank()) + throw new SecurityException("Authentication required"); + + credentialsValidator.validateCredentials(user, pass); + } + + /** + * Get DB names from the server. Adjust if your server exposes a different + * accessor. + */ + @SuppressWarnings("unchecked") + private Collection getDatabaseNames() { + // Replace with your server API: + // e.g., ((ArcadeDBServer)server).getDatabaseNames() + try { + var m = server.getClass().getMethod("getDatabaseNames"); + Object res = m.invoke(server); + if (res instanceof Collection c) { + return (Collection) c; + } + else if (res instanceof String[] arr) { + return Arrays.asList(arr); + } + } + catch (Throwable ignore) { + } + return Collections.emptyList(); + } + + private boolean containsDatabaseIgnoreCase(String name) { + for (String n : getDatabaseNames()) { + if (n.equalsIgnoreCase(name)) + return true; + } + return false; + } + + /** + * Create DB physically with READ_WRITE mode. Adjust to your server signature. + */ + private void createDatabasePhysical(String name) throws Exception { + // Typical signature: createDatabase(String, ComponentFile.MODE) + var m = server.getClass().getMethod("createDatabase", String.class, ComponentFile.MODE.class); + m.invoke(server, name, ComponentFile.MODE.READ_WRITE); + } + + /** Drop DB physically. Prefer API with 'removeFiles' boolean if available. */ + private void dropDatabasePhysical(String name) throws Exception { + try { + var m = server.getClass().getMethod("dropDatabase", String.class, boolean.class); + m.invoke(server, name, Boolean.TRUE); + } + catch (NoSuchMethodException nsme) { + // Fallback: dropDatabase(String) if present + var m2 = server.getClass().getMethod("dropDatabase", String.class); + m2.invoke(server, name); + } + } + + /** Open database for read ops. Adjust to your server's open/get method. */ + private Database openDatabase(String name) throws Exception { + // Commonly: server.getDatabase(name) or server.openDatabase(name) + try { + var m = server.getClass().getMethod("getDatabase", String.class); + Object db = m.invoke(server, name); + return (Database) db; + } + catch (NoSuchMethodException nsme) { + var m2 = server.getClass().getMethod("openDatabase", String.class); + Object db = m2.invoke(server, name); + return (Database) db; + } + } + + /** Approximate record count with a quick pass across types. */ + private long approximateRecordCount(Database db) { + long total = 0L; + try { + for (DocumentType t : db.getSchema().getTypes()) { + try { + // exact=false when supported; otherwise this counts exactly + total += db.countType(t.getName(), false); + } + catch (Throwable ignore) { + } + } + } + catch (Throwable ignore) { + } + return total; + } + + private boolean existsVertexType(Schema s, String name) { + + try { + return s.existsType(name); + } + catch (Throwable t) { + return false; + } + } + + private boolean existsEdgeType(Schema s, String name) { + + try { + + return s.existsType(name); + } + catch (Throwable t) { + return false; + } + } + + // ---------- safe server info fallbacks (optional; return sentinel values if + // not exposed) ---------- + + private String safeServerVersion() { + try { + var m = server.getClass().getMethod("getProductVersion"); + Object v = m.invoke(server); + return (v != null) ? v.toString() : "unknown"; + } + catch (Throwable t) { + return "unknown"; + } + } + + private long safeServerStartMs() { + try { + var m = server.getClass().getMethod("getStartTime"); + Object v = m.invoke(server); + if (v instanceof Number n) + return n.longValue(); + } + catch (Throwable t) { + // ignore + } + return 0L; + } + + private int safeHttpPort() { + try { + var m = server.getClass().getMethod("getHttpServer"); + Object http = m.invoke(server); + if (http != null) { + var pm = http.getClass().getMethod("getPort"); + Object p = pm.invoke(http); + if (p instanceof Number n) + return n.intValue(); + } + } + catch (Throwable ignore) { + } + return -1; + } + + private int safeGrpcPort() { + try { + var m = server.getClass().getMethod("getGrpcServer"); + Object g = m.invoke(server); + if (g != null) { + var pm = g.getClass().getMethod("getPort"); + Object p = pm.invoke(g); + if (p instanceof Number n) + return n.intValue(); + } + } + catch (Throwable ignore) { + } + return -1; + } + + private int safeBinaryPort() { + try { + var m = server.getClass().getMethod("getBinaryServer"); + Object b = m.invoke(server); + if (b != null) { + var pm = b.getClass().getMethod("getPort"); + Object p = pm.invoke(b); + if (p instanceof Number n) + return n.intValue(); + } + } + catch (Throwable ignore) { + } + return -1; + } +} \ No newline at end of file diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java new file mode 100644 index 0000000000..66ad20d64b --- /dev/null +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/ArcadeDbGrpcService.java @@ -0,0 +1,2976 @@ +package com.arcadedb.server.grpc; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; + +import javax.annotation.Nullable; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.arcadedb.database.Database; +import com.arcadedb.database.DatabaseFactory; +import com.arcadedb.database.Document; +import com.arcadedb.database.MutableDocument; +import com.arcadedb.database.MutableEmbeddedDocument; +import com.arcadedb.database.RID; +import com.arcadedb.engine.ComponentFile; +import com.arcadedb.graph.Edge; +import com.arcadedb.graph.MutableEdge; +import com.arcadedb.graph.MutableVertex; +import com.arcadedb.graph.Vertex; +import com.arcadedb.query.sql.executor.Result; +import com.arcadedb.query.sql.executor.ResultSet; +import com.arcadedb.schema.DocumentType; +import com.arcadedb.schema.EdgeType; +import com.arcadedb.schema.Schema; +import com.arcadedb.schema.VertexType; +import com.arcadedb.serializer.JsonSerializer; +import com.arcadedb.server.ArcadeDBServer; +import com.arcadedb.server.grpc.InsertOptions.ConflictMode; +import com.arcadedb.server.grpc.InsertOptions.TransactionMode; +import com.arcadedb.server.grpc.ProjectionSettings.ProjectionEncoding; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; +import com.google.protobuf.Timestamp; + +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.grpc.stub.ServerCallStreamObserver; +import io.grpc.stub.StreamObserver; + +/** + * gRPC Service implementation for ArcadeDB + */ +public class ArcadeDbGrpcService extends ArcadeDbServiceGrpc.ArcadeDbServiceImplBase { + + private static final Logger logger = LoggerFactory.getLogger(ArcadeDbGrpcService.class); + + // Pick serializer once + private static final JsonSerializer FAST = JsonSerializer.createJsonSerializer().setIncludeVertexEdges(false).setUseVertexEdgeSize(true) + .setUseCollectionSizeForEdges(true).setUseCollectionSize(false); + + private static final JsonSerializer SAFE = JsonSerializer.createJsonSerializer().setIncludeVertexEdges(false).setUseVertexEdgeSize(true) + .setUseCollectionSizeForEdges(false).setUseCollectionSize(false); + + // Transaction management + private final Map activeTransactions = new ConcurrentHashMap<>(); + + // Database connection pool + private final Map databasePool = new ConcurrentHashMap<>(); + + // Track highest acknowledged chunk per session for idempotency + private final Map sessionWatermark = new ConcurrentHashMap<>(); + + // ArcadeDB server reference (optional, for accessing existing databases) + private final ArcadeDBServer arcadeServer; + + // Database directory path + private final String databasePath; + + private static final int DEFAULT_MAX_COMMAND_ROWS = 1000; + + public ArcadeDbGrpcService(String databasePath, ArcadeDBServer server) { + this.databasePath = databasePath; + this.arcadeServer = server; + } + + public void close() { + // Close all open databases + for (Database db : databasePool.values()) { + try { + if (db != null && db.isOpen()) { + db.close(); + } + } + catch (Exception e) { + logger.error("Error closing database", e); + } + } + databasePool.clear(); + + // Clean up transactions + for (Database db : activeTransactions.values()) { + try { + if (db != null && db.isOpen()) { + db.rollback(); + db.close(); + } + } + catch (Exception e) { + logger.error("Error closing transaction database", e); + } + } + activeTransactions.clear(); + } + + @Override + public void executeCommand(ExecuteCommandRequest req, StreamObserver resp) { + + final long t0 = System.nanoTime(); + + Database db = null; + boolean beganHere = false; + + try { + + // Resolve DB + params + db = getDatabase(req.getDatabase(), req.getCredentials()); + final java.util.Map params = convertParameters(req.getParametersMap()); + + // Language defaults to "sql" when empty + final String language = (req.getLanguage() == null || req.getLanguage().isEmpty()) ? "sql" : req.getLanguage(); + + // Transaction: begin if requested + final boolean hasTx = req.hasTransaction(); + + final var tx = hasTx ? req.getTransaction() : null; + + logger.debug("executeCommand(): hasTx = {} tx ={}", hasTx, tx); + + if (hasTx && tx.getBegin()) { + db.begin(); + beganHere = true; + } + + long affected = 0L; + + final boolean returnRows = req.getReturnRows(); + final int maxRows = req.getMaxRows() > 0 ? req.getMaxRows() : DEFAULT_MAX_COMMAND_ROWS; + + ExecuteCommandResponse.Builder out = ExecuteCommandResponse.newBuilder().setSuccess(true).setMessage("OK"); + + // Execute the command + + logger.debug("executeCommand(): command = {}", req.getCommand()); + + try (ResultSet rs = db.command(language, req.getCommand(), params)) { + + if (rs != null) { + + logger.debug("executeCommand(): rs = {}", rs); + + if (returnRows) { + + logger.debug("executeCommand(): returning rows ..."); + + int emitted = 0; + + while (rs.hasNext()) { + + Result r = rs.next(); + + if (r.isElement()) { + affected++; // count modified/returned records + if (emitted < maxRows) { + out.addRecords(convertToGrpcRecord(r.getElement().get(), db)); + emitted++; + } + } + else { + + // Scalar / projection row (e.g., RETURN COUNT) + + if (emitted < maxRows) { + + GrpcRecord.Builder recB = GrpcRecord.newBuilder(); + + for (String p : r.getPropertyNames()) { + + recB.putProperties(p, convertPropToGrpcValue(p, r)); + } + + out.addRecords(recB.build()); + + emitted++; + } + + for (String p : r.getPropertyNames()) { + Object v = r.getProperty(p); + if (v instanceof Number n) + affected += n.longValue(); + } + } + } + } + else { + + logger.debug("executeCommand(): not returning rows ... rs = {}", rs); + + // Not returning rows: still consume to compute 'affected' + while (rs.hasNext()) { + Result r = rs.next(); + if (r.isElement()) { + affected++; + } + else { + for (String p : r.getPropertyNames()) { + Object v = r.getProperty(p); + if (v instanceof Number n) + affected += n.longValue(); + } + } + } + } + } + } + + logger.debug("executeCommand(): after - hasTx = {} tx = {}", hasTx, tx); + + // Transaction end — precedence: rollback > commit > begin-only⇒commit + if (hasTx) { + + if (tx.getRollback()) { + + logger.debug("executeCommand(): after - rolling back db={} tid={}", db.getName(), tx.getTransactionId()); + db.rollback(); + } + else if (tx.getCommit()) { + logger.debug("executeCommand(): after - committing [tx.getCommit() == true] db={} tid={}", db.getName(), + tx.getTransactionId()); + db.commit(); + } + else if (beganHere) { + // Began but no explicit commit/rollback flag — default to commit (HTTP parity) + logger.debug("executeCommand(): after - committing [beganHere == true] db={} tid={}", db.getName(), tx.getTransactionId()); + db.commit(); + } + } + + final long ms = (System.nanoTime() - t0) / 1_000_000L; + out.setAffectedRecords(affected).setExecutionTimeMs(ms); + + resp.onNext(out.build()); + resp.onCompleted(); + + } + catch (Exception e) { + + logger.error("ERROR", e); + + // Best-effort rollback if we began here and failed + try { + if (beganHere && db != null) + db.rollback(); + } + catch (Exception ignore) { + /* no-op */ } + + final long ms = (System.nanoTime() - t0) / 1_000_000L; + ExecuteCommandResponse err = ExecuteCommandResponse.newBuilder().setSuccess(false) + .setMessage(e.getMessage() == null ? e.toString() : e.getMessage()).setAffectedRecords(0L).setExecutionTimeMs(ms).build(); + + // Prefer returning a structured response so clients always get timing/message + resp.onNext(err); + resp.onCompleted(); + // If you prefer gRPC error codes: comment the two lines above and use: + // resp.onError(Status.INTERNAL.withDescription("ExecuteCommand: " + + // e.getMessage()).asException()); + } + } + + @Override + public void createRecord(CreateRecordRequest req, StreamObserver resp) { + + try { + + Database db = getDatabase(req.getDatabase(), req.getCredentials()); + + final String cls = req.getType(); // or req.getTargetClass() if that’s your proto + if (cls == null || cls.isEmpty()) + throw new IllegalArgumentException("targetClass is required"); + + Schema schema = db.getSchema(); + DocumentType dt = schema.getType(cls); + if (dt == null) + throw new IllegalArgumentException("Class not found: " + cls); + + // All properties from the request (proto map) — nested under "record" + final java.util.Map props = req.getRecord().getPropertiesMap(); + + // --- Vertex --- + if (dt instanceof com.arcadedb.schema.VertexType) { + com.arcadedb.graph.MutableVertex v = db.newVertex(cls); + + // apply properties with proper coercion + props.forEach((k, val) -> v.set(k, toJavaForProperty(db, v, dt, k, val))); + + v.save(); + + // Build response: proto has setRid(String) + CreateRecordResponse.Builder b = CreateRecordResponse.newBuilder().setRid(v.getIdentity().toString()); + + resp.onNext(b.build()); + resp.onCompleted(); + return; + } + + // --- Edge --- + if (dt instanceof EdgeType) { + + // Expect 'out' and 'in' as string RIDs in the properties map + String outStr = null, inStr = null; + + if (props.containsKey("out")) { + + GrpcValue pv = props.get("out"); + outStr = (pv.getKindCase() == GrpcValue.KindCase.STRING_VALUE) ? pv.getStringValue() : String.valueOf(fromGrpcValue(pv)); + } + + if (props.containsKey("in")) { + + GrpcValue pv = props.get("in"); + inStr = (pv.getKindCase() == GrpcValue.KindCase.STRING_VALUE) ? pv.getStringValue() : String.valueOf(fromGrpcValue(pv)); + } + + if (outStr == null || inStr == null) + throw new IllegalArgumentException("Edge requires 'out' and 'in' RIDs"); + + var outEl = db.lookupByRID(new RID(outStr), true); + var inEl = db.lookupByRID(new RID(inStr), true); + + if (outEl == null || inEl == null) + throw new IllegalArgumentException("Cannot resolve out/in vertices for edge"); + + // IMPORTANT: use MutableVertex to avoid newEdge(...) ambiguity + com.arcadedb.graph.MutableVertex outV = (MutableVertex) outEl.asVertex(true); + com.arcadedb.graph.MutableVertex inV = (MutableVertex) inEl.asVertex(true); + + com.arcadedb.graph.MutableEdge e = outV.newEdge(cls, inV); + + // apply remaining properties (skip endpoints) + + props.forEach((k, val) -> { + if (!"out".equals(k) && !"in".equals(k)) { + e.set(k, toJavaForProperty(db, e, dt, k, val)); + } + }); + + e.save(); + + CreateRecordResponse.Builder b = CreateRecordResponse.newBuilder(); + + b.setRid(e.getIdentity().toString()); + + resp.onNext(b.build()); + resp.onCompleted(); + + return; + } + + // --- Document --- + MutableDocument d = db.newDocument(cls); + props.forEach((k, val) -> d.set(k, toJavaForProperty(db, d, dt, k, val))); + d.save(); + + CreateRecordResponse.Builder b = CreateRecordResponse.newBuilder(); + + b.setRid(d.getIdentity().toString()); + + resp.onNext(b.build()); + resp.onCompleted(); + + } + catch (Exception e) { + + resp.onError(Status.INVALID_ARGUMENT.withDescription("CreateRecord: " + e.getMessage()).asException()); + } + } + + @Override + public void lookupByRid(LookupByRidRequest req, StreamObserver resp) { + try { + Database db = getDatabase(req.getDatabase(), req.getCredentials()); + + final String ridStr = req.getRid(); + if (ridStr == null || ridStr.isBlank()) + throw new IllegalArgumentException("rid is required"); + + var el = db.lookupByRID(new RID(ridStr), true); + if (el == null) { + resp.onNext(LookupByRidResponse.newBuilder().setFound(false).build()); + resp.onCompleted(); + return; + } + + resp.onNext(LookupByRidResponse.newBuilder().setFound(true).setRecord(convertToGrpcRecord(el.getRecord(), db)).build()); + resp.onCompleted(); + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("LookupByRid: " + e.getMessage()).asException()); + } + } + + @Override + public void updateRecord(UpdateRecordRequest req, StreamObserver resp) { + + Database db = null; + boolean beganHere = false; + + try { + + db = getDatabase(req.getDatabase(), req.getCredentials()); + + final String ridStr = req.getRid(); + + if (ridStr == null || ridStr.isEmpty()) + throw new IllegalArgumentException("rid is required"); + + // Begin transaction if requested + final boolean hasTx = req.hasTransaction(); + + final var tx = hasTx ? req.getTransaction() : null; + + if (hasTx && tx.getBegin()) { + db.begin(); + beganHere = true; + } + + // Lookup the record by RID + var el = db.lookupByRID(new RID(ridStr), true); + + if (el == null) { + resp.onError(Status.NOT_FOUND.withDescription("Record not found: " + ridStr).asException()); + return; + } + + final var dbRef = db; + + logger.debug("updateRecord(): el = {}; type = {}", el, el.getRecordType()); + + if (el instanceof Vertex elAsVertex) { + + logger.debug("updateRecord(): Processing Vertex ..."); + + // Get mutable view for updates (works for docs, vertices, edges) + MutableVertex mvertex = elAsVertex.modify(); + + var dtype = db.getSchema().getType(mvertex.getTypeName()); + + // Apply updates + + final java.util.Map props = req.hasRecord() ? req.getRecord().getPropertiesMap() + : req.hasPartial() ? req.getPartial().getPropertiesMap() : java.util.Collections.emptyMap(); + + // Exclude ArcadeDB system fields during update + + props.forEach((k, v) -> { + String key = k.trim().toLowerCase(); + if (key.equals("@rid") || key.equals("@type") || key.equals("@cat")) { + // Skip internal fields to prevent accidental overwrites + logger.debug("Skipping internal field during update: {}", k); + return; + } + // Perform the update for user-defined fields + mvertex.set(k, toJavaForProperty(dbRef, mvertex, dtype, k, v)); + }); + + mvertex.save(); + } + else if (el instanceof Document elAsDocument) { + + logger.debug("updateRecord(): Processing Document ..."); + + MutableDocument mdoc = elAsDocument.modify(); + + var dtype = db.getSchema().getType(mdoc.getTypeName()); + + // Apply updates + + final java.util.Map props = req.hasRecord() ? req.getRecord().getPropertiesMap() + : req.hasPartial() ? req.getPartial().getPropertiesMap() : java.util.Collections.emptyMap(); + + // Exclude ArcadeDB system fields during update + + props.forEach((k, v) -> { + String key = k.trim().toLowerCase(); + if (key.equals("@rid") || key.equals("@type") || key.equals("@cat")) { + // Skip internal fields to prevent accidental overwrites + logger.debug("Skipping internal field during update: {}", k); + return; + } + // Perform the update for user-defined fields + mdoc.set(k, toJavaForProperty(dbRef, mdoc, dtype, k, v)); + }); + + mdoc.save(); + } + + // Commit/rollback with proper precedence + if (hasTx) { + + if (tx.getRollback()) { + db.rollback(); + } + else if (tx.getCommit()) { + db.commit(); + } + else if (beganHere) { + db.commit(); + } + } + + resp.onNext(UpdateRecordResponse.newBuilder().setUpdated(true).setSuccess(true).build()); + + resp.onCompleted(); + } + catch (Exception e) { + + logger.error("ERROR in updateRecord", e); + + try { + if (beganHere && db != null) + db.rollback(); + } + catch (Exception ignore) { + // ignore rollback errors + } + + resp.onError(Status.INTERNAL.withDescription("UpdateRecord: " + e.getMessage()).asException()); + } + } + + @Override + public void deleteRecord(DeleteRecordRequest req, StreamObserver resp) { + Database db = null; + boolean beganHere = false; + + try { + db = getDatabase(req.getDatabase(), req.getCredentials()); + + final String ridStr = req.getRid(); + if (ridStr == null || ridStr.isBlank()) + throw new IllegalArgumentException("rid is required"); + + // Optional tx begin + if (req.hasTransaction() && req.getTransaction().getBegin()) { + db.begin(); + beganHere = true; + } + + var el = db.lookupByRID(new RID(ridStr), true); + if (el == null) { + // Soft "not found" + resp.onNext( + DeleteRecordResponse.newBuilder().setSuccess(true).setDeleted(false).setMessage("Record not found: " + ridStr).build()); + resp.onCompleted(); + // If we began here and no explicit rollback flag, default to commit (or + // rollback—your policy) + if (beganHere && req.hasTransaction() && !req.getTransaction().getRollback()) + db.commit(); + return; + } + + el.delete(); // deletes document/vertex/edge + + // Optional tx end controls + if (req.hasTransaction()) { + var tx = req.getTransaction(); + if (tx.getCommit()) + db.commit(); + else if (tx.getRollback()) + db.rollback(); + else if (beganHere) + db.commit(); // began here with no explicit end → commit by default + } + else if (beganHere) { + db.commit(); + } + + resp.onNext(DeleteRecordResponse.newBuilder().setSuccess(true).setDeleted(true).build()); + resp.onCompleted(); + + } + catch (Exception e) { + // Best-effort rollback if we started the tx + try { + if (beganHere && db != null) + db.rollback(); + } + catch (Exception ignore) { + } + + resp.onError( + Status.INTERNAL.withDescription("DeleteRecord: " + (e.getMessage() == null ? e.toString() : e.getMessage())).asException()); + } + } + + @Override + public void executeQuery(ExecuteQueryRequest request, StreamObserver responseObserver) { + + try { + + ProjectionConfig projectionConfig = null; + + if (request.hasProjectionSettings()) { + + var projectionSettings = request.getProjectionSettings(); + + final boolean includeProjections = projectionSettings.getIncludeProjections(); + final var projMode = projectionSettings.getProjectionEncoding(); + + int softLimit = projectionSettings.hasSoftLimitBytes() ? projectionSettings.getSoftLimitBytes().getValue() : 0; // choose your + // default + // or 0 = + // unlimited + projectionConfig = new ProjectionConfig(includeProjections, projMode, softLimit); + } + else { + + projectionConfig = new ProjectionConfig(true, ProjectionEncoding.PROJECTION_AS_JSON, 0); + } + + logger.debug("executeQuery(): projectionConfig.include = {} projectionConfig.mode = {}", projectionConfig.isInclude(), + projectionConfig.getEnc()); + + // Force compression for streaming (usually beneficial) + CompressionAwareService.setResponseCompression(responseObserver, "gzip"); + + Database database = getDatabase(request.getDatabase(), request.getCredentials()); + + // Check if this is part of a transaction + if (request.hasTransaction()) { + + logger.debug("executeQuery(): has Tx {}", request.getTransaction().getTransactionId()); + + database = activeTransactions.get(request.getTransaction().getTransactionId()); + if (database == null) { + throw new IllegalArgumentException("Invalid transaction ID"); + } + } + + // Execute the query + long startTime = System.currentTimeMillis(); + + logger.debug("executeQuery(): query = {}", request.getQuery()); + + ResultSet resultSet = database.query("sql", request.getQuery(), convertParameters(request.getParametersMap())); + + logger.debug("executeQuery(): to get resultSet = {}", (System.currentTimeMillis() - startTime)); + + // Build response + QueryResult.Builder resultBuilder = QueryResult.newBuilder(); + + // Process results + int count = 0; + + logger.debug("executeQuery(): resultSet.size = {}", resultSet.getExactSizeIfKnown()); + + while (resultSet.hasNext()) { + + Result result = resultSet.next(); + + logger.debug("executeQuery(): result = {}", result); + + if (result.isElement()) { + + logger.debug("executeQuery(): isElement"); + + com.arcadedb.database.Record dbRecord = result.getElement().get(); + + logger.debug("executeQuery(): dbRecord -> @rid = {}", dbRecord.getIdentity().toString()); + + GrpcRecord grpcRecord = convertToGrpcRecord(dbRecord, database); + + logger.debug("executeQuery(): grpcRecord -> @rid = {}", grpcRecord.getRid()); + + resultBuilder.addRecords(grpcRecord); + + count++; + + // Apply limit if specified + if (request.getLimit() > 0 && count >= request.getLimit()) { + break; + } + } + else { + + logger.debug("executeQuery(): NOT isElement"); + + // Scalar / projection row (e.g., RETURN COUNT) + + GrpcRecord.Builder recB = GrpcRecord.newBuilder(); + + for (String p : result.getPropertyNames()) { + + recB.putProperties(p, convertPropToGrpcValue(p, result, projectionConfig)); + } + + resultBuilder.addRecords(recB.build()); + } + } + + logger.debug("executeQuery(): count = {}", count); + + resultBuilder.setTotalRecordsInBatch(count); + + long executionTime = System.currentTimeMillis() - startTime; + + ExecuteQueryResponse response = ExecuteQueryResponse.newBuilder().addResults(resultBuilder.build()).setExecutionTimeMs(executionTime) + .build(); + + logger.debug("executeQuery(): executionTime + response generation = {}", executionTime); + + responseObserver.onNext(response); + responseObserver.onCompleted(); + + } + catch (Exception e) { + logger.error("Error executing query: {}", e.getMessage(), e); + responseObserver.onError(Status.INTERNAL.withDescription("Query execution failed: " + e.getMessage()).asException()); + } + } + + @Override + public void beginTransaction(BeginTransactionRequest request, StreamObserver responseObserver) { + final String reqDb = request.getDatabase(); + final String user = (request.hasCredentials() ? request.getCredentials().getUsername() : null); + + if (logger.isDebugEnabled()) { + logger.debug("beginTransaction(): received request db={} user={} activeTxCount(before)={}", reqDb, (user != null ? user : ""), + activeTransactions.size()); + } + + try { + final Database database = getDatabase(reqDb, request.getCredentials()); + + if (logger.isDebugEnabled()) { + logger.debug("beginTransaction(): resolved database instance dbName={} class={} hash={}", + (database != null ? database.getName() : ""), (database != null ? database.getClass().getSimpleName() : ""), + (database != null ? System.identityHashCode(database) : 0)); + logger.debug("beginTransaction(): calling database.begin()"); + } + + // Begin transaction + database.begin(); + + // Generate transaction ID and register + final String transactionId = generateTransactionId(); + activeTransactions.put(transactionId, database); + + if (logger.isDebugEnabled()) { + logger.debug("beginTransaction(): started txId={} for db={} activeTxCount(after)={}", transactionId, + (database != null ? database.getName() : ""), activeTransactions.size()); + } + + final BeginTransactionResponse response = BeginTransactionResponse.newBuilder().setTransactionId(transactionId) + .setTimestamp(System.currentTimeMillis()).build(); + + responseObserver.onNext(response); + responseObserver.onCompleted(); + } + catch (Throwable t) { + if (logger.isDebugEnabled()) { + logger.debug("beginTransaction(): FAILED db={} user={} err={}", reqDb, (user != null ? user : ""), t.toString(), t); + } + logger.error("Error beginning transaction: {}", t.getMessage(), t); + responseObserver.onError(Status.INTERNAL.withDescription("Failed to begin transaction: " + t.getMessage()).asException()); + } + } + + @Override + public void commitTransaction(CommitTransactionRequest req, StreamObserver rsp) { + final String txId = req.getTransaction().getTransactionId(); + + if (logger.isDebugEnabled()) { + logger.debug("commitTransaction(): received request txId={}", txId); + } + + if (txId == null || txId.isBlank()) { + if (logger.isDebugEnabled()) { + logger.debug("commitTransaction(): missing/blank txId"); + } + rsp.onError(Status.INVALID_ARGUMENT.withDescription("Missing transaction id").asException()); + return; + } + + // remove atomically to avoid double-commit races + final Database db = activeTransactions.remove(txId); + + if (logger.isDebugEnabled()) { + logger.debug("commitTransaction(): removed txId={}, presentPreviously={}, remainingActiveTx={}", txId, db != null, + activeTransactions.size()); + } + + if (db == null) { + // Idempotent no-op + if (logger.isDebugEnabled()) { + logger.debug("commitTransaction(): no active tx for id={}, responding committed=false", txId); + } + rsp.onNext(CommitTransactionResponse.newBuilder().setSuccess(true).setCommitted(false) + .setMessage("No active transaction for id=" + txId + " (already committed/rolled back?)").build()); + rsp.onCompleted(); + return; + } + + try { + if (logger.isDebugEnabled()) { + logger.debug("commitTransaction(): committing txId={} on db={}", txId, db.getName()); + } + db.commit(); + if (logger.isDebugEnabled()) { + logger.debug("commitTransaction(): commit OK txId={}", txId); + } + rsp.onNext(CommitTransactionResponse.newBuilder().setSuccess(true).setCommitted(true).build()); + rsp.onCompleted(); + } + catch (Throwable t) { + if (logger.isDebugEnabled()) { + logger.debug("commitTransaction(): commit FAILED txId={} err={}", txId, t.toString(), t); + } + // tx is unusable; do not reinsert into the map + rsp.onError(Status.ABORTED.withDescription("Commit failed: " + t.getMessage()).asException()); + } + } + + @Override + public void rollbackTransaction(RollbackTransactionRequest req, StreamObserver rsp) { + final String txId = req.getTransaction().getTransactionId(); + + if (logger.isDebugEnabled()) { + logger.debug("rollbackTransaction(): received request txId={}", txId); + } + + if (txId == null || txId.isBlank()) { + if (logger.isDebugEnabled()) { + logger.debug("rollbackTransaction(): missing/blank txId"); + } + rsp.onError(Status.INVALID_ARGUMENT.withDescription("Missing transaction id").asException()); + return; + } + + final Database db = activeTransactions.remove(txId); + + if (logger.isDebugEnabled()) { + logger.debug("rollbackTransaction(): removed txId={}, presentPreviously={}, remainingActiveTx={}", txId, db != null, + activeTransactions.size()); + } + + if (db == null) { + if (logger.isDebugEnabled()) { + logger.debug("rollbackTransaction(): no active tx for id={}, responding rolledBack=false", txId); + } + rsp.onNext(RollbackTransactionResponse.newBuilder().setSuccess(true).setRolledBack(false) + .setMessage("No active transaction for id=" + txId + " (already committed/rolled back?)").build()); + rsp.onCompleted(); + return; + } + + try { + if (logger.isDebugEnabled()) { + logger.debug("rollbackTransaction(): rolling back txId={} on db={}", txId, db.getName()); + } + db.rollback(); + if (logger.isDebugEnabled()) { + logger.debug("rollbackTransaction(): rollback OK txId={}", txId); + } + rsp.onNext(RollbackTransactionResponse.newBuilder().setSuccess(true).setRolledBack(true).build()); + rsp.onCompleted(); + } + catch (Throwable t) { + if (logger.isDebugEnabled()) { + logger.debug("rollbackTransaction(): rollback FAILED txId={} err={}", txId, t.toString(), t); + } + rsp.onError(Status.ABORTED.withDescription("Rollback failed: " + t.getMessage()).asException()); + } + } + + @Override + public void streamQuery(StreamQueryRequest request, StreamObserver responseObserver) { + + ProjectionConfig projectionConfig = getProjectionConfigFromRequest(request); + + final ServerCallStreamObserver scso = (ServerCallStreamObserver) responseObserver; + + final java.util.concurrent.atomic.AtomicBoolean cancelled = new java.util.concurrent.atomic.AtomicBoolean(false); + scso.setOnCancelHandler(() -> cancelled.set(true)); + + Database db = null; + boolean beganHere = false; + + try { + db = getDatabase(request.getDatabase(), request.getCredentials()); + final int batchSize = Math.max(1, request.getBatchSize()); + + // --- TX begin if requested --- + final boolean hasTx = request.hasTransaction(); + final var tx = hasTx ? request.getTransaction() : null; + if (hasTx && tx.getBegin()) { + db.begin(); + beganHere = true; + } + + // --- Dispatch on mode (helpers do NOT manage transactions) --- + switch (request.getRetrievalMode()) { + case MATERIALIZE_ALL -> streamMaterialized(db, request, batchSize, scso, cancelled, projectionConfig); + case PAGED -> streamPaged(db, request, batchSize, scso, cancelled, projectionConfig); + case CURSOR -> streamCursor(db, request, batchSize, scso, cancelled, projectionConfig); + default -> streamCursor(db, request, batchSize, scso, cancelled, projectionConfig); + } + + // If the client cancelled mid-stream, choose rollback unless caller explicitly + // asked to commit/rollback. + if (cancelled.get()) { + if (hasTx) { + if (tx.getRollback()) { + db.rollback(); + } + else if (tx.getCommit()) { + db.commit(); // caller explicitly wanted commit even if they cancelled + } + else if (beganHere) { + db.rollback(); // safe default on cancellation + } + } + return; // don't call onCompleted() + } + + // --- TX end (normal path) — precedence: rollback > commit > begin-only ⇒ + // commit --- + if (hasTx) { + if (tx.getRollback()) { + db.rollback(); + } + else if (tx.getCommit()) { + db.commit(); + } + else if (beganHere) { + db.commit(); + } + } + + scso.onCompleted(); + + } + catch (Exception e) { + // Best-effort rollback only if we began here and there wasn't an explicit + // commit/rollback + try { + if (beganHere && db != null) { + if (request.hasTransaction()) { + var tx = request.getTransaction(); + if (!tx.getCommit() && !tx.getRollback()) + db.rollback(); + } + else { + db.rollback(); + } + } + } + catch (Exception ignore) { + /* no-op */ } + + if (!cancelled.get()) { + responseObserver.onError(Status.INTERNAL.withDescription("Stream query failed: " + e.getMessage()).asException()); + } + } + } + + /** + * Mode 1 (existing behavior-ish): run once and iterate results, batching as we + * go. + */ + private void streamCursor(Database db, StreamQueryRequest request, int batchSize, ServerCallStreamObserver scso, + AtomicBoolean cancelled, ProjectionConfig projectionConfig) { + + long running = 0L; + + QueryResult.Builder batch = QueryResult.newBuilder(); + int inBatch = 0; + + try (ResultSet rs = db.query("sql", request.getQuery(), convertParameters(request.getParametersMap()))) { + + while (rs.hasNext()) { + + if (cancelled.get()) + return; + waitUntilReady(scso, cancelled); + + Result r = rs.next(); + + if (r.isElement()) { + + com.arcadedb.database.Record rec = r.getElement().get(); + + batch.addRecords(convertToGrpcRecord(rec, db)); + + inBatch++; + running++; + + if (inBatch >= batchSize) { + safeOnNext(scso, cancelled, + batch.setTotalRecordsInBatch(inBatch).setRunningTotalEmitted(running).setIsLastBatch(false).build()); + batch = QueryResult.newBuilder(); + inBatch = 0; + } + } + else { + + GrpcRecord.Builder rb = GrpcRecord.newBuilder(); + + for (String p : r.getPropertyNames()) { + rb.putProperties(p, convertPropToGrpcValue(p, r, projectionConfig)); // overload below + } + + batch.addRecords(rb.build()); + + inBatch++; + running++; + + if (inBatch >= batchSize) { + + safeOnNext(scso, cancelled, + batch.setTotalRecordsInBatch(inBatch).setRunningTotalEmitted(running).setIsLastBatch(false).build()); + + batch = QueryResult.newBuilder(); + inBatch = 0; + } + } + } + } + + if (!cancelled.get() && inBatch > 0) { + safeOnNext(scso, cancelled, batch.setTotalRecordsInBatch(inBatch).setRunningTotalEmitted(running).setIsLastBatch(true).build()); + } + } + + /** + * Mode 2: materialize everything first (simple, but can be memory-heavy). + * + * @param projectionConfig + */ + private void streamMaterialized(Database db, StreamQueryRequest request, int batchSize, ServerCallStreamObserver scso, + AtomicBoolean cancelled, ProjectionConfig projectionConfig) { + + final java.util.List all = new java.util.ArrayList<>(); + + try (ResultSet rs = db.query("sql", request.getQuery(), convertParameters(request.getParametersMap()))) { + + while (rs.hasNext()) { + if (cancelled.get()) + return; + + Result r = rs.next(); + + if (r.isElement()) { + + all.add(convertToGrpcRecord(r.getElement().get(), db)); + } + else { + + GrpcRecord.Builder rb = GrpcRecord.newBuilder(); + + for (String p : r.getPropertyNames()) { + rb.putProperties(p, convertPropToGrpcValue(p, r, projectionConfig)); // overload below + } + + all.add(rb.build()); + } + } + } + + long running = 0L; + for (int i = 0; i < all.size(); i += batchSize) { + if (cancelled.get()) + return; + waitUntilReady(scso, cancelled); + + int end = Math.min(i + batchSize, all.size()); + QueryResult.Builder b = QueryResult.newBuilder(); + for (int j = i; j < end; j++) + b.addRecords(all.get(j)); + + running += (end - i); + safeOnNext(scso, cancelled, + b.setTotalRecordsInBatch(end - i).setRunningTotalEmitted(running).setIsLastBatch(end == all.size()).build()); + } + } + + /** + * Mode 3: only fetch one page’s worth of rows per emission via LIMIT/SKIP. + * + * @param projectionConfig + */ + private void streamPaged(Database db, StreamQueryRequest request, int batchSize, ServerCallStreamObserver scso, + AtomicBoolean cancelled, ProjectionConfig projectionConfig) { + + final String pagedSql = wrapWithSkipLimit(request.getQuery()); // see helper below + int offset = 0; + long running = 0L; + + while (true) { + if (cancelled.get()) + return; + waitUntilReady(scso, cancelled); + + java.util.Map params = new java.util.HashMap<>(convertParameters(request.getParametersMap())); + params.put("_skip", offset); + params.put("_limit", batchSize); + + int count = 0; + QueryResult.Builder b = QueryResult.newBuilder(); + + try (ResultSet rs = db.query("sql", pagedSql, params)) { + while (rs.hasNext()) { + if (cancelled.get()) + return; + + Result r = rs.next(); + + if (r.isElement()) { + + b.addRecords(convertToGrpcRecord(r.getElement().get(), db)); + count++; + } + else { + + GrpcRecord.Builder rb = GrpcRecord.newBuilder(); + + for (String p : r.getPropertyNames()) { + rb.putProperties(p, convertPropToGrpcValue(p, r, projectionConfig)); // overload below + } + + b.addRecords(rb.build()); + count++; + } + } + } + + if (count == 0) + return; // no more rows + + running += count; + boolean last = count < batchSize; + + safeOnNext(scso, cancelled, b.setTotalRecordsInBatch(count).setRunningTotalEmitted(running).setIsLastBatch(last).build()); + + if (last) + return; + offset += batchSize; + } + } + + private ProjectionConfig getProjectionConfigFromRequest(StreamQueryRequest request) { + + ProjectionConfig projectionConfig = null; + + if (request.hasProjectionSettings()) { + + var projectionSettings = request.getProjectionSettings(); + + final boolean includeProjections = projectionSettings.getIncludeProjections(); + final var projMode = projectionSettings.getProjectionEncoding(); + + int softLimit = projectionSettings.hasSoftLimitBytes() ? projectionSettings.getSoftLimitBytes().getValue() : 0; // choose your + // default + // or 0 = unlimited + projectionConfig = new ProjectionConfig(includeProjections, projMode, softLimit); + } + else { + + projectionConfig = new ProjectionConfig(true, ProjectionEncoding.PROJECTION_AS_JSON, 0); + } + + return projectionConfig; + } + + /** Wrap arbitrary SQL so we can safely inject LIMIT/SKIP outside. */ + private String wrapWithSkipLimit(String originalSql) { + + // Minimal defensive approach; you can do a real parser if needed. + + // ArcadeDB: SELECT FROM (...) [ORDER BY ...] SKIP :_skip LIMIT :_limit + + return "SELECT FROM (" + originalSql + ") ORDER BY @rid SKIP :_skip LIMIT :_limit"; + } + + private void safeOnNext(ServerCallStreamObserver scso, AtomicBoolean cancelled, QueryResult payload) { + if (cancelled.get()) + return; + try { + scso.onNext(payload); + } + catch (StatusRuntimeException e) { + if (e.getStatus().getCode() == Status.Code.CANCELLED) { + cancelled.set(true); + return; + } + throw e; + } + } + + private void waitUntilReady(ServerCallStreamObserver scso, AtomicBoolean cancelled) { + // Skip if you’re okay with best-effort pushes; otherwise honor transport + // readiness + if (scso.isReady()) + return; + while (!scso.isReady()) { + if (cancelled.get()) + return; + // avoid burning CPU: + try { + Thread.sleep(1); + } + catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + return; + } + } + } + + // --- 1) Unary bulk --- + @Override + public void bulkInsert(BulkInsertRequest req, StreamObserver resp) { + + final InsertOptions opts = defaults(req.getOptions()); // apply defaults (batch size, tx mode, etc.) + final long started = System.currentTimeMillis(); + + try (InsertContext ctx = new InsertContext(opts)) { + + Counts totals = insertRows(ctx, req.getRowsList().iterator()); + + ctx.flushCommit(true); + + resp.onNext(ctx.summary(totals, started)); + resp.onCompleted(); + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("bulkInsert: " + e.getMessage()).asException()); + } + } + + // --- 2) Client-streaming; single summary at end --- + @Override + public StreamObserver insertStream(StreamObserver resp) { + final ServerCallStreamObserver call = (ServerCallStreamObserver) resp; + + call.disableAutoInboundFlowControl(); + + final long startedAt = System.currentTimeMillis(); + + final java.util.concurrent.atomic.AtomicBoolean cancelled = new java.util.concurrent.atomic.AtomicBoolean(false); + final java.util.concurrent.atomic.AtomicReference ctxRef = new java.util.concurrent.atomic.AtomicReference<>(); + + final Counts totals = new Counts(); + + // cache the first-chunk effective options to validate consistency + final java.util.concurrent.atomic.AtomicReference firstOptsRef = new java.util.concurrent.atomic.AtomicReference<>(); + + call.setOnCancelHandler(() -> cancelled.set(true)); + + // Pull the very first inbound message + call.request(1); + + return new StreamObserver<>() { + + @Override + public void onNext(InsertChunk c) { + + if (cancelled.get()) { + return; + } + + try { + + InsertContext ctx = ctxRef.get(); + + if (ctx == null) { + + // -------- First chunk: derive effective options and build context -------- + + // Options may be provided on the first chunk; otherwise use server defaults + InsertOptions sent = c.hasOptions() ? c.getOptions() : InsertOptions.getDefaultInstance(); + + InsertOptions effective = defaults(sent); // your existing default-merging helper + + // (Optional) If your chunk carries db/creds (recommended), you can validate/set + // them here + + // Otherwise InsertContext will resolve them via 'effective' or server defaults + // Example if your proto carries these on the chunk: + // String dbName = c.getDatabase(); + // DatabaseCredentials creds = c.getCredentials(); + // and pass them to the InsertContext ctor if it expects them. + + // Create and cache context + ctx = new InsertContext(effective); // begins tx if PER_STREAM / PER_BATCH per your logic + + ctxRef.set(ctx); + + firstOptsRef.set(effective); + + if (logger.isDebugEnabled()) + logger.debug("insertStream: initialized context with options:\n{}", effective); + } + else { + + // -------- Subsequent chunks: optionally validate option consistency -------- + if (c.hasOptions()) { + InsertOptions prev = firstOptsRef.get(); + InsertOptions cur = defaults(c.getOptions()); + // Minimal consistency check (customize as you wish) + if (!sameInsertContract(prev, cur)) { + // contract changed mid-stream -> fail fast + throw new IllegalArgumentException("insertStream: options changed after first chunk"); + } + } + } + + // -------- Insert rows for this chunk -------- + Counts cts = insertRows(ctxRef.get(), c.getRowsList().iterator()); + totals.add(cts); + } + catch (Exception e) { + // Register as an error on totals; keep stream alive unless you prefer to fail + // the call + totals.err(Math.max(0, (ctxRef.get() != null ? ctxRef.get().received : 0) - 1), "DB_ERROR", e.getMessage(), ""); + } + finally { + if (!cancelled.get()) + call.request(1); + } + } + + @Override + public void onError(Throwable t) { + InsertContext ctx = ctxRef.get(); + if (ctx != null) + ctx.closeQuietly(); + } + + @Override + public void onCompleted() { + + try { + + InsertContext ctx = ctxRef.get(); + + if (ctx == null) { + // Client closed without sending a first chunk → nothing to do + resp.onNext(InsertSummary.newBuilder().setReceived(0).setInserted(0).setUpdated(0).setIgnored(0).setFailed(0) + .setExecutionTimeMs(System.currentTimeMillis() - startedAt).build()); + + resp.onCompleted(); + return; + } + + ctx.flushCommit(true); // commit if not validate-only + + if (!cancelled.get()) { + resp.onNext(ctx.summary(totals, startedAt)); + resp.onCompleted(); + } + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("insertStream: " + e.getMessage()).asException()); + } + finally { + InsertContext ctx = ctxRef.get(); + if (ctx != null) + ctx.closeQuietly(); + } + } + }; + } + + /** Minimal consistency check: same "contract" for the stream. */ + private boolean sameInsertContract(InsertOptions a, InsertOptions b) { + if (a == null || b == null) + return a == b; + // lock down the knobs that must not change mid-stream + if (a.getTargetClass() != null && !a.getTargetClass().equals(b.getTargetClass())) + return false; + if (a.getConflictMode() != b.getConflictMode()) + return false; + if (a.getTransactionMode() != b.getTransactionMode()) + return false; + // You can also compare key/update columns if you require strict equality: + if (!a.getKeyColumnsList().equals(b.getKeyColumnsList())) + return false; + if (!a.getUpdateColumnsOnConflictList().equals(b.getUpdateColumnsOnConflictList())) + return false; + return true; + } + + @Override + public StreamObserver insertBidirectional(StreamObserver resp) { + final ServerCallStreamObserver call = (ServerCallStreamObserver) resp; + call.disableAutoInboundFlowControl(); + + final java.util.concurrent.atomic.AtomicReference ref = new java.util.concurrent.atomic.AtomicReference<>(); + final java.util.concurrent.atomic.AtomicBoolean cancelled = new java.util.concurrent.atomic.AtomicBoolean(false); + final java.util.concurrent.atomic.AtomicBoolean started = new java.util.concurrent.atomic.AtomicBoolean(false); + + final java.util.concurrent.ConcurrentLinkedQueue outQueue = new java.util.concurrent.ConcurrentLinkedQueue<>(); + + // drain helper + final Runnable drain = () -> { + while (call.isReady()) { + InsertResponse next = outQueue.poll(); + if (next == null) + break; + resp.onNext(next); + } + }; + + // enqueue or send immediately + final java.util.function.Consumer sendOrQueue = (ir) -> { + outQueue.offer(ir); + drain.run(); + }; + + call.setOnReadyHandler(drain); + + call.setOnCancelHandler(() -> { + cancelled.set(true); + final InsertContext ctx = ref.getAndSet(null); + if (ctx != null) { + sessionWatermark.remove(ctx.sessionId); + ctx.closeQuietly(); + } + }); + + call.request(1); + + return new StreamObserver<>() { + @Override + public void onNext(InsertRequest reqMsg) { + if (cancelled.get()) + return; + + try { + switch (reqMsg.getMsgCase()) { + + case START -> { + if (!started.compareAndSet(false, true)) { + resp.onError( + Status.FAILED_PRECONDITION.withDescription("insertBidirectional: START already received").asException()); + return; + } + + final InsertOptions opts = defaults(reqMsg.getStart().getOptions()); + final InsertContext ctx = new InsertContext(opts); + ctx.startedAt = System.currentTimeMillis(); + ctx.totals = new Counts(); + + ref.set(ctx); + sessionWatermark.put(ctx.sessionId, 0L); + + sendOrQueue.accept( + InsertResponse.newBuilder().setStarted(Started.newBuilder().setSessionId(ctx.sessionId).build()).build()); + + // pull next message + call.request(1); + } + + case CHUNK -> { + final InsertContext ctx = require(ref.get(), "session not started"); + final InsertChunk c = reqMsg.getChunk(); + + // idempotent replay guard + final long hi = sessionWatermark.getOrDefault(ctx.sessionId, 0L); + + if (c.getChunkSeq() <= hi) { + + resp.onNext(InsertResponse.newBuilder().setBatchAck(BatchAck.newBuilder().setSessionId(ctx.sessionId) + .setChunkSeq(c.getChunkSeq()).setInserted(0).setUpdated(0).setIgnored(0).setFailed(0).build()).build()); + + call.request(1); + return; + } + + Counts perChunk; + try { + perChunk = insertRows(ctx, c.getRowsList().iterator()); + ctx.totals.add(perChunk); + sessionWatermark.put(ctx.sessionId, c.getChunkSeq()); + + sendOrQueue.accept(InsertResponse.newBuilder() + .setBatchAck(BatchAck.newBuilder().setSessionId(ctx.sessionId).setChunkSeq(c.getChunkSeq()) + .setInserted(perChunk.inserted).setUpdated(perChunk.updated).setIgnored(perChunk.ignored) + .setFailed(perChunk.failed).addAllErrors(perChunk.errors).build()) + .build()); + + } + catch (Exception e) { + // surface as failed chunk and continue (or switch to resp.onError(...) if you + // want to fail fast) + + perChunk = new Counts(); + + perChunk.failed = c.getRowsCount(); + + perChunk.errors.add(InsertError.newBuilder().setRowIndex(Math.max(0, ctx.received - 1)).setCode("DB_ERROR") + .setMessage(String.valueOf(e.getMessage())).build()); + ctx.totals.add(perChunk); + // intentionally do not advance watermark on failure; client may replay safely + } + + sendOrQueue.accept(InsertResponse.newBuilder() + .setBatchAck(BatchAck.newBuilder().setSessionId(ctx.sessionId).setChunkSeq(c.getChunkSeq()) + .setInserted(perChunk.inserted).setUpdated(perChunk.updated).setIgnored(perChunk.ignored) + .setFailed(perChunk.failed).addAllErrors(perChunk.errors).build()) + .build()); + + call.request(1); + } + + case COMMIT -> { + final InsertContext ctx = require(ref.get(), "session not started"); + try { + ctx.flushCommit(true); // commit unless validate_only in your InsertContext logic + final InsertSummary sum = ctx.summary(ctx.totals, ctx.startedAt); + resp.onNext(InsertResponse.newBuilder().setCommitted(Committed.newBuilder().setSummary(sum).build()).build()); + resp.onCompleted(); + } + catch (Exception e) { + resp.onError(Status.INTERNAL.withDescription("commit: " + e.getMessage()).asException()); + } + finally { + sessionWatermark.remove(ctx.sessionId); + ctx.closeQuietly(); + ref.set(null); + } + } + + case MSG_NOT_SET -> { + // ignore + call.request(1); + } + } + } + catch (Exception unexpected) { + // defensive: fail fast on unexpected exceptions + resp.onError(Status.INTERNAL.withDescription("insertBidirectional: " + unexpected.getMessage()).asException()); + final InsertContext ctx = ref.getAndSet(null); + if (ctx != null) { + sessionWatermark.remove(ctx.sessionId); + ctx.closeQuietly(); + } + } + } + + @Override + public void onError(Throwable t) { + final InsertContext ctx = ref.getAndSet(null); + if (ctx != null) { + sessionWatermark.remove(ctx.sessionId); + ctx.closeQuietly(); + } + } + + @Override + public void onCompleted() { + // Define your policy for half-close without COMMIT: + final InsertContext ctx = ref.getAndSet(null); + if (ctx != null) { + try { + // Safer default is rollback; if you prefer auto-commit on close, call + // flushCommit(true) + ctx.flushCommit(false); // or ctx.rollbackQuietly() if you have a helper + } + catch (Exception ignore) { + } + sessionWatermark.remove(ctx.sessionId); + ctx.closeQuietly(); + } + } + }; + } + + private boolean tryUpsertVertex(final InsertContext ctx, final com.arcadedb.graph.MutableVertex incoming) { + + var keys = ctx.keyCols; + + if (keys.isEmpty()) + return false; + + // Read incoming values via the (read-only) document view + var inDoc = incoming.asDocument(); + + String where = String.join(" AND ", keys.stream().map(k -> k + " = ?").toList()); + Object[] params = keys.stream().map(inDoc::get).toArray(); + + // Prefer selecting the element so we can get a mutable vertex directly + final String sql = "SELECT FROM " + ctx.opts.getTargetClass() + " WHERE " + where; + + try (var rs = ctx.db.query("sql", sql, params)) { + if (!rs.hasNext()) + return false; + + var res = rs.next(); + if (!res.isElement()) + return false; + + Vertex v = res.getElement().get().asVertex(); + + MutableVertex existingV = v.modify(); + + // Apply the updates + + for (String col : ctx.updateCols) { + existingV.set(col, inDoc.get(col)); + } + + existingV.save(); + + return true; + } + } + + private boolean tryUpsertDocument(InsertContext ctx, MutableDocument incoming) { + + var keys = ctx.keyCols; + + if (keys.isEmpty()) + return false; + + String where = String.join(" AND ", keys.stream().map(k -> k + " = ?").toList()); + Object[] params = keys.stream().map(incoming::get).toArray(); + + try (ResultSet rs = ctx.db.query("sql", "SELECT FROM " + ctx.opts.getTargetClass() + " WHERE " + where, params)) { + + if (!rs.hasNext()) { + return false; + } + + var res = rs.next(); + if (!res.isElement()) + return false; + + Document d = res.getElement().get().asDocument(); + + var existing = d.modify(); + + // Apply the updates + + for (String col : ctx.updateCols) { + existing.set(col, incoming.get(col)); + } + + existing.save(); + + return true; + } + } + + // ---------- Core insert plumbing ---------- + + private static final class Counts { + + long received, inserted, updated, ignored, failed; + + final java.util.List errors = new java.util.ArrayList<>(); + + void add(Counts o) { + received += o.received; + inserted += o.inserted; + updated += o.updated; + ignored += o.ignored; + failed += o.failed; + errors.addAll(o.errors); + } + + void err(long rowIndex, String code, String msg, String field) { + failed++; + errors.add(InsertError.newBuilder().setRowIndex(rowIndex).setCode(code).setMessage(msg).setField(field).build()); + } + } + + private Counts insertRows(InsertContext ctx, java.util.Iterator it) { + + Counts c = new Counts(); + + int inBatch = 0; + + Schema schema = ctx.db.getSchema(); + DocumentType dt = schema.getType(ctx.opts.getTargetClass()); + boolean isVertex = dt instanceof VertexType; + boolean isEdge = dt instanceof EdgeType; + + while (it.hasNext()) { + + GrpcRecord r = it.next(); + + c.received++; + ctx.received++; + + try { + if (ctx.opts.getValidateOnly()) + continue; + + if (isVertex) { + MutableVertex v = ctx.db.newVertex(ctx.opts.getTargetClass()); + applyGrpcRecord(v, r); + if (ctx.opts.getConflictMode() == ConflictMode.CONFLICT_UPDATE && tryUpsertVertex(ctx, v)) { + c.updated++; + } + else { + v.save(); + c.inserted++; + } + } + else if (isEdge) { + + String outRid = getStringProp(r, "out"); // lookup helper you already have + String inRid = getStringProp(r, "in"); + + if (outRid == null || inRid == null) { + + c.failed++; + + c.errors.add(InsertError.newBuilder().setRowIndex(ctx.received - 1).setCode("MISSING_ENDPOINTS") + .setMessage("Edge requires 'out' and 'in'").build()); + } + else { + var outV = ctx.db.lookupByRID(new com.arcadedb.database.RID(outRid), true).asVertex(true); + var inV = ctx.db.lookupByRID(new com.arcadedb.database.RID(inRid), true).asVertex(true); + + // Create edge from the OUT vertex + com.arcadedb.graph.MutableEdge e = outV.newEdge(ctx.opts.getTargetClass(), inV); + applyGrpcRecord(e, r); // sets edge properties + e.save(); + c.inserted++; + } + } + else { + MutableDocument d = ctx.db.newDocument(ctx.opts.getTargetClass()); + applyGrpcRecord(d, r); + if (ctx.opts.getConflictMode() == ConflictMode.CONFLICT_UPDATE && tryUpsertDocument(ctx, d)) { + c.updated++; + } + else { + d.save(); + c.inserted++; + } + } + + } + catch (com.arcadedb.exception.DuplicatedKeyException dup) { + switch (ctx.opts.getConflictMode()) { + case CONFLICT_IGNORE -> c.ignored++; + case CONFLICT_ABORT, UNRECOGNIZED -> c.err(ctx.received - 1, "CONFLICT", dup.getMessage(), ""); + case CONFLICT_UPDATE -> c.err(ctx.received - 1, "CONFLICT", dup.getMessage(), ""); + case CONFLICT_ERROR -> c.err(ctx.received - 1, "CONFLICT", dup.getMessage(), ""); + } + } + catch (Exception e) { + c.err(ctx.received - 1, "DB_ERROR", e.getMessage(), ""); + } + + inBatch++; + if (ctx.opts.getTransactionMode() == TransactionMode.PER_BATCH && inBatch >= serverBatchSize(ctx)) { + ctx.flushCommit(false); + inBatch = 0; + } + else if (ctx.opts.getTransactionMode() == TransactionMode.PER_ROW) { + ctx.flushCommit(false); + inBatch = 0; + } + } + return c; + } + + private String getStringProp(GrpcRecord r, String key) { + if (r == null || key == null || key.isEmpty()) + return null; + GrpcValue v = r.getPropertiesMap().get(key); + if (v == null) + return null; + if (v.getKindCase() == GrpcValue.KindCase.STRING_VALUE) + return v.getStringValue(); + return String.valueOf(fromGrpcValue(v)); + } + + private int serverBatchSize(InsertContext ctx) { + return ctx.opts.getServerBatchSize() == 0 ? 1000 : ctx.opts.getServerBatchSize(); + } + + private void applyGrpcRecord(MutableDocument doc, GrpcRecord r) { + r.getPropertiesMap().forEach((k, grpcVal) -> { + // Skip system fields + if (k.startsWith("@")) + return; + Object javaVal = fromGrpcValue(grpcVal); + if (logger.isDebugEnabled()) + logger.debug("APPLY-DOC {} <= {} -> {}", k, summarizeGrpc(grpcVal), summarizeJava(javaVal)); + doc.set(k, javaVal); + }); + } + + private void applyGrpcRecord(MutableVertex vertex, GrpcRecord r) { + r.getPropertiesMap().forEach((k, grpcVal) -> { + // Skip system fields + if (k.startsWith("@")) + return; + Object javaVal = fromGrpcValue(grpcVal); + if (logger.isDebugEnabled()) + logger.debug("APPLY-VERTEX {} <= {} -> {}", k, summarizeGrpc(grpcVal), summarizeJava(javaVal)); + vertex.set(k, javaVal); + }); + } + + private void applyGrpcRecord(MutableEdge edge, GrpcRecord r) { + r.getPropertiesMap().forEach((k, grpcVal) -> { + // Skip system fields + if (k.startsWith("@")) + return; + Object javaVal = fromGrpcValue(grpcVal); + if (logger.isDebugEnabled()) + logger.debug("APPLY-EDGE {} <= {} -> {}", k, summarizeGrpc(grpcVal), summarizeJava(javaVal)); + edge.set(k, javaVal); + }); + } + + private static long tsToMillis(com.google.protobuf.Timestamp ts) { + return ts.getSeconds() * 1000L + ts.getNanos() / 1_000_000L; + } + + private Object fromGrpcValue(GrpcValue v) { + if (v == null) + return dbgDec("fromGrpcValue", v, null, null); + switch (v.getKindCase()) { + case BOOL_VALUE: + return dbgDec("fromGrpcValue", v, v.getBoolValue(), null); + case INT32_VALUE: + return dbgDec("fromGrpcValue", v, v.getInt32Value(), null); + case INT64_VALUE: + return dbgDec("fromGrpcValue", v, v.getInt64Value(), null); + case FLOAT_VALUE: + return dbgDec("fromGrpcValue", v, v.getFloatValue(), null); + case DOUBLE_VALUE: + return dbgDec("fromGrpcValue", v, v.getDoubleValue(), null); + case STRING_VALUE: + return dbgDec("fromGrpcValue", v, v.getStringValue(), null); + case BYTES_VALUE: + return dbgDec("fromGrpcValue", v, v.getBytesValue().toByteArray(), null); + case TIMESTAMP_VALUE: + return dbgDec("fromGrpcValue", v, new java.util.Date(tsToMillis(v.getTimestampValue())), null); + case LINK_VALUE: + return dbgDec("fromGrpcValue", v, new com.arcadedb.database.RID(v.getLinkValue().getRid()), null); + + case DECIMAL_VALUE: { + var d = v.getDecimalValue(); + return dbgDec("fromGrpcValue", v, new java.math.BigDecimal(java.math.BigInteger.valueOf(d.getUnscaled()), d.getScale()), null); + } + + case LIST_VALUE: { + var out = new java.util.ArrayList<>(); + for (GrpcValue e : v.getListValue().getValuesList()) + out.add(fromGrpcValue(e)); + return dbgDec("fromGrpcValue", v, out, null); + } + + case MAP_VALUE: { + var out = new java.util.LinkedHashMap(); + v.getMapValue().getEntriesMap().forEach((k, vv) -> out.put(k, fromGrpcValue(vv))); + return dbgDec("fromGrpcValue", v, out, null); + } + + case EMBEDDED_VALUE: { + var out = new java.util.LinkedHashMap(); + v.getEmbeddedValue().getFieldsMap().forEach((k, vv) -> out.put(k, fromGrpcValue(vv))); + return dbgDec("fromGrpcValue", v, out, null); + } + + case KIND_NOT_SET: + return dbgDec("fromGrpcValue", v, null, null); + } + return dbgDec("fromGrpcValue", v, null, null); + } + + private static com.google.protobuf.Timestamp msToTimestamp(long ms) { + long seconds = Math.floorDiv(ms, 1000L); + int nanos = (int) Math.floorMod(ms, 1000L) * 1_000_000; + return com.google.protobuf.Timestamp.newBuilder().setSeconds(seconds).setNanos(nanos).build(); + } + + private GrpcValue toGrpcValue(Object o) { + + return toGrpcValue(o, null); + } + + private GrpcValue toGrpcValue(Object o, ProjectionConfig pc) { + + if (logger.isDebugEnabled()) { + logger.debug("toGrpcValue(): Converting\n value = {}\n class = {}", o, (o == null ? "null" : o.getClass().getName())); + } + + if (o instanceof JsonElement je) { + return gsonToGrpc(je); + } + + GrpcValue.Builder b = GrpcValue.newBuilder(); + + if (o == null) + return dbgEnc("toGrpcValue", o, b.build(), null); + + if (o instanceof Boolean v) + return dbgEnc("toGrpcValue", o, b.setBoolValue(v).build(), null); + if (o instanceof Integer v) + return dbgEnc("toGrpcValue", o, b.setInt32Value(v).build(), null); + if (o instanceof Long v) + return dbgEnc("toGrpcValue", o, b.setInt64Value(v).build(), null); + if (o instanceof Float v) + return dbgEnc("toGrpcValue", o, b.setFloatValue(v).build(), null); + if (o instanceof Double v) + return dbgEnc("toGrpcValue", o, b.setDoubleValue(v).build(), null); + if (o instanceof CharSequence v) + return dbgEnc("toGrpcValue", o, b.setStringValue(v.toString()).build(), null); + if (o instanceof byte[] v) + return dbgEnc("toGrpcValue", o, b.setBytesValue(com.google.protobuf.ByteString.copyFrom(v)).build(), null); + + if (o instanceof java.util.Date v) { + return dbgEnc("toGrpcValue", o, + GrpcValue.newBuilder().setTimestampValue(msToTimestamp(v.getTime())).setLogicalType("datetime").build(), null); + } + + if (o instanceof com.arcadedb.database.RID rid) { + return dbgEnc("toGrpcValue", o, + GrpcValue.newBuilder().setLinkValue(GrpcLink.newBuilder().setRid(rid.toString()).build()).setLogicalType("rid").build(), + null); + } + + if (o instanceof java.math.BigDecimal v) { + var unscaled = v.unscaledValue(); + if (unscaled.bitLength() <= 63) { + return GrpcValue.newBuilder().setDecimalValue(GrpcDecimal.newBuilder().setUnscaled(unscaled.longValue()).setScale(v.scale())) + .setLogicalType("decimal").build(); + } + else { + // if you need >64-bit unscaled, switch GrpcDecimal.unscaled to bytes in the + // proto + return dbgEnc("toGrpcValue", o, GrpcValue.newBuilder().setStringValue(v.toPlainString()).setLogicalType("decimal").build(), + null); + } + } + + if (o instanceof com.arcadedb.database.Document edoc && edoc.getIdentity() == null) { + GrpcEmbedded.Builder eb = GrpcEmbedded.newBuilder(); + if (edoc.getType() != null) + eb.setType(edoc.getTypeName()); + for (String k : edoc.getPropertyNames()) { + eb.putFields(k, toGrpcValue(edoc.get(k))); + } + return dbgEnc("toGrpcValue", o, GrpcValue.newBuilder().setEmbeddedValue(eb.build()).build(), null); + } + + // ===== PROJECTION-AWARE Document handling ===== + if (o instanceof com.arcadedb.database.Document doc) { + + final boolean inProjection = (pc != null && pc.include); + + if (!inProjection) { + // Not a projection row: send as LINK if possible, else fall back + if (doc.getIdentity() != null && doc.getIdentity().isValid()) { + if (logger.isDebugEnabled()) { + logger.debug("GRPC-ENC [toGrpcValue] DOC-NON-PROJECTION -> LINK rid={}", doc.getIdentity()); + } + return GrpcValue.newBuilder().setLinkValue(GrpcLink.newBuilder().setRid(doc.getIdentity().toString()).build()) + .setLogicalType("rid").build(); + } + // No identity → treat as EMBEDDED-like MAP + if (logger.isDebugEnabled()) { + logger.debug("GRPC-ENC [toGrpcValue] DOC-NON-PROJECTION (no rid) -> MAP"); + } + GrpcMap.Builder mb = GrpcMap.newBuilder(); + if (doc.getType() != null) + mb.putEntries("@type", GrpcValue.newBuilder().setStringValue(doc.getTypeName()).build()); + for (String k : doc.getPropertyNames()) { + mb.putEntries(k, toGrpcValue(doc.get(k), pc)); + } + return GrpcValue.newBuilder().setMapValue(mb.build()).build(); + } + + // In a PROJECTION row: obey encoding mode + ProjectionEncoding enc = pc.enc; + + // 3.1 LINK mode + if (enc == ProjectionEncoding.PROJECTION_AS_LINK) { + if (doc.getIdentity() != null && doc.getIdentity().isValid()) { + if (logger.isDebugEnabled()) { + logger.debug("GRPC-ENC [toGrpcValue] PROJECTION -> LINK rid={}", doc.getIdentity()); + } + return GrpcValue.newBuilder().setLinkValue(GrpcLink.newBuilder().setRid(doc.getIdentity().toString()).build()) + .setLogicalType("rid").build(); + } + // No rid: fall back to MAP + if (logger.isDebugEnabled()) { + logger.debug("GRPC-ENC [toGrpcValue] PROJECTION LINK fallback -> MAP (no rid)"); + } + enc = ProjectionEncoding.PROJECTION_AS_MAP; + } + + // 3.2 MAP mode + if (enc == ProjectionEncoding.PROJECTION_AS_MAP) { + if (logger.isDebugEnabled()) { + logger.debug("GRPC-ENC [toGrpcValue] PROJECTION -> MAP rid={} type={}", + (doc.getIdentity() != null ? doc.getIdentity() : "null"), (doc.getType() != null ? doc.getTypeName() : "null")); + } + GrpcMap.Builder mb = GrpcMap.newBuilder(); + + // meta + if (doc.getIdentity() != null && doc.getIdentity().isValid()) { + GrpcValue ridVal = GrpcValue.newBuilder().setLinkValue(GrpcLink.newBuilder().setRid(doc.getIdentity().toString()).build()) + .setLogicalType("rid").build(); + // soft limit accounting (precise, using serialized sizes) + if (!pc.wouldExceed(bytesOf("@rid") + ridVal.getSerializedSize())) { + mb.putEntries("@rid", ridVal); + } + else { + pc.truncated = true; + } + } + if (doc.getType() != null) { + GrpcValue typeVal = GrpcValue.newBuilder().setStringValue(doc.getTypeName()).build(); + if (!pc.wouldExceed(bytesOf("@type") + typeVal.getSerializedSize())) { + mb.putEntries("@type", typeVal); + } + else { + pc.truncated = true; + } + } + + for (String k : doc.getPropertyNames()) { + GrpcValue child = toGrpcValue(doc.get(k), pc); // recurse with config + int add = bytesOf(k) + child.getSerializedSize(); + if (pc.wouldExceed(add)) { + if (logger.isDebugEnabled()) { + logger.debug("GRPC-ENC [toGrpcValue] PROJECTION MAP soft-limit hit; skipping '{}' (limit={}, used~{})", k, + pc.softLimitBytes, pc.used.get()); + } + pc.truncated = true; + break; + } + mb.putEntries(k, child); + } + + return GrpcValue.newBuilder().setMapValue(mb.build()).build(); + } + + // 3.3 JSON mode + if (enc == ProjectionEncoding.PROJECTION_AS_JSON) { + if (logger.isDebugEnabled()) { + logger.debug("GRPC-ENC [toGrpcValue] PROJECTION -> JSON rid={} type={}", + (doc.getIdentity() != null ? doc.getIdentity() : "null"), (doc.getType() != null ? doc.getTypeName() : "null")); + } + + try { + + // Let ArcadeDB serialize the document/result properly + + boolean hasEmptyCollection = false; + for (String k : doc.getPropertyNames()) { + Object v = doc.get(k); + if (v instanceof java.util.Collection c && c.isEmpty()) { + hasEmptyCollection = true; + break; + } + } + + var json = (hasEmptyCollection ? SAFE : FAST).serializeDocument(doc); + + byte[] jsonBytes = json.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8); + + // Soft limit handling + if (pc.softLimitBytes > 0 && jsonBytes.length > pc.softLimitBytes) { + pc.truncated = true; + if (logger.isDebugEnabled()) { + logger.debug("GRPC-ENC [toGrpcValue] PROJECTION JSON soft-limit hit; size={} limit={}", jsonBytes.length, + pc.softLimitBytes); + } + // Prefer a RID fallback if we have one + if (doc.getIdentity() != null && doc.getIdentity().isValid()) { + return GrpcValue.newBuilder().setLinkValue(GrpcLink.newBuilder().setRid(doc.getIdentity().toString()).build()) + .setLogicalType("rid").build(); + } + else { + jsonBytes = "{\"__truncated\":true}".getBytes(java.nio.charset.StandardCharsets.UTF_8); + } + } + else { + pc.used.addAndGet(jsonBytes.length); + } + + return GrpcValue.newBuilder().setBytesValue(com.google.protobuf.ByteString.copyFrom(jsonBytes)).setLogicalType("json") + .build(); + } + catch (Throwable t) { + // Safe fallback: send as MAP (typed GrpcValue tree) + logger.warn("GRPC-ENC [toGrpcValue] PROJECTION_AS_JSON failed for rid={} type={}, falling back to MAP; err={}", + (doc.getIdentity() != null ? doc.getIdentity() : "null"), (doc.getType() != null ? doc.getTypeName() : "null"), + t.toString()); + + GrpcMap.Builder mb = GrpcMap.newBuilder(); + + if (doc.getIdentity() != null && doc.getIdentity().isValid()) { + mb.putEntries("@rid", GrpcValue.newBuilder() + .setLinkValue(GrpcLink.newBuilder().setRid(doc.getIdentity().toString()).build()).setLogicalType("rid").build()); + } + if (doc.getType() != null) { + mb.putEntries("@type", GrpcValue.newBuilder().setStringValue(doc.getTypeName()).build()); + } + for (String k : doc.getPropertyNames()) { + mb.putEntries(k, toGrpcValue(doc.get(k))); + } + + return GrpcValue.newBuilder().setMapValue(mb.build()).build(); + } + } + + // Shouldn’t get here, but fall back + if (logger.isDebugEnabled()) { + logger.debug("GRPC-ENC [toGrpcValue] PROJECTION unknown encoding {}; falling back to LINK/STRING", enc.name()); + } + if (doc.getIdentity() != null && doc.getIdentity().isValid()) { + return GrpcValue.newBuilder().setLinkValue(GrpcLink.newBuilder().setRid(doc.getIdentity().toString()).build()) + .setLogicalType("rid").build(); + } + + // else small MAP fallback + GrpcMap.Builder mb = GrpcMap.newBuilder(); + for (String k : doc.getPropertyNames()) + mb.putEntries(k, toGrpcValue(doc.get(k), pc)); + return GrpcValue.newBuilder().setMapValue(mb.build()).build(); + } + + if (o instanceof java.util.Map m) { + GrpcMap.Builder mb = GrpcMap.newBuilder(); + for (var e : m.entrySet()) { + mb.putEntries(String.valueOf(e.getKey()), toGrpcValue(e.getValue())); + } + return dbgEnc("toGrpcValue", o, GrpcValue.newBuilder().setMapValue(mb.build()).build(), null); + } + + if (o instanceof java.util.Collection c) { + GrpcList.Builder lb = GrpcList.newBuilder(); + for (Object e : c) + lb.addValues(toGrpcValue(e)); + return dbgEnc("toGrpcValue", o, GrpcValue.newBuilder().setListValue(lb.build()).build(), null); + } + + if (o.getClass().isArray()) { + int len = java.lang.reflect.Array.getLength(o); + GrpcList.Builder lb = GrpcList.newBuilder(); + for (int i = 0; i < len; i++) { + lb.addValues(toGrpcValue(java.lang.reflect.Array.get(o, i))); + } + return dbgEnc("toGrpcValue", o, GrpcValue.newBuilder().setListValue(lb.build()).build(), null); + } + + // RID/Identifiable string fallback + if (o instanceof com.arcadedb.database.Identifiable id) + return dbgEnc("toGrpcValue", o, GrpcValue.newBuilder() + .setLinkValue(GrpcLink.newBuilder().setRid(id.getIdentity().toString()).build()).setLogicalType("rid").build(), null); + + // Support ArcadeDB Result/ResultInternal as structural MAP + if (o instanceof com.arcadedb.query.sql.executor.Result res) { + GrpcMap.Builder mb = GrpcMap.newBuilder(); + try { + for (String k : res.getPropertyNames()) { + mb.putEntries(k, toGrpcValue(res.getProperty(k))); + } + } + catch (Throwable ignore) { + } + + return dbgEnc("toGrpcValue", o, GrpcValue.newBuilder().setMapValue(mb.build()).build(), null); + } + + // Fallback + if (logger.isDebugEnabled()) { + logger.debug("GRPC-ENC [toGrpcValue] FALLBACK-TO-STRING for class={} value={}", o.getClass().getName(), String.valueOf(o)); + } + + return dbgEnc("toGrpcValue", o, GrpcValue.newBuilder().setStringValue(String.valueOf(o)).build(), null); + } + + private static int bytesOf(String s) { + return s == null ? 0 : s.getBytes(java.nio.charset.StandardCharsets.UTF_8).length; + } + + private InsertOptions defaults(InsertOptions in) { + InsertOptions.Builder b = in.toBuilder(); + if (in.getServerBatchSize() == 0) + b.setServerBatchSize(1000); + if (in.getTransactionMode() == TransactionMode.UNRECOGNIZED) + b.setTransactionMode(TransactionMode.PER_BATCH); + return b.build(); + } + + private static T require(T v, String msg) { + if (v == null) + throw Status.FAILED_PRECONDITION.withDescription(msg).asRuntimeException(); + return v; + } + + private static Timestamp ts(long ms) { + return Timestamp.newBuilder().setSeconds(ms / 1000).setNanos((int) ((ms % 1000) * 1_000_000)).build(); + } + + private final class InsertContext implements AutoCloseable { + + protected Counts totals; + + final InsertOptions opts; + + final Database db; + + final java.util.List keyCols; + final java.util.List updateCols; + + long startedAt; + + final String sessionId = java.util.UUID.randomUUID().toString(); + long received = 0; + + InsertContext(InsertOptions opts) { + + this.opts = opts; + + this.db = getDatabase(opts.getDatabase(), opts.getCredentials()); + + this.keyCols = opts.getKeyColumnsList(); + + this.updateCols = opts.getUpdateColumnsOnConflictList(); + + if (!opts.getValidateOnly()) { + if (opts.getTransactionMode() == TransactionMode.PER_STREAM || opts.getTransactionMode() == TransactionMode.PER_BATCH) { + db.begin(); + } + } + } + + void flushCommit(boolean end) { + + if (opts.getValidateOnly()) { + if (end) + db.rollback(); + return; + } + switch (opts.getTransactionMode()) { + case PER_ROW -> db.commit(); + case PER_REQUEST -> db.commit(); + case PER_BATCH -> { + db.commit(); + if (!end) + db.begin(); + } + case PER_STREAM -> { + if (end) + db.commit(); + } + default -> { + } + } + } + + InsertSummary summary(Counts c, long startedAtMs) { + + long now = System.currentTimeMillis(); + + return InsertSummary.newBuilder().setReceived(c.received).setInserted(c.inserted).setUpdated(c.updated).setIgnored(c.ignored) + .setFailed(c.failed).addAllErrors(c.errors).setStartedAt(ts(startedAtMs)).setFinishedAt(ts(now)).build(); + } + + @Override + public void close() { + } + + void closeQuietly() { + try { + close(); + } + catch (Exception ignore) { + } + } + } + + // Helper methods + + private Database getDatabase(String databaseName, DatabaseCredentials credentials) { + + // Validate credentials + validateCredentials(credentials); + + // Use the same approach as Postgres/Redis plugins + if (arcadeServer != null) { + + // This is how other plugins do it - get the already-open database + Database db = arcadeServer.getDatabase(databaseName); + + logger.debug("getDatabase(): db = {} isOpen = {}", db, db.isOpen()); + + if (db != null) { + return db; + } + } + + // Check if database is already in the pool + String poolKey = databaseName; + Database database = databasePool.get(poolKey); + + if (database != null && database.isOpen()) { + // Return existing open database + return database; + } + + // Create new database connection + synchronized (databasePool) { + // Double-check after acquiring lock + database = databasePool.get(poolKey); + if (database != null && database.isOpen()) { + return database; + } + + // Create database factory for the specific database + DatabaseFactory dbFactory = new DatabaseFactory(databasePath + "/" + databaseName); + + try { + // Open database - ArcadeDB requires MODE parameter + if (dbFactory.exists()) { + // Try READ_ONLY first to avoid conflicts + try { + database = dbFactory.open(ComponentFile.MODE.READ_ONLY); + } + catch (Exception e) { + // If READ_ONLY fails, try READ_WRITE + logger.debug("Opening database in READ_WRITE mode: {}", databaseName); + database = dbFactory.open(ComponentFile.MODE.READ_WRITE); + } + } + else { + // Create if it doesn't exist + database = dbFactory.create(); + } + + // Add to pool + databasePool.put(poolKey, database); + return database; + + } + catch (Exception e) { + logger.error("Failed to open database: {}", databaseName, e); + throw new RuntimeException("Cannot open database: " + databaseName + " - " + e.getMessage(), e); + } + } + } + + private void validateCredentials(DatabaseCredentials credentials) { + + // Implement credential validation logic + // This is a placeholder - integrate with ArcadeDB's security system + + if (credentials == null || credentials.getUsername().isEmpty()) { + throw new IllegalArgumentException("Invalid credentials"); + } + } + + private Map convertParameters(Map protoParams) { + + Map params = new HashMap<>(); + + for (Map.Entry entry : protoParams.entrySet()) { + + params.put(entry.getKey(), fromGrpcValue(entry.getValue())); + } + + return params; + } + + private GrpcRecord convertToGrpcRecord(com.arcadedb.database.Record dbRecord, Database db) { + + GrpcRecord.Builder builder = GrpcRecord.newBuilder().setRid(dbRecord.getIdentity().toString()); + + if (dbRecord instanceof Document doc) { + + if (doc.getType() != null) + builder.setType(doc.getTypeName()); + + if (doc.getIdentity() != null) + builder.setRid(doc.getIdentity().toString()); + + // set all properties + for (String propertyName : doc.getPropertyNames()) { + + Object value = doc.get(propertyName); + + if (value != null) { + + if (logger.isDebugEnabled()) { + logger.debug("convertToGrpcRecord(): Converting {}\n value = {}\n class = {}", propertyName, value, value.getClass()); + } + + GrpcValue gv = toGrpcValue(value); + + if (logger.isDebugEnabled()) + logger.debug("ENC-REC {}.{}: {} -> {}", builder.getRid(), propertyName, summarizeJava(value), summarizeGrpc(gv)); + + builder.putProperties(propertyName, gv); + } + } + + // If this is an Edge, include @out / @in for convenience (string rid or link) + if (dbRecord instanceof Edge edge) { + + if (!builder.getPropertiesMap().containsKey("@out")) { + builder.putProperties("@out", toGrpcValue(edge.getOut().getIdentity())); + } + + if (!builder.getPropertiesMap().containsKey("@in")) { + builder.putProperties("@in", toGrpcValue(edge.getIn().getIdentity())); + } + } + } + + if (logger.isDebugEnabled()) + logger.debug("ENC-REC DONE rid={} type={} props={}", builder.getRid(), builder.getType(), builder.getPropertiesCount()); + + return builder.build(); + } + + private GrpcValue convertPropToGrpcValue(String propName, Result result, ProjectionConfig pc) { + + Object propValue = result.getProperty(propName); + + if (logger.isDebugEnabled()) { + logger.debug("convertPropToGrpcValue(): Converting {}\n value = {}\n class = {}", propName, propValue, propValue.getClass()); + } + + return toGrpcValue(propValue, pc); + } + + private GrpcValue convertPropToGrpcValue(String propName, Result result) { + + Object propValue = result.getProperty(propName); + + if (logger.isDebugEnabled()) { + logger.debug("convertPropToGrpcValue(): Converting {}\n value = {}\n class = {}", propName, propValue, propValue.getClass()); + } + + return toGrpcValue(propValue); + } + + private Object toJavaForProperty(final Database db, final MutableDocument parent, final DocumentType dtype, final String propName, + final GrpcValue grpcValue) { + + if (grpcValue == null) + return null; + + // Try schema + com.arcadedb.schema.Property prop = null; + try { + prop = (dtype != null) ? dtype.getProperty(propName) : null; + } + catch (com.arcadedb.exception.SchemaException ignore) { + } + + if (prop != null) + return convertWithSchemaType(db, parent, prop, propName, grpcValue); + + // No schema → generic + return fromGrpcValue(grpcValue); + } + + private Object convertWithSchemaType(Database db, MutableDocument parent, com.arcadedb.schema.Property prop, String propName, GrpcValue v) { + + var t = prop.getType(); + + switch (t) { + + case BOOLEAN: + return switch (v.getKindCase()) { + case BOOL_VALUE -> v.getBoolValue(); + case STRING_VALUE -> Boolean.parseBoolean(v.getStringValue()); + default -> null; + }; + + case BYTE: + return switch (v.getKindCase()) { + case INT32_VALUE, INT64_VALUE, DOUBLE_VALUE, FLOAT_VALUE -> (byte) (long) fromGrpcValue(v); + case STRING_VALUE -> Byte.parseByte(v.getStringValue()); + default -> null; + }; + + case SHORT: + return switch (v.getKindCase()) { + case INT32_VALUE, INT64_VALUE, DOUBLE_VALUE, FLOAT_VALUE -> (short) (long) fromGrpcValue(v); + case STRING_VALUE -> Short.parseShort(v.getStringValue()); + default -> null; + }; + + case INTEGER: + return switch (v.getKindCase()) { + case INT32_VALUE -> v.getInt32Value(); + case INT64_VALUE -> (int) v.getInt64Value(); + case DOUBLE_VALUE -> (int) v.getDoubleValue(); + case FLOAT_VALUE -> (int) v.getFloatValue(); + case STRING_VALUE -> Integer.parseInt(v.getStringValue()); + default -> null; + }; + + case LONG: + return switch (v.getKindCase()) { + case INT64_VALUE -> v.getInt64Value(); + case INT32_VALUE -> (long) v.getInt32Value(); + case DOUBLE_VALUE -> (long) v.getDoubleValue(); + case FLOAT_VALUE -> (long) v.getFloatValue(); + case STRING_VALUE -> Long.parseLong(v.getStringValue()); + default -> null; + }; + + case FLOAT: + return switch (v.getKindCase()) { + case FLOAT_VALUE -> v.getFloatValue(); + case DOUBLE_VALUE -> (float) v.getDoubleValue(); + case INT32_VALUE -> (float) v.getInt32Value(); + case INT64_VALUE -> (float) v.getInt64Value(); + case STRING_VALUE -> Float.parseFloat(v.getStringValue()); + default -> null; + }; + + case DOUBLE: + return switch (v.getKindCase()) { + case DOUBLE_VALUE -> v.getDoubleValue(); + case FLOAT_VALUE -> (double) v.getFloatValue(); + case INT32_VALUE -> (double) v.getInt32Value(); + case INT64_VALUE -> (double) v.getInt64Value(); + case STRING_VALUE -> Double.parseDouble(v.getStringValue()); + default -> null; + }; + + case STRING: + return switch (v.getKindCase()) { + case STRING_VALUE -> v.getStringValue(); + default -> String.valueOf(fromGrpcValue(v)); + }; + + case DECIMAL: + return switch (v.getKindCase()) { + case DECIMAL_VALUE -> { + var d = v.getDecimalValue(); + yield new java.math.BigDecimal(java.math.BigInteger.valueOf(d.getUnscaled()), d.getScale()); + } + case STRING_VALUE -> new java.math.BigDecimal(v.getStringValue()); + case DOUBLE_VALUE -> java.math.BigDecimal.valueOf(v.getDoubleValue()); + case INT32_VALUE -> java.math.BigDecimal.valueOf(v.getInt32Value()); + case INT64_VALUE -> java.math.BigDecimal.valueOf(v.getInt64Value()); + default -> null; + }; + + case DATE: + case DATETIME: + // Prefer timestamp_value; else accept epoch ms in int64; else parse string + return switch (v.getKindCase()) { + case TIMESTAMP_VALUE -> new java.util.Date(tsToMillis(v.getTimestampValue())); + case INT64_VALUE -> new java.util.Date(v.getInt64Value()); + case STRING_VALUE -> new java.util.Date(Long.parseLong(v.getStringValue())); // or parse ISO if you emit it + default -> null; + }; + + case BINARY: + return switch (v.getKindCase()) { + case BYTES_VALUE -> v.getBytesValue().toByteArray(); + case STRING_VALUE -> v.getStringValue().getBytes(java.nio.charset.StandardCharsets.UTF_8); + default -> null; + }; + + case LINK: + return switch (v.getKindCase()) { + case LINK_VALUE -> new com.arcadedb.database.RID(v.getLinkValue().getRid()); + case STRING_VALUE -> new com.arcadedb.database.RID(v.getStringValue()); + default -> null; + }; + + case EMBEDDED: { + + // Use schema ofType if present; otherwise use embedded.type from the payload + if (logger.isDebugEnabled()) { + logger.debug("EMBEDDED: prop='{}', incoming kind={}, schema ofType='{}'", propName, v.getKindCase(), prop.getOfType()); + } + + String embeddedTypeName = (v.getKindCase() == GrpcValue.KindCase.EMBEDDED_VALUE && !v.getEmbeddedValue().getType().isEmpty()) + ? v.getEmbeddedValue().getType() + : prop.getOfType(); + + // If still unknown, try to discover from MAP payload keys + // (type/@type/empowerType) + if ((embeddedTypeName == null || embeddedTypeName.isEmpty()) && v.getKindCase() == GrpcValue.KindCase.MAP_VALUE) { + var entries = v.getMapValue().getEntriesMap(); + GrpcValue tv = entries.getOrDefault("type", entries.getOrDefault("@type", entries.get("empowerType"))); + if (tv != null && tv.getKindCase() == GrpcValue.KindCase.STRING_VALUE) { + embeddedTypeName = tv.getStringValue(); + if (logger.isDebugEnabled()) { + logger.debug("EMBEDDED: discovered embeddedTypeName='{}' from MAP payload", embeddedTypeName); + } + } + } + + // If we still don't know the type and payload isn't EMBEDDED, fall back to Map + if ((embeddedTypeName == null || embeddedTypeName.isEmpty()) && v.getKindCase() != GrpcValue.KindCase.EMBEDDED_VALUE) { + + if (logger.isDebugEnabled()) { + logger.debug("EMBEDDED: fallback to Map for prop='{}' (kind={}, embeddedTypeName='{}')", propName, v.getKindCase(), + embeddedTypeName); + } + return fromGrpcValue(v); + } + + if (logger.isDebugEnabled()) { + final String source = (v.getKindCase() == GrpcValue.KindCase.EMBEDDED_VALUE && !v.getEmbeddedValue().getType().isEmpty()) + ? "payload" + : "schema/discovered"; + int fields = (v.getKindCase() == GrpcValue.KindCase.EMBEDDED_VALUE) ? v.getEmbeddedValue().getFieldsCount() + : (v.getKindCase() == GrpcValue.KindCase.MAP_VALUE) ? v.getMapValue().getEntriesCount() : 0; + logger.debug("EMBEDDED: resolved embeddedTypeName='{}' (source={}), fields={}", embeddedTypeName, source, fields); + } + + // Build typed embedded document + MutableEmbeddedDocument ed = parent.newEmbeddedDocument(embeddedTypeName, propName); + DocumentType embeddedType = null; + try { + embeddedType = db.getSchema().getType(embeddedTypeName); + if (logger.isDebugEnabled()) { + logger.debug("EMBEDDED: schema type lookup '{}' -> {}", embeddedTypeName, (embeddedType != null ? "FOUND" : "NOT FOUND")); + } + } + catch (Exception e) { + if (logger.isDebugEnabled()) { + logger.debug("EMBEDDED: schema type lookup failed for '{}': {}", embeddedTypeName, e.toString()); + } + } + final DocumentType embeddedTypeFinal = embeddedType; + + // Populate from either EMBEDDED_VALUE.fields or MAP_VALUE.entries + switch (v.getKindCase()) { + case EMBEDDED_VALUE: + v.getEmbeddedValue().getFieldsMap().forEach((k, vv) -> { + if (logger.isDebugEnabled()) + logger.debug("EMBEDDED: field '{}' raw kind={}", k, vv.getKindCase()); + Object j = (embeddedTypeFinal != null) ? toJavaForProperty(db, ed, embeddedTypeFinal, k, vv) : fromGrpcValue(vv); + if (logger.isDebugEnabled()) + logger.debug("EMBEDDED: field '{}' converted -> {}", k, (j == null ? "null" : j.getClass().getSimpleName())); + ed.set(k, j); + }); + break; + + case MAP_VALUE: + v.getMapValue().getEntriesMap().forEach((k, vv) -> { + if (logger.isDebugEnabled()) + logger.debug("EMBEDDED: field '{}' raw kind={}", k, vv.getKindCase()); + Object j = (embeddedTypeFinal != null) ? toJavaForProperty(db, ed, embeddedTypeFinal, k, vv) : fromGrpcValue(vv); + if (logger.isDebugEnabled()) + logger.debug("EMBEDDED: field '{}' converted -> {}", k, (j == null ? "null" : j.getClass().getSimpleName())); + ed.set(k, j); + }); + break; + + default: + // Shouldn't happen; be safe + if (logger.isDebugEnabled()) { + logger.debug("EMBEDDED: unexpected kind={}, falling back to Map for prop='{}'", v.getKindCase(), propName); + } + return fromGrpcValue(v); + } + + if (logger.isDebugEnabled()) { + logger.debug("EMBEDDED: built embedded doc type='{}' for prop='{}'", embeddedTypeName, propName); + } + return ed; + } + + case MAP: + if (v.getKindCase() == GrpcValue.KindCase.MAP_VALUE) { + var m = new java.util.LinkedHashMap(); + v.getMapValue().getEntriesMap().forEach((k, vv) -> m.put(k, fromGrpcValue(vv))); + return m; + } + return null; + + case LIST: + if (v.getKindCase() == GrpcValue.KindCase.LIST_VALUE) { + var list = new java.util.ArrayList<>(); + for (GrpcValue item : v.getListValue().getValuesList()) { + list.add(fromGrpcValue(item)); + } + return list; + } + return null; + case ARRAY_OF_SHORTS: { + if (v.getKindCase() == GrpcValue.KindCase.LIST_VALUE) { + var out = new java.util.ArrayList(); + for (GrpcValue item : v.getListValue().getValuesList()) { + Object o = fromGrpcValue(item); + if (o instanceof Number n) + out.add(n.shortValue()); + else if (o instanceof String s) + out.add(Short.parseShort(s)); + } + return out; + } + return null; + } + + case ARRAY_OF_INTEGERS: { + if (v.getKindCase() == GrpcValue.KindCase.LIST_VALUE) { + var out = new java.util.ArrayList(); + for (GrpcValue item : v.getListValue().getValuesList()) { + Object o = fromGrpcValue(item); + if (o instanceof Number n) + out.add(n.intValue()); + else if (o instanceof String s) + out.add(Integer.parseInt(s)); + } + return out; + } + return null; + } + + case ARRAY_OF_LONGS: { + if (v.getKindCase() == GrpcValue.KindCase.LIST_VALUE) { + var out = new java.util.ArrayList(); + for (GrpcValue item : v.getListValue().getValuesList()) { + Object o = fromGrpcValue(item); + if (o instanceof Number n) + out.add(n.longValue()); + else if (o instanceof String s) + out.add(Long.parseLong(s)); + } + return out; + } + return null; + } + + case ARRAY_OF_FLOATS: { + if (v.getKindCase() == GrpcValue.KindCase.LIST_VALUE) { + var out = new java.util.ArrayList(); + for (GrpcValue item : v.getListValue().getValuesList()) { + Object o = fromGrpcValue(item); + if (o instanceof Number n) + out.add(n.floatValue()); + else if (o instanceof String s) + out.add(Float.parseFloat(s)); + } + return out; + } + return null; + } + + case ARRAY_OF_DOUBLES: { + if (v.getKindCase() == GrpcValue.KindCase.LIST_VALUE) { + var out = new java.util.ArrayList(); + for (GrpcValue item : v.getListValue().getValuesList()) { + Object o = fromGrpcValue(item); + if (o instanceof Number n) + out.add(n.doubleValue()); + else if (o instanceof String s) + out.add(Double.parseDouble(s)); + } + return out; + } + return null; + } + + case DATETIME_SECOND: + case DATETIME_MICROS: + case DATETIME_NANOS: { + // Same handling as DATETIME + return switch (v.getKindCase()) { + case TIMESTAMP_VALUE -> new java.util.Date(tsToMillis(v.getTimestampValue())); + case INT64_VALUE -> new java.util.Date(v.getInt64Value()); // epoch ms expected + case STRING_VALUE -> new java.util.Date(Long.parseLong(v.getStringValue())); + default -> null; + }; + } + } + + // default fallback + return fromGrpcValue(v); + } + + private String generateTransactionId() { + return "tx_" + System.nanoTime(); + } + + // ---- Debug helpers ---- + private static String summarizeJava(Object o) { + if (o == null) + return "null"; + try { + if (o instanceof CharSequence s) + return "String(" + s.length() + ")=\"" + (s.length() > 120 ? s.subSequence(0, 120) + "…" : s) + "\""; + if (o instanceof byte[] b) + return "bytes[" + b.length + "]"; + if (o instanceof java.util.Collection c) + return o.getClass().getSimpleName() + "[size=" + c.size() + "]"; + if (o instanceof java.util.Map m) + return o.getClass().getSimpleName() + "[size=" + m.size() + "]"; + return o.getClass().getSimpleName() + "(" + String.valueOf(o) + ")"; + } + catch (Exception e) { + return o.getClass().getSimpleName(); + } + } + + private static String summarizeGrpc(GrpcValue v) { + if (v == null) + return "GrpcValue(null)"; + switch (v.getKindCase()) { + case BOOL_VALUE: + return "BOOL(" + v.getBoolValue() + ")"; + case INT32_VALUE: + return "INT32(" + v.getInt32Value() + ")"; + case INT64_VALUE: + return "INT64(" + v.getInt64Value() + ")"; + case FLOAT_VALUE: + return "FLOAT(" + v.getFloatValue() + ")"; + case DOUBLE_VALUE: + return "DOUBLE(" + v.getDoubleValue() + ")"; + case STRING_VALUE: { + String s = v.getStringValue(); + return "STRING(" + s.length() + ")=\"" + (s.length() > 120 ? s.substring(0, 120) + "…" : s) + "\""; + } + case BYTES_VALUE: + return "BYTES[" + v.getBytesValue().size() + "]"; + case TIMESTAMP_VALUE: + return "TIMESTAMP(" + v.getTimestampValue().getSeconds() + "." + v.getTimestampValue().getNanos() + ")"; + case LIST_VALUE: + return "LIST[size=" + v.getListValue().getValuesCount() + "]"; + case MAP_VALUE: + return "MAP[size=" + v.getMapValue().getEntriesCount() + "]"; + case EMBEDDED_VALUE: + return "EMBEDDED[type=" + v.getEmbeddedValue().getType() + ", size=" + v.getEmbeddedValue().getFieldsCount() + "]"; + case LINK_VALUE: + return "LINK(" + v.getLinkValue().getRid() + ")"; + case DECIMAL_VALUE: + return "DECIMAL(unscaled=" + v.getDecimalValue().getUnscaled() + ", scale=" + v.getDecimalValue().getScale() + ")"; + case KIND_NOT_SET: + default: + return "GrpcValue(KIND_NOT_SET)"; + } + } + + private GrpcValue dbgEnc(String where, Object in, GrpcValue out, String ctx) { + if (logger.isDebugEnabled()) + logger.debug("GRPC-ENC [{}]{} in={} -> out={}", where, (ctx == null ? "" : " " + ctx), summarizeJava(in), summarizeGrpc(out)); + return out; + } + + private Object dbgDec(String where, GrpcValue in, Object out, String ctx) { + if (logger.isDebugEnabled()) + logger.debug("GRPC-DEC [{}]{} in={} -> out={}", where, (ctx == null ? "" : " " + ctx), summarizeGrpc(in), summarizeJava(out)); + return out; + } + + // --- Gson structural encoder: JsonElement -> GrpcValue --- + private GrpcValue gsonToGrpc(JsonElement n) { + + GrpcValue.Builder b = GrpcValue.newBuilder(); + + if (n == null || n.isJsonNull()) { + return b.build(); // KIND_NOT_SET => null on the client + } + + if (n.isJsonPrimitive()) { + JsonPrimitive p = n.getAsJsonPrimitive(); + if (p.isBoolean()) + return b.setBoolValue(p.getAsBoolean()).build(); + if (p.isNumber()) { + java.math.BigDecimal bd; + try { + bd = p.getAsBigDecimal(); + } + catch (Exception e) { + return b.setDoubleValue(p.getAsDouble()).build(); + } + if (bd.scale() == 0) { + try { + int i = bd.intValueExact(); + return b.setInt32Value(i).build(); + } + catch (ArithmeticException ignore) { + } + try { + long l = bd.longValueExact(); + return b.setInt64Value(l).build(); + } + catch (ArithmeticException ignore) { + } + java.math.BigInteger unscaled = bd.unscaledValue(); + if (unscaled.bitLength() <= 63) { + return b.setDecimalValue(GrpcDecimal.newBuilder().setUnscaled(unscaled.longValue()).setScale(bd.scale()).build()) + .build(); + } + return b.setStringValue(bd.toPlainString()).build(); + } + else { + java.math.BigInteger unscaled = bd.unscaledValue(); + if (unscaled.bitLength() <= 63) { + return b.setDecimalValue(GrpcDecimal.newBuilder().setUnscaled(unscaled.longValue()).setScale(bd.scale()).build()) + .build(); + } + return b.setStringValue(bd.toPlainString()).build(); + } + } + if (p.isString()) + return b.setStringValue(p.getAsString()).build(); + return b.setStringValue(p.getAsString()).build(); + } + + if (n.isJsonArray()) { + GrpcList.Builder lb = GrpcList.newBuilder(); + for (JsonElement e : n.getAsJsonArray()) { + lb.addValues(gsonToGrpc(e)); + } + return b.setListValue(lb.build()).build(); + } + + if (n.isJsonObject()) { + GrpcMap.Builder mb = GrpcMap.newBuilder(); + JsonObject obj = n.getAsJsonObject(); + for (var entry : obj.entrySet()) { + mb.putEntries(entry.getKey(), gsonToGrpc(entry.getValue())); + } + return b.setMapValue(mb.build()).build(); + } + + return b.setStringValue(n.toString()).build(); + } + + static final class ProjectionConfig { + final boolean include; + final ProjectionEncoding enc; + final int softLimitBytes; // 0 = no soft limit + + // Running counter for MAP/JSON builds: + final java.util.concurrent.atomic.AtomicInteger used = new java.util.concurrent.atomic.AtomicInteger(0); + volatile boolean truncated = false; + + ProjectionConfig(boolean include, ProjectionEncoding enc, int softLimitBytes) { + this.include = include; + this.enc = enc == null ? ProjectionEncoding.PROJECTION_AS_LINK : enc; + this.softLimitBytes = Math.max(softLimitBytes, 0); + } + + boolean wouldExceed(int add) { + if (softLimitBytes <= 0) + return false; + int after = used.addAndGet(add); + if (after > softLimitBytes) { + truncated = true; + } + return after > softLimitBytes; + } + + public boolean isInclude() { + return include; + } + + public ProjectionEncoding getEnc() { + return enc; + } + } +} \ No newline at end of file diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/CompressionAwareService.java b/grpcw/src/main/java/com/arcadedb/server/grpc/CompressionAwareService.java new file mode 100644 index 0000000000..860501d231 --- /dev/null +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/CompressionAwareService.java @@ -0,0 +1,86 @@ +package com.arcadedb.server.grpc; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.grpc.stub.ServerCallStreamObserver; +import io.grpc.stub.StreamObserver; + + +/** + * Enhanced compression-aware service wrapper + */ +class CompressionAwareService { + private static final Logger logger = LoggerFactory.getLogger(CompressionAwareService.class); + + /** + * Force compression for a specific response + */ + public static void setResponseCompression(StreamObserver responseObserver, String compression) { + if (responseObserver instanceof ServerCallStreamObserver) { + ServerCallStreamObserver serverObserver = (ServerCallStreamObserver) responseObserver; + serverObserver.setCompression(compression); + logger.debug("Set response compression to: {}", compression); + } + } + + /** + * Check if current request was compressed (call from service method) + */ + public static boolean isCurrentRequestCompressed() { + GrpcCompressionInterceptor.CompressionInfo info = + GrpcCompressionInterceptor.COMPRESSION_KEY.get(); + return info != null && info.requestCompressed; + } + + /** + * Get current request compression encoding + */ + public static String getCurrentRequestEncoding() { + GrpcCompressionInterceptor.CompressionInfo info = + GrpcCompressionInterceptor.COMPRESSION_KEY.get(); + return info != null ? info.requestEncoding : "identity"; + } + + /** + * Get compression statistics for monitoring + */ + public static class CompressionStats { + private long compressedRequests = 0; + private long uncompressedRequests = 0; + private long compressedResponses = 0; + private long uncompressedResponses = 0; + + public synchronized void recordRequest(boolean compressed) { + if (compressed) { + compressedRequests++; + } else { + uncompressedRequests++; + } + } + + public synchronized void recordResponse(boolean compressed) { + if (compressed) { + compressedResponses++; + } else { + uncompressedResponses++; + } + } + + public synchronized String getStats() { + long totalRequests = compressedRequests + uncompressedRequests; + long totalResponses = compressedResponses + uncompressedResponses; + + double reqCompressionRate = totalRequests > 0 ? + (compressedRequests * 100.0 / totalRequests) : 0; + double respCompressionRate = totalResponses > 0 ? + (compressedResponses * 100.0 / totalResponses) : 0; + + return String.format( + "Compression Stats - Requests: %.1f%% (%d/%d), Responses: %.1f%% (%d/%d)", + reqCompressionRate, compressedRequests, totalRequests, + respCompressionRate, compressedResponses, totalResponses + ); + } + } +} \ No newline at end of file diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcAuthInterceptor.java b/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcAuthInterceptor.java new file mode 100644 index 0000000000..06274dcab1 --- /dev/null +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcAuthInterceptor.java @@ -0,0 +1,142 @@ +package com.arcadedb.server.grpc; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.arcadedb.server.security.ServerSecurity; + +import io.grpc.Context; +import io.grpc.Contexts; +import io.grpc.Metadata; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; +import io.grpc.Status; + +/** + * Authentication interceptor for gRPC requests + */ +class GrpcAuthInterceptor implements ServerInterceptor { + private static final Logger logger = LoggerFactory.getLogger(GrpcAuthInterceptor.class); + + private static final String BEARER_TYPE = "Bearer"; + private static final Metadata.Key AUTHORIZATION_HEADER = + Metadata.Key.of("authorization", Metadata.ASCII_STRING_MARSHALLER); + private static final Metadata.Key USER_HEADER = + Metadata.Key.of("x-arcade-user", Metadata.ASCII_STRING_MARSHALLER); + private static final Metadata.Key PASSWORD_HEADER = + Metadata.Key.of("x-arcade-password", Metadata.ASCII_STRING_MARSHALLER); + private static final Metadata.Key DATABASE_HEADER = + Metadata.Key.of("x-arcade-database", Metadata.ASCII_STRING_MARSHALLER); + + private final ServerSecurity security; + private final boolean securityEnabled; + + public GrpcAuthInterceptor(ServerSecurity security) { + this.security = security; + // Check if security is enabled by checking if it's not null and has users configured + this.securityEnabled = (security != null && security.getUsers() != null && !security.getUsers().isEmpty()); + } + + @Override + public ServerCall.Listener interceptCall( + ServerCall call, + Metadata headers, + ServerCallHandler next) { + + String methodName = call.getMethodDescriptor().getFullMethodName(); + + // Skip auth for health check and reflection + if (methodName.startsWith("grpc.health.") || + methodName.startsWith("grpc.reflection.")) { + return next.startCall(call, headers); + } + + // If security is not enabled, allow all requests + if (!securityEnabled) { + return next.startCall(call, headers); + } + + try { + // Get database name from header (required for authentication) + String database = headers.get(DATABASE_HEADER); + if (database == null || database.isEmpty()) { + database = "default"; // Use default database if not specified + } + + // Try Bearer token authentication first + String authorization = headers.get(AUTHORIZATION_HEADER); + if (authorization != null && authorization.startsWith(BEARER_TYPE)) { + String token = authorization.substring(BEARER_TYPE.length()).trim(); + if (!validateToken(token, database)) { + call.close(Status.UNAUTHENTICATED.withDescription("Invalid token"), new Metadata()); + return new ServerCall.Listener() {}; + } + } else { + // Try basic authentication + String username = headers.get(USER_HEADER); + String password = headers.get(PASSWORD_HEADER); + + if (username == null || password == null) { + // No authentication provided for secured server + call.close(Status.UNAUTHENTICATED.withDescription("Authentication required"), new Metadata()); + return new ServerCall.Listener() {}; + } else { + // Validate credentials + if (!validateCredentials(username, password, database)) { + call.close(Status.UNAUTHENTICATED.withDescription("Invalid credentials"), new Metadata()); + return new ServerCall.Listener() {}; + } + // Add user to context + Context context = Context.current().withValue(USER_CONTEXT_KEY, username); + return Contexts.interceptCall(context, call, headers, next); + } + } + + // Add user context for downstream processing + Context context = Context.current(); + if (authorization != null) { + context = context.withValue(USER_CONTEXT_KEY, extractUserFromToken(authorization)); + } + + return Contexts.interceptCall(context, call, headers, next); + + } catch (Exception e) { + logger.error("Authentication error", e); + call.close(Status.INTERNAL.withDescription("Authentication error"), new Metadata()); + return new ServerCall.Listener() {}; + } + } + + private boolean validateToken(String token, String database) { + // Implement token validation logic + // This could integrate with JWT, OAuth2, or custom token validation + // For now, this is a placeholder + return true; + } + + private boolean validateCredentials(String username, String password, String database) { + if (security == null) { + return true; // No security configured + } + + try { + // ArcadeDB's authenticate method requires database name as well + // Returns a SecurityUser object if authentication succeeds, null otherwise + Object authenticatedUser = security.authenticate(username, password, database); + return authenticatedUser != null; + } catch (Exception e) { + logger.error("Failed to authenticate user: {} for database: {}", username, database, e); + return false; + } + } + + private String extractUserFromToken(String authorization) { + // Extract user information from token + // This is a placeholder - implement based on your token format + return "authenticated-user"; + } + + // Context key for storing authenticated user + public static final Context.Key USER_CONTEXT_KEY = Context.key("user"); +} \ No newline at end of file diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcCompressionInterceptor.java b/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcCompressionInterceptor.java new file mode 100644 index 0000000000..d40fff0e2f --- /dev/null +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcCompressionInterceptor.java @@ -0,0 +1,109 @@ +package com.arcadedb.server.grpc; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.grpc.Context; +import io.grpc.Contexts; +import io.grpc.ForwardingServerCall; +import io.grpc.Metadata; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; + +/** + * Compression interceptor that can force compression based on configuration + */ +class GrpcCompressionInterceptor implements ServerInterceptor { + private static final Logger logger = LoggerFactory.getLogger(GrpcCompressionInterceptor.class); + + // Context key to store compression info + public static final Context.Key COMPRESSION_KEY = Context.key("compression-info"); + + private static final Metadata.Key GRPC_ACCEPT_ENCODING = Metadata.Key.of("grpc-accept-encoding", Metadata.ASCII_STRING_MARSHALLER); + private static final Metadata.Key GRPC_ENCODING = Metadata.Key.of("grpc-encoding", Metadata.ASCII_STRING_MARSHALLER); + + private final boolean forceCompression; + private final String compressionType; + + private final int minMessageSizeForCompression; + + public GrpcCompressionInterceptor(boolean forceCompression, String compressionType, int minMessageSizeBytes) { + this.forceCompression = forceCompression; + this.compressionType = compressionType != null ? compressionType : "gzip"; + this.minMessageSizeForCompression = minMessageSizeBytes; + } + + @Override + public ServerCall.Listener interceptCall(ServerCall call, Metadata headers, + ServerCallHandler next) { + + String methodName = call.getMethodDescriptor().getFullMethodName(); + + // Check if request is compressed + String requestEncoding = headers.get(Metadata.Key.of("grpc-encoding", Metadata.ASCII_STRING_MARSHALLER)); + boolean requestCompressed = requestEncoding != null && !requestEncoding.equals("identity"); + + final boolean clientAcceptsGzip = headers.get(GRPC_ACCEPT_ENCODING) != null && headers.get(GRPC_ACCEPT_ENCODING).contains("gzip"); + + // Store compression info in context + CompressionInfo compressionInfo = new CompressionInfo(requestCompressed, requestEncoding); + Context context = Context.current().withValue(COMPRESSION_KEY, compressionInfo); + + // Wrap the call to control compression + ServerCall compressedCall = new ForwardingServerCall.SimpleForwardingServerCall(call) { + private boolean compressionSet = false; + + @Override + public void sendHeaders(Metadata responseHeaders) { + + if (clientAcceptsGzip && !compressionSet) { + + // Pick the compressor for this call before headers go out + super.setCompression("gzip"); + setMessageCompression(true); + + logger.debug("Forced {} compression for method: {}", compressionType, methodName); + + compressionSet = true; + } + + super.sendHeaders(responseHeaders); + } + + @Override + public void sendMessage(RespT message) { + + if (!compressionSet && forceCompression) { + + // Force compression for this response + + if (this.getMethodDescriptor().getType().serverSendsOneMessage()) { + // For unary calls, we can set compression + setMessageCompression(true); + compressionSet = true; + + logger.debug("Forced {} compression for method: {}", compressionType, methodName); + } + } + + super.sendMessage(message); + } + }; + + return Contexts.interceptCall(context, compressedCall, headers, next); + } + + /** + * Helper class to store compression information + */ + public static class CompressionInfo { + public final boolean requestCompressed; + public final String requestEncoding; + + public CompressionInfo(boolean requestCompressed, String requestEncoding) { + this.requestCompressed = requestCompressed; + this.requestEncoding = requestEncoding; + } + } +} \ No newline at end of file diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcLoggingInterceptor.java b/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcLoggingInterceptor.java new file mode 100644 index 0000000000..1dcdc0145f --- /dev/null +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcLoggingInterceptor.java @@ -0,0 +1,81 @@ +package com.arcadedb.server.grpc; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.grpc.ForwardingServerCall; +import io.grpc.Metadata; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; +import io.grpc.Status; + +/** + * Logging interceptor for gRPC requests with compression detection + */ +class GrpcLoggingInterceptor implements ServerInterceptor { + private static final Logger logger = LoggerFactory.getLogger(GrpcLoggingInterceptor.class); + + private static final Metadata.Key GRPC_ENCODING_KEY = + Metadata.Key.of("grpc-encoding", Metadata.ASCII_STRING_MARSHALLER); + private static final Metadata.Key GRPC_ACCEPT_ENCODING_KEY = + Metadata.Key.of("grpc-accept-encoding", Metadata.ASCII_STRING_MARSHALLER); + + @Override + public ServerCall.Listener interceptCall( + ServerCall call, + Metadata headers, + ServerCallHandler next) { + + String methodName = call.getMethodDescriptor().getFullMethodName(); + long startTime = System.currentTimeMillis(); + + // Check if client sent compressed request + String requestEncoding = headers.get(GRPC_ENCODING_KEY); + String acceptEncoding = headers.get(GRPC_ACCEPT_ENCODING_KEY); + boolean requestCompressed = requestEncoding != null && !requestEncoding.equals("identity"); + boolean clientAcceptsCompression = acceptEncoding != null && acceptEncoding.contains("gzip"); + + logger.debug("gRPC call started: {} (request compression: {}, client accepts: {})", + methodName, + requestCompressed ? requestEncoding : "none", + clientAcceptsCompression ? acceptEncoding : "none"); + + ServerCall wrappedCall = new ForwardingServerCall.SimpleForwardingServerCall(call) { + private String responseCompression = "none"; + + @Override + public void sendHeaders(Metadata headers) { + // Check if we're sending compressed response + String encoding = headers.get(GRPC_ENCODING_KEY); + if (encoding != null && !encoding.equals("identity")) { + responseCompression = encoding; + } + super.sendHeaders(headers); + } + + @Override + public void close(Status status, Metadata trailers) { + long duration = System.currentTimeMillis() - startTime; + + // Add compression info to trailers for client visibility + trailers.put(Metadata.Key.of("x-grpc-compression-used", Metadata.ASCII_STRING_MARSHALLER), + responseCompression); + trailers.put(Metadata.Key.of("x-grpc-request-compressed", Metadata.ASCII_STRING_MARSHALLER), + String.valueOf(requestCompressed)); + + if (status.isOk()) { + logger.debug("gRPC call completed: {} ({}ms, req-compression: {}, resp-compression: {})", + methodName, duration, + requestCompressed ? requestEncoding : "none", + responseCompression); + } else { + logger.warn("gRPC call failed: {} - {} ({}ms)", methodName, status, duration); + } + super.close(status, trailers); + } + }; + + return next.startCall(wrappedCall, headers); + } +} \ No newline at end of file diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcMetricsInterceptor.java b/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcMetricsInterceptor.java new file mode 100644 index 0000000000..100a1ee665 --- /dev/null +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcMetricsInterceptor.java @@ -0,0 +1,78 @@ +package com.arcadedb.server.grpc; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.arcadedb.server.ArcadeDBServer; + +import io.grpc.ForwardingServerCall; +import io.grpc.Metadata; +import io.grpc.ServerCall; +import io.grpc.ServerCallHandler; +import io.grpc.ServerInterceptor; +import io.grpc.Status; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Timer; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; + +/** + * Metrics interceptor for gRPC requests using Micrometer + */ +class GrpcMetricsInterceptor implements ServerInterceptor { + + private static final Logger logger = LoggerFactory.getLogger(GrpcMetricsInterceptor.class); + + private final MeterRegistry meterRegistry; + private final Counter requestCounter; + private final Counter errorCounter; + private final Timer requestTimer; + + public GrpcMetricsInterceptor(ArcadeDBServer server) { + // Try to get existing meter registry from server, or create a simple one + this.meterRegistry = new SimpleMeterRegistry(); // In production, integrate with server's meter registry + + this.requestCounter = Counter.builder("grpc.requests.total") + .description("Total number of gRPC requests") + .register(meterRegistry); + + this.errorCounter = Counter.builder("grpc.errors.total") + .description("Total number of gRPC errors") + .register(meterRegistry); + + this.requestTimer = Timer.builder("grpc.request.duration") + .description("gRPC request duration") + .register(meterRegistry); + } + + @Override + public ServerCall.Listener interceptCall( + ServerCall call, + Metadata headers, + ServerCallHandler next) { + + String methodName = call.getMethodDescriptor().getFullMethodName(); + Timer.Sample sample = Timer.start(meterRegistry); + + requestCounter.increment(); + + ServerCall wrappedCall = new ForwardingServerCall.SimpleForwardingServerCall(call) { + @Override + public void close(Status status, Metadata trailers) { + sample.stop(requestTimer); + + if (!status.isOk()) { + errorCounter.increment(); + } + + // Add metrics as trailers for observability + trailers.put(Metadata.Key.of("grpc-metrics-method", Metadata.ASCII_STRING_MARSHALLER), methodName); + trailers.put(Metadata.Key.of("grpc-metrics-status", Metadata.ASCII_STRING_MARSHALLER), status.getCode().toString()); + + super.close(status, trailers); + } + }; + + return next.startCall(wrappedCall, headers); + } +} \ No newline at end of file diff --git a/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcServerPlugin.java b/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcServerPlugin.java new file mode 100644 index 0000000000..d6fc8317dc --- /dev/null +++ b/grpcw/src/main/java/com/arcadedb/server/grpc/GrpcServerPlugin.java @@ -0,0 +1,405 @@ +package com.arcadedb.server.grpc; + +import java.io.File; +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.arcadedb.ContextConfiguration; +import com.arcadedb.server.ArcadeDBServer; +import com.arcadedb.server.ServerPlugin; +import com.arcadedb.server.security.ServerSecurity; +import com.arcadedb.server.security.credential.DefaultCredentialsValidator; + +import io.grpc.CompressorRegistry; +import io.grpc.DecompressorRegistry; +import io.grpc.Grpc; +import io.grpc.InsecureServerCredentials; +import io.grpc.Server; +import io.grpc.ServerBuilder; +import io.grpc.ServerCredentials; +import io.grpc.TlsServerCredentials; +import io.grpc.health.v1.HealthCheckResponse; +import io.grpc.protobuf.services.HealthStatusManager; +import io.grpc.protobuf.services.ProtoReflectionService; +import io.grpc.xds.XdsServerBuilder; + +/** + * ArcadeDB gRPC Server Plugin + * + * Configuration options: + * - grpc.enabled: Enable/disable gRPC server (default: true) + * - grpc.port: Port for standard gRPC server (default: 50051) + * - grpc.host: Host to bind (default: 0.0.0.0) + * - grpc.mode: Server mode - "standard", "xds", or "both" (default: standard) + * - grpc.xds.port: Port for XDS server (default: 50052) + * - grpc.tls.enabled: Enable TLS (default: false) + * - grpc.tls.cert: Path to TLS certificate + * - grpc.tls.key: Path to TLS private key + * - grpc.maxMessageSize: Max message size in MB (default: 100) + * - grpc.reflection.enabled: Enable gRPC reflection (default: true) + * - grpc.health.enabled: Enable health checking (default: true) + */ +public class GrpcServerPlugin implements ServerPlugin { + + private static final Logger logger = LoggerFactory.getLogger(GrpcServerPlugin.class); + + private ArcadeDBServer arcadeServer; + private Server grpcServer; + private Server xdsServer; + private HealthStatusManager healthManager; + private ArcadeDbGrpcService grpcService; // Keep reference for cleanup + private Thread shutdownHook; + + // Configuration keys as simple strings + private static final String CONFIG_PREFIX = "arcadedb.grpc."; + private static final String CONFIG_ENABLED = CONFIG_PREFIX + "enabled"; + private static final String CONFIG_PORT = CONFIG_PREFIX + "port"; + private static final String CONFIG_HOST = CONFIG_PREFIX + "host"; + private static final String CONFIG_MODE = CONFIG_PREFIX + "mode"; + private static final String CONFIG_XDS_PORT = CONFIG_PREFIX + "xds.port"; + private static final String CONFIG_TLS_ENABLED = CONFIG_PREFIX + "tls.enabled"; + private static final String CONFIG_TLS_CERT = CONFIG_PREFIX + "tls.cert"; + private static final String CONFIG_TLS_KEY = CONFIG_PREFIX + "tls.key"; + private static final String CONFIG_MAX_MESSAGE_SIZE = CONFIG_PREFIX + "maxMessageSize"; + private static final String CONFIG_REFLECTION_ENABLED = CONFIG_PREFIX + "reflection.enabled"; + private static final String CONFIG_HEALTH_ENABLED = CONFIG_PREFIX + "health.enabled"; + private static final String CONFIG_COMPRESSION_ENABLED = CONFIG_PREFIX + "compression.enabled"; + private static final String CONFIG_COMPRESSION_FORCE = CONFIG_PREFIX + "compression.force"; + private static final String CONFIG_COMPRESSION_TYPE = CONFIG_PREFIX + "compression.type"; + + @Override + public void configure(ArcadeDBServer server, ContextConfiguration configuration) { + this.arcadeServer = server; + } + + @Override + public void startService() { + ContextConfiguration config = arcadeServer.getConfiguration(); + + // Get configuration values with defaults + boolean enabled = getConfigBoolean(config, CONFIG_ENABLED, true); + if (!enabled) { + logger.info("gRPC server is disabled"); + return; + } + + String mode = getConfigString(config, CONFIG_MODE, "standard").toLowerCase(); + + try { + switch (mode) { + case "standard": + startStandardServer(config); + break; + case "xds": + startXdsServer(config); + break; + case "both": + startStandardServer(config); + startXdsServer(config); + break; + default: + logger.error("Invalid gRPC mode: {}. Use 'standard', 'xds', or 'both'", mode); + } + + registerShutdownHook(); + + } catch (IOException e) { + logger.error("Failed to start gRPC server", e); + throw new RuntimeException("Failed to start gRPC server", e); + } + } + + private void startStandardServer(ContextConfiguration config) throws IOException { + + int port = getConfigInt(config, CONFIG_PORT, 50051); + String host = getConfigString(config, CONFIG_HOST, "0.0.0.0"); + + ServerBuilder serverBuilder; + + // Configure TLS if enabled + if (getConfigBoolean(config, CONFIG_TLS_ENABLED, false)) { + serverBuilder = configureStandardTls(port, config); + } else { + serverBuilder = ServerBuilder.forPort(port); + } + + // Configure the server + configureServer(serverBuilder, config); + + grpcServer = serverBuilder + .maxInboundMessageSize(256 * 1024 * 1024) + .maxInboundMetadataSize(32 * 1024 * 1024) + .build().start(); + + // Build status message + StringBuilder status = new StringBuilder(); + status.append("gRPC server started on ").append(host).append(":").append(port); + status.append(" (mode: standard"); + + if (getConfigBoolean(config, CONFIG_TLS_ENABLED, false)) { + status.append(", TLS enabled"); + } + + if (getConfigBoolean(config, CONFIG_COMPRESSION_ENABLED, true)) { + + status.append(", compression: "); + + if (getConfigBoolean(config, CONFIG_COMPRESSION_FORCE, false)) { + status.append("forced-").append(getConfigString(config, CONFIG_COMPRESSION_TYPE, "gzip")); + } else { + status.append("available"); + } + } + + status.append(")"); + logger.info(status.toString()); + } + + private void startXdsServer(ContextConfiguration config) throws IOException { + int port = getConfigInt(config, CONFIG_XDS_PORT, 50052); + + // XDS server for service mesh integration + // XdsServerBuilder requires ServerCredentials + XdsServerBuilder xdsBuilder = XdsServerBuilder.forPort(port, InsecureServerCredentials.create()); + + // Configure the XDS server as a ServerBuilder + configureServer(xdsBuilder, config); + + xdsServer = xdsBuilder + .maxInboundMessageSize(256 * 1024 * 1024) + .maxInboundMetadataSize(32 * 1024 * 1024) + .build().start(); + + logger.info("gRPC XDS server started on port {} (xDS management enabled)", port); + } + + private void configureServer(ServerBuilder serverBuilder, ContextConfiguration config) { + + // Get database directory path + String databasePath = arcadeServer.getRootPath() + File.separator + "databases"; + + // Create the main service + ArcadeDbGrpcService mainService = new ArcadeDbGrpcService(databasePath, arcadeServer); + + // Add the main service + serverBuilder.addService(mainService); + + // Create the Admin service + ArcadeDbGrpcAdminService adminService = new ArcadeDbGrpcAdminService(arcadeServer, new DefaultCredentialsValidator()); + + // Add the Admin service + serverBuilder.addService(adminService); + + // Add health service if enabled + if (getConfigBoolean(config, CONFIG_HEALTH_ENABLED, true)) { + healthManager = new HealthStatusManager(); + serverBuilder.addService(healthManager.getHealthService()); + + // Set initial health status + healthManager.setStatus( + ArcadeDbGrpcService.class.getName(), + HealthCheckResponse.ServingStatus.SERVING + ); + } + + // Add reflection service if enabled + if (getConfigBoolean(config, CONFIG_REFLECTION_ENABLED, true)) { + serverBuilder.addService(ProtoReflectionService.newInstance()); + } + + serverBuilder.compressorRegistry(CompressorRegistry.getDefaultInstance()).decompressorRegistry(DecompressorRegistry.getDefaultInstance()); + + // Configure max message size + int maxMessageSizeMB = getConfigInt(config, CONFIG_MAX_MESSAGE_SIZE, 100); + + serverBuilder.maxInboundMessageSize(maxMessageSizeMB * 1024 * 1024); + + // Add interceptors for logging, metrics, auth, etc. + serverBuilder.intercept(new GrpcLoggingInterceptor()); + serverBuilder.intercept(new GrpcMetricsInterceptor(arcadeServer)); + + // Add compression interceptor if force compression is enabled + if (getConfigBoolean(config, CONFIG_COMPRESSION_FORCE, false)) { + String compressionType = getConfigString(config, CONFIG_COMPRESSION_TYPE, "gzip"); + serverBuilder.intercept(new GrpcCompressionInterceptor(true, compressionType, 1024)); + } + + // Add authentication interceptor if security is configured + ServerSecurity serverSecurity = arcadeServer.getSecurity(); + if (serverSecurity != null) { + serverBuilder.intercept(new GrpcAuthInterceptor(serverSecurity)); + } + } + + private ServerBuilder configureStandardTls(int port, ContextConfiguration config) { + String certPath = getConfigString(config, CONFIG_TLS_CERT, null); + String keyPath = getConfigString(config, CONFIG_TLS_KEY, null); + + if (certPath == null || keyPath == null) { + logger.warn("TLS enabled but certificate or key path not provided. Falling back to insecure."); + return ServerBuilder.forPort(port); + } + + File certFile = new File(certPath); + File keyFile = new File(keyPath); + + if (!certFile.exists() || !keyFile.exists()) { + logger.warn("TLS certificate or key file not found. Falling back to insecure."); + return ServerBuilder.forPort(port); + } + + try { + ServerCredentials credentials = TlsServerCredentials.create(certFile, keyFile); + // Use Grpc.newServerBuilderForPort for TLS + return Grpc.newServerBuilderForPort(port, credentials); + } catch (Exception e) { + logger.error("Failed to configure TLS", e); + return ServerBuilder.forPort(port); + } + } + + private ServerCredentials configureTlsCredentials(ContextConfiguration config) { + String certPath = getConfigString(config, CONFIG_TLS_CERT, null); + String keyPath = getConfigString(config, CONFIG_TLS_KEY, null); + + if (certPath == null || keyPath == null) { + logger.warn("TLS enabled but certificate or key path not provided. Using insecure credentials."); + return InsecureServerCredentials.create(); + } + + File certFile = new File(certPath); + File keyFile = new File(keyPath); + + if (!certFile.exists() || !keyFile.exists()) { + logger.warn("TLS certificate or key file not found. Using insecure credentials."); + return InsecureServerCredentials.create(); + } + + try { + return TlsServerCredentials.create(certFile, keyFile); + } catch (Exception e) { + logger.error("Failed to configure TLS credentials", e); + return InsecureServerCredentials.create(); + } + } + + private void registerShutdownHook() { + shutdownHook = new Thread(() -> { + logger.info("Shutting down gRPC server..."); + stopService(); + }); + Runtime.getRuntime().addShutdownHook(shutdownHook); + } + + @Override + public void stopService() { + try { + // Update health status to NOT_SERVING + if (healthManager != null) { + healthManager.setStatus( + ArcadeDbGrpcService.class.getName(), + HealthCheckResponse.ServingStatus.NOT_SERVING + ); + } + + // Close the gRPC service to release database connections + if (grpcService != null) { + try { + grpcService.close(); + logger.info("gRPC service closed and database connections released"); + } catch (Exception e) { + logger.error("Error closing gRPC service", e); + } + } + + // Shutdown servers gracefully + if (grpcServer != null) { + grpcServer.shutdown(); + if (!grpcServer.awaitTermination(30, TimeUnit.SECONDS)) { + grpcServer.shutdownNow(); + grpcServer.awaitTermination(5, TimeUnit.SECONDS); + } + logger.info("Standard gRPC server stopped"); + } + + if (xdsServer != null) { + xdsServer.shutdown(); + if (!xdsServer.awaitTermination(30, TimeUnit.SECONDS)) { + xdsServer.shutdownNow(); + xdsServer.awaitTermination(5, TimeUnit.SECONDS); + } + logger.info("XDS gRPC server stopped"); + } + + // Remove shutdown hook if it exists + if (shutdownHook != null) { + try { + Runtime.getRuntime().removeShutdownHook(shutdownHook); + } catch (IllegalStateException e) { + // Already shutting down + } + } + + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.error("Interrupted while shutting down gRPC server", e); + } + } + + /** + * Get the status of the gRPC servers + */ + public ServerStatus getStatus() { + return new ServerStatus( + grpcServer != null && !grpcServer.isShutdown(), + xdsServer != null && !xdsServer.isShutdown(), + grpcServer != null ? grpcServer.getPort() : -1, + xdsServer != null ? xdsServer.getPort() : -1 + ); + } + + // Configuration helper methods + private String getConfigString(ContextConfiguration config, String key, String defaultValue) { + + return config.getValueAsString(key, defaultValue); + } + + private int getConfigInt(ContextConfiguration config, String key, int defaultValue) { + + String value = getConfigString(config, key, null); + if (value != null) { + try { + return Integer.parseInt(value); + } catch (NumberFormatException e) { + logger.warn("Invalid integer value for {}: {}", key, value); + } + } + return defaultValue; + } + + private boolean getConfigBoolean(ContextConfiguration config, String key, boolean defaultValue) { + String value = getConfigString(config, key, null); + if (value != null) { + return Boolean.parseBoolean(value); + } + return defaultValue; + } + + public static class ServerStatus { + public final boolean standardServerRunning; + public final boolean xdsServerRunning; + public final int standardPort; + public final int xdsPort; + + public ServerStatus(boolean standardServerRunning, boolean xdsServerRunning, + int standardPort, int xdsPort) { + this.standardServerRunning = standardServerRunning; + this.xdsServerRunning = xdsServerRunning; + this.standardPort = standardPort; + this.xdsPort = xdsPort; + } + } +} \ No newline at end of file diff --git a/grpcw/src/test/resources/arcadedb-log.properties b/grpcw/src/test/resources/arcadedb-log.properties new file mode 100644 index 0000000000..536f217457 --- /dev/null +++ b/grpcw/src/test/resources/arcadedb-log.properties @@ -0,0 +1,31 @@ +# +# Copyright 2021-present Arcade Data Ltd +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Specify the handlers to create in the root logger +# (all loggers are children of the root logger) +# The following creates two handlers +handlers=java.util.logging.ConsoleHandler +# Set the default logging level for the root logger +.level=WARNING +com.arcadedb.level=WARNING +# Set the default logging level for new ConsoleHandler instances +java.util.logging.ConsoleHandler.level=WARNING +# Set the default formatter for new ConsoleHandler instances +java.util.logging.ConsoleHandler.formatter=com.arcadedb.utility.AnsiLogFormatter diff --git a/network/pom.xml b/network/pom.xml index 56aadbf55f..facea09538 100644 --- a/network/pom.xml +++ b/network/pom.xml @@ -32,18 +32,22 @@ arcadedb-network jar ArcadeDB Network + 5.15.2 + + com.arcadedb arcadedb-engine ${project.parent.version} compile + - + diff --git a/network/src/main/java/com/arcadedb/remote/RemoteDatabase.java b/network/src/main/java/com/arcadedb/remote/RemoteDatabase.java index 7174717d7a..27d0ea5314 100644 --- a/network/src/main/java/com/arcadedb/remote/RemoteDatabase.java +++ b/network/src/main/java/com/arcadedb/remote/RemoteDatabase.java @@ -506,7 +506,7 @@ String getSessionId() { return sessionId; } - void setSessionId(String sessionId) { + protected void setSessionId(String sessionId) { this.sessionId = sessionId; } @@ -557,7 +557,7 @@ protected Record json2Record(final JSONObject result) { return null; } - RID saveRecord(final MutableDocument record) { + protected RID saveRecord(final MutableDocument record) { stats.createRecord.incrementAndGet(); RID rid = record.getIdentity(); @@ -570,7 +570,7 @@ RID saveRecord(final MutableDocument record) { return rid; } - RID saveRecord(final MutableDocument record, final String bucketName) { + protected RID saveRecord(final MutableDocument record, final String bucketName) { stats.createRecord.incrementAndGet(); RID rid = record.getIdentity(); diff --git a/network/src/main/java/com/arcadedb/remote/RemoteDocumentType.java b/network/src/main/java/com/arcadedb/remote/RemoteDocumentType.java index 5116926829..98c3d7d280 100644 --- a/network/src/main/java/com/arcadedb/remote/RemoteDocumentType.java +++ b/network/src/main/java/com/arcadedb/remote/RemoteDocumentType.java @@ -64,8 +64,22 @@ public class RemoteDocumentType implements DocumentType { } void reload(final Result record) { - count = record.getProperty("records"); - buckets = record.getProperty("buckets"); + + Object recordsAsObject = record.getProperty("records"); + + if (recordsAsObject != null) { + + if (recordsAsObject instanceof Integer recordsAsInt) { + count = recordsAsInt; + } + else if (recordsAsObject instanceof Long recordsAsLong) { + count = recordsAsLong.intValue(); + } + } + + // count = record.getProperty("records"); + + buckets = record.getProperty("buckets"); bucketSelectionStrategy = record.getProperty("bucketSelectionStrategy"); parentTypes = record.getProperty("parentTypes"); @@ -122,7 +136,7 @@ public String getName() { return name; } - @Override +// @Override public void rename(final String newName) { remoteDatabase.command("sql", "alter type `" + name + "` name `" + newName + "`"); remoteDatabase.getSchema().reload(); @@ -254,12 +268,12 @@ public DocumentType removeSuperType(final DocumentType superType) { return this; } - @Override +// @Override public Set getAliases() { return aliases; } - @Override +// @Override public DocumentType setAliases(final Set aliases) { final String aliasesAsString = aliases.stream().map(a -> "`" + a + "`").collect(Collectors.joining(",")); remoteDatabase.command("sql", "alter type `" + name + "` aliases " + aliasesAsString); diff --git a/network/src/main/java/com/arcadedb/remote/RemoteImmutableDocument.java b/network/src/main/java/com/arcadedb/remote/RemoteImmutableDocument.java index 20e8cf13f8..213170d06b 100644 --- a/network/src/main/java/com/arcadedb/remote/RemoteImmutableDocument.java +++ b/network/src/main/java/com/arcadedb/remote/RemoteImmutableDocument.java @@ -41,7 +41,7 @@ public class RemoteImmutableDocument extends ImmutableDocument { protected final RemoteDatabase remoteDatabase; protected final Map map; - protected RemoteImmutableDocument(final RemoteDatabase remoteDatabase, final Map attributes) { + public RemoteImmutableDocument(final RemoteDatabase remoteDatabase, final Map attributes) { super(null, remoteDatabase.getSchema().getType((String) attributes.get(Property.TYPE_PROPERTY)), null, null); this.remoteDatabase = remoteDatabase; diff --git a/network/src/main/java/com/arcadedb/remote/RemoteImmutableVertex.java b/network/src/main/java/com/arcadedb/remote/RemoteImmutableVertex.java index 17da2fa0e8..569be119ee 100644 --- a/network/src/main/java/com/arcadedb/remote/RemoteImmutableVertex.java +++ b/network/src/main/java/com/arcadedb/remote/RemoteImmutableVertex.java @@ -37,7 +37,7 @@ public class RemoteImmutableVertex extends RemoteImmutableDocument implements Vertex { private final RemoteVertex internal; - protected RemoteImmutableVertex(final RemoteDatabase database, final Map properties) { + public RemoteImmutableVertex(final RemoteDatabase database, final Map properties) { super(database, properties); this.internal = new RemoteVertex(this, database); } diff --git a/network/src/main/java/com/arcadedb/remote/RemoteServer.java b/network/src/main/java/com/arcadedb/remote/RemoteServer.java index a8f0477939..0403484164 100644 --- a/network/src/main/java/com/arcadedb/remote/RemoteServer.java +++ b/network/src/main/java/com/arcadedb/remote/RemoteServer.java @@ -18,15 +18,16 @@ */ package com.arcadedb.remote; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import com.arcadedb.ContextConfiguration; import com.arcadedb.exception.DatabaseOperationException; import com.arcadedb.serializer.json.JSONObject; -import java.net.*; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.util.*; - /** * Remote Database implementation. It's not thread safe. For multi-thread usage create one instance of RemoteDatabase per thread. * diff --git a/network/src/main/proto/arcadedb-server.proto b/network/src/main/proto/arcadedb-server.proto new file mode 100644 index 0000000000..456c8fb35e --- /dev/null +++ b/network/src/main/proto/arcadedb-server.proto @@ -0,0 +1,281 @@ +syntax = "proto3"; + +package com.arcadedb.grpc; + +option java_multiple_files = true; +option java_package = "com.arcadedb.server.grpc"; +option java_outer_classname = "ArcadeDbProto"; + +import "google/protobuf/empty.proto"; +import "google/protobuf/struct.proto"; + +// Main ArcadeDB Service +service ArcadeDbService { + // Database operations + rpc CreateDatabase(CreateDatabaseRequest) returns (CreateDatabaseResponse); + rpc DropDatabase(DropDatabaseRequest) returns (DropDatabaseResponse); + rpc ListDatabases(ListDatabasesRequest) returns (ListDatabasesResponse); + rpc GetDatabaseInfo(GetDatabaseInfoRequest) returns (GetDatabaseInfoResponse); + + // Query operations + rpc ExecuteQuery(ExecuteQueryRequest) returns (ExecuteQueryResponse); + rpc ExecuteCommand(ExecuteCommandRequest) returns (ExecuteCommandResponse); + + // Transaction operations + rpc BeginTransaction(BeginTransactionRequest) returns (BeginTransactionResponse); + rpc CommitTransaction(CommitTransactionRequest) returns (CommitTransactionResponse); + rpc RollbackTransaction(RollbackTransactionRequest) returns (RollbackTransactionResponse); + + // Record operations + rpc CreateRecord(CreateRecordRequest) returns (CreateRecordResponse); + rpc GetRecord(GetRecordRequest) returns (GetRecordResponse); + rpc UpdateRecord(UpdateRecordRequest) returns (UpdateRecordResponse); + rpc DeleteRecord(DeleteRecordRequest) returns (DeleteRecordResponse); + + // Streaming operations + rpc StreamQuery(StreamQueryRequest) returns (stream QueryResult); + rpc BulkInsert(stream BulkInsertRequest) returns (BulkInsertResponse); + + // Server operations + rpc GetServerStatus(google.protobuf.Empty) returns (ServerStatusResponse); + rpc Ping(google.protobuf.Empty) returns (PingResponse); +} + +// Common messages +message DatabaseCredentials { + string username = 1; + string password = 2; +} + +message TransactionContext { + string transaction_id = 1; + string database = 2; +} + +// Database operations +message CreateDatabaseRequest { + string database_name = 1; + DatabaseCredentials credentials = 2; + map options = 3; +} + +message CreateDatabaseResponse { + bool success = 1; + string message = 2; + string database_id = 3; +} + +message DropDatabaseRequest { + string database_name = 1; + DatabaseCredentials credentials = 2; +} + +message DropDatabaseResponse { + bool success = 1; + string message = 2; +} + +message ListDatabasesRequest { + DatabaseCredentials credentials = 1; +} + +message ListDatabasesResponse { + repeated DatabaseInfo databases = 1; +} + +message DatabaseInfo { + string name = 1; + int64 size = 2; + string status = 3; + map properties = 4; +} + +message GetDatabaseInfoRequest { + string database_name = 1; + DatabaseCredentials credentials = 2; +} + +message GetDatabaseInfoResponse { + DatabaseInfo info = 1; +} + +// Query operations +message ExecuteQueryRequest { + string database = 1; + string query = 2; + map parameters = 3; + DatabaseCredentials credentials = 4; + TransactionContext transaction = 5; + int32 limit = 6; + int32 timeout_ms = 7; +} + +message ExecuteQueryResponse { + repeated QueryResult results = 1; + int64 execution_time_ms = 2; + string query_plan = 3; +} + +message ExecuteCommandRequest { + string database = 1; + string command = 2; + map parameters = 3; + DatabaseCredentials credentials = 4; + TransactionContext transaction = 5; +} + +message ExecuteCommandResponse { + bool success = 1; + string message = 2; + int64 affected_records = 3; + int64 execution_time_ms = 4; +} + +message QueryResult { + repeated Record records = 1; + repeated ColumnMetadata columns = 2; + int64 total_records = 3; +} + +message ColumnMetadata { + string name = 1; + string type = 2; + bool nullable = 3; +} + +// Transaction operations +message BeginTransactionRequest { + string database = 1; + DatabaseCredentials credentials = 2; + TransactionIsolation isolation = 3; +} + +enum TransactionIsolation { + READ_UNCOMMITTED = 0; + READ_COMMITTED = 1; + REPEATABLE_READ = 2; + SERIALIZABLE = 3; +} + +message BeginTransactionResponse { + string transaction_id = 1; + int64 timestamp = 2; +} + +message CommitTransactionRequest { + TransactionContext transaction = 1; + DatabaseCredentials credentials = 2; +} + +message CommitTransactionResponse { + bool success = 1; + string message = 2; + int64 timestamp = 3; +} + +message RollbackTransactionRequest { + TransactionContext transaction = 1; + DatabaseCredentials credentials = 2; +} + +message RollbackTransactionResponse { + bool success = 1; + string message = 2; +} + +// Record operations +message Record { + string rid = 1; + string type = 2; + map properties = 3; + int32 version = 4; +} + +message CreateRecordRequest { + string database = 1; + string type = 2; + map properties = 3; + DatabaseCredentials credentials = 4; + TransactionContext transaction = 5; +} + +message CreateRecordResponse { + Record record = 1; + bool success = 2; + string message = 3; +} + +message GetRecordRequest { + string database = 1; + string rid = 2; + DatabaseCredentials credentials = 3; + TransactionContext transaction = 4; +} + +message GetRecordResponse { + Record record = 1; + bool found = 2; +} + +message UpdateRecordRequest { + string database = 1; + string rid = 2; + map properties = 3; + DatabaseCredentials credentials = 4; + TransactionContext transaction = 5; + int32 expected_version = 6; +} + +message UpdateRecordResponse { + Record record = 1; + bool success = 2; + string message = 3; +} + +message DeleteRecordRequest { + string database = 1; + string rid = 2; + DatabaseCredentials credentials = 3; + TransactionContext transaction = 4; +} + +message DeleteRecordResponse { + bool success = 1; + string message = 2; +} + +// Streaming operations +message StreamQueryRequest { + string database = 1; + string query = 2; + map parameters = 3; + DatabaseCredentials credentials = 4; + int32 batch_size = 5; +} + +message BulkInsertRequest { + string database = 1; + string type = 2; + Record record = 3; + DatabaseCredentials credentials = 4; +} + +message BulkInsertResponse { + int64 total_inserted = 1; + int64 total_failed = 2; + repeated string errors = 3; +} + +// Server operations +message ServerStatusResponse { + string version = 1; + string status = 2; + int64 uptime_ms = 3; + int32 active_connections = 4; + map metrics = 5; +} + +message PingResponse { + int64 timestamp = 1; + string message = 2; +} \ No newline at end of file diff --git a/package/pom.xml b/package/pom.xml index 9e98840727..943eef1687 100644 --- a/package/pom.xml +++ b/package/pom.xml @@ -172,6 +172,18 @@ + + com.arcadedb + arcadedb-grpcw + ${project.parent.version} + shaded + + + * + * + + + com.arcadedb arcadedb-graphql diff --git a/pom.xml b/pom.xml index f47455ce79..8466fcecb7 100644 --- a/pom.xml +++ b/pom.xml @@ -88,6 +88,19 @@ 5.0.0 5.19.0 3.1.4 + + + + 3.8.0 + + 1.75.0 + + 4.32.0 + 4.32.0 + + 2.26.0 + 1.3.2 + @@ -111,6 +124,7 @@ + grpc engine network server @@ -122,11 +136,13 @@ mongodbw redisw postgresw + grpcw coverage studio package e2e e2e-perf + grpc-client @@ -350,6 +366,47 @@ + + + + io.github.ascopes + protobuf-maven-plugin + ${protobuf-maven-plugin.version} + + ${protoc.version} + true + + + io.grpc + protoc-gen-grpc-java + ${grpc.version} + + + + ${project.basedir}/src/main/proto + ${project.basedir}/src/generated/proto + + src/generated/protobuf/java + + *.proto + **/*.proto + + + + + + generate + + + + ${project.basedir}/src/generated/proto + ${project.basedir}/src/main/proto + + + + + + diff --git a/server/pom.xml b/server/pom.xml index e67e8d3b3e..c8dff53e2a 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -48,21 +48,6 @@ - - - - org.apache.maven.plugins - maven-jar-plugin - - - - test-jar - - - - - - com.arcadedb @@ -76,6 +61,12 @@ ${project.parent.version} compile + + com.arcadedb + arcadedb-grpc + ${project.parent.version} + compile + com.arcadedb arcadedb-integration @@ -103,6 +94,5 @@ micrometer-core ${micrometer.version} - diff --git a/server/src/main/java/com/arcadedb/server/monitor/ServerMonitor.java b/server/src/main/java/com/arcadedb/server/monitor/ServerMonitor.java index b4467b2da0..88d91626f7 100644 --- a/server/src/main/java/com/arcadedb/server/monitor/ServerMonitor.java +++ b/server/src/main/java/com/arcadedb/server/monitor/ServerMonitor.java @@ -23,8 +23,15 @@ import com.arcadedb.server.ArcadeDBServer; import com.arcadedb.server.event.ServerEventLog; -import java.io.*; -import java.util.concurrent.atomic.*; +import javax.management.MBeanServer; +import javax.management.ObjectName; +import java.io.File; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryMXBean; +import java.lang.management.MemoryUsage; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Level; +import java.util.logging.Logger; /** * Monitor ArcadeDB's server health. @@ -32,110 +39,242 @@ * @author Luca Garulli (l.garulli@arcadedata.com) */ public class ServerMonitor { - private static final int INTERVAL_TIME = 10_000; - private static final int MINS_30 = 30 * 60 * 1_000; - private static final int HOURS_24 = 24 * 60 * 60 * 1_000; - private final ArcadeDBServer server; - private volatile Thread checker; - private AtomicBoolean running = new AtomicBoolean(false); - private long lastHotspotSafepointTime = 0L; - private long lastHotspotSafepointCount = 0L; - private long lastHeapWarningReported = 0L; - private long lastDiskSpaceWarningReported = 0L; - - public ServerMonitor(final ArcadeDBServer server) { - this.server = server; - } - - public void start() { - running.set(true); - checker = new Thread(() -> monitor()); - checker.start(); - } - - private void monitor() { - while (running.get()) { - try { - checkDiskSpace(); - checkHeapRAM(); - checkJVMHotSpot(); - - Thread.sleep(INTERVAL_TIME); - } catch (InterruptedException e) { - break; - } catch (Exception e) { - // IGNORE IT - e.printStackTrace(); - } - } - } - - private void checkDiskSpace() { - if (System.currentTimeMillis() - lastDiskSpaceWarningReported < HOURS_24) - // REPORT ONLY EVERY 24H FROM THE LAST WARNING - return; - - final long freeSpace = new File(".").getFreeSpace(); - final long totalSpace = new File(".").getTotalSpace(); - final float freeSpacePerc = freeSpace * 100F / totalSpace; - if (freeSpacePerc < 20) { - // REPORT THE SPIKE - server.getEventLog().reportEvent(ServerEventLog.EVENT_TYPE.WARNING, "JVM", null, "Available space on disk is only " + ((int) freeSpacePerc) + "%"); - lastDiskSpaceWarningReported = System.currentTimeMillis(); - } - } - - private void checkHeapRAM() { - if (System.currentTimeMillis() - lastHeapWarningReported < MINS_30) - // REPORT ONLY EVERY 30 MINS FROM THE LAST WARNING - return; - - final Runtime runtime = Runtime.getRuntime(); - final long heapUsed = runtime.totalMemory() - runtime.freeMemory(); - final long heapMax = runtime.maxMemory(); - final float heapAvailablePerc = (heapMax - heapUsed) * 100F / heapMax; - - if (heapAvailablePerc < 20) { - // REPORT THE SPIKE - server.getEventLog() - .reportEvent(ServerEventLog.EVENT_TYPE.WARNING, "JVM", null, "Server overloaded: available heap RAM is only " + ((int) heapAvailablePerc) + "%"); - lastHeapWarningReported = System.currentTimeMillis(); - } - } - - private void checkJVMHotSpot() { - final sun.management.HotspotRuntimeMBean runtime = sun.management.ManagementFactoryHelper.getHotspotRuntimeMBean(); - - final long hotspotSafepointTime = runtime.getTotalSafepointTime(); - final long hotspotSafepointCount = runtime.getSafepointCount(); - - if (lastHotspotSafepointCount > 0) { - final float lastAvgSafepointTime = lastHotspotSafepointTime / (float) lastHotspotSafepointCount; - final float avgSafepointTime = hotspotSafepointTime / (float) hotspotSafepointCount; - final float deltaPerc = (avgSafepointTime - lastAvgSafepointTime) * 100 / lastAvgSafepointTime; - - if (deltaPerc > 20) { - // REPORT THE SPIKE - server.getEventLog().reportEvent(ServerEventLog.EVENT_TYPE.WARNING, "JVM", null, - "Server overloaded: JVM Safepoint spiked up " + ((int) deltaPerc) + "% from the last sampling"); - } - } - - lastHotspotSafepointTime = hotspotSafepointTime; - lastHotspotSafepointCount = hotspotSafepointCount; - } - - public void stop() { - running.set(false); - - if (checker != null) { - try { - checker.interrupt(); - checker.join(INTERVAL_TIME + 100); - } catch (Exception e) { - // IGNORE IT, ASSUMING THE THREAD STOPPED - } - } - } - -} + private static final Logger LOGGER = Logger.getLogger(ServerMonitor.class.getName()); + private static final int INTERVAL_TIME = 10_000; + private static final int MINS_30 = 30 * 60 * 1_000; + private static final int HOURS_24 = 24 * 60 * 60 * 1_000; + private final ArcadeDBServer server; + private volatile Thread checker; + private AtomicBoolean running = new AtomicBoolean(false); + private long lastHotspotSafepointTime = 0L; + private long lastHotspotSafepointCount = 0L; + private long lastHeapWarningReported = 0L; + private long lastDiskSpaceWarningReported = 0L; + + // JMX related fields + private MBeanServer mBeanServer; + private ObjectName hotspotRuntimeMBean; + private boolean safepointMonitoringAvailable = false; + private MemoryMXBean memoryMXBean; + + public ServerMonitor(final ArcadeDBServer server) { + this.server = server; + initializeJMXMonitoring(); + } + + private void initializeJMXMonitoring() { + try { + mBeanServer = ManagementFactory.getPlatformMBeanServer(); + memoryMXBean = ManagementFactory.getMemoryMXBean(); + + // Try to access HotSpot runtime MBean through standard JMX + try { + hotspotRuntimeMBean = new ObjectName("sun.management:type=HotspotRuntime"); + if (mBeanServer.isRegistered(hotspotRuntimeMBean)) { + safepointMonitoringAvailable = true; + LOGGER.log(Level.FINE, "HotSpot safepoint monitoring is available"); + } + } + catch (Exception e) { + LOGGER.log(Level.FINE, "HotSpot safepoint monitoring is not available: " + e.getMessage()); + } + } + catch (Exception e) { + LOGGER.log(Level.WARNING, "Failed to initialize JMX monitoring: " + e.getMessage(), e); + } + } + + public void start() { + running.set(true); + checker = new Thread(() -> monitor(), "ArcadeDB-ServerMonitor"); + checker.setDaemon(true); + checker.start(); + } + + private void monitor() { + while (running.get()) { + try { + checkDiskSpace(); + checkHeapRAM(); + checkJVMHotSpot(); + + Thread.sleep(INTERVAL_TIME); + } + catch (InterruptedException e) { + // Expected when stopping the monitor + Thread.currentThread().interrupt(); + break; + } + catch (Exception e) { + // Log the error but continue monitoring + LOGGER.log(Level.WARNING, "Error during server monitoring", e); + } + } + } + + private void checkDiskSpace() { + if (System.currentTimeMillis() - lastDiskSpaceWarningReported < HOURS_24) { + // REPORT ONLY EVERY 24H FROM THE LAST WARNING + return; + } + + try { + final File currentDir = new File("."); + final long freeSpace = currentDir.getUsableSpace(); // Better than getFreeSpace() + final long totalSpace = currentDir.getTotalSpace(); + + if (totalSpace > 0) { + final float freeSpacePerc = freeSpace * 100F / totalSpace; + if (freeSpacePerc < 20) { + // REPORT THE SPIKE + server.getEventLog().reportEvent(ServerEventLog.EVENT_TYPE.WARNING, "JVM", null, + String.format("Available space on disk is only %.1f%% (%.2f GB free of %.2f GB total)", freeSpacePerc, + freeSpace / (1024.0 * 1024.0 * 1024.0), totalSpace / (1024.0 * 1024.0 * 1024.0))); + lastDiskSpaceWarningReported = System.currentTimeMillis(); + } + } + } + catch (SecurityException e) { + LOGGER.log(Level.FINE, "Cannot check disk space due to security restrictions", e); + } + } + + private void checkHeapRAM() { + if (System.currentTimeMillis() - lastHeapWarningReported < MINS_30) { + // REPORT ONLY EVERY 30 MINS FROM THE LAST WARNING + return; + } + + try { + // Use MemoryMXBean for more accurate memory information + MemoryUsage heapUsage = memoryMXBean.getHeapMemoryUsage(); + + final long heapUsed = heapUsage.getUsed(); + final long heapMax = heapUsage.getMax(); + + if (heapMax > 0) { + final float heapAvailablePerc = (heapMax - heapUsed) * 100F / heapMax; + + if (heapAvailablePerc < 20) { + // REPORT THE SPIKE + server.getEventLog().reportEvent(ServerEventLog.EVENT_TYPE.WARNING, "JVM", null, + String.format("Server overloaded: available heap RAM is only %.1f%% (%.2f GB used of %.2f GB max)", + heapAvailablePerc, heapUsed / (1024.0 * 1024.0 * 1024.0), heapMax / (1024.0 * 1024.0 * 1024.0))); + lastHeapWarningReported = System.currentTimeMillis(); + + // Optionally suggest GC if memory is critically low + if (heapAvailablePerc < 10) { + LOGGER.log(Level.INFO, "Suggesting garbage collection due to low memory"); + System.gc(); + } + } + } + } + catch (Exception e) { + LOGGER.log(Level.FINE, "Error checking heap memory", e); + } + } + + private void checkJVMHotSpot() { + if (!safepointMonitoringAvailable) { + // Safepoint monitoring not available, skip this check + return; + } + + try { + // Access HotSpot metrics through JMX MBean + Long hotspotSafepointTime = (Long) mBeanServer.getAttribute(hotspotRuntimeMBean, "TotalSafepointTime"); + Long hotspotSafepointCount = (Long) mBeanServer.getAttribute(hotspotRuntimeMBean, "SafepointCount"); + + if (hotspotSafepointTime != null && hotspotSafepointCount != null && hotspotSafepointCount > 0) { + if (lastHotspotSafepointCount > 0) { + final float lastAvgSafepointTime = lastHotspotSafepointTime / (float) lastHotspotSafepointCount; + final float avgSafepointTime = hotspotSafepointTime / (float) hotspotSafepointCount; + + if (lastAvgSafepointTime > 0) { + final float deltaPerc = (avgSafepointTime - lastAvgSafepointTime) * 100 / lastAvgSafepointTime; + + if (deltaPerc > 20) { + // REPORT THE SPIKE + server.getEventLog().reportEvent(ServerEventLog.EVENT_TYPE.WARNING, "JVM", null, String.format( + "Server overloaded: JVM Safepoint spiked up %.1f%% from the last sampling (avg time: %.2fms -> %.2fms)", + deltaPerc, lastAvgSafepointTime, avgSafepointTime)); + } + } + } + + lastHotspotSafepointTime = hotspotSafepointTime; + lastHotspotSafepointCount = hotspotSafepointCount; + } + } + catch (Exception e) { + // If we can't access safepoint metrics, disable future attempts + safepointMonitoringAvailable = false; + LOGGER.log(Level.FINE, "Cannot access HotSpot safepoint metrics, disabling this monitoring", e); + } + } + + /** + * Alternative monitoring using GC metrics if safepoint monitoring is not + * available + */ + private void checkGCMetrics() { + try { + var gcBeans = ManagementFactory.getGarbageCollectorMXBeans(); + long totalGCTime = 0; + long totalGCCount = 0; + + for (var gcBean : gcBeans) { + totalGCTime += gcBean.getCollectionTime(); + totalGCCount += gcBean.getCollectionCount(); + } + + // You can add logic here to track and report on GC spikes + // similar to safepoint monitoring + + } + catch (Exception e) { + LOGGER.log(Level.FINE, "Error checking GC metrics", e); + } + } + + public void stop() { + running.set(false); + + if (checker != null) { + try { + checker.interrupt(); + checker.join(INTERVAL_TIME + 100); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + LOGGER.log(Level.FINE, "Interrupted while stopping monitor thread", e); + } + } + } + + /** + * Get current monitoring status + */ + public MonitoringStatus getStatus() { + return new MonitoringStatus(running.get(), safepointMonitoringAvailable, System.currentTimeMillis() - lastHeapWarningReported < MINS_30, + System.currentTimeMillis() - lastDiskSpaceWarningReported < HOURS_24); + } + + /** + * Status class for monitoring information + */ + public static class MonitoringStatus { + public final boolean isRunning; + public final boolean safepointMonitoringAvailable; + public final boolean recentHeapWarning; + public final boolean recentDiskWarning; + + public MonitoringStatus(boolean isRunning, boolean safepointMonitoringAvailable, boolean recentHeapWarning, boolean recentDiskWarning) { + this.isRunning = isRunning; + this.safepointMonitoringAvailable = safepointMonitoringAvailable; + this.recentHeapWarning = recentHeapWarning; + this.recentDiskWarning = recentDiskWarning; + } + } +} \ No newline at end of file diff --git a/server/src/main/proto/arcadedb-server.proto b/server/src/main/proto/arcadedb-server.proto new file mode 100644 index 0000000000..456c8fb35e --- /dev/null +++ b/server/src/main/proto/arcadedb-server.proto @@ -0,0 +1,281 @@ +syntax = "proto3"; + +package com.arcadedb.grpc; + +option java_multiple_files = true; +option java_package = "com.arcadedb.server.grpc"; +option java_outer_classname = "ArcadeDbProto"; + +import "google/protobuf/empty.proto"; +import "google/protobuf/struct.proto"; + +// Main ArcadeDB Service +service ArcadeDbService { + // Database operations + rpc CreateDatabase(CreateDatabaseRequest) returns (CreateDatabaseResponse); + rpc DropDatabase(DropDatabaseRequest) returns (DropDatabaseResponse); + rpc ListDatabases(ListDatabasesRequest) returns (ListDatabasesResponse); + rpc GetDatabaseInfo(GetDatabaseInfoRequest) returns (GetDatabaseInfoResponse); + + // Query operations + rpc ExecuteQuery(ExecuteQueryRequest) returns (ExecuteQueryResponse); + rpc ExecuteCommand(ExecuteCommandRequest) returns (ExecuteCommandResponse); + + // Transaction operations + rpc BeginTransaction(BeginTransactionRequest) returns (BeginTransactionResponse); + rpc CommitTransaction(CommitTransactionRequest) returns (CommitTransactionResponse); + rpc RollbackTransaction(RollbackTransactionRequest) returns (RollbackTransactionResponse); + + // Record operations + rpc CreateRecord(CreateRecordRequest) returns (CreateRecordResponse); + rpc GetRecord(GetRecordRequest) returns (GetRecordResponse); + rpc UpdateRecord(UpdateRecordRequest) returns (UpdateRecordResponse); + rpc DeleteRecord(DeleteRecordRequest) returns (DeleteRecordResponse); + + // Streaming operations + rpc StreamQuery(StreamQueryRequest) returns (stream QueryResult); + rpc BulkInsert(stream BulkInsertRequest) returns (BulkInsertResponse); + + // Server operations + rpc GetServerStatus(google.protobuf.Empty) returns (ServerStatusResponse); + rpc Ping(google.protobuf.Empty) returns (PingResponse); +} + +// Common messages +message DatabaseCredentials { + string username = 1; + string password = 2; +} + +message TransactionContext { + string transaction_id = 1; + string database = 2; +} + +// Database operations +message CreateDatabaseRequest { + string database_name = 1; + DatabaseCredentials credentials = 2; + map options = 3; +} + +message CreateDatabaseResponse { + bool success = 1; + string message = 2; + string database_id = 3; +} + +message DropDatabaseRequest { + string database_name = 1; + DatabaseCredentials credentials = 2; +} + +message DropDatabaseResponse { + bool success = 1; + string message = 2; +} + +message ListDatabasesRequest { + DatabaseCredentials credentials = 1; +} + +message ListDatabasesResponse { + repeated DatabaseInfo databases = 1; +} + +message DatabaseInfo { + string name = 1; + int64 size = 2; + string status = 3; + map properties = 4; +} + +message GetDatabaseInfoRequest { + string database_name = 1; + DatabaseCredentials credentials = 2; +} + +message GetDatabaseInfoResponse { + DatabaseInfo info = 1; +} + +// Query operations +message ExecuteQueryRequest { + string database = 1; + string query = 2; + map parameters = 3; + DatabaseCredentials credentials = 4; + TransactionContext transaction = 5; + int32 limit = 6; + int32 timeout_ms = 7; +} + +message ExecuteQueryResponse { + repeated QueryResult results = 1; + int64 execution_time_ms = 2; + string query_plan = 3; +} + +message ExecuteCommandRequest { + string database = 1; + string command = 2; + map parameters = 3; + DatabaseCredentials credentials = 4; + TransactionContext transaction = 5; +} + +message ExecuteCommandResponse { + bool success = 1; + string message = 2; + int64 affected_records = 3; + int64 execution_time_ms = 4; +} + +message QueryResult { + repeated Record records = 1; + repeated ColumnMetadata columns = 2; + int64 total_records = 3; +} + +message ColumnMetadata { + string name = 1; + string type = 2; + bool nullable = 3; +} + +// Transaction operations +message BeginTransactionRequest { + string database = 1; + DatabaseCredentials credentials = 2; + TransactionIsolation isolation = 3; +} + +enum TransactionIsolation { + READ_UNCOMMITTED = 0; + READ_COMMITTED = 1; + REPEATABLE_READ = 2; + SERIALIZABLE = 3; +} + +message BeginTransactionResponse { + string transaction_id = 1; + int64 timestamp = 2; +} + +message CommitTransactionRequest { + TransactionContext transaction = 1; + DatabaseCredentials credentials = 2; +} + +message CommitTransactionResponse { + bool success = 1; + string message = 2; + int64 timestamp = 3; +} + +message RollbackTransactionRequest { + TransactionContext transaction = 1; + DatabaseCredentials credentials = 2; +} + +message RollbackTransactionResponse { + bool success = 1; + string message = 2; +} + +// Record operations +message Record { + string rid = 1; + string type = 2; + map properties = 3; + int32 version = 4; +} + +message CreateRecordRequest { + string database = 1; + string type = 2; + map properties = 3; + DatabaseCredentials credentials = 4; + TransactionContext transaction = 5; +} + +message CreateRecordResponse { + Record record = 1; + bool success = 2; + string message = 3; +} + +message GetRecordRequest { + string database = 1; + string rid = 2; + DatabaseCredentials credentials = 3; + TransactionContext transaction = 4; +} + +message GetRecordResponse { + Record record = 1; + bool found = 2; +} + +message UpdateRecordRequest { + string database = 1; + string rid = 2; + map properties = 3; + DatabaseCredentials credentials = 4; + TransactionContext transaction = 5; + int32 expected_version = 6; +} + +message UpdateRecordResponse { + Record record = 1; + bool success = 2; + string message = 3; +} + +message DeleteRecordRequest { + string database = 1; + string rid = 2; + DatabaseCredentials credentials = 3; + TransactionContext transaction = 4; +} + +message DeleteRecordResponse { + bool success = 1; + string message = 2; +} + +// Streaming operations +message StreamQueryRequest { + string database = 1; + string query = 2; + map parameters = 3; + DatabaseCredentials credentials = 4; + int32 batch_size = 5; +} + +message BulkInsertRequest { + string database = 1; + string type = 2; + Record record = 3; + DatabaseCredentials credentials = 4; +} + +message BulkInsertResponse { + int64 total_inserted = 1; + int64 total_failed = 2; + repeated string errors = 3; +} + +// Server operations +message ServerStatusResponse { + string version = 1; + string status = 2; + int64 uptime_ms = 3; + int32 active_connections = 4; + map metrics = 5; +} + +message PingResponse { + int64 timestamp = 1; + string message = 2; +} \ No newline at end of file