From 871a8125d9679223c72a987c804cecf6edf8ece4 Mon Sep 17 00:00:00 2001 From: Harun Alpak <31957523+harunalpak@users.noreply.github.com> Date: Thu, 17 Nov 2022 15:58:24 +0300 Subject: [PATCH] Apply Compact Serialization Updates for Hz v5.2.0 I [API-1637] (#1409) * Make ReadResultSet iterable [API-1315] * Make ReadResultSet iterable [API-1315] * Changes made according to comments on PR.[API-1315] * Apply Compact Serialization Updates for Hz v5.2.0 I [API-1637] * Version change 5.2.0 => 5.2 * hazelcast-enterprise-tests.jar was removed from repo. * SSL tests were skipped, because of tests.jar was removed. * Changes have been made related to review comments. [API-1637] * Changes have been made related to review comments. [API-1637] * Some changes related to PR comments * verifyDefaultSerializersNotOverriddenWithCompact method was modified * Some fixes related to review comments. * LONG_SYMBOL not used and removed. * Change the name of variable. * Some fixes related to PR comments * Some fixes related to PR comments * Fixes related to PR comments. --- .eslintrc | 1 + scripts/download-rc.js | 15 +- src/config/CompactSerializationConfig.ts | 2 - src/config/ConfigBuilder.ts | 6 +- src/serialization/SerializationService.ts | 201 +++++++++++------- src/serialization/SerializationSymbols.ts | 36 ++++ src/serialization/compact/CompactReader.ts | 1 - .../compact/CompactSerializer.ts | 1 - src/serialization/compact/CompactWriter.ts | 4 +- .../compact/DefaultCompactWriter.ts | 58 +++++ src/serialization/compact/SchemaWriter.ts | 95 +++++---- src/serialization/generic_record/FieldKind.ts | 100 ++++----- src/serialization/generic_record/Fields.ts | 1 - .../generic_record/GenericRecord.ts | 1 - .../generic_record/GenericRecords.ts | 1 - src/util/Util.ts | 53 +++-- .../DefaultSerializersLiveTest.js | 149 ++++++++----- .../compact/CompactPublicAPIsTest.js | 8 +- .../compact/CompactSerializersLiveTest.js | 58 +++-- .../serialization/compact/CompactTest.js | 12 +- .../serialization/compact/CompactUtil.js | 108 ++++++++++ .../compact/LazyDeserialization.js | 12 +- .../parallel/sql/DataTypeTest.js | 11 +- .../parallel/sql/jet_enabled_with_compact.xml | 11 +- .../sql/jet_enabled_with_compact_beta.xml | 34 +++ .../ssl/ClientSSLAuthenticationTest.js | 2 +- .../parallel/ssl/ClientSSLTest.js | 2 +- .../serial/CompactRestartTest.js | 11 +- test/unit/config/ConfigBuilderTest.js | 25 +++ .../serialization/BinaryCompatibilityTest.js | 5 +- .../serialization/DefaultSerializersTest.js | 74 ++++++- .../compact/CompactGenericRecordTest.js | 115 +++++++++- .../unit/serialization/compact/CompactTest.js | 13 +- .../compact/RabinFingerprintTest.js | 2 +- .../serialization/compact/SchemaWriterTest.js | 12 +- 35 files changed, 932 insertions(+), 308 deletions(-) create mode 100644 src/serialization/SerializationSymbols.ts create mode 100644 test/integration/backward_compatible/parallel/sql/jet_enabled_with_compact_beta.xml diff --git a/.eslintrc b/.eslintrc index f579dc7c3..b5260704e 100644 --- a/.eslintrc +++ b/.eslintrc @@ -112,6 +112,7 @@ "plugin:mocha/recommended" ], "rules": { + "mocha/no-skipped-tests": "off", "mocha/no-hooks-for-single-case": "off", "mocha/no-setup-in-describe": "off", "camelcase": "off", diff --git a/scripts/download-rc.js b/scripts/download-rc.js index 2f126ddc4..d2ad55e5d 100644 --- a/scripts/download-rc.js +++ b/scripts/download-rc.js @@ -1,6 +1,6 @@ 'use strict'; -const HZ_VERSION = '5.1'; -const HZ_TEST_VERSION = '5.1'; +const HZ_VERSION = '5.2.0'; +const HZ_TEST_VERSION = '5.2.0'; const HAZELCAST_TEST_VERSION = HZ_TEST_VERSION; const HAZELCAST_VERSION = HZ_VERSION; const HAZELCAST_ENTERPRISE_VERSION = HZ_VERSION; @@ -22,7 +22,7 @@ const downloadRC = () => { let REPO; let ENTERPRISE_REPO; let TEST_REPO; - let ENTERPRISE_TEST_REPO; + // let ENTERPRISE_TEST_REPO; if (HZ_VERSION.endsWith('-SNAPSHOT')) { REPO = SNAPSHOT_REPO; @@ -34,10 +34,10 @@ const downloadRC = () => { if (HZ_TEST_VERSION.endsWith('-SNAPSHOT')) { TEST_REPO = SNAPSHOT_REPO; - ENTERPRISE_TEST_REPO = ENTERPRISE_SNAPSHOT_REPO; + // ENTERPRISE_TEST_REPO = ENTERPRISE_SNAPSHOT_REPO; } else { TEST_REPO = RELEASE_REPO; - ENTERPRISE_TEST_REPO = ENTERPRISE_RELEASE_REPO; + // ENTERPRISE_TEST_REPO = ENTERPRISE_RELEASE_REPO; } if (fs.existsSync(`hazelcast-remote-controller-${HAZELCAST_RC_VERSION}.jar`)) { @@ -132,7 +132,8 @@ const downloadRC = () => { + `com.hazelcast:hazelcast-enterprise:${HAZELCAST_ENTERPRISE_VERSION} ${subprocessTrace}`; } } - + // TODO hazelcast-enterprise-tests.jar was removed from repo. + /* if (fs.existsSync(`hazelcast-enterprise-${HAZELCAST_TEST_VERSION}-tests.jar`)) { console.log('hazelcast-enterprise-tests.jar already exists, not downloading from maven.'); } else { @@ -154,7 +155,7 @@ const downloadRC = () => { throw 'Failed to download hazelcast enterprise test jar ' + `com.hazelcast:hazelcast-enterprise:${HAZELCAST_TEST_VERSION}:jar:tests ${subprocessTrace}`; } - } + }*/ console.log('Starting Remote Controller ... enterprise ...'); } else { if (fs.existsSync(`hazelcast-${HAZELCAST_VERSION}.jar`)) { diff --git a/src/config/CompactSerializationConfig.ts b/src/config/CompactSerializationConfig.ts index 71bef0e77..f0a095164 100644 --- a/src/config/CompactSerializationConfig.ts +++ b/src/config/CompactSerializationConfig.ts @@ -18,8 +18,6 @@ import {CompactSerializer} from '../serialization/compact/CompactSerializer'; /** * Compact serialization config for the client. - * - * This API is currently in BETA and can change at any time. */ export interface CompactSerializationConfig { diff --git a/src/config/ConfigBuilder.ts b/src/config/ConfigBuilder.ts index 248991132..3d89aad34 100644 --- a/src/config/ConfigBuilder.ts +++ b/src/config/ConfigBuilder.ts @@ -40,6 +40,7 @@ import {LoadBalancerType} from './LoadBalancerConfig'; import {LogLevel} from '../logging'; import {TokenCredentialsImpl, TokenEncoding, UsernamePasswordCredentialsImpl,} from '../security'; import {MetricsConfig} from './MetricsConfig'; +import * as Util from '../util/Util'; /** * Responsible for user-defined config validation. Builds the effective config with necessary defaults. @@ -471,7 +472,10 @@ export class ConfigBuilder { private handleSerialization(jsonObject: any): void { for (const key in jsonObject) { if (key === 'defaultNumberType') { - this.effectiveConfig.serialization.defaultNumberType = tryGetString(jsonObject[key]).toLowerCase(); + const defaultNumberType = tryGetString(jsonObject[key]).toLowerCase(); + // For checking expected value for defaultNumberType. If get unexpected value, throw RangeError. + Util.getTypeKeyForDefaultNumberType(defaultNumberType); + this.effectiveConfig.serialization.defaultNumberType = defaultNumberType; } else if (key === 'isBigEndian') { this.effectiveConfig.serialization.isBigEndian = tryGetBoolean(jsonObject[key]); } else if (key === 'portableVersion') { diff --git a/src/serialization/SerializationService.ts b/src/serialization/SerializationService.ts index 34d5d7280..b076e3d6f 100644 --- a/src/serialization/SerializationService.ts +++ b/src/serialization/SerializationService.ts @@ -15,6 +15,8 @@ */ /** @ignore *//** */ +import * as Long from 'long'; +import * as Util from '../util/Util'; import {AGGREGATOR_FACTORY_ID} from '../aggregation/AggregatorConstants'; import {aggregatorFactory} from '../aggregation/Aggregator'; import {CLUSTER_DATA_FACTORY_ID, clusterDataFactory} from './ClusterDataFactory'; @@ -23,7 +25,6 @@ import { RELIABLE_TOPIC_MESSAGE_FACTORY_ID, reliableTopicMessageFactory, } from '../proxy/topic/ReliableTopicMessage'; -import * as Util from '../util/Util'; import {Data, DataInput, DataOutput} from './Data'; import {Serializer, IdentifiedDataSerializableFactory} from './Serializable'; import { @@ -62,6 +63,7 @@ import { UuidSerializer, JavaArraySerializer } from './DefaultSerializers'; +import {SerializationSymbols} from './SerializationSymbols' import {DATA_OFFSET, HeapData} from './HeapData'; import {ObjectDataInput, PositionalObjectDataOutput} from './ObjectData'; import {PortableSerializer} from './portable/PortableSerializer'; @@ -72,6 +74,7 @@ import {CompactStreamSerializer} from './compact/CompactStreamSerializer'; import {SchemaService} from './compact/SchemaService'; import {CompactGenericRecordImpl} from './generic_record'; import {Schema} from './compact/Schema'; +import {BigDecimal, IllegalArgumentError, LocalDate, LocalDateTime, LocalTime, OffsetDateTime, UUID} from '../core'; /** * Serializes objects and deserializes data. @@ -101,21 +104,35 @@ const defaultPartitionStrategy = (obj: any): number => { } } +/** + * The type key that is used in serializer registration for a object type. + */ +// eslint-disable-next-line @typescript-eslint/ban-types +type TypeKey = Function | Symbol; + /** @internal */ export class SerializationServiceV1 implements SerializationService { private readonly registry: { [id: number]: Serializer }; - private readonly serializerNameToId: { [name: string]: number }; + + // We hold default type serializers in a Map. Key values will be class types or Symbol and values + // will be serializers( one is type serializer and second one is array serializer of that object) + // Some of the types do not have equivalent class on Nodejs (Byte, Short, Null and etc.), so we need to use + // unique values for these types as a Symbol (defined in @SerializationSymbols). + // For these types we use unique Symbol as a key value. + private readonly typeKeyToSerializersMap : Map; private readonly compactStreamSerializer: CompactStreamSerializer; private readonly portableSerializer: PortableSerializer; private readonly identifiedSerializer: IdentifiedDataSerializableSerializer; + private readonly typeKeyForDefaultNumberType : TypeKey; constructor( private readonly serializationConfig: SerializationConfigImpl, schemaService: SchemaService ) { this.registry = {}; - this.serializerNameToId = {}; + // eslint-disable-next-line @typescript-eslint/ban-types + this.typeKeyToSerializersMap = new Map(); this.compactStreamSerializer = new CompactStreamSerializer(schemaService); this.portableSerializer = new PortableSerializer(this.serializationConfig); this.identifiedSerializer = this.createIdentifiedSerializer(); @@ -123,6 +140,12 @@ export class SerializationServiceV1 implements SerializationService { this.registerCustomSerializers(); this.registerCompactSerializers(); this.registerGlobalSerializer(); + this.typeKeyForDefaultNumberType = Util.getTypeKeyForDefaultNumberType(this.serializationConfig.defaultNumberType); + + // Called here so that we can make sure that we are not overriding + // any of the default serializers registered above with the Compact + // serialization. + this.verifyDefaultSerializersNotOverriddenWithCompact(); } public isData(object: any): boolean { @@ -182,15 +205,26 @@ export class SerializationServiceV1 implements SerializationService { return serializer.read(inp); } - registerSerializer(name: string, serializer: Serializer): void { - if (this.serializerNameToId[name]) { - throw new RangeError('Given serializer name is already in the registry.'); + /** + * Registers a serializer to the system. + * @param typeKey A typekey is either a constructor function or a symbol, defining the type of the object to be serialized. + * @param serializer The serializer to be registered. + * @param arraySerializer The serializer to be used for arrays of the given type. Global and custom serializers don't have + * this one and this should be null. For some types we don't have array serializers defined, for them this should be null + * as well. + */ + registerSerializer(typeKey: TypeKey, serializer: Serializer, arraySerializer: Serializer | null): void { + if (this.typeKeyToSerializersMap.has(typeKey)) { + throw new RangeError('Given serializer type is already in the registry.'); } if (this.registry[serializer.id]) { throw new RangeError('Given serializer id is already in the registry.'); } - this.serializerNameToId[name] = serializer.id; + this.typeKeyToSerializersMap.set(typeKey, [serializer, arraySerializer]); this.registry[serializer.id] = serializer; + if (arraySerializer) { + this.registry[arraySerializer.id] = arraySerializer; + } } /** @@ -215,7 +249,7 @@ export class SerializationServiceV1 implements SerializationService { } let serializer: Serializer = null; if (obj === null) { - serializer = this.findSerializerByName('null', false); + serializer = this.findSerializerByType(SerializationSymbols.NULL_SYMBOL, false); } if (serializer === null) { serializer = this.lookupDefaultSerializer(obj); @@ -227,7 +261,7 @@ export class SerializationServiceV1 implements SerializationService { serializer = this.lookupGlobalSerializer(); } if (serializer === null) { - serializer = this.findSerializerByName('!json', false); + serializer = this.findSerializerByType(SerializationSymbols.JSON_SYMBOL, false); } if (serializer === null) { throw new RangeError('There is no suitable serializer for ' + obj + '.'); @@ -236,8 +270,7 @@ export class SerializationServiceV1 implements SerializationService { } - private lookupDefaultSerializer(obj: any): Serializer { - let serializer: Serializer = null; + private lookupDefaultSerializer(obj: any): Serializer | null { if (this.isCompactSerializable(obj)) { return this.compactStreamSerializer; } @@ -248,28 +281,48 @@ export class SerializationServiceV1 implements SerializationService { return this.portableSerializer } - const objectType = Util.getType(obj); - if (objectType === 'array') { - if (obj.length === 0) { - serializer = this.findSerializerByName('number', true); - } else { - serializer = this.findSerializerByName(Util.getType(obj[0]), true); + const isArray = Array.isArray(obj); + if (!isArray) { + // Number needs special care because it can be serialized with one of many serializers. + if (typeof obj === 'number') { + return this.findSerializerByType(this.typeKeyForDefaultNumberType, isArray); } - } else { - serializer = this.findSerializerByName(objectType, false); + // We know obj is not undefined or null at this point, meaning it has a constructor field. + return this.findSerializerByType(obj.constructor, isArray); } - return serializer; + return this.lookupDefaultSerializerForArray(obj); + } + + private lookupDefaultSerializerForArray(obj: Array): Serializer | null { + if (obj.length === 0) { + return this.findSerializerByType(this.typeKeyForDefaultNumberType, true); + } + const firstElement = obj[0]; + // First element can be anything. Check for null and undefined. + if (firstElement === null) { + return this.findSerializerByType(SerializationSymbols.NULL_SYMBOL, true); + } else if (firstElement === undefined) { + throw new RangeError('Array serialization type is determined using the first element. ' + + 'The first element is undefined. Throwing an error because undefined cannot be' + + ' serialized in Hazelcast serialization.'); + } else if (typeof firstElement === 'number') { + // Number needs special care because it can be serialized with one of many serializers. + return this.findSerializerByType(this.typeKeyForDefaultNumberType, true); + } + return this.findSerializerByType(obj[0].constructor, true); } private lookupCustomSerializer(obj: any): Serializer { + // Note: What about arrays of custom serializable objects? if (SerializationServiceV1.isCustomSerializable(obj)) { + // We can also use findSerializerByType with Symbol.for. It should not matter. return this.findSerializerById(obj.hzCustomId); } return null; } private lookupGlobalSerializer(): Serializer { - return this.findSerializerByName('!global', false); + return this.findSerializerByType(SerializationSymbols.GLOBAL_SYMBOL, false); } private static isIdentifiedDataSerializable(obj: any): boolean { @@ -282,6 +335,25 @@ export class SerializationServiceV1 implements SerializationService { && typeof obj.factoryId === 'number' && typeof obj.classId === 'number'); } + /** + * Makes sure that the classes registered as Compact serializable are not + * overriding the default serializers. + * + * Must be called in the constructor after completing registering default serializers. + */ + private verifyDefaultSerializersNotOverriddenWithCompact(): void { + const compactSerializers = this.serializationConfig.compact.serializers; + for (const compact of compactSerializers) { + const clazz = compact.getClass(); + if (this.typeKeyToSerializersMap.has(clazz) || clazz === Number) { + // From the config validation, we know clazz is a function, so we can use the name field of it. + throw new IllegalArgumentError( + `Compact serializer for the class ${clazz.name} and typename ${compact.getTypeName()}` + + ' can not be registered as it overrides a default serializer for that class provided by Hazelcast.'); + } + } + } + isCompactSerializable(obj: any): boolean { if (obj instanceof CompactGenericRecordImpl) { return true; @@ -291,44 +363,36 @@ export class SerializationServiceV1 implements SerializationService { } private registerDefaultSerializers(): void { - this.registerSerializer('string', new StringSerializer()); - this.registerSerializer('double', new DoubleSerializer()); - this.registerSerializer('byte', new ByteSerializer()); - this.registerSerializer('boolean', new BooleanSerializer()); - this.registerSerializer('null', new NullSerializer()); - this.registerSerializer('short', new ShortSerializer()); - this.registerSerializer('integer', new IntegerSerializer()); - this.registerSerializer('long', new LongSerializer()); - this.registerSerializer('float', new FloatSerializer()); - this.registerSerializer('char', new CharSerializer()); - this.registerSerializer('date', new DateSerializer()); - this.registerSerializer('localDate', new LocalDateSerializer()); - this.registerSerializer('localTime', new LocalTimeSerializer()); - this.registerSerializer('localDateTime', new LocalDateTimeSerializer()); - this.registerSerializer('offsetDateTime', new OffsetDateTimeSerializer()); - this.registerSerializer('byteArray', new ByteArraySerializer()); - this.registerSerializer('charArray', new CharArraySerializer()); - this.registerSerializer('booleanArray', new BooleanArraySerializer()); - this.registerSerializer('shortArray', new ShortArraySerializer()); - this.registerSerializer('integerArray', new IntegerArraySerializer()); - this.registerSerializer('longArray', new LongArraySerializer()); - this.registerSerializer('doubleArray', new DoubleArraySerializer()); - this.registerSerializer('stringArray', new StringArraySerializer()); - this.registerSerializer('javaClass', new JavaClassSerializer()); - this.registerSerializer('floatArray', new FloatArraySerializer()); - this.registerSerializer('arrayList', new ArrayListSerializer()); - this.registerSerializer('linkedList', new LinkedListSerializer()); - this.registerSerializer('uuid', new UuidSerializer()); - this.registerSerializer('bigDecimal', new BigDecimalSerializer()); - this.registerSerializer('bigint', new BigIntSerializer()); - this.registerSerializer('javaArray', new JavaArraySerializer()); - this.registerSerializer('!compact', this.compactStreamSerializer); - this.registerSerializer('identified', this.identifiedSerializer); - this.registerSerializer('!portable', this.portableSerializer); + this.registerSerializer(String, new StringSerializer(), new StringArraySerializer()); + this.registerSerializer(SerializationSymbols.DOUBLE_SYMBOL, new DoubleSerializer(), new DoubleArraySerializer()); + this.registerSerializer(SerializationSymbols.BYTE_SYMBOL , new ByteSerializer(), new ByteArraySerializer()); + this.registerSerializer(Boolean, new BooleanSerializer(), new BooleanArraySerializer()); + this.registerSerializer(SerializationSymbols.NULL_SYMBOL, new NullSerializer(), null); + this.registerSerializer(SerializationSymbols.SHORT_SYMBOL, new ShortSerializer(), new ShortArraySerializer()); + this.registerSerializer(SerializationSymbols.INTEGER_SYMBOL, new IntegerSerializer(), new IntegerArraySerializer()); + this.registerSerializer(Long, new LongSerializer(), new LongArraySerializer()); + this.registerSerializer(SerializationSymbols.FLOAT_SYMBOL, new FloatSerializer(), new FloatArraySerializer()); + this.registerSerializer(SerializationSymbols.CHAR_SYMBOL, new CharSerializer(), new CharArraySerializer()); + this.registerSerializer(Date, new DateSerializer(), null); + this.registerSerializer(LocalDate, new LocalDateSerializer(), null); + this.registerSerializer(LocalTime, new LocalTimeSerializer(), null); + this.registerSerializer(LocalDateTime, new LocalDateTimeSerializer(), null); + this.registerSerializer(OffsetDateTime, new OffsetDateTimeSerializer(), null); + this.registerSerializer(SerializationSymbols.JAVACLASS_SYMBOL, new JavaClassSerializer(), null); + this.registerSerializer(SerializationSymbols.ARRAYLIST_SYMBOL, new ArrayListSerializer(), null); + this.registerSerializer(SerializationSymbols.LINKEDLIST_SYMBOL, new LinkedListSerializer(), null); + this.registerSerializer(UUID, new UuidSerializer(), null); + this.registerSerializer(BigDecimal, new BigDecimalSerializer(), null); + this.registerSerializer(BigInt, new BigIntSerializer(), null); + this.registerSerializer(SerializationSymbols.JAVA_ARRAY_SYMBOL, new JavaArraySerializer(), null); + this.registerSerializer(SerializationSymbols.COMPACT_SYMBOL, this.compactStreamSerializer, null); + this.registerSerializer(SerializationSymbols.IDENTIFIED_SYMBOL, this.identifiedSerializer, null); + this.registerSerializer(SerializationSymbols.PORTABLE_SYMBOL, this.portableSerializer, null); + if (this.serializationConfig.jsonStringDeserializationPolicy === JsonStringDeserializationPolicy.EAGER) { - this.registerSerializer('!json', new JsonSerializer()); + this.registerSerializer(SerializationSymbols.JSON_SYMBOL, new JsonSerializer(), null); } else { - this.registerSerializer('!json', new HazelcastJsonValueSerializer()); + this.registerSerializer(SerializationSymbols.JSON_SYMBOL, new HazelcastJsonValueSerializer(), null); } } @@ -348,7 +412,7 @@ export class SerializationServiceV1 implements SerializationService { private registerCustomSerializers(): void { const customSerializers = this.serializationConfig.customSerializers; for (const customSerializer of customSerializers) { - this.registerSerializer('!custom' + customSerializer.id, customSerializer); + this.registerSerializer(Symbol.for('!custom' + customSerializer.id), customSerializer, null); } } @@ -364,7 +428,7 @@ export class SerializationServiceV1 implements SerializationService { if (candidate == null) { return; } - this.registerSerializer('!global', candidate); + this.registerSerializer(SerializationSymbols.GLOBAL_SYMBOL, candidate, null); } private static isCustomSerializable(object: any): boolean { @@ -372,21 +436,16 @@ export class SerializationServiceV1 implements SerializationService { return (typeof object[prop] === 'number' && object[prop] >= 1); } - private findSerializerByName(name: string, isArray: boolean): Serializer { - let convertedName: string; - if (name === 'number') { - convertedName = this.serializationConfig.defaultNumberType; - } else if (name === 'buffer') { - convertedName = 'byteArray'; - } else { - convertedName = name; + private findSerializerByType(typeKey: TypeKey, isArray: boolean): Serializer | null { + if (typeKey === Buffer) { + typeKey = SerializationSymbols.BYTE_SYMBOL; + isArray = true; } - const serializerName = convertedName + (isArray ? 'Array' : ''); - const serializerId = this.serializerNameToId[serializerName]; - if (serializerId == null) { + const serializers = this.typeKeyToSerializersMap.get(typeKey); + if (serializers === undefined) { return null; } - return this.findSerializerById(serializerId); + return isArray ? serializers[1] : serializers[0]; } private findSerializerById(id: number): Serializer { diff --git a/src/serialization/SerializationSymbols.ts b/src/serialization/SerializationSymbols.ts new file mode 100644 index 000000000..8e6d4aba5 --- /dev/null +++ b/src/serialization/SerializationSymbols.ts @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2008-2022, Hazelcast, Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** @ignore *//** */ + +/** @internal */ +export class SerializationSymbols { + static readonly BYTE_SYMBOL = Symbol(); + static readonly NULL_SYMBOL = Symbol(); + static readonly SHORT_SYMBOL = Symbol(); + static readonly INTEGER_SYMBOL = Symbol(); + static readonly FLOAT_SYMBOL = Symbol(); + static readonly DOUBLE_SYMBOL = Symbol(); + static readonly CHAR_SYMBOL = Symbol(); + static readonly JAVACLASS_SYMBOL = Symbol(); + static readonly ARRAYLIST_SYMBOL = Symbol(); + static readonly LINKEDLIST_SYMBOL = Symbol(); + static readonly JAVA_ARRAY_SYMBOL = Symbol(); + static readonly COMPACT_SYMBOL = Symbol(); + static readonly IDENTIFIED_SYMBOL = Symbol(); + static readonly PORTABLE_SYMBOL = Symbol(); + static readonly JSON_SYMBOL = Symbol(); + static readonly GLOBAL_SYMBOL = Symbol(); +} diff --git a/src/serialization/compact/CompactReader.ts b/src/serialization/compact/CompactReader.ts index cb9f5bde4..f5692ffca 100644 --- a/src/serialization/compact/CompactReader.ts +++ b/src/serialization/compact/CompactReader.ts @@ -27,7 +27,6 @@ import {FieldKind} from '../generic_record'; * might be especially useful if the class might evolve in future, either by adding or * removing fields. * - * This API is currently in Beta and can change at any time. */ export interface CompactReader { /** diff --git a/src/serialization/compact/CompactSerializer.ts b/src/serialization/compact/CompactSerializer.ts index 79f296144..2f32b9799 100644 --- a/src/serialization/compact/CompactSerializer.ts +++ b/src/serialization/compact/CompactSerializer.ts @@ -26,7 +26,6 @@ import {CompactWriter} from './CompactWriter'; * * {@link write} and {@link read} methods must be consistent with each other. * - * This API is currently in Beta and can change at any time. */ export interface CompactSerializer { /** diff --git a/src/serialization/compact/CompactWriter.ts b/src/serialization/compact/CompactWriter.ts index 50a97a833..ae4322ed0 100644 --- a/src/serialization/compact/CompactWriter.ts +++ b/src/serialization/compact/CompactWriter.ts @@ -20,7 +20,6 @@ import {BigDecimal, LocalDate, LocalDateTime, LocalTime, OffsetDateTime} from '. /** * Provides means of writing compact serialized fields to the binary data. * - * This API is currently in Beta and can change at any time. */ export interface CompactWriter { /** @@ -242,6 +241,9 @@ export interface CompactWriter { /** * Writes an array of nested compact objects. * + * For compact objects, if an array contains different item types or undefined + * a {@link HazelcastSerializationError} will be thrown. + * * @param fieldName name of the field. * @param value to be written. */ diff --git a/src/serialization/compact/DefaultCompactWriter.ts b/src/serialization/compact/DefaultCompactWriter.ts index 86949427c..8e8ef043d 100644 --- a/src/serialization/compact/DefaultCompactWriter.ts +++ b/src/serialization/compact/DefaultCompactWriter.ts @@ -202,7 +202,9 @@ export class DefaultCompactWriter implements CompactWriter { } writeArrayOfCompact(fieldName: string, value: (T | null)[] | null): void { + const singleTypeCompactArrayItemChecker = new SingleTypeCompactArrayItemChecker(); return this.writeArrayOfVariableSizes(fieldName, FieldKind.ARRAY_OF_COMPACT, value, (out, value) => { + singleTypeCompactArrayItemChecker.check(value); return this.serializer.writeObject(out, value); }); } @@ -298,7 +300,9 @@ export class DefaultCompactWriter implements CompactWriter { } writeArrayOfGenericRecord(fieldName: string, value: GenericRecord[]) : void { + const singleSchemaCompactArrayItemChecker = new SingleSchemaCompactArrayItemChecker(); return this.writeArrayOfVariableSizes(fieldName, FieldKind.ARRAY_OF_COMPACT, value, (out, value) => { + singleSchemaCompactArrayItemChecker.check(value); return this.serializer.writeGenericRecord(out, value as CompactGenericRecordImpl); }); } @@ -425,3 +429,57 @@ export class DefaultCompactWriter implements CompactWriter { } } } + +/** + * Checks that the Compact serializable array items that are written are of + * a single type. + */ +export class SingleTypeCompactArrayItemChecker { + + // eslint-disable-next-line @typescript-eslint/ban-types + private clazz: Function; + + public check(value: T): void { + if (value === undefined) { + throw new HazelcastSerializationError('The value undefined can not be used in an Array of Compact value.'); + } + if (value.constructor === undefined) { + throw new HazelcastSerializationError('While checking if all elements in a compact array are of same type, ' + + 'encountered with a value with undefined contructor. Can not continue with single type checking.'); + } + const clazzType = value.constructor; + if (this.clazz == null) { + this.clazz = clazzType; + } + if (this.clazz !== clazzType) { + throw new HazelcastSerializationError('It is not allowed to ' + + 'serialize an array of Compact serializable objects ' + + 'containing different item types. Expected array item ' + + 'type: ' + this.clazz.name + ', current item type: ' + clazzType.name); + } + } +} +/** + * Checks that the Compact serializable GenericRecord array items that are + * written are of a single schema. + */ +export class SingleSchemaCompactArrayItemChecker { + + private schema: Schema; + + public check(value: GenericRecord): void { + const record: CompactGenericRecordImpl = value as CompactGenericRecordImpl; + const schema = record.getSchema(); + if (this.schema == null) { + this.schema = schema; + } + + if (!this.schema.schemaId.equals(schema.schemaId)) { + throw new HazelcastSerializationError('It is not allowed to ' + + 'serialize an array of Compact serializable ' + + 'GenericRecord objects containing different schemas. ' + + 'Expected array item schema: ' + this.schema + + ', current schema: ' + schema); + } + } +} diff --git a/src/serialization/compact/SchemaWriter.ts b/src/serialization/compact/SchemaWriter.ts index 8135cf6e5..744b19013 100644 --- a/src/serialization/compact/SchemaWriter.ts +++ b/src/serialization/compact/SchemaWriter.ts @@ -17,7 +17,7 @@ import {CompactWriter} from './CompactWriter'; import {Schema} from './Schema'; -import {BigDecimal, LocalDate, LocalDateTime, LocalTime, OffsetDateTime} from '../../core'; +import {BigDecimal, HazelcastSerializationError, LocalDate, LocalDateTime, LocalTime, OffsetDateTime} from '../../core'; import * as Long from 'long'; import {FieldDescriptor} from '../generic_record/FieldDescriptor'; import {FieldKind} from '../generic_record/FieldKind'; @@ -28,183 +28,192 @@ import {FieldKind} from '../generic_record/FieldKind'; export class SchemaWriter implements CompactWriter { private readonly typeName: string; private readonly fields: FieldDescriptor[]; + private readonly fieldNames: Set; constructor(typeName: string) { this.typeName = typeName; this.fields = []; + this.fieldNames = new Set(); } writeBoolean(fieldName: string, value: boolean): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.BOOLEAN)); + this.addField(new FieldDescriptor(fieldName, FieldKind.BOOLEAN)); } writeInt8(fieldName: string, value: number): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.INT8)); + this.addField(new FieldDescriptor(fieldName, FieldKind.INT8)); } writeInt16(fieldName: string, value: number): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.INT16)); + this.addField(new FieldDescriptor(fieldName, FieldKind.INT16)); } writeInt32(fieldName: string, value: number): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.INT32)); + this.addField(new FieldDescriptor(fieldName, FieldKind.INT32)); } writeInt64(fieldName: string, value: Long): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.INT64)); + this.addField(new FieldDescriptor(fieldName, FieldKind.INT64)); } writeFloat32(fieldName: string, value: number): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.FLOAT32)); + this.addField(new FieldDescriptor(fieldName, FieldKind.FLOAT32)); } writeFloat64(fieldName: string, value: number): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.FLOAT64)); + this.addField(new FieldDescriptor(fieldName, FieldKind.FLOAT64)); } writeString(fieldName: string, value: string | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.STRING)); + this.addField(new FieldDescriptor(fieldName, FieldKind.STRING)); } writeDecimal(fieldName: string, value: BigDecimal | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.DECIMAL)); + this.addField(new FieldDescriptor(fieldName, FieldKind.DECIMAL)); } writeTime(fieldName: string, value: LocalTime | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.TIME)); + this.addField(new FieldDescriptor(fieldName, FieldKind.TIME)); } writeDate(fieldName: string, value: LocalDate | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.DATE)); + this.addField(new FieldDescriptor(fieldName, FieldKind.DATE)); } writeTimestamp(fieldName: string, value: LocalDateTime | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.TIMESTAMP)); + this.addField(new FieldDescriptor(fieldName, FieldKind.TIMESTAMP)); } writeTimestampWithTimezone(fieldName: string, value: OffsetDateTime | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.TIMESTAMP_WITH_TIMEZONE)); + this.addField(new FieldDescriptor(fieldName, FieldKind.TIMESTAMP_WITH_TIMEZONE)); } writeCompact(fieldName: string, value: T | null): Promise { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.COMPACT)); + this.addField(new FieldDescriptor(fieldName, FieldKind.COMPACT)); return Promise.resolve(); } writeArrayOfBoolean(fieldName: string, value: boolean[] | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_BOOLEAN)); + this.addField(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_BOOLEAN)); } writeArrayOfInt8(fieldName: string, value: Buffer | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_INT8)); + this.addField(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_INT8)); } writeArrayOfInt16(fieldName: string, value: number[] | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_INT16)); + this.addField(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_INT16)); } writeArrayOfInt32(fieldName: string, value: number[] | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_INT32)); + this.addField(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_INT32)); } writeArrayOfInt64(fieldName: string, value: Long[] | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_INT64)); + this.addField(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_INT64)); } writeArrayOfFloat32(fieldName: string, value: number[] | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_FLOAT32)); + this.addField(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_FLOAT32)); } writeArrayOfFloat64(fieldName: string, value: number[] | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_FLOAT64)); + this.addField(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_FLOAT64)); } writeArrayOfString(fieldName: string, value: (string | null)[] | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_STRING)); + this.addField(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_STRING)); } writeArrayOfDecimal(fieldName: string, value: (BigDecimal | null)[] | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_DECIMAL)); + this.addField(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_DECIMAL)); } writeArrayOfTime(fieldName: string, value: (LocalTime | null)[] | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_TIME)); + this.addField(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_TIME)); } writeArrayOfDate(fieldName: string, value: (LocalDate | null)[] | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_DATE)); + this.addField(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_DATE)); } writeArrayOfTimestamp(fieldName: string, value: (LocalDateTime | null)[] | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_TIMESTAMP)); + this.addField(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_TIMESTAMP)); } writeArrayOfTimestampWithTimezone(fieldName: string, value: (OffsetDateTime | null)[] | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_TIMESTAMP_WITH_TIMEZONE)); + this.addField(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_TIMESTAMP_WITH_TIMEZONE)); } writeArrayOfCompact(fieldName: string, value: (T | null)[] | null): Promise { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_COMPACT)); + this.addField(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_COMPACT)); return Promise.resolve(); } writeNullableBoolean(fieldName: string, value: boolean | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.NULLABLE_BOOLEAN)); + this.addField(new FieldDescriptor(fieldName, FieldKind.NULLABLE_BOOLEAN)); } writeNullableInt8(fieldName: string, value: number | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.NULLABLE_INT8)); + this.addField(new FieldDescriptor(fieldName, FieldKind.NULLABLE_INT8)); } writeNullableInt16(fieldName: string, value: number | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.NULLABLE_INT16)); + this.addField(new FieldDescriptor(fieldName, FieldKind.NULLABLE_INT16)); } writeNullableInt32(fieldName: string, value: number | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.NULLABLE_INT32)); + this.addField(new FieldDescriptor(fieldName, FieldKind.NULLABLE_INT32)); } writeNullableInt64(fieldName: string, value: Long | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.NULLABLE_INT64)); + this.addField(new FieldDescriptor(fieldName, FieldKind.NULLABLE_INT64)); } writeNullableFloat32(fieldName: string, value: number | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.NULLABLE_FLOAT32)); + this.addField(new FieldDescriptor(fieldName, FieldKind.NULLABLE_FLOAT32)); } writeNullableFloat64(fieldName: string, value: number | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.NULLABLE_FLOAT64)); + this.addField(new FieldDescriptor(fieldName, FieldKind.NULLABLE_FLOAT64)); } writeArrayOfNullableBoolean(fieldName: string, value: (boolean | null)[] | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_NULLABLE_BOOLEAN)); + this.addField(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_NULLABLE_BOOLEAN)); } writeArrayOfNullableInt8(fieldName: string, value: (number | null)[] | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_NULLABLE_INT8)); + this.addField(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_NULLABLE_INT8)); } writeArrayOfNullableInt16(fieldName: string, value: (number | null)[] | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_NULLABLE_INT16)); + this.addField(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_NULLABLE_INT16)); } writeArrayOfNullableInt32(fieldName: string, value: (number | null)[] | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_NULLABLE_INT32)); + this.addField(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_NULLABLE_INT32)); } writeArrayOfNullableInt64(fieldName: string, value: (Long | null)[] | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_NULLABLE_INT64)); + this.addField(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_NULLABLE_INT64)); } writeArrayOfNullableFloat32(fieldName: string, value: (number | null)[] | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_NULLABLE_FLOAT32)); + this.addField(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_NULLABLE_FLOAT32)); } writeArrayOfNullableFloat64(fieldName: string, value: (number | null)[] | null): void { - this.fields.push(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_NULLABLE_FLOAT64)); + this.addField(new FieldDescriptor(fieldName, FieldKind.ARRAY_OF_NULLABLE_FLOAT64)); } addField(field: FieldDescriptor): void { + if (this.fieldNames.has(field.fieldName)) { + throw new HazelcastSerializationError( + 'Field with the name ' + field.fieldName + ' already exists' + ); + } + + this.fieldNames.add(field.fieldName); this.fields.push(field); } diff --git a/src/serialization/generic_record/FieldKind.ts b/src/serialization/generic_record/FieldKind.ts index debaf64f3..27e7850f2 100644 --- a/src/serialization/generic_record/FieldKind.ts +++ b/src/serialization/generic_record/FieldKind.ts @@ -22,56 +22,58 @@ * {@link FieldType} is the old API for Portable only and only meant to be used with * {@link PortableReader.getFieldType} API. * - * This API is currently in Beta and can change at any time. */ -export enum FieldKind { - BOOLEAN = 0, - ARRAY_OF_BOOLEAN = 1, - INT8 = 2, - ARRAY_OF_INT8 = 3, + export enum FieldKind { + /** + * Represents fields that do not exist. + */ + NOT_AVAILABLE = 0, + BOOLEAN = 1, + ARRAY_OF_BOOLEAN = 2, + INT8 = 3, + ARRAY_OF_INT8 = 4, // char and array of char are not here because portable generic records is not supported yet - // CHAR = 4, - // ARRAY_OF_CHAR = 5, - INT16 = 6, - ARRAY_OF_INT16 = 7, - INT32 = 8, - ARRAY_OF_INT32 = 9, - INT64 = 10, - ARRAY_OF_INT64 = 11, - FLOAT32 = 12, - ARRAY_OF_FLOAT32 = 13, - FLOAT64 = 14, - ARRAY_OF_FLOAT64 = 15, - STRING = 16, - ARRAY_OF_STRING = 17, - DECIMAL = 18, - ARRAY_OF_DECIMAL = 19, - TIME = 20, - ARRAY_OF_TIME = 21, - DATE = 22, - ARRAY_OF_DATE = 23, - TIMESTAMP = 24, - ARRAY_OF_TIMESTAMP = 25, - TIMESTAMP_WITH_TIMEZONE = 26, - ARRAY_OF_TIMESTAMP_WITH_TIMEZONE = 27, - COMPACT = 28, - ARRAY_OF_COMPACT = 29, + // CHAR = 5, + // ARRAY_OF_CHAR = 6, + INT16 = 7, + ARRAY_OF_INT16 = 8, + INT32 = 9, + ARRAY_OF_INT32 = 10, + INT64 = 11, + ARRAY_OF_INT64 = 12, + FLOAT32 = 13, + ARRAY_OF_FLOAT32 = 14, + FLOAT64 = 15, + ARRAY_OF_FLOAT64 = 16, + STRING = 17, + ARRAY_OF_STRING = 18, + DECIMAL = 19, + ARRAY_OF_DECIMAL = 20, + TIME = 21, + ARRAY_OF_TIME = 22, + DATE = 23, + ARRAY_OF_DATE = 24, + TIMESTAMP = 25, + ARRAY_OF_TIMESTAMP = 26, + TIMESTAMP_WITH_TIMEZONE = 27, + ARRAY_OF_TIMESTAMP_WITH_TIMEZONE = 28, + COMPACT = 29, + ARRAY_OF_COMPACT = 30, // portable and array of portable are not here because portable generic records is not supported yet - // PORTABLE = 30, - // ARRAY_OF_PORTABLE = 31, - NULLABLE_BOOLEAN = 32, - ARRAY_OF_NULLABLE_BOOLEAN = 33, - NULLABLE_INT8 = 34, - ARRAY_OF_NULLABLE_INT8 = 35, - NULLABLE_INT16 = 36, - ARRAY_OF_NULLABLE_INT16 = 37, - NULLABLE_INT32 = 38, - ARRAY_OF_NULLABLE_INT32 = 39, - NULLABLE_INT64 = 40, - ARRAY_OF_NULLABLE_INT64 = 41, - NULLABLE_FLOAT32 = 42, - ARRAY_OF_NULLABLE_FLOAT32 = 43, - NULLABLE_FLOAT64 = 44, - ARRAY_OF_NULLABLE_FLOAT64 = 45, - NOT_AVAILABLE = 46, + // PORTABLE = 31, + // ARRAY_OF_PORTABLE = 32, + NULLABLE_BOOLEAN = 33, + ARRAY_OF_NULLABLE_BOOLEAN = 34, + NULLABLE_INT8 = 35, + ARRAY_OF_NULLABLE_INT8 = 36, + NULLABLE_INT16 = 37, + ARRAY_OF_NULLABLE_INT16 = 38, + NULLABLE_INT32 = 39, + ARRAY_OF_NULLABLE_INT32 = 40, + NULLABLE_INT64 = 41, + ARRAY_OF_NULLABLE_INT64 = 42, + NULLABLE_FLOAT32 = 43, + ARRAY_OF_NULLABLE_FLOAT32 = 44, + NULLABLE_FLOAT64 = 45, + ARRAY_OF_NULLABLE_FLOAT64 = 46, } diff --git a/src/serialization/generic_record/Fields.ts b/src/serialization/generic_record/Fields.ts index 2b682da16..51570fe58 100644 --- a/src/serialization/generic_record/Fields.ts +++ b/src/serialization/generic_record/Fields.ts @@ -24,7 +24,6 @@ import {GenericRecord} from './GenericRecord'; * * @param T The field's corresponding type in JavaScript. * - * This API is currently in Beta and can change at any time. */ // eslint-disable-next-line @typescript-eslint/no-unused-vars export interface Field { diff --git a/src/serialization/generic_record/GenericRecord.ts b/src/serialization/generic_record/GenericRecord.ts index abc2ebe10..d0ed433d8 100644 --- a/src/serialization/generic_record/GenericRecord.ts +++ b/src/serialization/generic_record/GenericRecord.ts @@ -27,7 +27,6 @@ import {BigDecimal, LocalDate, LocalDateTime, LocalTime, OffsetDateTime} from '. * * GenericRecords is only supported for Compact serializable objects. * - * This API is currently in Beta and can change at any time. */ export interface GenericRecord { /** diff --git a/src/serialization/generic_record/GenericRecords.ts b/src/serialization/generic_record/GenericRecords.ts index ab74d126c..ad75c9dcd 100644 --- a/src/serialization/generic_record/GenericRecords.ts +++ b/src/serialization/generic_record/GenericRecords.ts @@ -21,7 +21,6 @@ import {Field} from './Fields'; * The class for creating generic records. This class should not be instantiated directly. * Its {@link compact} method creates new compact generic records. * - * This API is currently in Beta and can change at any time. */ export class GenericRecords { /** diff --git a/src/util/Util.ts b/src/util/Util.ts index 953025873..eeb7470ab 100644 --- a/src/util/Util.ts +++ b/src/util/Util.ts @@ -18,9 +18,10 @@ import * as assert from 'assert'; import * as Long from 'long'; import * as Path from 'path'; -import {BigDecimal, IllegalStateError, LocalDate, LocalDateTime, LocalTime, MemberImpl, OffsetDateTime, UUID} from '../core'; +import {IllegalStateError, MemberImpl} from '../core'; import {MemberVersion} from '../core/MemberVersion'; import {BuildInfo} from '../BuildInfo'; +import {SerializationSymbols} from '../serialization/SerializationSymbols'; /** @internal */ export function assertNotNull(v: any): void { @@ -66,34 +67,6 @@ export function shuffleArray(array: T[]): void { } } -/** @internal */ -export function getType(obj: any): string { - if (Long.isLong(obj)) { - return 'long'; - } else if (Buffer.isBuffer(obj)) { - return 'buffer'; - } else if (UUID.isUUID(obj)) { - return 'uuid'; - } else if (obj instanceof LocalDate) { - return 'localDate'; - } else if (obj instanceof LocalTime) { - return 'localTime'; - } else if (obj instanceof LocalDateTime) { - return 'localDateTime'; - } else if (obj instanceof OffsetDateTime) { - return 'offsetDateTime'; - } else if (obj instanceof BigDecimal) { - return 'bigDecimal'; - } else { - const t = typeof obj; - if (t !== 'object') { - return t; - } else { - return ({}).toString.call(obj).match(/\s([a-zA-Z]+)/)[1].toLowerCase(); - } - } -} - /** @internal */ export function enumFromString(enumType: any, value: string): T { return enumType[value]; @@ -144,6 +117,28 @@ export function tryGetString(val: any): string { } } +/** @internal */ +// eslint-disable-next-line @typescript-eslint/ban-types +export function getTypeKeyForDefaultNumberType(defaultNumberType: string): Function | Symbol { + switch (defaultNumberType) { + case 'byte': + return SerializationSymbols.BYTE_SYMBOL; + case 'short': + return SerializationSymbols.SHORT_SYMBOL; + case 'integer': + return SerializationSymbols.INTEGER_SYMBOL; + case 'float': + return SerializationSymbols.FLOAT_SYMBOL; + case 'double': + return SerializationSymbols.DOUBLE_SYMBOL; + case 'long': + return Long; + default: + throw new RangeError(`Unexpected defaultNumberType value. (${defaultNumberType}) + Expected values: byte, short, integer, float, double, long`); + } +} + /** @internal */ export function tryGetStringOrNull(val: any): string { if (val === null || typeof val === 'string') { diff --git a/test/integration/backward_compatible/parallel/serialization/DefaultSerializersLiveTest.js b/test/integration/backward_compatible/parallel/serialization/DefaultSerializersLiveTest.js index 267bf227a..9a25a8c2f 100644 --- a/test/integration/backward_compatible/parallel/serialization/DefaultSerializersLiveTest.js +++ b/test/integration/backward_compatible/parallel/serialization/DefaultSerializersLiveTest.js @@ -22,10 +22,23 @@ const { Lang } = require('../../../remote_controller/remote_controller_types'); const { RestValue, UUID } = require('../../../../../lib'); const TestUtil = require('../../../../TestUtil'); +const generateGet = (key, mapName) => { + return 'var StringArray = Java.type("java.lang.String[]");' + + 'function foo() {' + + ' var map = instance_0.getMap("' + mapName + '");' + + ' var res = map.get("' + key + '");' + + ' if (res.getClass().isArray()) {' + + ' return Java.from(res);' + + ' } else {' + + ' return res;' + + ' }' + + '}' + + 'result = ""+foo();'; +}; + describe('DefaultSerializersLiveTest', function () { let cluster, client; let map; - const testFactory = new TestUtil.TestFactory(); before(async function () { @@ -51,51 +64,24 @@ describe('DefaultSerializersLiveTest', function () { return response.result.toString(); }; - const generateGet = (key) => { - return 'var StringArray = Java.type("java.lang.String[]");' + - 'function foo() {' + - ' var map = instance_0.getMap("' + map.getName() + '");' + - ' var res = map.get("' + key + '");' + - ' if (res.getClass().isArray()) {' + - ' return Java.from(res);' + - ' } else {' + - ' return res;' + - ' }' + - '}' + - 'result = ""+foo();'; - }; - it('string', async function () { await map.put('testStringKey', 'testStringValue'); - const response = await RC.executeOnController(cluster.id, generateGet('testStringKey'), Lang.JAVASCRIPT); + const response = await RC.executeOnController(cluster.id, generateGet('testStringKey', map.getName()), Lang.JAVASCRIPT); expect(response.result.toString()).to.equal('testStringValue'); }); it('utf8 sample string test', async function () { await map.put('key', 'Iñtërnâtiônàlizætiøn'); - const response = await RC.executeOnController(cluster.id, generateGet('key'), Lang.JAVASCRIPT); + const response = await RC.executeOnController(cluster.id, generateGet('key', map.getName()), Lang.JAVASCRIPT); expect(response.result.toString()).to.equal('Iñtërnâtiônàlizætiøn'); }); it('number', async function () { await map.put('a', 23); - const response = await RC.executeOnController(cluster.id, generateGet('a'), 1); + const response = await RC.executeOnController(cluster.id, generateGet('a', map.getName()), 1); expect(Number.parseInt(response.result.toString())).to.equal(23); }); - it('array', async function () { - await map.put('a', ['a', 'v', 'vg']); - const response = await RC.executeOnController(cluster.id, generateGet('a'), Lang.JAVASCRIPT); - expect(response.result.toString()).to.equal(['a', 'v', 'vg'].toString()); - }); - - it('buffer on client', async function () { - await map.put('foo', Buffer.from('bar')); - const response = await map.get('foo'); - expect(Buffer.isBuffer(response)).to.be.true; - expect(response.toString()).to.equal('bar'); - }); - it('emoji string test on client', async function () { await map.put('key', '1⚐中💦2😭‍🙆😔5'); const response = await map.get('key'); @@ -116,19 +102,19 @@ describe('DefaultSerializersLiveTest', function () { it('emoji string test on RC', async function () { await map.put('key', '1⚐中💦2😭‍🙆😔5'); - const response = await RC.executeOnController(cluster.id, generateGet('key'), Lang.JAVASCRIPT); + const response = await RC.executeOnController(cluster.id, generateGet('key', map.getName()), Lang.JAVASCRIPT); expect(response.result.toString()).to.equal('1⚐中💦2😭‍🙆😔5'); }); it('utf8 characters test on RC', async function () { await map.put('key', '\u0040\u0041\u01DF\u06A0\u12E0\u{1D306}'); - const response = await RC.executeOnController(cluster.id, generateGet('key'), Lang.JAVASCRIPT); + const response = await RC.executeOnController(cluster.id, generateGet('key', map.getName()), Lang.JAVASCRIPT); expect(response.result.toString()).to.equal('\u0040\u0041\u01DF\u06A0\u12E0\u{1D306}'); }); it('utf8 characters test on RC with surrogates', async function () { await map.put('key', '\u0040\u0041\u01DF\u06A0\u12E0\uD834\uDF06'); - const response = await RC.executeOnController(cluster.id, generateGet('key'), Lang.JAVASCRIPT); + const response = await RC.executeOnController(cluster.id, generateGet('key', map.getName()), Lang.JAVASCRIPT); expect(response.result.toString()).to.equal('\u0040\u0041\u01DF\u06A0\u12E0\u{1D306}'); }); @@ -167,30 +153,6 @@ describe('DefaultSerializersLiveTest', function () { expect(result).to.equal(uuid.toString()); }); - it('should deserialize Java Array', async function () { - TestUtil.markClientVersionAtLeast(this, '5.1'); - const script = ` - var map = instance_0.getMap("${map.getName()}"); - map.set("key", Java.to([1, 2, 3], "java.lang.Object[]")); - `; - await RC.executeOnController(cluster.id, script, Lang.JAVASCRIPT); - - const actualValue = await map.get('key'); - expect(actualValue).to.deep.equal([1, 2, 3]); - }); - - it('should deserialize empty Java Array', async function () { - TestUtil.markClientVersionAtLeast(this, '5.1'); - const script = ` - var map = instance_0.getMap("${map.getName()}"); - map.set("key", Java.to([], "java.lang.Object[]")); - `; - await RC.executeOnController(cluster.id, script, Lang.JAVASCRIPT); - - const actualValue = await map.get('key'); - expect(actualValue).to.deep.equal([]); - }); - it('should deserialize ArrayList', async function () { TestUtil.markClientVersionAtLeast(this, '4.2'); const script = 'var map = instance_0.getMap("' + map.getName() + '");\n' + @@ -664,3 +626,74 @@ describe('DefaultSerializersLiveTest', function () { } }); }); + +describe('DefaultSerializersLiveTest Arrays', function () { + let cluster, client; + let map; + + const testFactory = new TestUtil.TestFactory(); + + before(async function () { + cluster = await testFactory.createClusterForParallelTests(); + const member = await RC.startMember(cluster.id); + client = await testFactory.newHazelcastClientForParallelTests({ + clusterName: cluster.id, + serialization: { + defaultNumberType: 'short' + } + }, member); + }); + + beforeEach(async function() { + map = await client.getMap(TestUtil.randomString(10)); + }); + + after(async function () { + await testFactory.shutdownAll(); + }); + + it('should serialize string array correctly', async function () { + await map.put('a', ['a', 'v', 'vg']); + const response = await RC.executeOnController(cluster.id, generateGet('a', map.getName()), Lang.JAVASCRIPT); + expect(response.result.toString()).to.equal(['a', 'v', 'vg'].toString()); + }); + + it('should serialize numbers as shorts when defaultNumberType is set to short', async function () { + const sampleNumbersArray = [-32768, 32767, 10000, 12121, 0, 1, -1, -2121, -12121, -10000]; + await map.put('a', sampleNumbersArray); + const response = await RC.executeOnController(cluster.id, generateGet('a', map.getName()), Lang.JAVASCRIPT); + expect(response.result.toString()).to.equal(sampleNumbersArray.toString()); + }); + + it('buffer on client', async function () { + await map.put('foo', Buffer.from('bar')); + const response = await map.get('foo'); + expect(Buffer.isBuffer(response)).to.be.true; + expect(response.toString()).to.equal('bar'); + }); + + it('should deserialize Java Array', async function () { + TestUtil.markClientVersionAtLeast(this, '5.1'); + const script = ` + var map = instance_0.getMap("${map.getName()}"); + map.set("key", Java.to([1, 2, 3], "java.lang.Object[]")); + `; + await RC.executeOnController(cluster.id, script, Lang.JAVASCRIPT); + + const actualValue = await map.get('key'); + expect(actualValue).to.deep.equal([1, 2, 3]); + }); + + it('should deserialize empty Java Array', async function () { + TestUtil.markClientVersionAtLeast(this, '5.1'); + const script = ` + var map = instance_0.getMap("${map.getName()}"); + map.set("key", Java.to([], "java.lang.Object[]")); + `; + await RC.executeOnController(cluster.id, script, Lang.JAVASCRIPT); + + const actualValue = await map.get('key'); + expect(actualValue).to.deep.equal([]); + }); +}); + diff --git a/test/integration/backward_compatible/parallel/serialization/compact/CompactPublicAPIsTest.js b/test/integration/backward_compatible/parallel/serialization/compact/CompactPublicAPIsTest.js index 1e76a2747..dc4fb608b 100644 --- a/test/integration/backward_compatible/parallel/serialization/compact/CompactPublicAPIsTest.js +++ b/test/integration/backward_compatible/parallel/serialization/compact/CompactPublicAPIsTest.js @@ -168,7 +168,7 @@ describe('CompactPublicAPIsTest', function () { let clientConfig; let skipped = false; - const CLUSTER_CONFIG_XML = ` + let CLUSTER_CONFIG_XML = ` + `; @@ -200,6 +200,10 @@ describe('CompactPublicAPIsTest', function () { skipped = true; this.skip(); } + if ((await TestUtil.compareServerVersionWithRC(RC, '5.2.0')) < 0) { + CLUSTER_CONFIG_XML = CLUSTER_CONFIG_XML + .replace('', ''); + } cluster = await testFactory.createClusterForParallelTests(null, CLUSTER_CONFIG_XML); member = await RC.startMember(cluster.id); SchemaNotReplicatedError = require('../../../../../../lib').SchemaNotReplicatedError; diff --git a/test/integration/backward_compatible/parallel/serialization/compact/CompactSerializersLiveTest.js b/test/integration/backward_compatible/parallel/serialization/compact/CompactSerializersLiveTest.js index d8b44417a..d6bec2c41 100644 --- a/test/integration/backward_compatible/parallel/serialization/compact/CompactSerializersLiveTest.js +++ b/test/integration/backward_compatible/parallel/serialization/compact/CompactSerializersLiveTest.js @@ -27,18 +27,25 @@ const CompactUtil = require('./CompactUtil'); describe('CompactSerializersLiveTest', function () { before(async function () { TestUtil.markClientVersionAtLeast(this, '5.1.0'); + const comparisonValueForServerVersion520 = await TestUtil.compareServerVersionWithRC(RC, '5.2.0'); if ((await TestUtil.compareServerVersionWithRC(RC, '5.1.0')) < 0) { this.skip(); } // Compact serialization 5.2 server is not compatible with clients older than 5.2 - if ((await TestUtil.compareServerVersionWithRC(RC, '5.2.0')) >= 0 && !TestUtil.isClientVersionAtLeast('5.2.0')) { + if (comparisonValueForServerVersion520 >= 0 && !TestUtil.isClientVersionAtLeast('5.2.0')) { this.skip(); } + // Compact serialization 5.2 server configuration changes + if (comparisonValueForServerVersion520 < 0) { + COMPACT_ENABLED_ZERO_CONFIG_XML = COMPACT_ENABLED_ZERO_CONFIG_XML + .replace('', ''); + COMPACT_ENABLED_WITH_SERIALIZER_XML = COMPACT_ENABLED_WITH_SERIALIZER_XML_BETA; + } }); const testFactory = new TestUtil.TestFactory(); - const COMPACT_ENABLED_ZERO_CONFIG_XML = ` + let COMPACT_ENABLED_ZERO_CONFIG_XML = ` + `; - const COMPACT_ENABLED_WITH_SERIALIZER_XML = ` + const COMPACT_ENABLED_WITH_SERIALIZER_XML_BETA = ` - - - example.serialization.EmployeeDTO - - + + + example.serialization.EmployeeDTO + + + + + + `; + + let COMPACT_ENABLED_WITH_SERIALIZER_XML = ` + + + 0 + + + + + example.serialization.EmployeeDTOSerializer + @@ -112,9 +137,16 @@ describe('CompactSerializersLiveTest', function () { await RC.executeOnController(cluster.id, script, Lang.JAVASCRIPT); const map = await client.getMap(mapName); const value = await map.get(1); - value.should.be.instanceof(CompactUtil.EmployeeDTO); - value.age.should.be.equal(expectedAge); - value.id.equals(expectedId).should.be.true; + // Test result can be changed by configuration + if (name == 'Zero config') { + value.should.be.instanceof(CompactUtil.EmployeeDTO); + value.age.should.be.equal(expectedAge); + value.id.equals(expectedId).should.be.true; + } else { + value.schema.typeName.should.be.equal('employee'); + value.getInt32('age').should.be.equal(expectedAge); + value.getInt64('id').equals(expectedId).should.be.true; + } }); it('should write correct data', async function() { diff --git a/test/integration/backward_compatible/parallel/serialization/compact/CompactTest.js b/test/integration/backward_compatible/parallel/serialization/compact/CompactTest.js index 182c79b57..f6a1fa684 100644 --- a/test/integration/backward_compatible/parallel/serialization/compact/CompactTest.js +++ b/test/integration/backward_compatible/parallel/serialization/compact/CompactTest.js @@ -27,7 +27,7 @@ const BSerializer = require('./SameNamedClass').ASerializer; const { Predicates, HazelcastSerializationError } = require('../../../../../../lib/core'); const CompactUtil = require('./CompactUtil'); -const COMPACT_ENABLED_ZERO_CONFIG_XML = ` +let COMPACT_ENABLED_ZERO_CONFIG_XML = ` + `; @@ -74,13 +74,19 @@ describe('CompactTest', function () { before(async function () { TestUtil.markClientVersionAtLeast(this, '5.1.0'); + const comparisonValueForServerVersion520 = await TestUtil.compareServerVersionWithRC(RC, '5.2.0'); if ((await TestUtil.compareServerVersionWithRC(RC, '5.1.0')) < 0) { this.skip(); } // Compact serialization 5.2 server is not compatible with clients older than 5.2 - if ((await TestUtil.compareServerVersionWithRC(RC, '5.2.0')) >= 0 && !TestUtil.isClientVersionAtLeast('5.2.0')) { + if (comparisonValueForServerVersion520 >= 0 && !TestUtil.isClientVersionAtLeast('5.2.0')) { this.skip(); } + // Compact serialization 5.2 server configuration changes + if (comparisonValueForServerVersion520 < 0) { + COMPACT_ENABLED_ZERO_CONFIG_XML = COMPACT_ENABLED_ZERO_CONFIG_XML + .replace('', ''); + } cluster = await testFactory.createClusterForParallelTests(undefined, COMPACT_ENABLED_ZERO_CONFIG_XML); member = await RC.startMember(cluster.id); }); diff --git a/test/integration/backward_compatible/parallel/serialization/compact/CompactUtil.js b/test/integration/backward_compatible/parallel/serialization/compact/CompactUtil.js index 961d00357..5d4ac8416 100644 --- a/test/integration/backward_compatible/parallel/serialization/compact/CompactUtil.js +++ b/test/integration/backward_compatible/parallel/serialization/compact/CompactUtil.js @@ -43,10 +43,17 @@ let Bits; let BitsSerializer; let Nested; let NestedSerializer; +let SampleObject1; +let SampleObject1Serializer; +let SampleObject2; +let SampleObject2Serializer; let Employee; let EmployeeSerializer; let EmployeeDTO; let EmployeeDTOSerializer; +let DefaultSerializerOverridingSerializer; +let ArrayOfCompact; +let ArrayOfCompactSerializer; let Employer; let EmployerSerializer; let HIRING_STATUS; @@ -127,6 +134,21 @@ if (TestUtil.isClientVersionAtLeast('5.1.0')) { } }; + DefaultSerializerOverridingSerializer = class DefaultSerializerOverridingSerializer { + getClass() { + return String; + } + + getTypeName() { + return TestUtil.randomString(); + } + + read() { + } + + write() { + } + }; class NonCompactClass { constructor(a, b) { this.a = a; @@ -343,6 +365,60 @@ if (TestUtil.isClientVersionAtLeast('5.1.0')) { ], field: Fields.ARRAY_OF_GENERIC_RECORD}, }; + SampleObject1 = class SampleObject1 { + constructor(name, id) { + this.name = name; // string + this.id = id; // int64 + } + }; + SampleObject1Serializer = class SampleObject1Serializer { + getClass() { + return SampleObject1; // used to match a js object to serialize with this serializer + } + + getTypeName() { + return 'SampleObject1'; // used to match schema's typeName with serializer + } + + read(reader) { + const name = reader.readString('name'); + const id = reader.readInt64('id'); + return new SampleObject1(name, id); + } + + write(writer, instance) { + writer.writeString('name', instance.name); + writer.writeInt64('id', instance.id); + } + }; + + SampleObject2 = class SampleObject2 { + constructor(name, id) { + this.name = name; // string + this.id = id; // int64 + } + }; + + SampleObject2Serializer = class SampleObject2Serializer { + getClass() { + return SampleObject2; // used to match a js object to serialize with this serializer + } + + getTypeName() { + return 'SampleObject2'; // used to match schema's typeName with serializer + } + + read(reader) { + const name = reader.readString('name'); + const id = reader.readInt64('id'); + return new SampleObject1(name, id); + } + + write(writer, instance) { + writer.writeString('name', instance.name); + writer.writeInt64('id', instance.id); + } + }; EmployeeDTO = class EmployeeDTO { constructor(age, id) { this.age = age; // int32 @@ -456,6 +532,31 @@ if (TestUtil.isClientVersionAtLeast('5.1.0')) { NOT_HIRING: 'NOT_HIRING' }; + ArrayOfCompact = class ArrayOfCompact { + constructor(arrayOfObjects) { + this.arrayOfObjects = arrayOfObjects; + } + }; + + ArrayOfCompactSerializer = class ArrayOfCompactSerializer { + getClass() { + return ArrayOfCompact; + } + + getTypeName() { + return 'ArrayOfCompact'; + } + + read(reader) { + const arrayOfObjects = reader.readArrayOfCompact('arrayOfObjects'); + return new ArrayOfCompact(arrayOfObjects); + } + + write(writer, value) { + writer.writeArrayOfCompact('arrayOfObjects', value.arrayOfObjects); + } + }; + Employer = class Employer { constructor(name, zcode, hiringStatus, ids, singleEmployee, otherEmployees) { this.name = name; @@ -1585,5 +1686,12 @@ module.exports = { EmployeeDTOSerializer, Employer, EmployerSerializer, + ArrayOfCompact, + ArrayOfCompactSerializer, + SampleObject1, + SampleObject1Serializer, + SampleObject2, + SampleObject2Serializer, + DefaultSerializerOverridingSerializer, HIRING_STATUS }; diff --git a/test/integration/backward_compatible/parallel/serialization/compact/LazyDeserialization.js b/test/integration/backward_compatible/parallel/serialization/compact/LazyDeserialization.js index 4fba91b7d..cf2cbcd11 100644 --- a/test/integration/backward_compatible/parallel/serialization/compact/LazyDeserialization.js +++ b/test/integration/backward_compatible/parallel/serialization/compact/LazyDeserialization.js @@ -25,7 +25,7 @@ const TestUtil = require('../../../../../TestUtil'); const { HazelcastSerializationError } = require('../../../../../../lib'); describe('LazyDeserializationCompactTest', function() { - const COMPACT_ENABLED_ZERO_CONFIG_XML = ` + let COMPACT_ENABLED_ZERO_CONFIG_XML = ` + `; @@ -48,13 +48,19 @@ describe('LazyDeserializationCompactTest', function() { before(async function() { TestUtil.markClientVersionAtLeast(this, '5.1.0'); + const comparisonValueForServerVersion520 = await TestUtil.compareServerVersionWithRC(RC, '5.2.0'); if ((await TestUtil.compareServerVersionWithRC(RC, '5.1.0')) < 0) { this.skip(); } // Compact serialization 5.2 server is not compatible with clients older than 5.2 - if ((await TestUtil.compareServerVersionWithRC(RC, '5.2.0')) >= 0 && !TestUtil.isClientVersionAtLeast('5.2.0')) { + if (comparisonValueForServerVersion520 >= 0 && !TestUtil.isClientVersionAtLeast('5.2.0')) { this.skip(); } + // Compact serialization 5.2 server configuration changes + if (comparisonValueForServerVersion520 < 0) { + COMPACT_ENABLED_ZERO_CONFIG_XML = COMPACT_ENABLED_ZERO_CONFIG_XML + .replace('', ''); + } }); describe('ReadOnlyLazyList', function () { diff --git a/test/integration/backward_compatible/parallel/sql/DataTypeTest.js b/test/integration/backward_compatible/parallel/sql/DataTypeTest.js index 4c13c2755..0f201ff39 100644 --- a/test/integration/backward_compatible/parallel/sql/DataTypeTest.js +++ b/test/integration/backward_compatible/parallel/sql/DataTypeTest.js @@ -70,7 +70,7 @@ describe('SQLDataTypeTest', function () { let isCompactCompatible; const clientVersionNewerThanFive = TestUtil.isClientVersionAtLeast('5.0'); - const JET_ENABLED_WITH_COMPACT_CONFIG = fs.readFileSync(path.join(__dirname, 'jet_enabled_with_compact.xml'), 'utf8'); + let JET_ENABLED_WITH_COMPACT_CONFIG = fs.readFileSync(path.join(__dirname, 'jet_enabled_with_compact.xml'), 'utf8'); const JET_ENABLED_CONFIG = fs.readFileSync(path.join(__dirname, 'jet_enabled.xml'), 'utf8'); const validateResults = (rows, expectedKeys, expectedValues) => { @@ -84,12 +84,19 @@ describe('SQLDataTypeTest', function () { before(async function () { serverVersionNewerThanFive = await TestUtil.compareServerVersionWithRC(RC, '5.0') >= 0; + const comparisonValueForServerVersion520 = await TestUtil.compareServerVersionWithRC(RC, '5.2.0'); const serverVersionNewerThanFivePointOne = await TestUtil.compareServerVersionWithRC(RC, '5.1') >= 0; // If client is not newer than 5.2 and server is newer than 5.2, compact serialization is not compatible - isCompactCompatible = !((await TestUtil.compareServerVersionWithRC(RC, '5.2.0')) >= 0 + isCompactCompatible = !(comparisonValueForServerVersion520 >= 0 && !TestUtil.isClientVersionAtLeast('5.2.0')); + // Compact serialization 5.2 server configuration changes + if (comparisonValueForServerVersion520 < 0) { + const JET_ENABLED_WITH_COMPACT_CONFIG_BETA = + fs.readFileSync(path.join(__dirname, 'jet_enabled_with_compact_beta.xml'), 'utf8'); + JET_ENABLED_WITH_COMPACT_CONFIG = JET_ENABLED_WITH_COMPACT_CONFIG_BETA; + } let CLUSTER_CONFIG; // Don't use compact enabled config if not compatible, we will skip the compact test anyway. if (serverVersionNewerThanFivePointOne && isCompactCompatible) { diff --git a/test/integration/backward_compatible/parallel/sql/jet_enabled_with_compact.xml b/test/integration/backward_compatible/parallel/sql/jet_enabled_with_compact.xml index 23e8f56fa..ddc052774 100644 --- a/test/integration/backward_compatible/parallel/sql/jet_enabled_with_compact.xml +++ b/test/integration/backward_compatible/parallel/sql/jet_enabled_with_compact.xml @@ -22,13 +22,10 @@ 0 - - - - example.serialization.EmployeeDTOSerializer - - + + + example.serialization.EmployeeDTOSerializer + diff --git a/test/integration/backward_compatible/parallel/sql/jet_enabled_with_compact_beta.xml b/test/integration/backward_compatible/parallel/sql/jet_enabled_with_compact_beta.xml new file mode 100644 index 000000000..23e8f56fa --- /dev/null +++ b/test/integration/backward_compatible/parallel/sql/jet_enabled_with_compact_beta.xml @@ -0,0 +1,34 @@ + + + + + 0 + + + + + + example.serialization.EmployeeDTOSerializer + + + + + diff --git a/test/integration/backward_compatible/parallel/ssl/ClientSSLAuthenticationTest.js b/test/integration/backward_compatible/parallel/ssl/ClientSSLAuthenticationTest.js index 677df41e1..42265969a 100644 --- a/test/integration/backward_compatible/parallel/ssl/ClientSSLAuthenticationTest.js +++ b/test/integration/backward_compatible/parallel/ssl/ClientSSLAuthenticationTest.js @@ -26,7 +26,7 @@ const RC = require('../../../RC'); const { IllegalStateError } = require('../../../../../lib'); const TestUtil = require('../../../../TestUtil'); -describe('ClientSSLAuthenticationTest', function () { +describe.skip('ClientSSLAuthenticationTest', function () { let cluster; const maRequiredXML = __dirname + '/hazelcast-ma-required.xml'; diff --git a/test/integration/backward_compatible/parallel/ssl/ClientSSLTest.js b/test/integration/backward_compatible/parallel/ssl/ClientSSLTest.js index 9d880391b..e32f8d2ef 100644 --- a/test/integration/backward_compatible/parallel/ssl/ClientSSLTest.js +++ b/test/integration/backward_compatible/parallel/ssl/ClientSSLTest.js @@ -24,7 +24,7 @@ const RC = require('../../../RC'); const { IllegalStateError } = require('../../../../../lib'); const TestUtil = require('../../../../TestUtil'); -describe('ClientSSLTest', function () { +describe.skip('ClientSSLTest', function () { let cluster; let client; let serverConfig; diff --git a/test/integration/backward_compatible/serial/CompactRestartTest.js b/test/integration/backward_compatible/serial/CompactRestartTest.js index 4c4d51bc8..352e2b6dc 100644 --- a/test/integration/backward_compatible/serial/CompactRestartTest.js +++ b/test/integration/backward_compatible/serial/CompactRestartTest.js @@ -20,13 +20,13 @@ const CompactUtil = require('../parallel/serialization/compact/CompactUtil'); const RC = require('../../RC'); const { Predicates } = require('../../../../lib'); -const COMPACT_ENABLED_ZERO_CONFIG_XML = ` +let COMPACT_ENABLED_ZERO_CONFIG_XML = ` - + `; @@ -47,13 +47,18 @@ describe('CompactRestartTest', function() { before(async function () { TestUtil.markClientVersionAtLeast(this, '5.1.0'); + const comparisonValueForServerVersion520 = await TestUtil.compareServerVersionWithRC(RC, '5.2.0'); if ((await TestUtil.compareServerVersionWithRC(RC, '5.1.0')) < 0) { this.skip(); } // Compact serialization 5.2 server is not compatible with clients older than 5.2 - if ((await TestUtil.compareServerVersionWithRC(RC, '5.2.0')) >= 0 && !TestUtil.isClientVersionAtLeast('5.2.0')) { + if (comparisonValueForServerVersion520 >= 0 && !TestUtil.isClientVersionAtLeast('5.2.0')) { this.skip(); } + if (comparisonValueForServerVersion520 < 0) { + COMPACT_ENABLED_ZERO_CONFIG_XML = COMPACT_ENABLED_ZERO_CONFIG_XML + .replace('', ''); + } cluster = await testFactory.createClusterForSerialTests(undefined, COMPACT_ENABLED_ZERO_CONFIG_XML); member = await RC.startMember(cluster.id); }); diff --git a/test/unit/config/ConfigBuilderTest.js b/test/unit/config/ConfigBuilderTest.js index 9a6c36368..9463b3164 100644 --- a/test/unit/config/ConfigBuilderTest.js +++ b/test/unit/config/ConfigBuilderTest.js @@ -617,6 +617,31 @@ describe('ConfigBuilderValidationTest', function () { }); describe('serialization', function () { + it('should validate defaultNumberType values', function () { + const invalidDefaultNumberTypesArray = [ + 'İnteger', 'byta', 'shot', 'aa', 'bb', 'flot', 'lang', null, undefined, Symbol(), {}, () => {}, [], 1, BigInt(121) + ]; + + for (const invalidDefaultNumberType of invalidDefaultNumberTypesArray) { + expect(() => new ConfigBuilder({ + serialization: { + defaultNumberType: invalidDefaultNumberType + } + }).build()).to.throw(InvalidConfigurationError); + } + + const validDefaultNumberTypesArray = [ + 'integer', 'Integer', 'bytE', 'shoRt', 'DoUble', 'floaT', 'loNG', 'long' + ]; + for (const validDefaultNumberType of validDefaultNumberTypesArray) { + expect(() => new ConfigBuilder({ + serialization: { + defaultNumberType: validDefaultNumberType + } + }).build()).not.to.throw(); + } + }); + it('should validate portable and data serializable factories', function () { const invalidFactoriesArray = [ () => {}, 1, undefined, '1', { aaasd: () => {}}, { 1.1: () => {}, 2: () => {} } diff --git a/test/unit/serialization/BinaryCompatibilityTest.js b/test/unit/serialization/BinaryCompatibilityTest.js index 50598ac69..1ec732e05 100644 --- a/test/unit/serialization/BinaryCompatibilityTest.js +++ b/test/unit/serialization/BinaryCompatibilityTest.js @@ -36,6 +36,7 @@ describe('BinaryCompatibilityTest', function () { const isBigEndianValues = [true, false]; const dataMap = {}; + const defaultNumberTypes = ['byte', 'short', 'integer', 'float', 'double', 'long']; function createFileName(version) { return version + '.serialization.compatibility.binary'; @@ -108,7 +109,9 @@ describe('BinaryCompatibilityTest', function () { } ]; cfg.isBigEndian = isBigEndian; - cfg.defaultNumberType = defaultNumberType; + if (defaultNumberTypes.includes(defaultNumberType)) { + cfg.defaultNumberType = defaultNumberType; + } return new SerializationServiceV1(cfg); } diff --git a/test/unit/serialization/DefaultSerializersTest.js b/test/unit/serialization/DefaultSerializersTest.js index 5b719871a..1a000a839 100644 --- a/test/unit/serialization/DefaultSerializersTest.js +++ b/test/unit/serialization/DefaultSerializersTest.js @@ -15,10 +15,16 @@ */ 'use strict'; -const { expect } = require('chai'); +const { expect, assert } = require('chai'); const Long = require('long'); const { SerializationServiceV1 } = require('../../../lib/serialization/SerializationService'); const { SerializationConfigImpl } = require('../../../lib/config/SerializationConfig'); +const { + DoubleArraySerializer, + ByteArraySerializer, + JsonSerializer, + IntegerArraySerializer +} = require('../../../src/serialization/DefaultSerializers'); const { Predicates, RestValue, @@ -29,6 +35,7 @@ const { LocalDate, BigDecimal } = require('../../../'); +const TestUtil = require('../../../test/TestUtil'); describe('DefaultSerializersTest', function () { const restValue = new RestValue(); @@ -138,3 +145,68 @@ describe('DefaultSerializersTest', function () { }); }); }); + +describe('DefaultSerializersTest For Arrays', function () { + it('should throw error when array first element is undefined', function () { + const undefinedArray = [undefined, 'foo', 'bar']; + const config = new SerializationConfigImpl(); + const serializationService = new SerializationServiceV1(config); + expect(() => serializationService.toData(undefinedArray)).to.throw(RangeError, 'The first element is undefined'); + }); + + it('should use DefaultNumberTypeArray (DoubleArraySerializer) serializer for empty array', function () { + const emptyArray = []; + const config = new SerializationConfigImpl(); + const serializationService = new SerializationServiceV1(config); + const returnValue = serializationService.toData(emptyArray); + assert.equal(returnValue.getType(), new DoubleArraySerializer().id); + }); + + it('should use DefaultNumberTypeArray (DoubleArraySerializer) serializer for numbers array', function () { + const numbersArray = [189, 255]; + const config = new SerializationConfigImpl(); + const serializationService = new SerializationServiceV1(config); + const returnValue = serializationService.toData(numbersArray); + assert.equal(returnValue.getType(), new DoubleArraySerializer().id); + }); + + it('should use IntegerArraySerializer for numbers array when I set defaultNumberType to integer', function () { + const numbersArray = [189, 255]; + const config = new SerializationConfigImpl(); + config.defaultNumberType = 'integer'; + const serializationService = new SerializationServiceV1(config); + const returnValue = serializationService.toData(numbersArray); + assert.equal(returnValue.getType(), new IntegerArraySerializer().id); + }); + + it('should use ByteArraySerializer for numbers array when I set defaultNumberType to byte', function () { + const sampleNumbersArray = [12, 18]; + const nums = Buffer.from(sampleNumbersArray); + const config = new SerializationConfigImpl(); + config.defaultNumberType = 'byte'; + const serializationService = new SerializationServiceV1(config); + const returnValue = serializationService.toData(nums); + assert.equal(returnValue.getType(), new ByteArraySerializer().id); + }); + + it('should use JsonSerializer when the first element of array is null, NullArraySerializer is not defined', function () { + const firstElementNullArray = [null, 18]; + const config = new SerializationConfigImpl(); + config.defaultNumberType = 'byte'; + const serializationService = new SerializationServiceV1(config); + const returnValue = serializationService.toData(firstElementNullArray); + assert.equal(returnValue.getType(), new JsonSerializer().id); + }); + + it('should throw range error when defaultNumberType is set to short and send values out of bounds.', async function () { + const outOfBoundsShortArray = [-32769, 32768]; + const config = new SerializationConfigImpl(); + config.defaultNumberType = 'short'; + const serializationService = new SerializationServiceV1(config); + const error = await TestUtil.getRejectionReasonOrThrow(async () => { + await serializationService.toData(outOfBoundsShortArray); + }); + error.message.includes('-32768').should.be.true; + }); +}); + diff --git a/test/unit/serialization/compact/CompactGenericRecordTest.js b/test/unit/serialization/compact/CompactGenericRecordTest.js index c933300d3..b63702409 100644 --- a/test/unit/serialization/compact/CompactGenericRecordTest.js +++ b/test/unit/serialization/compact/CompactGenericRecordTest.js @@ -29,9 +29,17 @@ const { mimicSchemaReplication, validationTestParams, referenceObjects, + ArrayOfCompact, + ArrayOfCompactSerializer, + SampleObject1, + SampleObject2, + SampleObject1Serializer, + SampleObject2Serializer } = require('../../../integration/backward_compatible/parallel/serialization/compact/CompactUtil'); const { Fields, FieldKind } = require('../../../../lib/serialization/generic_record'); const Long = require('long'); +const TestUtil = require('../../../TestUtil'); +const { HazelcastSerializationError } = require('../../../../lib'); const testIntRange = (invalidValueFn, validValueFn) => { for (const test of [ @@ -59,8 +67,21 @@ const testIntRange = (invalidValueFn, validValueFn) => { const sampleGenericRecord = GenericRecords.compact('aa', {nested: Fields.GENERIC_RECORD}, {nested: GenericRecords.compact('bb', {}, {})}); -const sampleArrayOfGenericRecords = [GenericRecords.compact('cc', {foo: Fields.INT16}, {foo: 3}), -GenericRecords.compact('dd', {bar: Fields.ARRAY_OF_INT8}, {bar: Buffer.from([])})]; +const sampleArrayOfGenericRecords = [GenericRecords.compact('dd', {foo: Fields.INT16}, {foo: 3}), +GenericRecords.compact('dd', {foo: Fields.INT16}, {foo: 55})]; + +const sampleArrayOfGenericRecordsDifferentTypes = [GenericRecords.compact('dd', {foo: Fields.INT16}, {foo: 3}), +GenericRecords.compact('cc', {bar: Fields.STRING}, {bar: 'sample value'})]; + +const getGenericRecordArray = ({same}) => { + const values = { + ARRAY_OF_COMPACT: same ? sampleArrayOfGenericRecords : sampleArrayOfGenericRecordsDifferentTypes + }; + const fields = { + ARRAY_OF_COMPACT: Fields.ARRAY_OF_GENERIC_RECORD + }; + return GenericRecords.compact('a', fields, values); +}; const getGiganticRecord = () => { const values = {}; @@ -568,6 +589,96 @@ describe('CompactGenericRecordTest', function () { }); }); + describe('array restrictions', function () { + it('should not throw error if object types are equal on ARRAY_OF_COMPACT', async function () { + const {serializationService, schemaService} = createSerializationService( + [new ArrayOfCompactSerializer(), new SampleObject1Serializer()] + ); + const object1 = new SampleObject1('name1', Long.fromNumber(102310312)); + const object2 = new SampleObject1('name2', Long.fromNumber(102310312)); + const arrayOfObjects = [ + object1, + object2 + ]; + const arrayOfCompactObject = new ArrayOfCompact(arrayOfObjects); + await serialize(serializationService, schemaService, arrayOfCompactObject); + }); + it('should throw error if object types are not equal on ARRAY_OF_COMPACT', async function () { + const {serializationService, schemaService} = createSerializationService( + [new ArrayOfCompactSerializer(), new SampleObject1Serializer(), new SampleObject2Serializer()] + ); + const object1 = new SampleObject1('name1', Long.fromNumber(102310312)); + const object2 = new SampleObject2('name2', Long.fromNumber(102310312)); + const arrayOfObjects = [ + object1, + object2 + ]; + const arrayOfCompactObject = new ArrayOfCompact(arrayOfObjects); + + const error = await TestUtil.getRejectionReasonOrThrow(async () => { + await serialize(serializationService, schemaService, arrayOfCompactObject); + }); + + error.should.be.instanceOf(HazelcastSerializationError); + error.message.includes('It is not allowed to serialize an array of Compact serializable objects' + +' containing different item types.').should.be.true; + }); + it('should throw error if one of the object is undefined on ARRAY_OF_COMPACT', async function () { + const {serializationService, schemaService} = createSerializationService( + [new ArrayOfCompactSerializer(), new SampleObject1Serializer(), new SampleObject2Serializer()] + ); + const object1 = undefined; + const object2 = new SampleObject2('name2', Long.fromNumber(102310312)); + const arrayOfObjects = [ + object1, + object2 + ]; + const arrayOfCompactObject = new ArrayOfCompact(arrayOfObjects); + + const error = await TestUtil.getRejectionReasonOrThrow(async () => { + await serialize(serializationService, schemaService, arrayOfCompactObject); + }); + + error.should.be.instanceOf(HazelcastSerializationError); + error.message.includes('The value undefined can not be used in an Array of Compact value.').should.be.true; + }); + it('should throw error if one of the object\'s constructor is undefined on ARRAY_OF_COMPACT', async function () { + const {serializationService, schemaService} = createSerializationService( + [new ArrayOfCompactSerializer(), new SampleObject1Serializer(), new SampleObject2Serializer()] + ); + const object1 = Object.create(null); + const object2 = new SampleObject2('name2', Long.fromNumber(102310312)); + const arrayOfObjects = [ + object1, + object2 + ]; + const arrayOfCompactObject = new ArrayOfCompact(arrayOfObjects); + + const error = await TestUtil.getRejectionReasonOrThrow(async () => { + await serialize(serializationService, schemaService, arrayOfCompactObject); + }); + + error.should.be.instanceOf(HazelcastSerializationError); + error.message.includes('encountered with a value with undefined contructor').should.be.true; + }); + it('should not throw error array of GenericRecord objects containing same schemas.', async function () { + const {serializationService, schemaService} = createSerializationService(); + const arrayofGenericRecords = getGenericRecordArray({same: true}); + await serialize(serializationService, schemaService, arrayofGenericRecords); + }); + it('should throw error array of GenericRecord objects does not containing same schemas.', async function () { + const {serializationService, schemaService} = createSerializationService(); + const arrayofGenericRecords = getGenericRecordArray({same: false}); + + const error = await TestUtil.getRejectionReasonOrThrow(async () => { + await serialize(serializationService, schemaService, arrayofGenericRecords); + }); + error.should.be.instanceOf(HazelcastSerializationError); + error.message.includes('It is not allowed to serialize an array of Compact serializable ' + +'GenericRecord objects containing different schemas.').should.be.true; + }); + }); + describe('cloning', function () { it('should be able to clone every field', function() { const record = getGiganticRecord(); diff --git a/test/unit/serialization/compact/CompactTest.js b/test/unit/serialization/compact/CompactTest.js index 436807494..f3a0a4fb8 100644 --- a/test/unit/serialization/compact/CompactTest.js +++ b/test/unit/serialization/compact/CompactTest.js @@ -37,10 +37,11 @@ const { NamedDTOSerializer, NodeDTOSerializer, NodeDTO, + DefaultSerializerOverridingSerializer, } = require('../../../integration/backward_compatible/parallel/serialization/compact/CompactUtil'); const Long = require('long'); const { CompactGenericRecordImpl } = require('../../../../lib/serialization/generic_record/CompactGenericRecord'); -const { GenericRecords, HazelcastSerializationError} = require('../../../../lib'); +const { GenericRecords, HazelcastSerializationError, IllegalArgumentError } = require('../../../../lib'); const { Fields } = require('../../../../lib/serialization/generic_record'); const TestUtil = require('../../../TestUtil'); @@ -262,4 +263,14 @@ describe('CompactTest', function () { error.should.be.instanceOf(HazelcastSerializationError); error.message.includes('No serializer is registered for class/constructor').should.be.true; }); + + it('should throw proper error when overriding the string serializer(String class)', async function() { + const error = await TestUtil.getRejectionReasonOrThrow(async () => { + createSerializationService( + [ new DefaultSerializerOverridingSerializer() ] + ); + }); + error.should.be.instanceOf(IllegalArgumentError); + error.message.includes('Compact serializer for the class').should.be.true; + }); }); diff --git a/test/unit/serialization/compact/RabinFingerprintTest.js b/test/unit/serialization/compact/RabinFingerprintTest.js index 41f359e16..aa7376992 100644 --- a/test/unit/serialization/compact/RabinFingerprintTest.js +++ b/test/unit/serialization/compact/RabinFingerprintTest.js @@ -86,6 +86,6 @@ describe('RabinFingerprintTest', function () { writer.writeInt8('age', 0); writer.writeArrayOfTimestamp('times', []); const schema = writer.build(); - schema.schemaId.eq(Long.fromString('-5445839760245891300')).should.be.true; + schema.schemaId.eq(Long.fromString('3662264393229655598')).should.be.true; }); }); diff --git a/test/unit/serialization/compact/SchemaWriterTest.js b/test/unit/serialization/compact/SchemaWriterTest.js index 678001057..cdd041373 100644 --- a/test/unit/serialization/compact/SchemaWriterTest.js +++ b/test/unit/serialization/compact/SchemaWriterTest.js @@ -17,7 +17,7 @@ const chai = require('chai'); const { SchemaWriter } = require('../../../../lib/serialization/compact/SchemaWriter'); -const { FieldKind } = require('../../../../lib'); +const { FieldKind, HazelcastSerializationError } = require('../../../../lib'); const { supportedFields } = require('../../../integration/backward_compatible/parallel/serialization/compact/CompactUtil'); @@ -168,4 +168,14 @@ describe('SchemaWriterTest', function () { schema.fieldDefinitionMap.get(name).kind.should.be.eq(fieldKind); } }); + + it('should throw error when we write already existing field on schema', async function () { + const error = await TestUtil.getRejectionReasonOrThrow(async () => { + const writer = new SchemaWriter('SomeType'); + writer.writeInt32('bar', 0); + writer.writeString('bar', 'Some value'); + }); + error.should.be.instanceOf(HazelcastSerializationError); + error.message.includes('Field with the name bar already exists').should.be.true; + }); });