diff --git a/sdk/build.gradle.kts b/sdk/build.gradle.kts index 6c5abe50e0..8c83d79a50 100644 --- a/sdk/build.gradle.kts +++ b/sdk/build.gradle.kts @@ -46,6 +46,7 @@ dependencies { } } api(libs.jackson.databind) + api(libs.jackson.yaml) api(libs.slf4j) api(libs.elasticsearch.java) api(libs.freemarker) diff --git a/sdk/src/main/java/com/atlan/cache/CustomMetadataCache.java b/sdk/src/main/java/com/atlan/cache/CustomMetadataCache.java index 3236e30df1..431e713eb4 100644 --- a/sdk/src/main/java/com/atlan/cache/CustomMetadataCache.java +++ b/sdk/src/main/java/com/atlan/cache/CustomMetadataCache.java @@ -772,7 +772,7 @@ public Map getCustomMetadataFromSearchResult( return map; } - private static Object deserializePrimitive(JsonNode jsonValue) throws LogicException { + public static Object deserializePrimitive(JsonNode jsonValue) throws LogicException { if (jsonValue.isValueNode()) { if (jsonValue.isTextual()) { return jsonValue.asText(); diff --git a/sdk/src/main/java/com/atlan/model/contracts/DCS_V_0_0_2.java b/sdk/src/main/java/com/atlan/model/contracts/DCS_V_0_0_2.java new file mode 100644 index 0000000000..2b85201ac6 --- /dev/null +++ b/sdk/src/main/java/com/atlan/model/contracts/DCS_V_0_0_2.java @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: Apache-2.0 + Copyright 2024 Atlan Pte. Ltd. */ +package com.atlan.model.contracts; + +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.experimental.SuperBuilder; +import lombok.extern.jackson.Jacksonized; + +/** + * Capture the detailed specification of a data contract for an asset. + */ +@Getter +@Jacksonized +@SuperBuilder(toBuilder = true) +@EqualsAndHashCode(callSuper = false) +@SuppressWarnings("cast") +public class DCS_V_0_0_2 extends DataContractSpec { + private static final long serialVersionUID = 2L; + + /** Fixed typeName for Tables. */ + @Getter(onMethod_ = {@Override}) + @Builder.Default + String templateVersion = "0.0.2"; +} diff --git a/sdk/src/main/java/com/atlan/model/contracts/DataContractSpec.java b/sdk/src/main/java/com/atlan/model/contracts/DataContractSpec.java new file mode 100644 index 0000000000..059db65f70 --- /dev/null +++ b/sdk/src/main/java/com/atlan/model/contracts/DataContractSpec.java @@ -0,0 +1,289 @@ +/* SPDX-License-Identifier: Apache-2.0 + Copyright 2024 Atlan Pte. Ltd. */ +package com.atlan.model.contracts; + +import com.atlan.model.core.AtlanObject; +import com.atlan.model.core.CustomMetadataAttributes; +import com.atlan.model.enums.AtlanAnnouncementType; +import com.atlan.model.enums.CertificateStatus; +import com.atlan.serde.ReadableCustomMetadataDeserializer; +import com.atlan.serde.ReadableCustomMetadataSerializer; +import com.atlan.serde.Serde; +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Singular; +import lombok.experimental.SuperBuilder; +import lombok.extern.jackson.Jacksonized; +import lombok.extern.slf4j.Slf4j; + +/** + * Capture the detailed specification of a data contract for an asset. + */ +@Getter +@SuperBuilder(toBuilder = true, builderMethodName = "_internal") +@EqualsAndHashCode(callSuper = false) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.EXISTING_PROPERTY, property = "template_version") +@JsonSubTypes({ + @JsonSubTypes.Type(value = DCS_V_0_0_2.class, name = "0.0.2"), +}) +@SuppressWarnings("cast") +@Slf4j +public class DataContractSpec extends AtlanObject { + private static final long serialVersionUID = 2L; + + /** Controls the specification as one for a data contract. */ + @Builder.Default + String kind = "DataContract"; + + /** State of the contract. */ + DataContractStatus status; + + /** Version of the template for the data contract. */ + @JsonProperty("template_version") + String templateVersion; + + /** Name of the asset as it exists inside Atlan. */ + String dataset; + + /** Type of the dataset in Atlan. */ + String type; + + /** Description of this dataset. */ + String description; + + /** Name that must match a data source defined in your config file. */ + String datasource; + + /** Owners of the dataset, which can include users (by username) and / or groups (by internal Atlan alias). */ + Owners owners; + + /** Certification to apply to the dataset. */ + Certification certification; + + /** Announcement to apply to the dataset. */ + Announcement announcement; + + /** Glossary terms to assign to the dataset. */ + @Singular + List terms; + + /** Atlan tags for the dataset. */ + @Singular + List tags; + + /** Custom metadata for the dataset. */ + @Singular + @JsonProperty("custom_metadata") + @JsonSerialize(using = ReadableCustomMetadataSerializer.class) + @JsonDeserialize(using = ReadableCustomMetadataDeserializer.class) + Map customMetadataSets; + + /** Details of each column in the dataset to be governed. */ + @Singular + List columns; + + /** List of checks to run to verify data quality of the dataset. */ + @Singular + List checks; + + /** Any extra properties provided in the specification (but unknown to this version of the template). */ + @Singular + @JsonAnySetter + Map extraProperties; + + @JsonAnyGetter + public Map getExtraProperties() { + return extraProperties; + } + + @Getter + @Jacksonized + @SuperBuilder(toBuilder = true) + @EqualsAndHashCode(callSuper = false) + public static final class Owners { + /** Individual users who own the dataset. */ + @Singular + List users; + + /** Groups that own the dataset. */ + @Singular + List groups; + } + + @Getter + @Jacksonized + @SuperBuilder(toBuilder = true) + @EqualsAndHashCode(callSuper = false) + public static final class Certification { + /** State of the certification. */ + CertificateStatus status; + + /** Message to accompany the certification. */ + String message; + } + + @Getter + @Jacksonized + @SuperBuilder(toBuilder = true) + @EqualsAndHashCode(callSuper = false) + public static final class Announcement { + /** Type of announcement. */ + AtlanAnnouncementType type; + + /** Title to use for the announcement. */ + String title; + + /** Message to accompany the announcement. */ + String description; + } + + @Getter + @Jacksonized + @SuperBuilder(toBuilder = true) + @EqualsAndHashCode(callSuper = false) + public static final class DCTag { + /** Human-readable name of the Atlan tag. */ + String name; + + /** Whether to propagate the tag at all (true) or not (false). */ + Boolean propagate; + + /** Whether to propagate the tag through lineage (true) or not (false). */ + @JsonProperty("restrict_propagation_through_lineage") + Boolean propagateThroughLineage; + + /** Whether to propagate the tag through asset's containment hierarchy (true) or not (false). */ + @JsonProperty("restrict_propagation_through_hierarchy") + Boolean propagateThroughHierarchy; + } + + @Getter + @Jacksonized + @SuperBuilder(toBuilder = true) + @EqualsAndHashCode(callSuper = false) + public static final class DCColumn { + /** Name of the column as it is defined in the source system (often technical). */ + String name; + + /** Alias for the column, to make its name more readable. */ + @JsonProperty("business_name") + String displayName; + + /** Description of this column, for documentation purposes. */ + String description; + + /** When true, this column is the primary key for the table. */ + @JsonProperty("is_primary") + Boolean isPrimary; + + /** Physical data type of values in this column (e.g. {@code varchar(20)}). */ + @JsonProperty("data_type") + String dataType; + + /** Logical data type of values in this column (e.g. {@code string}). */ + @JsonProperty("logical_type") + String logicalType; + + /** Format of data to consider invalid. */ + @JsonProperty("invalid_format") + String invalidFormat; + + /** Format of data to consider valid. */ + @JsonProperty("valid_format") + String validFormat; + + /** Regular expression to match invalid values. */ + @JsonProperty("invalid_regex") + String invalidRegex; + + /** Regular expression to match valid values. */ + @JsonProperty("valid_regex") + String validRegex; + + /** Regular expression to match missing values. */ + @JsonProperty("missing_regex") + String missingRegex; + + /** Enumeration of values that should be considered invalid. */ + @Singular + @JsonProperty("invalid_values") + List invalidValues; + + /** Enumeration of values that should be considered valid. */ + @Singular + @JsonProperty("valid_values") + List validValues; + + /** Enumeration of values that should be considered missing. */ + @Singular + @JsonProperty("missing_values") + List missingValues; + + /** When true, this column cannot be empty (without values). */ + @JsonProperty("not_null") + Boolean notNull; + + /** Fixed length for a string to be considered valid. */ + @JsonProperty("valid_length") + Long validLength; + + /** Maximum length for a string to be considered valid. */ + @JsonProperty("valid_max_length") + Long validMaxLength; + + /** Minimum numeric value considered valid. */ + @JsonProperty("valid_min") + Double validMin; + + /** Maximum numeric value considered valid. */ + @JsonProperty("valid_max") + Double validMax; + + /** Minimum length for a string to be considered valid. */ + @JsonProperty("valid_min_length") + Long validMinLength; + + /** When true, this column must have unique values. */ + Boolean unique; + } + + /** + * Parse a DataContractSpec object from the provided string form of the specification. + * Note: all comments and empty fields will be lost during conversion to an object. + * + * @param spec YAML string form of the data contract specification + * @return object representing the data contract + * @throws IOException on any errors parsing the provided string as a data contract specification + */ + public static DataContractSpec fromString(String spec) throws IOException { + return Serde.yamlMapper.readValue(spec, DataContractSpec.class); + } + + /** + * Translate this DataContractSpec object into a YAML string representation. + * Note: will not contain any comments or empty fields. + * + * @return YAML string representation of the data contract specification + */ + @Override + public String toString() { + try { + return Serde.yamlMapper.writeValueAsString(this); + } catch (JsonProcessingException e) { + log.error("Error translating DataContractSpec into string.", e); + } + return ""; + } +} diff --git a/sdk/src/main/java/com/atlan/model/contracts/DataContractStatus.java b/sdk/src/main/java/com/atlan/model/contracts/DataContractStatus.java new file mode 100644 index 0000000000..150a48fae0 --- /dev/null +++ b/sdk/src/main/java/com/atlan/model/contracts/DataContractStatus.java @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: Apache-2.0 + Copyright 2022 Atlan Pte. Ltd. */ +package com.atlan.model.contracts; + +import com.atlan.model.enums.AtlanEnum; +import com.fasterxml.jackson.annotation.JsonValue; +import lombok.Getter; + +public enum DataContractStatus implements AtlanEnum { + DRAFT("draft"), + VERIFIED("verified"); + + @JsonValue + @Getter(onMethod_ = {@Override}) + private final String value; + + DataContractStatus(String value) { + this.value = value; + } + + public static DataContractStatus fromValue(String value) { + for (DataContractStatus b : DataContractStatus.values()) { + if (b.value.equals(value)) { + return b; + } + } + return null; + } +} diff --git a/sdk/src/main/java/com/atlan/serde/ReadableCustomMetadataDeserializer.java b/sdk/src/main/java/com/atlan/serde/ReadableCustomMetadataDeserializer.java new file mode 100644 index 0000000000..a1a4071b3b --- /dev/null +++ b/sdk/src/main/java/com/atlan/serde/ReadableCustomMetadataDeserializer.java @@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: Apache-2.0 + Copyright 2022 Atlan Pte. Ltd. */ +package com.atlan.serde; + +import com.atlan.cache.CustomMetadataCache; +import com.atlan.exception.AtlanException; +import com.atlan.exception.ErrorCode; +import com.atlan.exception.LogicException; +import com.atlan.model.core.CustomMetadataAttributes; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import com.fasterxml.jackson.databind.jsontype.TypeDeserializer; +import com.fasterxml.jackson.databind.node.ArrayNode; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import lombok.extern.slf4j.Slf4j; + +/** + * Custom deserialization of a map from custom metadata set name to its values. + * In particular, this retains human-readable names throughout. + */ +@Slf4j +public class ReadableCustomMetadataDeserializer extends StdDeserializer> { + private static final long serialVersionUID = 2L; + + public ReadableCustomMetadataDeserializer() { + super(CustomMetadataMapDeserializer.class); + } + + /** + * {@inheritDoc} + */ + @Override + public Object deserializeWithType( + JsonParser parser, DeserializationContext context, TypeDeserializer typeDeserializer) throws IOException { + return deserialize(parser, context); + } + + /** + * {@inheritDoc} + */ + @Override + public Map deserialize(JsonParser parser, DeserializationContext context) + throws IOException { + return deserialize(parser.getCodec().readTree(parser)); + } + + /** + * Actually do the work of deserializing a custom metadata map. + * + * @param root of the parsed JSON tree + * @return the deserialized custom metadata details + * @throws IOException on any issues parsing the JSON + */ + Map deserialize(JsonNode root) throws IOException { + Map map = new HashMap<>(); + + // Iterate through all the top-level field names of the object... + for (Iterator it = root.fieldNames(); it.hasNext(); ) { + String cmName = it.next(); + CustomMetadataAttributes.CustomMetadataAttributesBuilder builder = CustomMetadataAttributes.builder(); + JsonNode attributeNames = root.get(cmName); + try { + for (Iterator attrs = attributeNames.fieldNames(); attrs.hasNext(); ) { + String attrName = attrs.next(); + JsonNode jsonValue = attributeNames.get(attrName); + if (jsonValue.isArray()) { + Set values = new HashSet<>(); + ArrayNode array = (ArrayNode) jsonValue; + for (JsonNode element : array) { + Object primitive = CustomMetadataCache.deserializePrimitive(element); + values.add(primitive); + } + if (!values.isEmpty()) { + builder.attribute(attrName, values); + } + } else if (jsonValue.isValueNode()) { + Object primitive = CustomMetadataCache.deserializePrimitive(jsonValue); + builder.attribute(attrName, primitive); + } else { + throw new LogicException(ErrorCode.UNABLE_TO_DESERIALIZE, jsonValue.toString()); + } + } + } catch (AtlanException e) { + log.error("Unable to translate one of the provided custom metadata attributes of: {}.", cmName, e); + } + map.put(cmName, builder.build()); + } + return map.isEmpty() ? Collections.emptyMap() : Collections.unmodifiableMap(map); + } +} diff --git a/sdk/src/main/java/com/atlan/serde/ReadableCustomMetadataSerializer.java b/sdk/src/main/java/com/atlan/serde/ReadableCustomMetadataSerializer.java new file mode 100644 index 0000000000..161725a6d7 --- /dev/null +++ b/sdk/src/main/java/com/atlan/serde/ReadableCustomMetadataSerializer.java @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: Apache-2.0 + Copyright 2022 Atlan Pte. Ltd. */ +package com.atlan.serde; + +import com.atlan.model.core.CustomMetadataAttributes; +import com.atlan.util.JacksonUtils; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.jsontype.TypeSerializer; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import com.fasterxml.jackson.databind.type.TypeFactory; +import java.io.IOException; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; + +/** + * Custom serialization of a map of custom metadata. + * In particular, this retains human-readable names throughout. + */ +@Slf4j +public class ReadableCustomMetadataSerializer extends StdSerializer> { + private static final long serialVersionUID = 2L; + + // TODO: Pass the Map to the other constructor, rather than null + public ReadableCustomMetadataSerializer() { + super(TypeFactory.defaultInstance().constructType(Map.class)); + } + + public ReadableCustomMetadataSerializer(Class> t) { + super(t); + } + + /** + * {@inheritDoc} + */ + @Override + public void serializeWithType( + Map value, + JsonGenerator gen, + SerializerProvider serializers, + TypeSerializer typeSer) + throws IOException { + serialize(value, gen, serializers); + } + + /** + * {@inheritDoc} + */ + @Override + public void serialize(Map cmMap, JsonGenerator gen, SerializerProvider sp) + throws IOException, JsonProcessingException { + gen.writeStartObject(); + if (cmMap != null) { + for (Map.Entry entry : cmMap.entrySet()) { + String cmName = entry.getKey(); + if (cmName != null) { + CustomMetadataAttributes cma = entry.getValue(); + if (cma != null && !cma.isEmpty()) { + JacksonUtils.serializeObject(gen, cmName, cma.getAttributes()); + } + } + } + } + gen.writeEndObject(); + } +} diff --git a/sdk/src/main/java/com/atlan/serde/Serde.java b/sdk/src/main/java/com/atlan/serde/Serde.java index adffa2f4a4..88c6466e0c 100644 --- a/sdk/src/main/java/com/atlan/serde/Serde.java +++ b/sdk/src/main/java/com/atlan/serde/Serde.java @@ -24,6 +24,8 @@ import com.fasterxml.jackson.databind.deser.BeanDeserializerModifier; import com.fasterxml.jackson.databind.module.SimpleModule; import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; import io.github.classgraph.ClassGraph; import io.github.classgraph.ClassInfo; import io.github.classgraph.ScanResult; @@ -52,6 +54,9 @@ public class Serde { /** JSONP mapper through which to do Jackson-based (de-)serialization of Elastic objects. */ static final JsonpMapper jsonpMapper = new JacksonJsonpMapper(); + /** Singular ObjectMapper through which to (de-)serialize raw POJOs and YAML. */ + public static final ObjectMapper yamlMapper = createMapperYAML(); + private static final Map> deserializerCache = new ConcurrentHashMap<>(); private static final Map> assetClasses; private static final Map> builderClasses; @@ -199,6 +204,23 @@ public static ObjectMapper createMapper(AtlanClient client) { return om; } + /** + * Set up the serialization and deserialization of tenant-agnostic YAML. + * @return an ObjectMapper for tenant-agnostic YAML transformations + */ + public static ObjectMapper createMapperYAML() { + // Set default options, using client-aware deserialization + ObjectMapper om = new ObjectMapper(new YAMLFactory().disable(YAMLGenerator.Feature.USE_NATIVE_TYPE_ID)) + .setSerializationInclusion(JsonInclude.Include.NON_EMPTY) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + .configure(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL, true); + // Set standard (non-tenant-specific) modules + for (Module m : SIMPLE_MODULES) { + om.registerModule(m); + } + return om; + } + /** * Deserialize a value direct to an object. *