diff --git a/build.gradle b/build.gradle index 99085787a..5bb23bb78 100644 --- a/build.gradle +++ b/build.gradle @@ -163,6 +163,7 @@ dependencies { testImplementation 'org.junit.jupiter:junit-jupiter-api:5.7.2' testImplementation 'org.junit.jupiter:junit-jupiter-params:5.7.2' testImplementation 'org.junit.jupiter:junit-jupiter-engine:5.7.2' + testImplementation "org.opensearch:opensearch-core:${opensearch_version}" testRuntimeOnly 'org.junit.vintage:junit-vintage-engine:5.7.2' testCompileOnly 'junit:junit:4.13.2' } diff --git a/src/main/java/org/opensearch/ad/AnomalyDetectorPlugin.java b/src/main/java/org/opensearch/ad/AnomalyDetectorPlugin.java index 90a68339c..d91f86466 100644 --- a/src/main/java/org/opensearch/ad/AnomalyDetectorPlugin.java +++ b/src/main/java/org/opensearch/ad/AnomalyDetectorPlugin.java @@ -43,10 +43,6 @@ import org.opensearch.ad.cluster.ClusterManagerEventListener; import org.opensearch.ad.cluster.HashRing; import org.opensearch.ad.constant.ADCommonName; -import org.opensearch.ad.dataprocessor.IntegerSensitiveSingleFeatureLinearUniformInterpolator; -import org.opensearch.ad.dataprocessor.Interpolator; -import org.opensearch.ad.dataprocessor.LinearUniformInterpolator; -import org.opensearch.ad.dataprocessor.SingleFeatureLinearUniformInterpolator; import org.opensearch.ad.feature.FeatureManager; import org.opensearch.ad.feature.SearchFeatureDao; import org.opensearch.ad.indices.AnomalyDetectionIndices; @@ -195,6 +191,8 @@ import org.opensearch.threadpool.ScalingExecutorBuilder; import org.opensearch.threadpool.ThreadPool; import org.opensearch.timeseries.constant.CommonName; +import org.opensearch.timeseries.dataprocessor.Imputer; +import org.opensearch.timeseries.dataprocessor.LinearUniformImputer; import org.opensearch.timeseries.stats.StatNames; import org.opensearch.watcher.ResourceWatcherService; @@ -344,9 +342,7 @@ public Collection createComponents( ); this.clusterService = clusterService; - SingleFeatureLinearUniformInterpolator singleFeatureLinearUniformInterpolator = - new IntegerSensitiveSingleFeatureLinearUniformInterpolator(); - Interpolator interpolator = new LinearUniformInterpolator(singleFeatureLinearUniformInterpolator); + Imputer imputer = new LinearUniformImputer(true); stateManager = new NodeStateManager( client, xContentRegistry, @@ -360,7 +356,7 @@ public Collection createComponents( SearchFeatureDao searchFeatureDao = new SearchFeatureDao( client, xContentRegistry, - interpolator, + imputer, securityClientUtil, settings, clusterService, @@ -388,7 +384,7 @@ public Collection createComponents( FeatureManager featureManager = new FeatureManager( searchFeatureDao, - interpolator, + imputer, getClock(), AnomalyDetectorSettings.MAX_TRAIN_SAMPLE, AnomalyDetectorSettings.MAX_SAMPLE_STRIDE, @@ -534,7 +530,7 @@ public PooledObject wrap(LinkedBuffer obj) { AnomalyDetectorSettings.NUM_MIN_SAMPLES, AnomalyDetectorSettings.MAX_SAMPLE_STRIDE, AnomalyDetectorSettings.MAX_TRAIN_SAMPLE, - interpolator, + imputer, searchFeatureDao, AnomalyDetectorSettings.THRESHOLD_MIN_PVALUE, featureManager, @@ -779,8 +775,7 @@ public PooledObject wrap(LinkedBuffer obj) { anomalyDetectionIndices, anomalyDetectorRunner, searchFeatureDao, - singleFeatureLinearUniformInterpolator, - interpolator, + imputer, gson, jvmService, hashRing, diff --git a/src/main/java/org/opensearch/ad/caching/PriorityCache.java b/src/main/java/org/opensearch/ad/caching/PriorityCache.java index 31c183ca8..1130a57c5 100644 --- a/src/main/java/org/opensearch/ad/caching/PriorityCache.java +++ b/src/main/java/org/opensearch/ad/caching/PriorityCache.java @@ -57,10 +57,10 @@ import org.opensearch.ad.settings.EnabledSetting; import org.opensearch.ad.util.DateUtils; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.Strings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.common.Strings; import org.opensearch.threadpool.ThreadPool; import org.opensearch.timeseries.constant.CommonMessages; diff --git a/src/main/java/org/opensearch/ad/caching/PriorityTracker.java b/src/main/java/org/opensearch/ad/caching/PriorityTracker.java index 05304912f..439d67679 100644 --- a/src/main/java/org/opensearch/ad/caching/PriorityTracker.java +++ b/src/main/java/org/opensearch/ad/caching/PriorityTracker.java @@ -27,7 +27,7 @@ import org.apache.commons.lang.builder.ToStringBuilder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.ad.annotation.Generated; +import org.opensearch.timeseries.annotation.Generated; /** * A priority tracker for entities. Read docs/entity-priority.pdf for details. diff --git a/src/main/java/org/opensearch/ad/common/exception/NotSerializedADExceptionName.java b/src/main/java/org/opensearch/ad/common/exception/NotSerializedADExceptionName.java index 0631d29d3..31668cb5e 100644 --- a/src/main/java/org/opensearch/ad/common/exception/NotSerializedADExceptionName.java +++ b/src/main/java/org/opensearch/ad/common/exception/NotSerializedADExceptionName.java @@ -16,7 +16,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.opensearch.OpenSearchException; +import org.opensearch.BaseExceptionsHelper; import org.opensearch.common.io.stream.NotSerializableExceptionWrapper; /** @@ -29,16 +29,16 @@ */ public enum NotSerializedADExceptionName { - RESOURCE_NOT_FOUND_EXCEPTION_NAME_UNDERSCORE(OpenSearchException.getExceptionName(new ResourceNotFoundException("", ""))), - LIMIT_EXCEEDED_EXCEPTION_NAME_UNDERSCORE(OpenSearchException.getExceptionName(new LimitExceededException("", "", false))), - END_RUN_EXCEPTION_NAME_UNDERSCORE(OpenSearchException.getExceptionName(new EndRunException("", "", false))), - ANOMALY_DETECTION_EXCEPTION_NAME_UNDERSCORE(OpenSearchException.getExceptionName(new AnomalyDetectionException("", ""))), - INTERNAL_FAILURE_NAME_UNDERSCORE(OpenSearchException.getExceptionName(new InternalFailure("", ""))), - CLIENT_EXCEPTION_NAME_UNDERSCORE(OpenSearchException.getExceptionName(new ClientException("", ""))), - CANCELLATION_EXCEPTION_NAME_UNDERSCORE(OpenSearchException.getExceptionName(new ADTaskCancelledException("", ""))), - DUPLICATE_TASK_EXCEPTION_NAME_UNDERSCORE(OpenSearchException.getExceptionName(new DuplicateTaskException(""))), - AD_VERSION_EXCEPTION_NAME_UNDERSCORE(OpenSearchException.getExceptionName(new ADVersionException(""))), - AD_VALIDATION_EXCEPTION_NAME_UNDERSCORE(OpenSearchException.getExceptionName(new ADValidationException("", null, null))); + RESOURCE_NOT_FOUND_EXCEPTION_NAME_UNDERSCORE(BaseExceptionsHelper.getExceptionName(new ResourceNotFoundException("", ""))), + LIMIT_EXCEEDED_EXCEPTION_NAME_UNDERSCORE(BaseExceptionsHelper.getExceptionName(new LimitExceededException("", "", false))), + END_RUN_EXCEPTION_NAME_UNDERSCORE(BaseExceptionsHelper.getExceptionName(new EndRunException("", "", false))), + ANOMALY_DETECTION_EXCEPTION_NAME_UNDERSCORE(BaseExceptionsHelper.getExceptionName(new AnomalyDetectionException("", ""))), + INTERNAL_FAILURE_NAME_UNDERSCORE(BaseExceptionsHelper.getExceptionName(new InternalFailure("", ""))), + CLIENT_EXCEPTION_NAME_UNDERSCORE(BaseExceptionsHelper.getExceptionName(new ClientException("", ""))), + CANCELLATION_EXCEPTION_NAME_UNDERSCORE(BaseExceptionsHelper.getExceptionName(new ADTaskCancelledException("", ""))), + DUPLICATE_TASK_EXCEPTION_NAME_UNDERSCORE(BaseExceptionsHelper.getExceptionName(new DuplicateTaskException(""))), + AD_VERSION_EXCEPTION_NAME_UNDERSCORE(BaseExceptionsHelper.getExceptionName(new ADVersionException(""))), + AD_VALIDATION_EXCEPTION_NAME_UNDERSCORE(BaseExceptionsHelper.getExceptionName(new ADValidationException("", null, null))); private static final Logger LOG = LogManager.getLogger(NotSerializedADExceptionName.class); private final String name; diff --git a/src/main/java/org/opensearch/ad/dataprocessor/IntegerSensitiveSingleFeatureLinearUniformInterpolator.java b/src/main/java/org/opensearch/ad/dataprocessor/IntegerSensitiveSingleFeatureLinearUniformInterpolator.java deleted file mode 100644 index cc187d7f8..000000000 --- a/src/main/java/org/opensearch/ad/dataprocessor/IntegerSensitiveSingleFeatureLinearUniformInterpolator.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.ad.dataprocessor; - -import java.util.Arrays; - -import com.google.common.math.DoubleMath; - -/** - * Interpolator sensitive to integral values. - */ -public class IntegerSensitiveSingleFeatureLinearUniformInterpolator extends SingleFeatureLinearUniformInterpolator { - - /** - * Interpolates integral/floating-point results. - * - * If all samples are integral, the results are integral. - * Else, the results are floating points. - * - * @param samples integral/floating-point samples - * @param numInterpolants the number of interpolants - * @return {code numInterpolants} interpolated results - */ - public double[] interpolate(double[] samples, int numInterpolants) { - double[] interpolants = super.interpolate(samples, numInterpolants); - if (Arrays.stream(samples).allMatch(DoubleMath::isMathematicalInteger)) { - interpolants = Arrays.stream(interpolants).map(Math::rint).toArray(); - } - return interpolants; - } -} diff --git a/src/main/java/org/opensearch/ad/dataprocessor/Interpolator.java b/src/main/java/org/opensearch/ad/dataprocessor/Interpolator.java deleted file mode 100644 index 752498e88..000000000 --- a/src/main/java/org/opensearch/ad/dataprocessor/Interpolator.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.ad.dataprocessor; - -/* - * An object for interpolating feature vectors. - * - * In certain situations, due to time and compute cost, we are only allowed to - * query a sparse sample of data points / feature vectors from a cluster. - * However, we need a large sample of feature vectors in order to train our - * anomaly detection algorithms. An Interpolator approximates the data points - * between a given, ordered list of samples. - */ -public interface Interpolator { - - /* - * Interpolates the given sample feature vectors. - * - * Computes a list `numInterpolants` feature vectors using the ordered list - * of `numSamples` input sample vectors where each sample vector has size - * `numFeatures`. - * - * @param samples A `numFeatures x numSamples` list of feature vectors. - * @param numInterpolants The desired number of interpolating vectors. - * @return A `numFeatures x numInterpolants` list of feature vectors. - */ - double[][] interpolate(double[][] samples, int numInterpolants); -} diff --git a/src/main/java/org/opensearch/ad/dataprocessor/LinearUniformInterpolator.java b/src/main/java/org/opensearch/ad/dataprocessor/LinearUniformInterpolator.java deleted file mode 100644 index b62aeda9b..000000000 --- a/src/main/java/org/opensearch/ad/dataprocessor/LinearUniformInterpolator.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.ad.dataprocessor; - -/* - * A piecewise linear interpolator with uniformly spaced points. - * - * The LinearUniformInterpolator constructs a piecewise linear interpolation on - * the input list of sample feature vectors. That is, between every consecutive - * pair of points we construct a linear interpolation. The linear interpolation - * is computed on a per-feature basis. - * - * This class uses the helper class SingleFeatureLinearUniformInterpolator to - * compute per-feature interpolants. - * - * @see SingleFeatureLinearUniformInterpolator - */ -public class LinearUniformInterpolator implements Interpolator { - - private SingleFeatureLinearUniformInterpolator singleFeatureLinearUniformInterpolator; - - public LinearUniformInterpolator(SingleFeatureLinearUniformInterpolator singleFeatureLinearUniformInterpolator) { - this.singleFeatureLinearUniformInterpolator = singleFeatureLinearUniformInterpolator; - } - - /* - * Piecewise linearly interpolates the given sample feature vectors. - * - * Computes a list `numInterpolants` feature vectors using the ordered list - * of `numSamples` input sample vectors where each sample vector has size - * `numFeatures`. The feature vectors are computing using a piecewise linear - * interpolation. - * - * @param samples A `numFeatures x numSamples` list of feature vectors. - * @param numInterpolants The desired number of interpolating feature vectors. - * @return A `numFeatures x numInterpolants` list of feature vectors. - * @see SingleFeatureLinearUniformInterpolator - */ - public double[][] interpolate(double[][] samples, int numInterpolants) { - int numFeatures = samples.length; - double[][] interpolants = new double[numFeatures][numInterpolants]; - - for (int featureIndex = 0; featureIndex < numFeatures; featureIndex++) { - interpolants[featureIndex] = this.singleFeatureLinearUniformInterpolator.interpolate(samples[featureIndex], numInterpolants); - } - return interpolants; - } -} diff --git a/src/main/java/org/opensearch/ad/dataprocessor/SingleFeatureLinearUniformInterpolator.java b/src/main/java/org/opensearch/ad/dataprocessor/SingleFeatureLinearUniformInterpolator.java deleted file mode 100644 index 6349f29d3..000000000 --- a/src/main/java/org/opensearch/ad/dataprocessor/SingleFeatureLinearUniformInterpolator.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.ad.dataprocessor; - -import java.util.Arrays; - -/* - * A piecewise linear interpolator in a single feature dimension. - * - * A utility class for LinearUniformInterpolator. Constructs uniformly spaced - * piecewise linear interpolations within a single feature dimension. - * - * @see LinearUniformInterpolator - */ -public class SingleFeatureLinearUniformInterpolator { - - /* - * Piecewise linearly interpolates the given sample of one-dimensional - * features. - * - * Computes a list `numInterpolants` features using the ordered list of - * `numSamples` input one-dimensional samples. The interpolant features are - * computing using a piecewise linear interpolation. - * - * @param samples A `numSamples` sized list of sample features. - * @param numInterpolants The desired number of interpolating features. - * @return A `numInterpolants` sized array of interpolant features. - * @see LinearUniformInterpolator - */ - public double[] interpolate(double[] samples, int numInterpolants) { - int numSamples = samples.length; - double[] interpolants = new double[numInterpolants]; - - if (numSamples == 0) { - interpolants = new double[0]; - } else if (numSamples == 1) { - Arrays.fill(interpolants, samples[0]); - } else { - /* assume the piecewise linear interpolation between the samples is a - parameterized curve f(t) for t in [0, 1]. Each pair of samples - determines a interval [t_i, t_(i+1)]. For each interpolant we determine - which interval it lies inside and then scale the value of t, - accordingly to compute the interpolant value. - - for numerical stability reasons we omit processing the final - interpolant in this loop since this last interpolant is always equal - to the last sample. - */ - for (int interpolantIndex = 0; interpolantIndex < (numInterpolants - 1); interpolantIndex++) { - double tGlobal = ((double) interpolantIndex) / (numInterpolants - 1.0); - double tInterval = tGlobal * (numSamples - 1.0); - int intervalIndex = (int) Math.floor(tInterval); - tInterval -= intervalIndex; - - double leftSample = samples[intervalIndex]; - double rightSample = samples[intervalIndex + 1]; - double interpolant = (1.0 - tInterval) * leftSample + tInterval * rightSample; - interpolants[interpolantIndex] = interpolant; - } - - // the final interpolant is always the final sample - interpolants[numInterpolants - 1] = samples[numSamples - 1]; - } - return interpolants; - } -} diff --git a/src/main/java/org/opensearch/ad/feature/FeatureManager.java b/src/main/java/org/opensearch/ad/feature/FeatureManager.java index cb86d2b86..07d5aef55 100644 --- a/src/main/java/org/opensearch/ad/feature/FeatureManager.java +++ b/src/main/java/org/opensearch/ad/feature/FeatureManager.java @@ -42,11 +42,11 @@ import org.opensearch.action.support.ThreadedActionListener; import org.opensearch.ad.CleanState; import org.opensearch.ad.common.exception.EndRunException; -import org.opensearch.ad.dataprocessor.Interpolator; import org.opensearch.ad.model.AnomalyDetector; import org.opensearch.ad.model.Entity; import org.opensearch.threadpool.ThreadPool; import org.opensearch.timeseries.constant.CommonMessages; +import org.opensearch.timeseries.dataprocessor.Imputer; /** * A facade managing feature data operations and buffers. @@ -59,7 +59,7 @@ public class FeatureManager implements CleanState { private final Map>>> detectorIdsToTimeShingles; private final SearchFeatureDao searchFeatureDao; - private final Interpolator interpolator; + private final Imputer imputer; private final Clock clock; private final int maxTrainSamples; @@ -78,7 +78,7 @@ public class FeatureManager implements CleanState { * Constructor with dependencies and configuration. * * @param searchFeatureDao DAO of features from search - * @param interpolator interpolator of samples + * @param imputer imputer of samples * @param clock clock for system time * @param maxTrainSamples max number of samples from search * @param maxSampleStride max stride between uninterpolated train samples @@ -94,7 +94,7 @@ public class FeatureManager implements CleanState { */ public FeatureManager( SearchFeatureDao searchFeatureDao, - Interpolator interpolator, + Imputer imputer, Clock clock, int maxTrainSamples, int maxSampleStride, @@ -109,7 +109,7 @@ public FeatureManager( String adThreadPoolName ) { this.searchFeatureDao = searchFeatureDao; - this.interpolator = interpolator; + this.imputer = imputer; this.clock = clock; this.maxTrainSamples = maxTrainSamples; this.maxSampleStride = maxSampleStride; @@ -592,8 +592,8 @@ void getSamplesForRanges( private List> getPreviewRanges(List> ranges, int stride, int shingleSize) { double[] rangeStarts = ranges.stream().mapToDouble(Entry::getKey).toArray(); double[] rangeEnds = ranges.stream().mapToDouble(Entry::getValue).toArray(); - double[] previewRangeStarts = interpolator.interpolate(new double[][] { rangeStarts }, stride * (ranges.size() - 1) + 1)[0]; - double[] previewRangeEnds = interpolator.interpolate(new double[][] { rangeEnds }, stride * (ranges.size() - 1) + 1)[0]; + double[] previewRangeStarts = imputer.impute(new double[][] { rangeStarts }, stride * (ranges.size() - 1) + 1)[0]; + double[] previewRangeEnds = imputer.impute(new double[][] { rangeEnds }, stride * (ranges.size() - 1) + 1)[0]; List> previewRanges = IntStream .range(shingleSize - 1, previewRangeStarts.length) .mapToObj(i -> new SimpleImmutableEntry<>((long) previewRangeStarts[i], (long) previewRangeEnds[i])) @@ -614,7 +614,7 @@ private Entry getPreviewFeatures(double[][] samples, int Entry unprocessedAndProcessed = Optional .of(samples) .map(m -> transpose(m)) - .map(m -> interpolator.interpolate(m, stride * (samples.length - 1) + 1)) + .map(m -> imputer.impute(m, stride * (samples.length - 1) + 1)) .map(m -> transpose(m)) .map(m -> new SimpleImmutableEntry<>(copyOfRange(m, shingleSize - 1, m.length), batchShingle(m, shingleSize))) .get(); diff --git a/src/main/java/org/opensearch/ad/feature/SearchFeatureDao.java b/src/main/java/org/opensearch/ad/feature/SearchFeatureDao.java index cdbab8c3b..8c73f3538 100644 --- a/src/main/java/org/opensearch/ad/feature/SearchFeatureDao.java +++ b/src/main/java/org/opensearch/ad/feature/SearchFeatureDao.java @@ -42,7 +42,6 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.ad.common.exception.AnomalyDetectionException; import org.opensearch.ad.constant.ADCommonName; -import org.opensearch.ad.dataprocessor.Interpolator; import org.opensearch.ad.model.AnomalyDetector; import org.opensearch.ad.model.Entity; import org.opensearch.ad.model.IntervalTimeConfiguration; @@ -72,6 +71,7 @@ import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.sort.FieldSortBuilder; import org.opensearch.search.sort.SortOrder; +import org.opensearch.timeseries.dataprocessor.Imputer; /** * DAO for features from search. @@ -86,7 +86,7 @@ public class SearchFeatureDao extends AbstractRetriever { // Dependencies private final Client client; private final NamedXContentRegistry xContent; - private final Interpolator interpolator; + private final Imputer imputer; private final SecurityClientUtil clientUtil; private volatile int maxEntitiesForPreview; private volatile int pageSize; @@ -98,7 +98,7 @@ public class SearchFeatureDao extends AbstractRetriever { public SearchFeatureDao( Client client, NamedXContentRegistry xContent, - Interpolator interpolator, + Imputer imputer, SecurityClientUtil clientUtil, Settings settings, ClusterService clusterService, @@ -110,7 +110,7 @@ public SearchFeatureDao( ) { this.client = client; this.xContent = xContent; - this.interpolator = interpolator; + this.imputer = imputer; this.clientUtil = clientUtil; this.maxEntitiesForPreview = maxEntitiesForPreview; @@ -130,7 +130,7 @@ public SearchFeatureDao( * * @param client ES client for queries * @param xContent ES XContentRegistry - * @param interpolator interpolator for missing values + * @param imputer imputer for missing values * @param clientUtil utility for ES client * @param settings ES settings * @param clusterService ES ClusterService @@ -140,7 +140,7 @@ public SearchFeatureDao( public SearchFeatureDao( Client client, NamedXContentRegistry xContent, - Interpolator interpolator, + Imputer imputer, SecurityClientUtil clientUtil, Settings settings, ClusterService clusterService, @@ -149,7 +149,7 @@ public SearchFeatureDao( this( client, xContent, - interpolator, + imputer, clientUtil, settings, clusterService, @@ -825,7 +825,7 @@ private Optional toMatrix(ArrayDeque sampledFeatures) { } private double[] getInterpolants(double[] previous, double[] next) { - return transpose(interpolator.interpolate(transpose(new double[][] { previous, next }), 3))[1]; + return transpose(imputer.impute(transpose(new double[][] { previous, next }), 3))[1]; } private double[][] transpose(double[][] matrix) { diff --git a/src/main/java/org/opensearch/ad/ml/EntityColdStarter.java b/src/main/java/org/opensearch/ad/ml/EntityColdStarter.java index 2c9095eb8..a7591752b 100644 --- a/src/main/java/org/opensearch/ad/ml/EntityColdStarter.java +++ b/src/main/java/org/opensearch/ad/ml/EntityColdStarter.java @@ -44,7 +44,6 @@ import org.opensearch.ad.caching.DoorKeeper; import org.opensearch.ad.common.exception.AnomalyDetectionException; import org.opensearch.ad.common.exception.EndRunException; -import org.opensearch.ad.dataprocessor.Interpolator; import org.opensearch.ad.feature.FeatureManager; import org.opensearch.ad.feature.SearchFeatureDao; import org.opensearch.ad.model.AnomalyDetector; @@ -57,6 +56,7 @@ import org.opensearch.ad.util.ExceptionUtil; import org.opensearch.common.settings.Settings; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.timeseries.dataprocessor.Imputer; import com.amazon.randomcutforest.config.Precision; import com.amazon.randomcutforest.parkservices.ThresholdedRandomCutForest; @@ -77,7 +77,7 @@ public class EntityColdStarter implements MaintenanceState, CleanState { private final double thresholdMinPvalue; private final int defaulStrideLength; private final int defaultNumberOfSamples; - private final Interpolator interpolator; + private final Imputer imputer; private final SearchFeatureDao searchFeatureDao; private Instant lastThrottledColdStartTime; private final FeatureManager featureManager; @@ -108,7 +108,7 @@ public class EntityColdStarter implements MaintenanceState, CleanState { * results are returned. * @param defaultSampleStride default sample distances measured in detector intervals. * @param defaultTrainSamples Default train samples to collect. - * @param interpolator Used to generate data points between samples. + * @param imputer Used to generate data points between samples. * @param searchFeatureDao Used to issue ES queries. * @param thresholdMinPvalue min P-value for thresholding * @param featureManager Used to create features for models. @@ -130,7 +130,7 @@ public EntityColdStarter( int numMinSamples, int defaultSampleStride, int defaultTrainSamples, - Interpolator interpolator, + Imputer imputer, SearchFeatureDao searchFeatureDao, double thresholdMinPvalue, FeatureManager featureManager, @@ -150,7 +150,7 @@ public EntityColdStarter( this.numMinSamples = numMinSamples; this.defaulStrideLength = defaultSampleStride; this.defaultNumberOfSamples = defaultTrainSamples; - this.interpolator = interpolator; + this.imputer = imputer; this.searchFeatureDao = searchFeatureDao; this.thresholdMinPvalue = thresholdMinPvalue; this.featureManager = featureManager; @@ -173,7 +173,7 @@ public EntityColdStarter( int numMinSamples, int maxSampleStride, int maxTrainSamples, - Interpolator interpolator, + Imputer imputer, SearchFeatureDao searchFeatureDao, double thresholdMinPvalue, FeatureManager featureManager, @@ -192,7 +192,7 @@ public EntityColdStarter( numMinSamples, maxSampleStride, maxTrainSamples, - interpolator, + imputer, searchFeatureDao, thresholdMinPvalue, featureManager, @@ -501,8 +501,8 @@ private void getFeatures( int numInterpolants = (i - lastSample.getLeft()) * stride + 1; double[][] points = featureManager .transpose( - interpolator - .interpolate( + imputer + .impute( featureManager.transpose(new double[][] { lastSample.getRight(), featuresOptional.get() }), numInterpolants ) diff --git a/src/main/java/org/opensearch/ad/model/ADTask.java b/src/main/java/org/opensearch/ad/model/ADTask.java index 9a9bcdbc5..283ff80de 100644 --- a/src/main/java/org/opensearch/ad/model/ADTask.java +++ b/src/main/java/org/opensearch/ad/model/ADTask.java @@ -17,7 +17,6 @@ import java.io.IOException; import java.time.Instant; -import org.opensearch.ad.annotation.Generated; import org.opensearch.ad.util.ParseUtils; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; @@ -26,6 +25,7 @@ import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.timeseries.annotation.Generated; import com.google.common.base.Objects; diff --git a/src/main/java/org/opensearch/ad/model/ADTaskProfile.java b/src/main/java/org/opensearch/ad/model/ADTaskProfile.java index 79c6d90c0..d95c6c579 100644 --- a/src/main/java/org/opensearch/ad/model/ADTaskProfile.java +++ b/src/main/java/org/opensearch/ad/model/ADTaskProfile.java @@ -19,13 +19,13 @@ import java.util.Objects; import org.opensearch.Version; -import org.opensearch.ad.annotation.Generated; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.timeseries.annotation.Generated; /** * One anomaly detection task means one detector starts to run until stopped. diff --git a/src/main/java/org/opensearch/ad/model/AnomalyDetector.java b/src/main/java/org/opensearch/ad/model/AnomalyDetector.java index ab6f8a74c..ae948f304 100644 --- a/src/main/java/org/opensearch/ad/model/AnomalyDetector.java +++ b/src/main/java/org/opensearch/ad/model/AnomalyDetector.java @@ -30,7 +30,6 @@ import java.util.stream.Collectors; import org.apache.logging.log4j.util.Strings; -import org.opensearch.ad.annotation.Generated; import org.opensearch.ad.common.exception.ADValidationException; import org.opensearch.ad.constant.ADCommonMessages; import org.opensearch.ad.constant.CommonValue; @@ -52,6 +51,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; +import org.opensearch.timeseries.annotation.Generated; import org.opensearch.timeseries.constant.CommonMessages; import org.opensearch.timeseries.constant.CommonName; diff --git a/src/main/java/org/opensearch/ad/model/AnomalyDetectorExecutionInput.java b/src/main/java/org/opensearch/ad/model/AnomalyDetectorExecutionInput.java index 6e636d00c..b98fdbb58 100644 --- a/src/main/java/org/opensearch/ad/model/AnomalyDetectorExecutionInput.java +++ b/src/main/java/org/opensearch/ad/model/AnomalyDetectorExecutionInput.java @@ -16,12 +16,12 @@ import java.io.IOException; import java.time.Instant; -import org.opensearch.ad.annotation.Generated; import org.opensearch.ad.util.ParseUtils; -import org.opensearch.common.Strings; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.timeseries.annotation.Generated; import com.google.common.base.Objects; diff --git a/src/main/java/org/opensearch/ad/model/AnomalyResult.java b/src/main/java/org/opensearch/ad/model/AnomalyResult.java index 8eea5b394..d14371f07 100644 --- a/src/main/java/org/opensearch/ad/model/AnomalyResult.java +++ b/src/main/java/org/opensearch/ad/model/AnomalyResult.java @@ -24,7 +24,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.opensearch.ad.annotation.Generated; import org.opensearch.ad.constant.CommonValue; import org.opensearch.ad.ml.ThresholdingResult; import org.opensearch.ad.util.ParseUtils; @@ -37,6 +36,7 @@ import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.timeseries.annotation.Generated; import org.opensearch.timeseries.constant.CommonName; import com.google.common.base.Objects; diff --git a/src/main/java/org/opensearch/ad/model/AnomalyResultBucket.java b/src/main/java/org/opensearch/ad/model/AnomalyResultBucket.java index 415e21481..8f91f34b5 100644 --- a/src/main/java/org/opensearch/ad/model/AnomalyResultBucket.java +++ b/src/main/java/org/opensearch/ad/model/AnomalyResultBucket.java @@ -15,7 +15,6 @@ import java.util.Map; import org.apache.commons.lang.builder.ToStringBuilder; -import org.opensearch.ad.annotation.Generated; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.io.stream.Writeable; @@ -23,6 +22,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.aggregations.bucket.composite.CompositeAggregation.Bucket; import org.opensearch.search.aggregations.metrics.InternalMax; +import org.opensearch.timeseries.annotation.Generated; import com.google.common.base.Objects; diff --git a/src/main/java/org/opensearch/ad/model/DetectionDateRange.java b/src/main/java/org/opensearch/ad/model/DetectionDateRange.java index 717e2cd62..856253379 100644 --- a/src/main/java/org/opensearch/ad/model/DetectionDateRange.java +++ b/src/main/java/org/opensearch/ad/model/DetectionDateRange.java @@ -17,7 +17,6 @@ import java.time.Instant; import org.apache.commons.lang.builder.ToStringBuilder; -import org.opensearch.ad.annotation.Generated; import org.opensearch.ad.util.ParseUtils; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; @@ -25,6 +24,7 @@ import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.timeseries.annotation.Generated; import com.google.common.base.Objects; diff --git a/src/main/java/org/opensearch/ad/model/DetectorInternalState.java b/src/main/java/org/opensearch/ad/model/DetectorInternalState.java index 0ab35cfbd..d8c8c47b1 100644 --- a/src/main/java/org/opensearch/ad/model/DetectorInternalState.java +++ b/src/main/java/org/opensearch/ad/model/DetectorInternalState.java @@ -16,13 +16,13 @@ import java.io.IOException; import java.time.Instant; -import org.opensearch.ad.annotation.Generated; import org.opensearch.ad.util.ParseUtils; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.timeseries.annotation.Generated; import com.google.common.base.Objects; diff --git a/src/main/java/org/opensearch/ad/model/Entity.java b/src/main/java/org/opensearch/ad/model/Entity.java index d346d4034..f5e0085a3 100644 --- a/src/main/java/org/opensearch/ad/model/Entity.java +++ b/src/main/java/org/opensearch/ad/model/Entity.java @@ -25,7 +25,6 @@ import java.util.TreeMap; import org.apache.lucene.util.SetOnce; -import org.opensearch.ad.annotation.Generated; import org.opensearch.ad.settings.AnomalyDetectorSettings; import org.opensearch.common.Numbers; import org.opensearch.common.bytes.BytesReference; @@ -41,6 +40,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.xcontent.XContentParser.Token; import org.opensearch.index.query.TermQueryBuilder; +import org.opensearch.timeseries.annotation.Generated; import org.opensearch.timeseries.constant.CommonName; import com.google.common.base.Joiner; diff --git a/src/main/java/org/opensearch/ad/model/Feature.java b/src/main/java/org/opensearch/ad/model/Feature.java index 2b9248271..7ec86742a 100644 --- a/src/main/java/org/opensearch/ad/model/Feature.java +++ b/src/main/java/org/opensearch/ad/model/Feature.java @@ -16,7 +16,6 @@ import java.io.IOException; import org.apache.logging.log4j.util.Strings; -import org.opensearch.ad.annotation.Generated; import org.opensearch.ad.util.ParseUtils; import org.opensearch.common.UUIDs; import org.opensearch.common.io.stream.StreamInput; @@ -26,6 +25,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.search.aggregations.AggregationBuilder; +import org.opensearch.timeseries.annotation.Generated; import com.google.common.base.Objects; diff --git a/src/main/java/org/opensearch/ad/model/FeatureData.java b/src/main/java/org/opensearch/ad/model/FeatureData.java index 3b223bbdf..9df55d35d 100644 --- a/src/main/java/org/opensearch/ad/model/FeatureData.java +++ b/src/main/java/org/opensearch/ad/model/FeatureData.java @@ -15,11 +15,11 @@ import java.io.IOException; -import org.opensearch.ad.annotation.Generated; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.timeseries.annotation.Generated; import com.google.common.base.Objects; diff --git a/src/main/java/org/opensearch/ad/model/IntervalTimeConfiguration.java b/src/main/java/org/opensearch/ad/model/IntervalTimeConfiguration.java index e8e124587..cffb00b5f 100644 --- a/src/main/java/org/opensearch/ad/model/IntervalTimeConfiguration.java +++ b/src/main/java/org/opensearch/ad/model/IntervalTimeConfiguration.java @@ -17,11 +17,11 @@ import java.util.Locale; import java.util.Set; -import org.opensearch.ad.annotation.Generated; import org.opensearch.ad.constant.ADCommonMessages; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.timeseries.annotation.Generated; import org.opensearch.timeseries.constant.CommonMessages; import com.google.common.base.Objects; diff --git a/src/main/java/org/opensearch/ad/ratelimit/CheckPointMaintainRequestAdapter.java b/src/main/java/org/opensearch/ad/ratelimit/CheckPointMaintainRequestAdapter.java index 8723e3d35..072855069 100644 --- a/src/main/java/org/opensearch/ad/ratelimit/CheckPointMaintainRequestAdapter.java +++ b/src/main/java/org/opensearch/ad/ratelimit/CheckPointMaintainRequestAdapter.java @@ -27,10 +27,10 @@ import org.opensearch.ad.ml.ModelState; import org.opensearch.ad.util.DateUtils; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.Strings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.common.Strings; public class CheckPointMaintainRequestAdapter { private static final Logger LOG = LogManager.getLogger(CheckPointMaintainRequestAdapter.class); diff --git a/src/main/java/org/opensearch/ad/ratelimit/CheckpointWriteWorker.java b/src/main/java/org/opensearch/ad/ratelimit/CheckpointWriteWorker.java index 56caddd54..fc7fc0a07 100644 --- a/src/main/java/org/opensearch/ad/ratelimit/CheckpointWriteWorker.java +++ b/src/main/java/org/opensearch/ad/ratelimit/CheckpointWriteWorker.java @@ -39,9 +39,9 @@ import org.opensearch.ad.model.AnomalyDetector; import org.opensearch.ad.util.ExceptionUtil; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.Strings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.Strings; import org.opensearch.threadpool.ThreadPool; public class CheckpointWriteWorker extends BatchWorker { diff --git a/src/main/java/org/opensearch/ad/rest/RestGetAnomalyDetectorAction.java b/src/main/java/org/opensearch/ad/rest/RestGetAnomalyDetectorAction.java index 46cb52843..330a3a68e 100644 --- a/src/main/java/org/opensearch/ad/rest/RestGetAnomalyDetectorAction.java +++ b/src/main/java/org/opensearch/ad/rest/RestGetAnomalyDetectorAction.java @@ -30,7 +30,7 @@ import org.opensearch.ad.transport.GetAnomalyDetectorAction; import org.opensearch.ad.transport.GetAnomalyDetectorRequest; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.Strings; +import org.opensearch.core.common.Strings; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.RestActions; diff --git a/src/main/java/org/opensearch/ad/rest/RestPreviewAnomalyDetectorAction.java b/src/main/java/org/opensearch/ad/rest/RestPreviewAnomalyDetectorAction.java index e300689d4..abae558cc 100644 --- a/src/main/java/org/opensearch/ad/rest/RestPreviewAnomalyDetectorAction.java +++ b/src/main/java/org/opensearch/ad/rest/RestPreviewAnomalyDetectorAction.java @@ -28,7 +28,7 @@ import org.opensearch.ad.transport.PreviewAnomalyDetectorAction; import org.opensearch.ad.transport.PreviewAnomalyDetectorRequest; import org.opensearch.ad.util.RestHandlerUtils; -import org.opensearch.common.Strings; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; diff --git a/src/main/java/org/opensearch/ad/rest/RestStatsAnomalyDetectorAction.java b/src/main/java/org/opensearch/ad/rest/RestStatsAnomalyDetectorAction.java index 5fa9bf686..ae0886604 100644 --- a/src/main/java/org/opensearch/ad/rest/RestStatsAnomalyDetectorAction.java +++ b/src/main/java/org/opensearch/ad/rest/RestStatsAnomalyDetectorAction.java @@ -29,7 +29,7 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.Strings; +import org.opensearch.core.common.Strings; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.RestToXContentListener; diff --git a/src/main/java/org/opensearch/ad/transport/ADBatchAnomalyResultRequest.java b/src/main/java/org/opensearch/ad/transport/ADBatchAnomalyResultRequest.java index 53cdb4a76..7d2d86f90 100644 --- a/src/main/java/org/opensearch/ad/transport/ADBatchAnomalyResultRequest.java +++ b/src/main/java/org/opensearch/ad/transport/ADBatchAnomalyResultRequest.java @@ -19,9 +19,9 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.ad.model.ADTask; import org.opensearch.ad.model.AnomalyDetector; -import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.Strings; public class ADBatchAnomalyResultRequest extends ActionRequest { private ADTask adTask; diff --git a/src/main/java/org/opensearch/ad/transport/ADCancelTaskRequest.java b/src/main/java/org/opensearch/ad/transport/ADCancelTaskRequest.java index 4b0d02250..f289e6fc1 100644 --- a/src/main/java/org/opensearch/ad/transport/ADCancelTaskRequest.java +++ b/src/main/java/org/opensearch/ad/transport/ADCancelTaskRequest.java @@ -19,9 +19,9 @@ import org.opensearch.action.support.nodes.BaseNodesRequest; import org.opensearch.ad.constant.ADCommonMessages; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.Strings; public class ADCancelTaskRequest extends BaseNodesRequest { diff --git a/src/main/java/org/opensearch/ad/transport/ADTaskProfileRequest.java b/src/main/java/org/opensearch/ad/transport/ADTaskProfileRequest.java index 4fe5e5bec..c2c05ff05 100644 --- a/src/main/java/org/opensearch/ad/transport/ADTaskProfileRequest.java +++ b/src/main/java/org/opensearch/ad/transport/ADTaskProfileRequest.java @@ -19,9 +19,9 @@ import org.opensearch.action.support.nodes.BaseNodesRequest; import org.opensearch.ad.constant.ADCommonMessages; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.Strings; public class ADTaskProfileRequest extends BaseNodesRequest { diff --git a/src/main/java/org/opensearch/ad/transport/AnomalyResultRequest.java b/src/main/java/org/opensearch/ad/transport/AnomalyResultRequest.java index bb9b3f110..b9efc10b3 100644 --- a/src/main/java/org/opensearch/ad/transport/AnomalyResultRequest.java +++ b/src/main/java/org/opensearch/ad/transport/AnomalyResultRequest.java @@ -22,11 +22,11 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.ad.constant.ADCommonMessages; import org.opensearch.ad.constant.ADCommonName; -import org.opensearch.common.Strings; import org.opensearch.common.io.stream.InputStreamStreamInput; import org.opensearch.common.io.stream.OutputStreamStreamOutput; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.timeseries.constant.CommonMessages; diff --git a/src/main/java/org/opensearch/ad/transport/DeleteAnomalyDetectorRequest.java b/src/main/java/org/opensearch/ad/transport/DeleteAnomalyDetectorRequest.java index 624dbc6c0..22686616b 100644 --- a/src/main/java/org/opensearch/ad/transport/DeleteAnomalyDetectorRequest.java +++ b/src/main/java/org/opensearch/ad/transport/DeleteAnomalyDetectorRequest.java @@ -18,9 +18,9 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.ad.constant.ADCommonMessages; -import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.Strings; public class DeleteAnomalyDetectorRequest extends ActionRequest { diff --git a/src/main/java/org/opensearch/ad/transport/DeleteModelRequest.java b/src/main/java/org/opensearch/ad/transport/DeleteModelRequest.java index 080df95d6..2710fca8b 100644 --- a/src/main/java/org/opensearch/ad/transport/DeleteModelRequest.java +++ b/src/main/java/org/opensearch/ad/transport/DeleteModelRequest.java @@ -20,9 +20,9 @@ import org.opensearch.ad.constant.ADCommonMessages; import org.opensearch.ad.constant.ADCommonName; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/src/main/java/org/opensearch/ad/transport/EntityProfileRequest.java b/src/main/java/org/opensearch/ad/transport/EntityProfileRequest.java index a4950334a..76addebcf 100644 --- a/src/main/java/org/opensearch/ad/transport/EntityProfileRequest.java +++ b/src/main/java/org/opensearch/ad/transport/EntityProfileRequest.java @@ -23,9 +23,9 @@ import org.opensearch.ad.constant.ADCommonName; import org.opensearch.ad.model.Entity; import org.opensearch.ad.model.EntityProfileName; -import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/src/main/java/org/opensearch/ad/transport/EntityResultRequest.java b/src/main/java/org/opensearch/ad/transport/EntityResultRequest.java index beb9acd62..bafab7f80 100644 --- a/src/main/java/org/opensearch/ad/transport/EntityResultRequest.java +++ b/src/main/java/org/opensearch/ad/transport/EntityResultRequest.java @@ -24,9 +24,9 @@ import org.opensearch.ad.constant.ADCommonMessages; import org.opensearch.ad.constant.ADCommonName; import org.opensearch.ad.model.Entity; -import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.timeseries.constant.CommonMessages; diff --git a/src/main/java/org/opensearch/ad/transport/GetAnomalyDetectorTransportAction.java b/src/main/java/org/opensearch/ad/transport/GetAnomalyDetectorTransportAction.java index d870a02d7..3eda10a94 100644 --- a/src/main/java/org/opensearch/ad/transport/GetAnomalyDetectorTransportAction.java +++ b/src/main/java/org/opensearch/ad/transport/GetAnomalyDetectorTransportAction.java @@ -60,11 +60,11 @@ import org.opensearch.client.Client; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.CheckedConsumer; -import org.opensearch.common.Strings; import org.opensearch.common.inject.Inject; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.commons.authuser.User; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.rest.RestStatus; diff --git a/src/main/java/org/opensearch/ad/transport/RCFPollingRequest.java b/src/main/java/org/opensearch/ad/transport/RCFPollingRequest.java index 339980f40..a0e2c1e49 100644 --- a/src/main/java/org/opensearch/ad/transport/RCFPollingRequest.java +++ b/src/main/java/org/opensearch/ad/transport/RCFPollingRequest.java @@ -19,9 +19,9 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.ad.constant.ADCommonMessages; import org.opensearch.ad.constant.ADCommonName; -import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/src/main/java/org/opensearch/ad/transport/RCFResultRequest.java b/src/main/java/org/opensearch/ad/transport/RCFResultRequest.java index 114fb6fca..d06f1eae8 100644 --- a/src/main/java/org/opensearch/ad/transport/RCFResultRequest.java +++ b/src/main/java/org/opensearch/ad/transport/RCFResultRequest.java @@ -19,9 +19,9 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.ad.constant.ADCommonMessages; import org.opensearch.ad.constant.ADCommonName; -import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.timeseries.constant.CommonName; diff --git a/src/main/java/org/opensearch/ad/transport/SearchTopAnomalyResultTransportAction.java b/src/main/java/org/opensearch/ad/transport/SearchTopAnomalyResultTransportAction.java index a56458337..f9a837dbd 100644 --- a/src/main/java/org/opensearch/ad/transport/SearchTopAnomalyResultTransportAction.java +++ b/src/main/java/org/opensearch/ad/transport/SearchTopAnomalyResultTransportAction.java @@ -39,8 +39,8 @@ import org.opensearch.ad.model.AnomalyResultBucket; import org.opensearch.ad.transport.handler.ADSearchHandler; import org.opensearch.client.Client; -import org.opensearch.common.Strings; import org.opensearch.common.inject.Inject; +import org.opensearch.core.common.Strings; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.ExistsQueryBuilder; import org.opensearch.index.query.QueryBuilder; diff --git a/src/main/java/org/opensearch/ad/transport/StopDetectorRequest.java b/src/main/java/org/opensearch/ad/transport/StopDetectorRequest.java index 7bfc2c973..787bb851a 100644 --- a/src/main/java/org/opensearch/ad/transport/StopDetectorRequest.java +++ b/src/main/java/org/opensearch/ad/transport/StopDetectorRequest.java @@ -21,11 +21,11 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.ad.constant.ADCommonMessages; import org.opensearch.ad.constant.ADCommonName; -import org.opensearch.common.Strings; import org.opensearch.common.io.stream.InputStreamStreamInput; import org.opensearch.common.io.stream.OutputStreamStreamOutput; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/src/main/java/org/opensearch/ad/transport/ThresholdResultRequest.java b/src/main/java/org/opensearch/ad/transport/ThresholdResultRequest.java index a71e9116f..72751bf9a 100644 --- a/src/main/java/org/opensearch/ad/transport/ThresholdResultRequest.java +++ b/src/main/java/org/opensearch/ad/transport/ThresholdResultRequest.java @@ -19,9 +19,9 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.ad.constant.ADCommonMessages; import org.opensearch.ad.constant.ADCommonName; -import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.timeseries.constant.CommonName; diff --git a/src/main/java/org/opensearch/ad/util/ADSafeSecurityInjector.java b/src/main/java/org/opensearch/ad/util/ADSafeSecurityInjector.java index 8fcd11ce0..b38455d36 100644 --- a/src/main/java/org/opensearch/ad/util/ADSafeSecurityInjector.java +++ b/src/main/java/org/opensearch/ad/util/ADSafeSecurityInjector.java @@ -19,10 +19,10 @@ import org.opensearch.ad.NodeStateManager; import org.opensearch.ad.common.exception.EndRunException; import org.opensearch.ad.model.AnomalyDetector; -import org.opensearch.common.Strings; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.commons.authuser.User; +import org.opensearch.core.common.Strings; public class ADSafeSecurityInjector extends SafeSecurityInjector { private static final Logger LOG = LogManager.getLogger(ADSafeSecurityInjector.class); diff --git a/src/main/java/org/opensearch/ad/annotation/Generated.java b/src/main/java/org/opensearch/timeseries/annotation/Generated.java similarity index 94% rename from src/main/java/org/opensearch/ad/annotation/Generated.java rename to src/main/java/org/opensearch/timeseries/annotation/Generated.java index 08156483a..d3812c9ca 100644 --- a/src/main/java/org/opensearch/ad/annotation/Generated.java +++ b/src/main/java/org/opensearch/timeseries/annotation/Generated.java @@ -9,7 +9,7 @@ * GitHub history for details. */ -package org.opensearch.ad.annotation; +package org.opensearch.timeseries.annotation; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; diff --git a/src/main/java/org/opensearch/timeseries/dataprocessor/FixedValueImputer.java b/src/main/java/org/opensearch/timeseries/dataprocessor/FixedValueImputer.java new file mode 100644 index 000000000..9b8f6bf21 --- /dev/null +++ b/src/main/java/org/opensearch/timeseries/dataprocessor/FixedValueImputer.java @@ -0,0 +1,47 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.timeseries.dataprocessor; + +import java.util.Arrays; + +/** + * fixing missing value (denoted using Double.NaN) using a fixed set of specified values. + * The 2nd parameter of interpolate is ignored as we infer the number of imputed values + * using the number of Double.NaN. + */ +public class FixedValueImputer extends Imputer { + private double[] fixedValue; + + public FixedValueImputer(double[] fixedValue) { + this.fixedValue = fixedValue; + } + + /** + * Given an array of samples, fill with given value. + * We will ignore the rest of samples beyond the 2nd element. + * + * @return an imputed array of size numImputed + */ + @Override + public double[][] impute(double[][] samples, int numImputed) { + int numFeatures = samples.length; + double[][] imputed = new double[numFeatures][numImputed]; + + for (int featureIndex = 0; featureIndex < numFeatures; featureIndex++) { + imputed[featureIndex] = singleFeatureInterpolate(samples[featureIndex], numImputed, fixedValue[featureIndex]); + } + return imputed; + } + + private double[] singleFeatureInterpolate(double[] samples, int numInterpolants, double defaultVal) { + return Arrays.stream(samples).map(d -> Double.isNaN(d) ? defaultVal : d).toArray(); + } + + @Override + protected double[] singleFeatureImpute(double[] samples, int numInterpolants) { + throw new UnsupportedOperationException("The operation is not supported"); + } +} diff --git a/src/main/java/org/opensearch/timeseries/dataprocessor/ImputationMethod.java b/src/main/java/org/opensearch/timeseries/dataprocessor/ImputationMethod.java new file mode 100644 index 000000000..90494862c --- /dev/null +++ b/src/main/java/org/opensearch/timeseries/dataprocessor/ImputationMethod.java @@ -0,0 +1,25 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.timeseries.dataprocessor; + +public enum ImputationMethod { + /** + * This method replaces all missing values with 0's. It's a simple approach, but it may introduce bias if the data is not centered around zero. + */ + ZERO, + /** + * This method replaces missing values with a predefined set of values. The values are the same for each input dimension, and they need to be specified by the user. + */ + FIXED_VALUES, + /** + * This method replaces missing values with the last known value in the respective input dimension. It's a commonly used method for time series data, where temporal continuity is expected. + */ + PREVIOUS, + /** + * This method estimates missing values by interpolating linearly between known values in the respective input dimension. This method assumes that the data follows a linear trend. + */ + LINEAR +} diff --git a/src/main/java/org/opensearch/timeseries/dataprocessor/ImputationOption.java b/src/main/java/org/opensearch/timeseries/dataprocessor/ImputationOption.java new file mode 100644 index 000000000..9943171e5 --- /dev/null +++ b/src/main/java/org/opensearch/timeseries/dataprocessor/ImputationOption.java @@ -0,0 +1,128 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.timeseries.dataprocessor; + +import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.Optional; + +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; + +public class ImputationOption implements Writeable, ToXContent { + // field name in toXContent + public static final String METHOD_FIELD = "method"; + public static final String DEFAULT_FILL_FIELD = "defaultFill"; + + private final ImputationMethod method; + private final Optional defaultFill; + + public ImputationOption(ImputationMethod method, Optional defaultFill) { + this.method = method; + this.defaultFill = defaultFill; + } + + public ImputationOption(ImputationMethod method) { + this(method, Optional.empty()); + } + + public ImputationOption(StreamInput in) throws IOException { + this.method = in.readEnum(ImputationMethod.class); + if (in.readBoolean()) { + this.defaultFill = Optional.of(in.readDoubleArray()); + } else { + this.defaultFill = Optional.empty(); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeEnum(method); + if (defaultFill.isEmpty()) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + out.writeDoubleArray(defaultFill.get()); + } + } + + public static ImputationOption parse(XContentParser parser) throws IOException { + ImputationMethod method = ImputationMethod.ZERO; + List defaultFill = new ArrayList<>(); + + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = parser.currentName(); + parser.nextToken(); + switch (fieldName) { + case METHOD_FIELD: + method = ImputationMethod.valueOf(parser.text().toUpperCase(Locale.ROOT)); + break; + case DEFAULT_FILL_FIELD: + ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + defaultFill.add(parser.doubleValue()); + } + break; + default: + break; + } + } + return new ImputationOption(method, Optional.of(defaultFill.stream().mapToDouble(Double::doubleValue).toArray())); + } + + public XContentBuilder toXContent(XContentBuilder builder) throws IOException { + return toXContent(builder, ToXContent.EMPTY_PARAMS); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + XContentBuilder xContentBuilder = builder.startObject(); + + xContentBuilder.field(METHOD_FIELD, method); + + if (!defaultFill.isEmpty()) { + builder.array(DEFAULT_FILL_FIELD, defaultFill.get()); + } + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (o == this) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + ImputationOption other = (ImputationOption) o; + return method == other.method + && (defaultFill.isEmpty() ? other.defaultFill.isEmpty() : Arrays.equals(defaultFill.get(), other.defaultFill.get())); + } + + @Override + public int hashCode() { + return Objects.hash(method, (defaultFill.isEmpty() ? 0 : Arrays.hashCode(defaultFill.get()))); + } + + public ImputationMethod getMethod() { + return method; + } + + public Optional getDefaultFill() { + return defaultFill; + } +} diff --git a/src/main/java/org/opensearch/timeseries/dataprocessor/Imputer.java b/src/main/java/org/opensearch/timeseries/dataprocessor/Imputer.java new file mode 100644 index 000000000..4e885421c --- /dev/null +++ b/src/main/java/org/opensearch/timeseries/dataprocessor/Imputer.java @@ -0,0 +1,48 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.timeseries.dataprocessor; + +/* + * An object for imputing feature vectors. + * + * In certain situations, due to time and compute cost, we are only allowed to + * query a sparse sample of data points / feature vectors from a cluster. + * However, we need a large sample of feature vectors in order to train our + * anomaly detection algorithms. An Imputer approximates the data points + * between a given, ordered list of samples. + */ +public abstract class Imputer { + + /** + * Imputes the given sample feature vectors. + * + * Computes a list `numImputed` feature vectors using the ordered list + * of `numSamples` input sample vectors where each sample vector has size + * `numFeatures`. + * + * + * @param samples A `numFeatures x numSamples` list of feature vectors. + * @param numImputed The desired number of imputed vectors. + * @return A `numFeatures x numImputed` list of feature vectors. + */ + public double[][] impute(double[][] samples, int numImputed) { + int numFeatures = samples.length; + double[][] interpolants = new double[numFeatures][numImputed]; + + for (int featureIndex = 0; featureIndex < numFeatures; featureIndex++) { + interpolants[featureIndex] = singleFeatureImpute(samples[featureIndex], numImputed); + } + return interpolants; + } + + /** + * compute per-feature impute value + * @param samples input array + * @param numImputed number of elements in the return array + * @return input array with missing values imputed + */ + protected abstract double[] singleFeatureImpute(double[] samples, int numImputed); +} diff --git a/src/main/java/org/opensearch/timeseries/dataprocessor/LinearUniformImputer.java b/src/main/java/org/opensearch/timeseries/dataprocessor/LinearUniformImputer.java new file mode 100644 index 000000000..2fa3fd651 --- /dev/null +++ b/src/main/java/org/opensearch/timeseries/dataprocessor/LinearUniformImputer.java @@ -0,0 +1,82 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.timeseries.dataprocessor; + +import java.util.Arrays; + +import com.google.common.math.DoubleMath; + +/** + * A piecewise linear imputer with uniformly spaced points. + * + * The LinearUniformImputer constructs a piecewise linear imputation on + * the input list of sample feature vectors. That is, between every consecutive + * pair of points we construct a linear imputation. The linear imputation + * is computed on a per-feature basis. + * + */ +public class LinearUniformImputer extends Imputer { + // if true, impute integral/floating-point results: when all samples are integral, + // the results are integral. Else, the results are floating points. + private boolean integerSensitive; + + public LinearUniformImputer(boolean integerSensitive) { + this.integerSensitive = integerSensitive; + } + + /* + * Piecewise linearly impute the given sample of one-dimensional + * features. + * + * Computes a list `numImputed` features using the ordered list of + * `numSamples` input one-dimensional samples. The imputed features are + * computing using a piecewise linear imputation. + * + * @param samples A `numSamples` sized list of sample features. + * @param numImputed The desired number of imputed features. + * @return A `numImputed` sized array of imputed features. + */ + @Override + public double[] singleFeatureImpute(double[] samples, int numImputed) { + int numSamples = samples.length; + double[] imputedValues = new double[numImputed]; + + if (numSamples == 0) { + imputedValues = new double[0]; + } else if (numSamples == 1) { + Arrays.fill(imputedValues, samples[0]); + } else { + /* assume the piecewise linear imputation between the samples is a + parameterized curve f(t) for t in [0, 1]. Each pair of samples + determines a interval [t_i, t_(i+1)]. For each imputed value we + determine which interval it lies inside and then scale the value of t, + accordingly to compute the imputed value. + + for numerical stability reasons we omit processing the final + imputed value in this loop since this last imputed value is always equal + to the last sample. + */ + for (int imputedIndex = 0; imputedIndex < (numImputed - 1); imputedIndex++) { + double tGlobal = (imputedIndex) / (numImputed - 1.0); + double tInterval = tGlobal * (numSamples - 1.0); + int intervalIndex = (int) Math.floor(tInterval); + tInterval -= intervalIndex; + + double leftSample = samples[intervalIndex]; + double rightSample = samples[intervalIndex + 1]; + double imputed = (1.0 - tInterval) * leftSample + tInterval * rightSample; + imputedValues[imputedIndex] = imputed; + } + + // the final imputed value is always the final sample + imputedValues[numImputed - 1] = samples[numSamples - 1]; + } + if (integerSensitive && Arrays.stream(samples).allMatch(DoubleMath::isMathematicalInteger)) { + imputedValues = Arrays.stream(imputedValues).map(Math::rint).toArray(); + } + return imputedValues; + } +} diff --git a/src/main/java/org/opensearch/timeseries/dataprocessor/PreviousValueImputer.java b/src/main/java/org/opensearch/timeseries/dataprocessor/PreviousValueImputer.java new file mode 100644 index 000000000..e91c90814 --- /dev/null +++ b/src/main/java/org/opensearch/timeseries/dataprocessor/PreviousValueImputer.java @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.timeseries.dataprocessor; + +/** + * Given an array of samples, fill missing values (represented using Double.NaN) + * with previous value. + * The return array may be smaller than the input array as we remove leading missing + * values after interpolation. If the first sample is Double.NaN + * as there is no last known value to fill in. + * The 2nd parameter of interpolate is ignored as we infer the number of imputed values + * using the number of Double.NaN. + * + */ +public class PreviousValueImputer extends Imputer { + + @Override + protected double[] singleFeatureImpute(double[] samples, int numInterpolants) { + int numSamples = samples.length; + double[] interpolants = new double[numSamples]; + + if (numSamples > 0) { + System.arraycopy(samples, 0, interpolants, 0, samples.length); + if (numSamples > 1) { + double lastKnownValue = Double.NaN; + for (int interpolantIndex = 0; interpolantIndex < numSamples; interpolantIndex++) { + if (Double.isNaN(interpolants[interpolantIndex])) { + if (!Double.isNaN(lastKnownValue)) { + interpolants[interpolantIndex] = lastKnownValue; + } + } else { + lastKnownValue = interpolants[interpolantIndex]; + } + } + } + } + return interpolants; + } +} diff --git a/src/main/java/org/opensearch/timeseries/dataprocessor/ZeroImputer.java b/src/main/java/org/opensearch/timeseries/dataprocessor/ZeroImputer.java new file mode 100644 index 000000000..1d0656de1 --- /dev/null +++ b/src/main/java/org/opensearch/timeseries/dataprocessor/ZeroImputer.java @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.timeseries.dataprocessor; + +import java.util.Arrays; + +/** + * fixing missing value (denoted using Double.NaN) using 0. + * The 2nd parameter of impute is ignored as we infer the number + * of imputed values using the number of Double.NaN. + */ +public class ZeroImputer extends Imputer { + + @Override + public double[] singleFeatureImpute(double[] samples, int numInterpolants) { + return Arrays.stream(samples).map(d -> Double.isNaN(d) ? 0.0 : d).toArray(); + } +} diff --git a/src/main/java/org/opensearch/timeseries/util/DataUtil.java b/src/main/java/org/opensearch/timeseries/util/DataUtil.java new file mode 100644 index 000000000..4f417e4f7 --- /dev/null +++ b/src/main/java/org/opensearch/timeseries/util/DataUtil.java @@ -0,0 +1,48 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.timeseries.util; + +import java.util.Arrays; + +public class DataUtil { + /** + * Removes leading rows in a 2D array that contain Double.NaN values. + * + * This method iterates over the rows of the provided 2D array. If a row is found + * where all elements are not Double.NaN, it removes this row and all rows before it + * from the array. The modified array, which may be smaller than the original, is then returned. + * + * Note: If all rows contain at least one Double.NaN, the method will return an empty array. + * + * @param arr The 2D array from which leading rows containing Double.NaN are to be removed. + * @return A possibly smaller 2D array with leading rows containing Double.NaN removed. + */ + public static double[][] ltrim(double[][] arr) { + int numRows = arr.length; + if (numRows == 0) { + return new double[0][0]; + } + + int numCols = arr[0].length; + int startIndex = numRows; // Initialized to numRows + for (int i = 0; i < numRows; i++) { + boolean hasNaN = false; + for (int j = 0; j < numCols; j++) { + if (Double.isNaN(arr[i][j])) { + hasNaN = true; + break; + } + } + if (!hasNaN) { + startIndex = i; + break; // Stop the loop as soon as a row without NaN is found + } + } + + return Arrays.copyOfRange(arr, startIndex, arr.length); + } + +} diff --git a/src/test/java/org/opensearch/ad/dataprocessor/SingleFeatureLinearUniformInterpolatorTests.java b/src/test/java/org/opensearch/ad/dataprocessor/SingleFeatureLinearUniformInterpolatorTests.java deleted file mode 100644 index e668b51a0..000000000 --- a/src/test/java/org/opensearch/ad/dataprocessor/SingleFeatureLinearUniformInterpolatorTests.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.ad.dataprocessor; - -import static org.junit.Assert.assertArrayEquals; - -import java.util.Arrays; -import java.util.Collection; - -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameters; - -@RunWith(Parameterized.class) -public class SingleFeatureLinearUniformInterpolatorTests { - - @Parameters - public static Collection data() { - double[] singleComponent = { -1.0, 2.0 }; - double[] multiComponent = { 0.0, 1.0, -1. }; - double oneThird = 1.0 / 3.0; - - return Arrays - .asList( - new Object[][] { - { new double[0], 1, new double[0] }, - { new double[] { 1 }, 2, new double[] { 1, 1 } }, - { singleComponent, 2, singleComponent }, - { singleComponent, 3, new double[] { -1.0, 0.5, 2.0 } }, - { singleComponent, 4, new double[] { -1.0, 0.0, 1.0, 2.0 } }, - { multiComponent, 3, multiComponent }, - { multiComponent, 4, new double[] { 0.0, 2 * oneThird, oneThird, -1.0 } }, - { multiComponent, 5, new double[] { 0.0, 0.5, 1.0, 0.0, -1.0 } }, - { multiComponent, 6, new double[] { 0.0, 0.4, 0.8, 0.6, -0.2, -1.0 } } } - ); - } - - private double[] input; - private int numInterpolants; - private double[] expected; - private SingleFeatureLinearUniformInterpolator interpolator; - - public SingleFeatureLinearUniformInterpolatorTests(double[] input, int numInterpolants, double[] expected) { - this.input = input; - this.numInterpolants = numInterpolants; - this.expected = expected; - } - - @Before - public void setUp() { - this.interpolator = new SingleFeatureLinearUniformInterpolator(); - } - - @Test - public void testInterpolation() { - double[] actual = interpolator.interpolate(input, numInterpolants); - double delta = 1e-8; - assertArrayEquals(expected, actual, delta); - } -} diff --git a/src/test/java/org/opensearch/ad/e2e/AbstractSyntheticDataTest.java b/src/test/java/org/opensearch/ad/e2e/AbstractSyntheticDataTest.java index c0a7676fa..02bf8dbd4 100644 --- a/src/test/java/org/opensearch/ad/e2e/AbstractSyntheticDataTest.java +++ b/src/test/java/org/opensearch/ad/e2e/AbstractSyntheticDataTest.java @@ -35,8 +35,8 @@ import org.opensearch.client.Response; import org.opensearch.client.RestClient; import org.opensearch.client.WarningsHandler; -import org.opensearch.common.Strings; import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.XContentBuilder; import com.google.common.collect.ImmutableList; @@ -66,7 +66,7 @@ protected void disableResourceNotFoundFaultTolerence() throws IOException { settingCommand.endObject(); settingCommand.endObject(); Request request = new Request("PUT", "/_cluster/settings"); - request.setJsonEntity(Strings.toString(settingCommand)); + request.setJsonEntity(org.opensearch.common.Strings.toString(settingCommand)); adminClient().performRequest(request); } @@ -192,6 +192,7 @@ protected String createDetector( return detectorId; } + @Override protected void waitAllSyncheticDataIngested(int expectedSize, String datasetName, RestClient client) throws Exception { int maxWaitCycles = 3; do { diff --git a/src/test/java/org/opensearch/ad/feature/FeatureManagerTests.java b/src/test/java/org/opensearch/ad/feature/FeatureManagerTests.java index 64ca51936..445669153 100644 --- a/src/test/java/org/opensearch/ad/feature/FeatureManagerTests.java +++ b/src/test/java/org/opensearch/ad/feature/FeatureManagerTests.java @@ -58,14 +58,13 @@ import org.opensearch.action.ActionListener; import org.opensearch.ad.AnomalyDetectorPlugin; import org.opensearch.ad.common.exception.EndRunException; -import org.opensearch.ad.dataprocessor.Interpolator; -import org.opensearch.ad.dataprocessor.LinearUniformInterpolator; -import org.opensearch.ad.dataprocessor.SingleFeatureLinearUniformInterpolator; import org.opensearch.ad.model.AnomalyDetector; import org.opensearch.ad.model.Entity; import org.opensearch.ad.model.IntervalTimeConfiguration; import org.opensearch.ad.util.ArrayEqMatcher; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.timeseries.dataprocessor.Imputer; +import org.opensearch.timeseries.dataprocessor.LinearUniformImputer; @RunWith(JUnitParamsRunner.class) @SuppressWarnings("unchecked") @@ -91,7 +90,7 @@ public class FeatureManagerTests { private SearchFeatureDao searchFeatureDao; @Mock - private Interpolator interpolator; + private Imputer imputer; @Mock private Clock clock; @@ -125,7 +124,7 @@ public void setup() { intervalInMilliseconds = detectorIntervalTimeConfig.toDuration().toMillis(); when(detector.getDetectorIntervalInMilliseconds()).thenReturn(intervalInMilliseconds); - Interpolator interpolator = new LinearUniformInterpolator(new SingleFeatureLinearUniformInterpolator()); + Imputer imputer = new LinearUniformImputer(false); ExecutorService executorService = mock(ExecutorService.class); @@ -139,7 +138,7 @@ public void setup() { this.featureManager = spy( new FeatureManager( searchFeatureDao, - interpolator, + imputer, clock, maxTrainSamples, maxSampleStride, @@ -215,7 +214,7 @@ public void getColdStartData_returnExpectedToListener( featureManager = spy( new FeatureManager( searchFeatureDao, - interpolator, + imputer, clock, maxTrainSamples, maxSampleStride, @@ -449,11 +448,10 @@ private void getPreviewFeaturesTemplate(List> samplesResults, return null; }).when(searchFeatureDao).getFeatureSamplesForPeriods(eq(detector), eq(sampleRanges), any()); - when(interpolator.interpolate(argThat(new ArrayEqMatcher<>(new double[][] { { 1, 3 } })), eq(3))) - .thenReturn(new double[][] { { 1, 2, 3 } }); - when(interpolator.interpolate(argThat(new ArrayEqMatcher<>(new double[][] { { 0, 120000 } })), eq(3))) + when(imputer.impute(argThat(new ArrayEqMatcher<>(new double[][] { { 1, 3 } })), eq(3))).thenReturn(new double[][] { { 1, 2, 3 } }); + when(imputer.impute(argThat(new ArrayEqMatcher<>(new double[][] { { 0, 120000 } })), eq(3))) .thenReturn(new double[][] { { 0, 60000, 120000 } }); - when(interpolator.interpolate(argThat(new ArrayEqMatcher<>(new double[][] { { 60000, 180000 } })), eq(3))) + when(imputer.impute(argThat(new ArrayEqMatcher<>(new double[][] { { 60000, 180000 } })), eq(3))) .thenReturn(new double[][] { { 60000, 120000, 180000 } }); ActionListener listener = mock(ActionListener.class); diff --git a/src/test/java/org/opensearch/ad/feature/NoPowermockSearchFeatureDaoTests.java b/src/test/java/org/opensearch/ad/feature/NoPowermockSearchFeatureDaoTests.java index 67e73cdb5..60fb5e273 100644 --- a/src/test/java/org/opensearch/ad/feature/NoPowermockSearchFeatureDaoTests.java +++ b/src/test/java/org/opensearch/ad/feature/NoPowermockSearchFeatureDaoTests.java @@ -58,8 +58,6 @@ import org.opensearch.ad.AbstractADTest; import org.opensearch.ad.NodeStateManager; import org.opensearch.ad.TestHelpers; -import org.opensearch.ad.dataprocessor.LinearUniformInterpolator; -import org.opensearch.ad.dataprocessor.SingleFeatureLinearUniformInterpolator; import org.opensearch.ad.model.AnomalyDetector; import org.opensearch.ad.model.Entity; import org.opensearch.ad.model.Feature; @@ -100,6 +98,8 @@ import org.opensearch.search.aggregations.metrics.InternalMax; import org.opensearch.search.aggregations.metrics.SumAggregationBuilder; import org.opensearch.search.internal.InternalSearchResponse; +import org.opensearch.timeseries.dataprocessor.Imputer; +import org.opensearch.timeseries.dataprocessor.LinearUniformImputer; import com.carrotsearch.hppc.BitMixer; import com.google.common.collect.ImmutableList; @@ -115,7 +115,7 @@ public class NoPowermockSearchFeatureDaoTests extends AbstractADTest { private AnomalyDetector detector; private Client client; private SearchFeatureDao searchFeatureDao; - private LinearUniformInterpolator interpolator; + private Imputer imputer; private SecurityClientUtil clientUtil; private Settings settings; private ClusterService clusterService; @@ -154,7 +154,7 @@ public void setUp() throws Exception { client = mock(Client.class); when(client.threadPool()).thenReturn(threadPool); - interpolator = new LinearUniformInterpolator(new SingleFeatureLinearUniformInterpolator()); + imputer = new LinearUniformImputer(false); settings = Settings.EMPTY; ClusterSettings clusterSettings = new ClusterSettings( @@ -178,7 +178,7 @@ public void setUp() throws Exception { searchFeatureDao = new SearchFeatureDao( client, xContentRegistry(), // Important. Without this, ParseUtils cannot parse anything - interpolator, + imputer, clientUtil, settings, clusterService, @@ -365,7 +365,7 @@ public void testGetHighestCountEntitiesExhaustedPages() throws InterruptedExcept searchFeatureDao = new SearchFeatureDao( client, xContentRegistry(), - interpolator, + imputer, clientUtil, settings, clusterService, @@ -411,7 +411,7 @@ public void testGetHighestCountEntitiesNotEnoughTime() throws InterruptedExcepti searchFeatureDao = new SearchFeatureDao( client, xContentRegistry(), - interpolator, + imputer, clientUtil, settings, clusterService, diff --git a/src/test/java/org/opensearch/ad/feature/SearchFeatureDaoParamTests.java b/src/test/java/org/opensearch/ad/feature/SearchFeatureDaoParamTests.java index a6380b8cf..74b1d030b 100644 --- a/src/test/java/org/opensearch/ad/feature/SearchFeatureDaoParamTests.java +++ b/src/test/java/org/opensearch/ad/feature/SearchFeatureDaoParamTests.java @@ -55,9 +55,6 @@ import org.opensearch.ad.AnomalyDetectorPlugin; import org.opensearch.ad.NodeStateManager; import org.opensearch.ad.constant.ADCommonName; -import org.opensearch.ad.dataprocessor.Interpolator; -import org.opensearch.ad.dataprocessor.LinearUniformInterpolator; -import org.opensearch.ad.dataprocessor.SingleFeatureLinearUniformInterpolator; import org.opensearch.ad.model.AnomalyDetector; import org.opensearch.ad.model.IntervalTimeConfiguration; import org.opensearch.ad.settings.AnomalyDetectorSettings; @@ -82,6 +79,8 @@ import org.opensearch.search.aggregations.metrics.Percentile; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.timeseries.dataprocessor.Imputer; +import org.opensearch.timeseries.dataprocessor.LinearUniformImputer; import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; @@ -150,7 +149,7 @@ public class SearchFeatureDaoParamTests { private MultiSearchRequest multiSearchRequest; private IntervalTimeConfiguration detectionInterval; private String detectorId; - private Interpolator interpolator; + private Imputer imputer; private Settings settings; @Before @@ -158,7 +157,7 @@ public void setup() throws Exception { MockitoAnnotations.initMocks(this); PowerMockito.mockStatic(ParseUtils.class); - interpolator = new LinearUniformInterpolator(new SingleFeatureLinearUniformInterpolator()); + imputer = new LinearUniformImputer(false); ExecutorService executorService = mock(ExecutorService.class); when(threadPool.executor(AnomalyDetectorPlugin.AD_THREAD_POOL_NAME)).thenReturn(executorService); @@ -179,7 +178,7 @@ public void setup() throws Exception { }).when(nodeStateManager).getAnomalyDetector(any(String.class), any(ActionListener.class)); clientUtil = new SecurityClientUtil(nodeStateManager, settings); searchFeatureDao = spy( - new SearchFeatureDao(client, xContent, interpolator, clientUtil, settings, null, AnomalyDetectorSettings.NUM_SAMPLES_PER_TREE) + new SearchFeatureDao(client, xContent, imputer, clientUtil, settings, null, AnomalyDetectorSettings.NUM_SAMPLES_PER_TREE) ); detectionInterval = new IntervalTimeConfiguration(1, ChronoUnit.MINUTES); diff --git a/src/test/java/org/opensearch/ad/feature/SearchFeatureDaoTests.java b/src/test/java/org/opensearch/ad/feature/SearchFeatureDaoTests.java index 25541cf3b..407ccfce5 100644 --- a/src/test/java/org/opensearch/ad/feature/SearchFeatureDaoTests.java +++ b/src/test/java/org/opensearch/ad/feature/SearchFeatureDaoTests.java @@ -60,9 +60,6 @@ import org.opensearch.ad.AnomalyDetectorPlugin; import org.opensearch.ad.NodeStateManager; import org.opensearch.ad.constant.ADCommonName; -import org.opensearch.ad.dataprocessor.Interpolator; -import org.opensearch.ad.dataprocessor.LinearUniformInterpolator; -import org.opensearch.ad.dataprocessor.SingleFeatureLinearUniformInterpolator; import org.opensearch.ad.model.AnomalyDetector; import org.opensearch.ad.model.Entity; import org.opensearch.ad.model.IntervalTimeConfiguration; @@ -98,6 +95,8 @@ import org.opensearch.search.aggregations.metrics.Percentile; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.timeseries.dataprocessor.Imputer; +import org.opensearch.timeseries.dataprocessor.LinearUniformImputer; import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; @@ -153,7 +152,7 @@ public class SearchFeatureDaoTests { private Map aggsMap; private IntervalTimeConfiguration detectionInterval; private String detectorId; - private Interpolator interpolator; + private Imputer imputer; private Settings settings; @Before @@ -161,7 +160,7 @@ public void setup() throws Exception { MockitoAnnotations.initMocks(this); PowerMockito.mockStatic(ParseUtils.class); - interpolator = new LinearUniformInterpolator(new SingleFeatureLinearUniformInterpolator()); + imputer = new LinearUniformImputer(false); ExecutorService executorService = mock(ExecutorService.class); when(threadPool.executor(AnomalyDetectorPlugin.AD_THREAD_POOL_NAME)).thenReturn(executorService); @@ -182,7 +181,7 @@ public void setup() throws Exception { }).when(nodeStateManager).getAnomalyDetector(any(String.class), any(ActionListener.class)); clientUtil = new SecurityClientUtil(nodeStateManager, settings); searchFeatureDao = spy( - new SearchFeatureDao(client, xContent, interpolator, clientUtil, settings, null, AnomalyDetectorSettings.NUM_SAMPLES_PER_TREE) + new SearchFeatureDao(client, xContent, imputer, clientUtil, settings, null, AnomalyDetectorSettings.NUM_SAMPLES_PER_TREE) ); detectionInterval = new IntervalTimeConfiguration(1, ChronoUnit.MINUTES); diff --git a/src/test/java/org/opensearch/ad/ml/AbstractCosineDataTest.java b/src/test/java/org/opensearch/ad/ml/AbstractCosineDataTest.java index 2107d4e59..0e4952600 100644 --- a/src/test/java/org/opensearch/ad/ml/AbstractCosineDataTest.java +++ b/src/test/java/org/opensearch/ad/ml/AbstractCosineDataTest.java @@ -38,10 +38,6 @@ import org.opensearch.ad.MemoryTracker; import org.opensearch.ad.NodeStateManager; import org.opensearch.ad.TestHelpers; -import org.opensearch.ad.dataprocessor.IntegerSensitiveSingleFeatureLinearUniformInterpolator; -import org.opensearch.ad.dataprocessor.Interpolator; -import org.opensearch.ad.dataprocessor.LinearUniformInterpolator; -import org.opensearch.ad.dataprocessor.SingleFeatureLinearUniformInterpolator; import org.opensearch.ad.feature.FeatureManager; import org.opensearch.ad.feature.SearchFeatureDao; import org.opensearch.ad.model.AnomalyDetector; @@ -61,6 +57,8 @@ import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.ThreadPool; import org.opensearch.timeseries.constant.CommonName; +import org.opensearch.timeseries.dataprocessor.Imputer; +import org.opensearch.timeseries.dataprocessor.LinearUniformImputer; import com.google.common.collect.ImmutableList; @@ -75,7 +73,7 @@ public class AbstractCosineDataTest extends AbstractADTest { EntityColdStarter entityColdStarter; NodeStateManager stateManager; SearchFeatureDao searchFeatureDao; - Interpolator interpolator; + Imputer imputer; CheckpointDao checkpoint; FeatureManager featureManager; Settings settings; @@ -152,16 +150,14 @@ public void setUp() throws Exception { clusterService ); - SingleFeatureLinearUniformInterpolator singleFeatureLinearUniformInterpolator = - new IntegerSensitiveSingleFeatureLinearUniformInterpolator(); - interpolator = new LinearUniformInterpolator(singleFeatureLinearUniformInterpolator); + imputer = new LinearUniformImputer(true); searchFeatureDao = mock(SearchFeatureDao.class); checkpoint = mock(CheckpointDao.class); featureManager = new FeatureManager( searchFeatureDao, - interpolator, + imputer, clock, AnomalyDetectorSettings.MAX_TRAIN_SAMPLE, AnomalyDetectorSettings.MAX_SAMPLE_STRIDE, @@ -189,7 +185,7 @@ public void setUp() throws Exception { numMinSamples, AnomalyDetectorSettings.MAX_SAMPLE_STRIDE, AnomalyDetectorSettings.MAX_TRAIN_SAMPLE, - interpolator, + imputer, searchFeatureDao, AnomalyDetectorSettings.THRESHOLD_MIN_PVALUE, featureManager, diff --git a/src/test/java/org/opensearch/ad/ml/EntityColdStarterTests.java b/src/test/java/org/opensearch/ad/ml/EntityColdStarterTests.java index a6d5d34c1..dc77931f8 100644 --- a/src/test/java/org/opensearch/ad/ml/EntityColdStarterTests.java +++ b/src/test/java/org/opensearch/ad/ml/EntityColdStarterTests.java @@ -166,9 +166,9 @@ public void testColdStart() throws InterruptedException, IOException { // for function interpolate: // 1st parameter is a matrix of size numFeatures * numSamples // 2nd parameter is the number of interpolants including two samples - double[][] interval1 = interpolator.interpolate(new double[][] { new double[] { sample1[0], sample2[0] } }, 61); + double[][] interval1 = imputer.impute(new double[][] { new double[] { sample1[0], sample2[0] } }, 61); expectedColdStartData.addAll(convertToFeatures(interval1, 60)); - double[][] interval2 = interpolator.interpolate(new double[][] { new double[] { sample2[0], sample3[0] } }, 61); + double[][] interval2 = imputer.impute(new double[][] { new double[] { sample2[0], sample3[0] } }, 61); expectedColdStartData.addAll(convertToFeatures(interval2, 61)); assertEquals(121, expectedColdStartData.size()); @@ -302,11 +302,11 @@ public void testTwoSegmentsWithSingleSample() throws InterruptedException, IOExc // for function interpolate: // 1st parameter is a matrix of size numFeatures * numSamples // 2nd parameter is the number of interpolants including two samples - double[][] interval1 = interpolator.interpolate(new double[][] { new double[] { sample1[0], sample2[0] } }, 61); + double[][] interval1 = imputer.impute(new double[][] { new double[] { sample1[0], sample2[0] } }, 61); expectedColdStartData.addAll(convertToFeatures(interval1, 60)); - double[][] interval2 = interpolator.interpolate(new double[][] { new double[] { sample2[0], sample3[0] } }, 61); + double[][] interval2 = imputer.impute(new double[][] { new double[] { sample2[0], sample3[0] } }, 61); expectedColdStartData.addAll(convertToFeatures(interval2, 60)); - double[][] interval3 = interpolator.interpolate(new double[][] { new double[] { sample3[0], sample5[0] } }, 121); + double[][] interval3 = imputer.impute(new double[][] { new double[] { sample3[0], sample5[0] } }, 121); expectedColdStartData.addAll(convertToFeatures(interval3, 121)); assertTrue("size: " + model.getSamples().size(), model.getSamples().isEmpty()); assertEquals(241, expectedColdStartData.size()); @@ -358,13 +358,13 @@ public void testTwoSegments() throws InterruptedException, IOException { // for function interpolate: // 1st parameter is a matrix of size numFeatures * numSamples // 2nd parameter is the number of interpolants including two samples - double[][] interval1 = interpolator.interpolate(new double[][] { new double[] { sample1[0], sample2[0] } }, 61); + double[][] interval1 = imputer.impute(new double[][] { new double[] { sample1[0], sample2[0] } }, 61); expectedColdStartData.addAll(convertToFeatures(interval1, 60)); - double[][] interval2 = interpolator.interpolate(new double[][] { new double[] { sample2[0], sample3[0] } }, 61); + double[][] interval2 = imputer.impute(new double[][] { new double[] { sample2[0], sample3[0] } }, 61); expectedColdStartData.addAll(convertToFeatures(interval2, 60)); - double[][] interval3 = interpolator.interpolate(new double[][] { new double[] { sample3[0], sample5[0] } }, 121); + double[][] interval3 = imputer.impute(new double[][] { new double[] { sample3[0], sample5[0] } }, 121); expectedColdStartData.addAll(convertToFeatures(interval3, 120)); - double[][] interval4 = interpolator.interpolate(new double[][] { new double[] { sample5[0], sample6[0] } }, 61); + double[][] interval4 = imputer.impute(new double[][] { new double[] { sample5[0], sample6[0] } }, 61); expectedColdStartData.addAll(convertToFeatures(interval4, 61)); assertEquals(301, expectedColdStartData.size()); assertTrue("size: " + model.getSamples().size(), model.getSamples().isEmpty()); @@ -701,7 +701,7 @@ public void testAccuracyOneMinuteIntervalNoInterpolation() throws Exception { numMinSamples, AnomalyDetectorSettings.MAX_SAMPLE_STRIDE, AnomalyDetectorSettings.MAX_TRAIN_SAMPLE, - interpolator, + imputer, searchFeatureDao, AnomalyDetectorSettings.THRESHOLD_MIN_PVALUE, featureManager, diff --git a/src/test/java/org/opensearch/ad/ml/HCADModelPerfTests.java b/src/test/java/org/opensearch/ad/ml/HCADModelPerfTests.java index c51bc11b5..359d47e5f 100644 --- a/src/test/java/org/opensearch/ad/ml/HCADModelPerfTests.java +++ b/src/test/java/org/opensearch/ad/ml/HCADModelPerfTests.java @@ -113,7 +113,7 @@ private void averageAccuracyTemplate( featureManager = new FeatureManager( searchFeatureDao, - interpolator, + imputer, clock, AnomalyDetectorSettings.MAX_TRAIN_SAMPLE, AnomalyDetectorSettings.MAX_SAMPLE_STRIDE, @@ -138,7 +138,7 @@ private void averageAccuracyTemplate( numMinSamples, AnomalyDetectorSettings.MAX_SAMPLE_STRIDE, AnomalyDetectorSettings.MAX_TRAIN_SAMPLE, - interpolator, + imputer, searchFeatureDao, AnomalyDetectorSettings.THRESHOLD_MIN_PVALUE, featureManager, diff --git a/src/test/java/org/opensearch/ad/ml/ModelManagerTests.java b/src/test/java/org/opensearch/ad/ml/ModelManagerTests.java index 1b4797eb6..8fbb7b75f 100644 --- a/src/test/java/org/opensearch/ad/ml/ModelManagerTests.java +++ b/src/test/java/org/opensearch/ad/ml/ModelManagerTests.java @@ -62,9 +62,6 @@ import org.opensearch.ad.caching.EntityCache; import org.opensearch.ad.common.exception.LimitExceededException; import org.opensearch.ad.common.exception.ResourceNotFoundException; -import org.opensearch.ad.dataprocessor.IntegerSensitiveSingleFeatureLinearUniformInterpolator; -import org.opensearch.ad.dataprocessor.LinearUniformInterpolator; -import org.opensearch.ad.dataprocessor.SingleFeatureLinearUniformInterpolator; import org.opensearch.ad.feature.FeatureManager; import org.opensearch.ad.feature.SearchFeatureDao; import org.opensearch.ad.ml.ModelManager.ModelType; @@ -80,6 +77,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.monitor.jvm.JvmService; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.timeseries.dataprocessor.LinearUniformImputer; import org.powermock.modules.junit4.PowerMockRunner; import org.powermock.modules.junit4.PowerMockRunnerDelegate; @@ -914,9 +912,7 @@ public void getNullState() { public void getEmptyStateFullSamples() { SearchFeatureDao searchFeatureDao = mock(SearchFeatureDao.class); - SingleFeatureLinearUniformInterpolator singleFeatureLinearUniformInterpolator = - new IntegerSensitiveSingleFeatureLinearUniformInterpolator(); - LinearUniformInterpolator interpolator = new LinearUniformInterpolator(singleFeatureLinearUniformInterpolator); + LinearUniformImputer interpolator = new LinearUniformImputer(true); NodeStateManager stateManager = mock(NodeStateManager.class); featureManager = new FeatureManager( diff --git a/src/test/java/org/opensearch/ad/rest/AnomalyDetectorRestApiIT.java b/src/test/java/org/opensearch/ad/rest/AnomalyDetectorRestApiIT.java index b7605ce90..3e439f213 100644 --- a/src/test/java/org/opensearch/ad/rest/AnomalyDetectorRestApiIT.java +++ b/src/test/java/org/opensearch/ad/rest/AnomalyDetectorRestApiIT.java @@ -1673,7 +1673,7 @@ public void testSearchTopAnomalyResultsOnNonExistentResultIndex() throws IOExcep // Delete any existing result index if (indexExistsWithAdminClient(ADCommonName.ANOMALY_RESULT_INDEX_ALIAS)) { - deleteIndexWithAdminClient(ADCommonName.ANOMALY_RESULT_INDEX_ALIAS); + deleteIndexWithAdminClient(ADCommonName.ANOMALY_RESULT_INDEX_ALIAS + "*"); } Response response = searchTopAnomalyResults( detector.getDetectorId(), @@ -1710,12 +1710,8 @@ public void testSearchTopAnomalyResultsOnEmptyResultIndex() throws IOException { client() ); - // Clear any existing result index, create an empty one - // Use the DELETE API to delete the index or indices associated with the alias first - // then Use the DELETE API to remove the alias. if (indexExistsWithAdminClient(ADCommonName.ANOMALY_RESULT_INDEX_ALIAS)) { deleteIndexWithAdminClient(ADCommonName.ANOMALY_RESULT_INDEX_ALIAS + "*"); - deleteIndexWithAdminClient(ADCommonName.ANOMALY_RESULT_INDEX_ALIAS); } TestHelpers.createEmptyAnomalyResultIndex(adminClient()); Response response = searchTopAnomalyResults( diff --git a/src/test/java/org/opensearch/ad/rest/SecureADRestIT.java b/src/test/java/org/opensearch/ad/rest/SecureADRestIT.java index 0a3ec8c76..a96ba20f1 100644 --- a/src/test/java/org/opensearch/ad/rest/SecureADRestIT.java +++ b/src/test/java/org/opensearch/ad/rest/SecureADRestIT.java @@ -17,6 +17,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Map; +import java.util.Random; import org.apache.hc.core5.http.HttpHeaders; import org.apache.hc.core5.http.HttpHost; @@ -60,49 +61,74 @@ public class SecureADRestIT extends AnomalyDetectorRestTestCase { private String indexAllAccessRole = "index_all_access"; private String indexSearchAccessRole = "index_all_search"; + /** + * Create an unguessable password. Simple password are weak due to https://tinyurl.com/383em9zk + * @return a random password. + */ + public static String generatePassword() { + String characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + + Random rng = new Random(); + + char[] password = new char[10]; + for (int i = 0; i < 10; i++) { + password[i] = characters.charAt(rng.nextInt(characters.length())); + } + + return new String(password); + } + @Before public void setupSecureTests() throws IOException { if (!isHttps()) throw new IllegalArgumentException("Secure Tests are running but HTTPS is not set"); createIndexRole(indexAllAccessRole, "*"); createSearchRole(indexSearchAccessRole, "*"); - createUser(aliceUser, aliceUser, new ArrayList<>(Arrays.asList("odfe"))); - aliceClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), aliceUser, aliceUser) + String alicePassword = generatePassword(); + createUser(aliceUser, alicePassword, new ArrayList<>(Arrays.asList("odfe"))); + aliceClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), aliceUser, alicePassword) .setSocketTimeout(60000) .build(); - createUser(bobUser, bobUser, new ArrayList<>(Arrays.asList("odfe"))); - bobClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), bobUser, bobUser) + String bobPassword = generatePassword(); + createUser(bobUser, bobPassword, new ArrayList<>(Arrays.asList("odfe"))); + bobClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), bobUser, bobPassword) .setSocketTimeout(60000) .build(); - createUser(catUser, catUser, new ArrayList<>(Arrays.asList("aes"))); - catClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), catUser, catUser) + String catPassword = generatePassword(); + createUser(catUser, catPassword, new ArrayList<>(Arrays.asList("aes"))); + catClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), catUser, catPassword) .setSocketTimeout(60000) .build(); - createUser(dogUser, dogUser, new ArrayList<>(Arrays.asList())); - dogClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), dogUser, dogUser) + String dogPassword = generatePassword(); + createUser(dogUser, dogPassword, new ArrayList<>(Arrays.asList())); + dogClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), dogUser, dogPassword) .setSocketTimeout(60000) .build(); - createUser(elkUser, elkUser, new ArrayList<>(Arrays.asList("odfe"))); - elkClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), elkUser, elkUser) + String elkPassword = generatePassword(); + createUser(elkUser, elkPassword, new ArrayList<>(Arrays.asList("odfe"))); + elkClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), elkUser, elkPassword) .setSocketTimeout(60000) .build(); - createUser(fishUser, fishUser, new ArrayList<>(Arrays.asList("odfe", "aes"))); - fishClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), fishUser, fishUser) + String fishPassword = generatePassword(); + createUser(fishUser, fishPassword, new ArrayList<>(Arrays.asList("odfe", "aes"))); + fishClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), fishUser, fishPassword) .setSocketTimeout(60000) .build(); - createUser(goatUser, goatUser, new ArrayList<>(Arrays.asList("opensearch"))); - goatClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), goatUser, goatUser) + String goatPassword = generatePassword(); + createUser(goatUser, goatPassword, new ArrayList<>(Arrays.asList("opensearch"))); + goatClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), goatUser, goatPassword) .setSocketTimeout(60000) .build(); - createUser(lionUser, lionUser, new ArrayList<>(Arrays.asList("opensearch"))); - lionClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), lionUser, lionUser) + String lionPassword = generatePassword(); + createUser(lionUser, lionPassword, new ArrayList<>(Arrays.asList("opensearch"))); + lionClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), lionUser, lionPassword) .setSocketTimeout(60000) .build(); diff --git a/src/test/java/org/opensearch/ad/util/ExceptionUtilsTests.java b/src/test/java/org/opensearch/ad/util/ExceptionUtilsTests.java index 94af37770..00c9d1a2d 100644 --- a/src/test/java/org/opensearch/ad/util/ExceptionUtilsTests.java +++ b/src/test/java/org/opensearch/ad/util/ExceptionUtilsTests.java @@ -56,7 +56,7 @@ public void testCountInStats() { public void testGetErrorMessage() { assertEquals("test", ExceptionUtil.getErrorMessage(new AnomalyDetectionException("test"))); assertEquals("test", ExceptionUtil.getErrorMessage(new IllegalArgumentException("test"))); - assertEquals("org.opensearch.OpenSearchException: test", ExceptionUtil.getErrorMessage(new OpenSearchException("test"))); + assertEquals("OpenSearchException[test]", ExceptionUtil.getErrorMessage(new OpenSearchException("test"))); assertTrue( ExceptionUtil .getErrorMessage(new RuntimeException("test")) diff --git a/src/test/java/org/opensearch/timeseries/dataprocessor/FixedValueImputerTests.java b/src/test/java/org/opensearch/timeseries/dataprocessor/FixedValueImputerTests.java new file mode 100644 index 000000000..81b9b5bfb --- /dev/null +++ b/src/test/java/org/opensearch/timeseries/dataprocessor/FixedValueImputerTests.java @@ -0,0 +1,34 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.timeseries.dataprocessor; + +import static org.junit.Assert.assertArrayEquals; + +import org.junit.Test; + +public class FixedValueImputerTests { + + @Test + public void testImpute() { + // Initialize the FixedValueImputer with some fixed values + double[] fixedValues = { 2.0, 3.0 }; + FixedValueImputer imputer = new FixedValueImputer(fixedValues); + + // Create a sample array with some missing values (Double.NaN) + double[][] samples = { { 1.0, Double.NaN, 3.0 }, { Double.NaN, 2.0, 3.0 } }; + + // Call the impute method + double[][] imputed = imputer.impute(samples, 3); + + // Check the results + double[][] expected = { { 1.0, 2.0, 3.0 }, { 3.0, 2.0, 3.0 } }; + double delta = 0.0001; + + for (int i = 0; i < expected.length; i++) { + assertArrayEquals("The arrays are not equal", expected[i], imputed[i], delta); + } + } +} diff --git a/src/test/java/org/opensearch/timeseries/dataprocessor/ImputationOptionTests.java b/src/test/java/org/opensearch/timeseries/dataprocessor/ImputationOptionTests.java new file mode 100644 index 000000000..a0df85ead --- /dev/null +++ b/src/test/java/org/opensearch/timeseries/dataprocessor/ImputationOptionTests.java @@ -0,0 +1,124 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.timeseries.dataprocessor; + +import java.io.IOException; +import java.util.Optional; + +import org.opensearch.common.bytes.BytesReference; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.test.OpenSearchTestCase; + +public class ImputationOptionTests extends OpenSearchTestCase { + + public void testStreamInputAndOutput() throws IOException { + // Prepare the data to be read by the StreamInput object. + ImputationMethod method = ImputationMethod.PREVIOUS; + double[] defaultFill = { 1.0, 2.0, 3.0 }; + + ImputationOption option = new ImputationOption(method, Optional.of(defaultFill)); + + // Write the ImputationOption to the StreamOutput. + BytesStreamOutput out = new BytesStreamOutput(); + option.writeTo(out); + + StreamInput in = out.bytes().streamInput(); + + // Create an ImputationOption using the mocked StreamInput. + ImputationOption inOption = new ImputationOption(in); + + // Check that the created ImputationOption has the correct values. + assertEquals(method, inOption.getMethod()); + assertArrayEquals(defaultFill, inOption.getDefaultFill().get(), 1e-6); + } + + public void testToXContent() throws IOException { + double[] defaultFill = { 1.0, 2.0, 3.0 }; + ImputationOption imputationOption = new ImputationOption(ImputationMethod.FIXED_VALUES, Optional.of(defaultFill)); + + String xContent = "{" + "\"method\":\"FIXED_VALUES\"," + "\"defaultFill\":[1.0,2.0,3.0]" + "}"; + + XContentBuilder builder = imputationOption.toXContent(JsonXContent.contentBuilder(), ToXContent.EMPTY_PARAMS); + String actualJson = BytesReference.bytes(builder).utf8ToString(); + + assertEquals(xContent, actualJson); + } + + public void testParse() throws IOException { + String xContent = "{" + "\"method\":\"FIXED_VALUES\"," + "\"defaultFill\":[1.0,2.0,3.0]" + "}"; + + double[] defaultFill = { 1.0, 2.0, 3.0 }; + ImputationOption imputationOption = new ImputationOption(ImputationMethod.FIXED_VALUES, Optional.of(defaultFill)); + + try ( + XContentParser parser = JsonXContent.jsonXContent + .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, xContent) + ) { + // advance to first token + XContentParser.Token token = parser.nextToken(); + if (token != XContentParser.Token.START_OBJECT) { + throw new IOException("Expected data to start with an Object"); + } + + ImputationOption parsedOption = ImputationOption.parse(parser); + + assertEquals(imputationOption.getMethod(), parsedOption.getMethod()); + assertTrue(imputationOption.getDefaultFill().isPresent()); + assertTrue(parsedOption.getDefaultFill().isPresent()); + assertEquals(imputationOption.getDefaultFill().get().length, parsedOption.getDefaultFill().get().length); + for (int i = 0; i < imputationOption.getDefaultFill().get().length; i++) { + assertEquals(imputationOption.getDefaultFill().get()[i], parsedOption.getDefaultFill().get()[i], 0); + } + } + } + + public void testEqualsAndHashCode() { + double[] defaultFill1 = { 1.0, 2.0, 3.0 }; + double[] defaultFill2 = { 4.0, 5.0, 6.0 }; + + ImputationOption option1 = new ImputationOption(ImputationMethod.FIXED_VALUES, Optional.of(defaultFill1)); + ImputationOption option2 = new ImputationOption(ImputationMethod.FIXED_VALUES, Optional.of(defaultFill1)); + ImputationOption option3 = new ImputationOption(ImputationMethod.LINEAR, Optional.of(defaultFill2)); + + // Test reflexivity + assertTrue(option1.equals(option1)); + + // Test symmetry + assertTrue(option1.equals(option2)); + assertTrue(option2.equals(option1)); + + // Test transitivity + ImputationOption option2Clone = new ImputationOption(ImputationMethod.FIXED_VALUES, Optional.of(defaultFill1)); + assertTrue(option1.equals(option2)); + assertTrue(option2.equals(option2Clone)); + assertTrue(option1.equals(option2Clone)); + + // Test consistency: ultiple invocations of a.equals(b) consistently return true or consistently return false. + assertTrue(option1.equals(option2)); + assertTrue(option1.equals(option2)); + + // Test non-nullity + assertFalse(option1.equals(null)); + + // Test hashCode consistency + assertEquals(option1.hashCode(), option1.hashCode()); + + // Test hashCode equality + assertTrue(option1.equals(option2)); + assertEquals(option1.hashCode(), option2.hashCode()); + + // Test inequality + assertFalse(option1.equals(option3)); + assertNotEquals(option1.hashCode(), option3.hashCode()); + } +} diff --git a/src/test/java/org/opensearch/timeseries/dataprocessor/IntegerSensitiveLinearUniformImputerTests.java b/src/test/java/org/opensearch/timeseries/dataprocessor/IntegerSensitiveLinearUniformImputerTests.java new file mode 100644 index 000000000..03e8b6cb3 --- /dev/null +++ b/src/test/java/org/opensearch/timeseries/dataprocessor/IntegerSensitiveLinearUniformImputerTests.java @@ -0,0 +1,72 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.timeseries.dataprocessor; + +import static org.junit.Assert.assertArrayEquals; + +import java.util.Arrays; +import java.util.Collection; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +/** + * Compared to MultiFeatureLinearUniformImputerTests, outputs are different and + * integerSensitive is enabled + * + */ +@RunWith(Parameterized.class) +public class IntegerSensitiveLinearUniformImputerTests { + + @Parameters + public static Collection data() { + double[][] singleComponent = { { -1.0, 2.0 }, { 1.0, 1.0 } }; + double[][] multiComponent = { { 0.0, 1.0, -1.0 }, { 1.0, 1.0, 1.0 } }; + + return Arrays + .asList( + new Object[][] { + // after integer sensitive rint rounding + { singleComponent, 2, singleComponent }, + { singleComponent, 3, new double[][] { { -1.0, 0, 2.0 }, { 1.0, 1.0, 1.0 } } }, + { singleComponent, 4, new double[][] { { -1.0, 0.0, 1.0, 2.0 }, { 1.0, 1.0, 1.0, 1.0 } } }, + { multiComponent, 3, multiComponent }, + { multiComponent, 4, new double[][] { { 0.0, 1.0, 0.0, -1.0 }, { 1.0, 1.0, 1.0, 1.0 } } }, + { multiComponent, 5, new double[][] { { 0.0, 0.0, 1.0, 0.0, -1.0 }, { 1.0, 1.0, 1.0, 1.0, 1.0 } } }, + { multiComponent, 6, new double[][] { { 0.0, 0.0, 1.0, 1.0, -0.0, -1.0 }, { 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 } } }, } + ); + } + + private double[][] input; + private int numInterpolants; + private double[][] expected; + private Imputer imputer; + + public IntegerSensitiveLinearUniformImputerTests(double[][] input, int numInterpolants, double[][] expected) { + this.input = input; + this.numInterpolants = numInterpolants; + this.expected = expected; + } + + @Before + public void setUp() { + this.imputer = new LinearUniformImputer(true); + } + + @Test + public void testImputation() { + double[][] actual = imputer.impute(input, numInterpolants); + double delta = 1e-8; + int numFeatures = expected.length; + + for (int i = 0; i < numFeatures; i++) { + assertArrayEquals(expected[i], actual[i], delta); + } + } +} diff --git a/src/test/java/org/opensearch/ad/dataprocessor/LinearUniformInterpolatorTests.java b/src/test/java/org/opensearch/timeseries/dataprocessor/MultiFeatureLinearUniformImputerTests.java similarity index 81% rename from src/test/java/org/opensearch/ad/dataprocessor/LinearUniformInterpolatorTests.java rename to src/test/java/org/opensearch/timeseries/dataprocessor/MultiFeatureLinearUniformImputerTests.java index 2d39566ca..3656be278 100644 --- a/src/test/java/org/opensearch/ad/dataprocessor/LinearUniformInterpolatorTests.java +++ b/src/test/java/org/opensearch/timeseries/dataprocessor/MultiFeatureLinearUniformImputerTests.java @@ -9,7 +9,7 @@ * GitHub history for details. */ -package org.opensearch.ad.dataprocessor; +package org.opensearch.timeseries.dataprocessor; import static org.junit.Assert.assertArrayEquals; @@ -23,7 +23,7 @@ import org.junit.runners.Parameterized.Parameters; @RunWith(Parameterized.class) -public class LinearUniformInterpolatorTests { +public class MultiFeatureLinearUniformImputerTests { @Parameters public static Collection data() { @@ -34,6 +34,7 @@ public static Collection data() { return Arrays .asList( new Object[][] { + // no integer sensitive rint rounding at the end of singleFeatureImpute. { singleComponent, 2, singleComponent }, { singleComponent, 3, new double[][] { { -1.0, 0.5, 2.0 }, { 1.0, 1.0, 1.0 } } }, { singleComponent, 4, new double[][] { { -1.0, 0.0, 1.0, 2.0 }, { 1.0, 1.0, 1.0, 1.0 } } }, @@ -47,9 +48,9 @@ public static Collection data() { private double[][] input; private int numInterpolants; private double[][] expected; - private LinearUniformInterpolator interpolator; + private Imputer imputer; - public LinearUniformInterpolatorTests(double[][] input, int numInterpolants, double[][] expected) { + public MultiFeatureLinearUniformImputerTests(double[][] input, int numInterpolants, double[][] expected) { this.input = input; this.numInterpolants = numInterpolants; this.expected = expected; @@ -57,12 +58,12 @@ public LinearUniformInterpolatorTests(double[][] input, int numInterpolants, dou @Before public void setUp() { - this.interpolator = new LinearUniformInterpolator(new SingleFeatureLinearUniformInterpolator()); + this.imputer = new LinearUniformImputer(false); } @Test - public void testInterpolation() { - double[][] actual = interpolator.interpolate(input, numInterpolants); + public void testImputation() { + double[][] actual = imputer.impute(input, numInterpolants); double delta = 1e-8; int numFeatures = expected.length; diff --git a/src/test/java/org/opensearch/timeseries/dataprocessor/PreviousValueImputerTests.java b/src/test/java/org/opensearch/timeseries/dataprocessor/PreviousValueImputerTests.java new file mode 100644 index 000000000..ff0cfbba2 --- /dev/null +++ b/src/test/java/org/opensearch/timeseries/dataprocessor/PreviousValueImputerTests.java @@ -0,0 +1,28 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.timeseries.dataprocessor; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.jupiter.api.Test; + +public class PreviousValueImputerTests { + @Test + void testSingleFeatureImpute() { + PreviousValueImputer imputer = new PreviousValueImputer(); + + double[] samples = { 1.0, Double.NaN, 3.0, Double.NaN, 5.0 }; + double[] expected = { 1.0, 1.0, 3.0, 3.0, 5.0 }; + + assertArrayEquals(expected, imputer.singleFeatureImpute(samples, 0), "Imputation failed"); + + // The second test checks whether the method removes leading Double.NaN values from the array + samples = new double[] { Double.NaN, 2.0, Double.NaN, 4.0 }; + expected = new double[] { Double.NaN, 2.0, 2.0, 4.0 }; + + assertArrayEquals(expected, imputer.singleFeatureImpute(samples, 0), "Imputation failed with leading NaN"); + } +} diff --git a/src/test/java/org/opensearch/ad/dataprocessor/IntegerSensitiveSingleFeatureLinearUniformInterpolatorTests.java b/src/test/java/org/opensearch/timeseries/dataprocessor/SingleFeatureLinearUniformImputerTests.java similarity index 54% rename from src/test/java/org/opensearch/ad/dataprocessor/IntegerSensitiveSingleFeatureLinearUniformInterpolatorTests.java rename to src/test/java/org/opensearch/timeseries/dataprocessor/SingleFeatureLinearUniformImputerTests.java index 045c05297..17aae0422 100644 --- a/src/test/java/org/opensearch/ad/dataprocessor/IntegerSensitiveSingleFeatureLinearUniformInterpolatorTests.java +++ b/src/test/java/org/opensearch/timeseries/dataprocessor/SingleFeatureLinearUniformImputerTests.java @@ -9,7 +9,7 @@ * GitHub history for details. */ -package org.opensearch.ad.dataprocessor; +package org.opensearch.timeseries.dataprocessor; import static org.junit.Assert.assertTrue; @@ -23,25 +23,28 @@ import org.junit.runner.RunWith; @RunWith(JUnitParamsRunner.class) -public class IntegerSensitiveSingleFeatureLinearUniformInterpolatorTests { +public class SingleFeatureLinearUniformImputerTests { - private IntegerSensitiveSingleFeatureLinearUniformInterpolator interpolator; + private Imputer imputer; @Before public void setup() { - interpolator = new IntegerSensitiveSingleFeatureLinearUniformInterpolator(); + imputer = new LinearUniformImputer(false); } - private Object[] interpolateData() { + private Object[] imputeData() { return new Object[] { new Object[] { new double[] { 25.25, 25.75 }, 3, new double[] { 25.25, 25.5, 25.75 } }, new Object[] { new double[] { 25, 75 }, 3, new double[] { 25, 50, 75 } }, - new Object[] { new double[] { 25, 75.5 }, 3, new double[] { 25, 50.25, 75.5 } }, }; + new Object[] { new double[] { 25, 75.5 }, 3, new double[] { 25, 50.25, 75.5 } }, + new Object[] { new double[] { 25.25, 25.75 }, 3, new double[] { 25.25, 25.5, 25.75 } }, + new Object[] { new double[] { 25, 75 }, 3, new double[] { 25, 50, 75 } }, + new Object[] { new double[] { 25, 75.5 }, 3, new double[] { 25, 50.25, 75.5 } } }; } @Test - @Parameters(method = "interpolateData") - public void interpolate_returnExpected(double[] samples, int num, double[] expected) { - assertTrue(Arrays.equals(expected, interpolator.interpolate(samples, num))); + @Parameters(method = "imputeData") + public void impute_returnExpected(double[] samples, int num, double[] expected) { + assertTrue(Arrays.equals(expected, imputer.singleFeatureImpute(samples, num))); } } diff --git a/src/test/java/org/opensearch/timeseries/dataprocessor/ZeroImputerTests.java b/src/test/java/org/opensearch/timeseries/dataprocessor/ZeroImputerTests.java new file mode 100644 index 000000000..a13189db2 --- /dev/null +++ b/src/test/java/org/opensearch/timeseries/dataprocessor/ZeroImputerTests.java @@ -0,0 +1,39 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.timeseries.dataprocessor; + +import static org.junit.Assert.assertArrayEquals; + +import junitparams.JUnitParamsRunner; +import junitparams.Parameters; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(JUnitParamsRunner.class) +public class ZeroImputerTests { + + private Imputer imputer; + + @Before + public void setup() { + imputer = new ZeroImputer(); + } + + private Object[] imputeData() { + return new Object[] { + new Object[] { new double[] { 25.25, Double.NaN, 25.75 }, 3, new double[] { 25.25, 0, 25.75 } }, + new Object[] { new double[] { Double.NaN, 25, 75 }, 3, new double[] { 0, 25, 75 } }, + new Object[] { new double[] { 25, 75.5, Double.NaN }, 3, new double[] { 25, 75.5, 0 } }, }; + } + + @Test + @Parameters(method = "imputeData") + public void impute_returnExpected(double[] samples, int num, double[] expected) { + assertArrayEquals("The arrays are not equal", expected, imputer.singleFeatureImpute(samples, num), 0.001); + } +} diff --git a/src/test/java/org/opensearch/timeseries/util/LTrimTests.java b/src/test/java/org/opensearch/timeseries/util/LTrimTests.java new file mode 100644 index 000000000..384982828 --- /dev/null +++ b/src/test/java/org/opensearch/timeseries/util/LTrimTests.java @@ -0,0 +1,43 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.timeseries.util; + +import org.opensearch.test.OpenSearchTestCase; + +public class LTrimTests extends OpenSearchTestCase { + + public void testLtrimEmptyArray() { + + double[][] input = {}; + double[][] expectedOutput = {}; + + assertArrayEquals(expectedOutput, DataUtil.ltrim(input)); + } + + public void testLtrimAllNaN() { + + double[][] input = { { Double.NaN, Double.NaN }, { Double.NaN, Double.NaN }, { Double.NaN, Double.NaN } }; + double[][] expectedOutput = {}; + + assertArrayEquals(expectedOutput, DataUtil.ltrim(input)); + } + + public void testLtrimSomeNaN() { + + double[][] input = { { Double.NaN, Double.NaN }, { 1.0, 2.0 }, { 3.0, 4.0 } }; + double[][] expectedOutput = { { 1.0, 2.0 }, { 3.0, 4.0 } }; + + assertArrayEquals(expectedOutput, DataUtil.ltrim(input)); + } + + public void testLtrimNoNaN() { + + double[][] input = { { 1.0, 2.0 }, { 3.0, 4.0 }, { 5.0, 6.0 } }; + double[][] expectedOutput = { { 1.0, 2.0 }, { 3.0, 4.0 }, { 5.0, 6.0 } }; + + assertArrayEquals(expectedOutput, DataUtil.ltrim(input)); + } +}