From a39bfcc5f3424a90dcf538b212d57590fb562eae Mon Sep 17 00:00:00 2001 From: Kaituo Li Date: Wed, 17 May 2023 11:33:09 -0700 Subject: [PATCH 1/2] Implementing Multiple Imputation Methods for Missing Values This Pull Request introduces multiple imputation methods to handle missing data in Anomaly Detection (AD) and forecasting. The ability to handle missing data is crucial to improve the robustness and accuracy of our models. The following imputation methods have been implemented: * Zero Imputation (ZERO): This method replaces all missing values with 0's. It's a simple approach, but it may introduce bias if the data is not centered around zero. * Fixed Values Imputation (FIXED_VALUES): This method replaces missing values with a predefined set of values. The values are the same for each input dimension, and they need to be specified by the user. * Previous Value Imputation (PREVIOUS): This method replaces missing values with the last known value in the respective input dimension. It's a commonly used method for time series data, where temporal continuity is expected. * Linear Interpolation (LINEAR): This method estimates missing values by interpolating linearly between known values in the respective input dimension. This method assumes that the data follows a linear trend. These methods are designed to provide a set of options for users to handle missing data based on their specific needs and the nature of their data. Testing Done: The code changes in this pull request have been validated through a gradle build to ensure that all new and existing tests pass successfully. Signed-off-by: Kaituo Li --- .../opensearch/ad/AnomalyDetectorPlugin.java | 19 +-- .../ad/caching/PriorityTracker.java | 2 +- ...ingleFeatureLinearUniformInterpolator.java | 40 ------ .../ad/dataprocessor/Interpolator.java | 37 ----- .../LinearUniformInterpolator.java | 57 -------- ...ingleFeatureLinearUniformInterpolator.java | 75 ---------- .../opensearch/ad/feature/FeatureManager.java | 16 +-- .../ad/feature/SearchFeatureDao.java | 16 +-- .../opensearch/ad/ml/EntityColdStarter.java | 18 +-- .../java/org/opensearch/ad/model/ADTask.java | 2 +- .../opensearch/ad/model/ADTaskProfile.java | 2 +- .../opensearch/ad/model/AnomalyDetector.java | 2 +- .../model/AnomalyDetectorExecutionInput.java | 2 +- .../opensearch/ad/model/AnomalyResult.java | 2 +- .../ad/model/AnomalyResultBucket.java | 2 +- .../ad/model/DetectionDateRange.java | 2 +- .../ad/model/DetectorInternalState.java | 2 +- .../java/org/opensearch/ad/model/Entity.java | 2 +- .../java/org/opensearch/ad/model/Feature.java | 2 +- .../org/opensearch/ad/model/FeatureData.java | 2 +- .../ad/model/IntervalTimeConfiguration.java | 2 +- .../annotation/Generated.java | 2 +- .../dataprocessor/FixedValueImputer.java | 53 ++++++++ .../dataprocessor/ImputationMethod.java | 25 ++++ .../dataprocessor/ImputationOption.java | 128 ++++++++++++++++++ .../timeseries/dataprocessor/Imputer.java | 54 ++++++++ .../dataprocessor/LinearUniformImputer.java | 88 ++++++++++++ .../dataprocessor/PreviousValueImputer.java | 48 +++++++ .../timeseries/dataprocessor/ZeroImputer.java | 27 ++++ .../opensearch/timeseries/util/DataUtil.java | 54 ++++++++ ...FeatureLinearUniformInterpolatorTests.java | 71 ---------- .../ad/feature/FeatureManagerTests.java | 20 ++- .../NoPowermockSearchFeatureDaoTests.java | 14 +- .../feature/SearchFeatureDaoParamTests.java | 11 +- .../ad/feature/SearchFeatureDaoTests.java | 11 +- .../ad/ml/AbstractCosineDataTest.java | 16 +-- .../ad/ml/EntityColdStarterTests.java | 20 +-- .../opensearch/ad/ml/HCADModelPerfTests.java | 4 +- .../opensearch/ad/ml/ModelManagerTests.java | 8 +- .../ad/rest/AnomalyDetectorRestApiIT.java | 6 +- .../opensearch/ad/rest/SecureADRestIT.java | 58 +++++--- .../dataprocessor/FixedValueImputerTests.java | 40 ++++++ .../dataprocessor/ImputationOptionTests.java | 124 +++++++++++++++++ ...gerSensitiveLinearUniformImputerTests.java | 78 +++++++++++ ...ultiFeatureLinearUniformImputerTests.java} | 15 +- .../PreviousValueImputerTests.java | 28 ++++ ...ngleFeatureLinearUniformImputerTests.java} | 21 +-- .../dataprocessor/ZeroImputerTests.java | 45 ++++++ .../timeseries/util/LTrimTests.java | 49 +++++++ 49 files changed, 996 insertions(+), 426 deletions(-) delete mode 100644 src/main/java/org/opensearch/ad/dataprocessor/IntegerSensitiveSingleFeatureLinearUniformInterpolator.java delete mode 100644 src/main/java/org/opensearch/ad/dataprocessor/Interpolator.java delete mode 100644 src/main/java/org/opensearch/ad/dataprocessor/LinearUniformInterpolator.java delete mode 100644 src/main/java/org/opensearch/ad/dataprocessor/SingleFeatureLinearUniformInterpolator.java rename src/main/java/org/opensearch/{ad => timeseries}/annotation/Generated.java (94%) create mode 100644 src/main/java/org/opensearch/timeseries/dataprocessor/FixedValueImputer.java create mode 100644 src/main/java/org/opensearch/timeseries/dataprocessor/ImputationMethod.java create mode 100644 src/main/java/org/opensearch/timeseries/dataprocessor/ImputationOption.java create mode 100644 src/main/java/org/opensearch/timeseries/dataprocessor/Imputer.java create mode 100644 src/main/java/org/opensearch/timeseries/dataprocessor/LinearUniformImputer.java create mode 100644 src/main/java/org/opensearch/timeseries/dataprocessor/PreviousValueImputer.java create mode 100644 src/main/java/org/opensearch/timeseries/dataprocessor/ZeroImputer.java create mode 100644 src/main/java/org/opensearch/timeseries/util/DataUtil.java delete mode 100644 src/test/java/org/opensearch/ad/dataprocessor/SingleFeatureLinearUniformInterpolatorTests.java create mode 100644 src/test/java/org/opensearch/timeseries/dataprocessor/FixedValueImputerTests.java create mode 100644 src/test/java/org/opensearch/timeseries/dataprocessor/ImputationOptionTests.java create mode 100644 src/test/java/org/opensearch/timeseries/dataprocessor/IntegerSensitiveLinearUniformImputerTests.java rename src/test/java/org/opensearch/{ad/dataprocessor/LinearUniformInterpolatorTests.java => timeseries/dataprocessor/MultiFeatureLinearUniformImputerTests.java} (81%) create mode 100644 src/test/java/org/opensearch/timeseries/dataprocessor/PreviousValueImputerTests.java rename src/test/java/org/opensearch/{ad/dataprocessor/IntegerSensitiveSingleFeatureLinearUniformInterpolatorTests.java => timeseries/dataprocessor/SingleFeatureLinearUniformImputerTests.java} (54%) create mode 100644 src/test/java/org/opensearch/timeseries/dataprocessor/ZeroImputerTests.java create mode 100644 src/test/java/org/opensearch/timeseries/util/LTrimTests.java diff --git a/src/main/java/org/opensearch/ad/AnomalyDetectorPlugin.java b/src/main/java/org/opensearch/ad/AnomalyDetectorPlugin.java index 90a68339c..d91f86466 100644 --- a/src/main/java/org/opensearch/ad/AnomalyDetectorPlugin.java +++ b/src/main/java/org/opensearch/ad/AnomalyDetectorPlugin.java @@ -43,10 +43,6 @@ import org.opensearch.ad.cluster.ClusterManagerEventListener; import org.opensearch.ad.cluster.HashRing; import org.opensearch.ad.constant.ADCommonName; -import org.opensearch.ad.dataprocessor.IntegerSensitiveSingleFeatureLinearUniformInterpolator; -import org.opensearch.ad.dataprocessor.Interpolator; -import org.opensearch.ad.dataprocessor.LinearUniformInterpolator; -import org.opensearch.ad.dataprocessor.SingleFeatureLinearUniformInterpolator; import org.opensearch.ad.feature.FeatureManager; import org.opensearch.ad.feature.SearchFeatureDao; import org.opensearch.ad.indices.AnomalyDetectionIndices; @@ -195,6 +191,8 @@ import org.opensearch.threadpool.ScalingExecutorBuilder; import org.opensearch.threadpool.ThreadPool; import org.opensearch.timeseries.constant.CommonName; +import org.opensearch.timeseries.dataprocessor.Imputer; +import org.opensearch.timeseries.dataprocessor.LinearUniformImputer; import org.opensearch.timeseries.stats.StatNames; import org.opensearch.watcher.ResourceWatcherService; @@ -344,9 +342,7 @@ public Collection createComponents( ); this.clusterService = clusterService; - SingleFeatureLinearUniformInterpolator singleFeatureLinearUniformInterpolator = - new IntegerSensitiveSingleFeatureLinearUniformInterpolator(); - Interpolator interpolator = new LinearUniformInterpolator(singleFeatureLinearUniformInterpolator); + Imputer imputer = new LinearUniformImputer(true); stateManager = new NodeStateManager( client, xContentRegistry, @@ -360,7 +356,7 @@ public Collection createComponents( SearchFeatureDao searchFeatureDao = new SearchFeatureDao( client, xContentRegistry, - interpolator, + imputer, securityClientUtil, settings, clusterService, @@ -388,7 +384,7 @@ public Collection createComponents( FeatureManager featureManager = new FeatureManager( searchFeatureDao, - interpolator, + imputer, getClock(), AnomalyDetectorSettings.MAX_TRAIN_SAMPLE, AnomalyDetectorSettings.MAX_SAMPLE_STRIDE, @@ -534,7 +530,7 @@ public PooledObject wrap(LinkedBuffer obj) { AnomalyDetectorSettings.NUM_MIN_SAMPLES, AnomalyDetectorSettings.MAX_SAMPLE_STRIDE, AnomalyDetectorSettings.MAX_TRAIN_SAMPLE, - interpolator, + imputer, searchFeatureDao, AnomalyDetectorSettings.THRESHOLD_MIN_PVALUE, featureManager, @@ -779,8 +775,7 @@ public PooledObject wrap(LinkedBuffer obj) { anomalyDetectionIndices, anomalyDetectorRunner, searchFeatureDao, - singleFeatureLinearUniformInterpolator, - interpolator, + imputer, gson, jvmService, hashRing, diff --git a/src/main/java/org/opensearch/ad/caching/PriorityTracker.java b/src/main/java/org/opensearch/ad/caching/PriorityTracker.java index 05304912f..439d67679 100644 --- a/src/main/java/org/opensearch/ad/caching/PriorityTracker.java +++ b/src/main/java/org/opensearch/ad/caching/PriorityTracker.java @@ -27,7 +27,7 @@ import org.apache.commons.lang.builder.ToStringBuilder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.ad.annotation.Generated; +import org.opensearch.timeseries.annotation.Generated; /** * A priority tracker for entities. Read docs/entity-priority.pdf for details. diff --git a/src/main/java/org/opensearch/ad/dataprocessor/IntegerSensitiveSingleFeatureLinearUniformInterpolator.java b/src/main/java/org/opensearch/ad/dataprocessor/IntegerSensitiveSingleFeatureLinearUniformInterpolator.java deleted file mode 100644 index cc187d7f8..000000000 --- a/src/main/java/org/opensearch/ad/dataprocessor/IntegerSensitiveSingleFeatureLinearUniformInterpolator.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.ad.dataprocessor; - -import java.util.Arrays; - -import com.google.common.math.DoubleMath; - -/** - * Interpolator sensitive to integral values. - */ -public class IntegerSensitiveSingleFeatureLinearUniformInterpolator extends SingleFeatureLinearUniformInterpolator { - - /** - * Interpolates integral/floating-point results. - * - * If all samples are integral, the results are integral. - * Else, the results are floating points. - * - * @param samples integral/floating-point samples - * @param numInterpolants the number of interpolants - * @return {code numInterpolants} interpolated results - */ - public double[] interpolate(double[] samples, int numInterpolants) { - double[] interpolants = super.interpolate(samples, numInterpolants); - if (Arrays.stream(samples).allMatch(DoubleMath::isMathematicalInteger)) { - interpolants = Arrays.stream(interpolants).map(Math::rint).toArray(); - } - return interpolants; - } -} diff --git a/src/main/java/org/opensearch/ad/dataprocessor/Interpolator.java b/src/main/java/org/opensearch/ad/dataprocessor/Interpolator.java deleted file mode 100644 index 752498e88..000000000 --- a/src/main/java/org/opensearch/ad/dataprocessor/Interpolator.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.ad.dataprocessor; - -/* - * An object for interpolating feature vectors. - * - * In certain situations, due to time and compute cost, we are only allowed to - * query a sparse sample of data points / feature vectors from a cluster. - * However, we need a large sample of feature vectors in order to train our - * anomaly detection algorithms. An Interpolator approximates the data points - * between a given, ordered list of samples. - */ -public interface Interpolator { - - /* - * Interpolates the given sample feature vectors. - * - * Computes a list `numInterpolants` feature vectors using the ordered list - * of `numSamples` input sample vectors where each sample vector has size - * `numFeatures`. - * - * @param samples A `numFeatures x numSamples` list of feature vectors. - * @param numInterpolants The desired number of interpolating vectors. - * @return A `numFeatures x numInterpolants` list of feature vectors. - */ - double[][] interpolate(double[][] samples, int numInterpolants); -} diff --git a/src/main/java/org/opensearch/ad/dataprocessor/LinearUniformInterpolator.java b/src/main/java/org/opensearch/ad/dataprocessor/LinearUniformInterpolator.java deleted file mode 100644 index b62aeda9b..000000000 --- a/src/main/java/org/opensearch/ad/dataprocessor/LinearUniformInterpolator.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.ad.dataprocessor; - -/* - * A piecewise linear interpolator with uniformly spaced points. - * - * The LinearUniformInterpolator constructs a piecewise linear interpolation on - * the input list of sample feature vectors. That is, between every consecutive - * pair of points we construct a linear interpolation. The linear interpolation - * is computed on a per-feature basis. - * - * This class uses the helper class SingleFeatureLinearUniformInterpolator to - * compute per-feature interpolants. - * - * @see SingleFeatureLinearUniformInterpolator - */ -public class LinearUniformInterpolator implements Interpolator { - - private SingleFeatureLinearUniformInterpolator singleFeatureLinearUniformInterpolator; - - public LinearUniformInterpolator(SingleFeatureLinearUniformInterpolator singleFeatureLinearUniformInterpolator) { - this.singleFeatureLinearUniformInterpolator = singleFeatureLinearUniformInterpolator; - } - - /* - * Piecewise linearly interpolates the given sample feature vectors. - * - * Computes a list `numInterpolants` feature vectors using the ordered list - * of `numSamples` input sample vectors where each sample vector has size - * `numFeatures`. The feature vectors are computing using a piecewise linear - * interpolation. - * - * @param samples A `numFeatures x numSamples` list of feature vectors. - * @param numInterpolants The desired number of interpolating feature vectors. - * @return A `numFeatures x numInterpolants` list of feature vectors. - * @see SingleFeatureLinearUniformInterpolator - */ - public double[][] interpolate(double[][] samples, int numInterpolants) { - int numFeatures = samples.length; - double[][] interpolants = new double[numFeatures][numInterpolants]; - - for (int featureIndex = 0; featureIndex < numFeatures; featureIndex++) { - interpolants[featureIndex] = this.singleFeatureLinearUniformInterpolator.interpolate(samples[featureIndex], numInterpolants); - } - return interpolants; - } -} diff --git a/src/main/java/org/opensearch/ad/dataprocessor/SingleFeatureLinearUniformInterpolator.java b/src/main/java/org/opensearch/ad/dataprocessor/SingleFeatureLinearUniformInterpolator.java deleted file mode 100644 index 6349f29d3..000000000 --- a/src/main/java/org/opensearch/ad/dataprocessor/SingleFeatureLinearUniformInterpolator.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.ad.dataprocessor; - -import java.util.Arrays; - -/* - * A piecewise linear interpolator in a single feature dimension. - * - * A utility class for LinearUniformInterpolator. Constructs uniformly spaced - * piecewise linear interpolations within a single feature dimension. - * - * @see LinearUniformInterpolator - */ -public class SingleFeatureLinearUniformInterpolator { - - /* - * Piecewise linearly interpolates the given sample of one-dimensional - * features. - * - * Computes a list `numInterpolants` features using the ordered list of - * `numSamples` input one-dimensional samples. The interpolant features are - * computing using a piecewise linear interpolation. - * - * @param samples A `numSamples` sized list of sample features. - * @param numInterpolants The desired number of interpolating features. - * @return A `numInterpolants` sized array of interpolant features. - * @see LinearUniformInterpolator - */ - public double[] interpolate(double[] samples, int numInterpolants) { - int numSamples = samples.length; - double[] interpolants = new double[numInterpolants]; - - if (numSamples == 0) { - interpolants = new double[0]; - } else if (numSamples == 1) { - Arrays.fill(interpolants, samples[0]); - } else { - /* assume the piecewise linear interpolation between the samples is a - parameterized curve f(t) for t in [0, 1]. Each pair of samples - determines a interval [t_i, t_(i+1)]. For each interpolant we determine - which interval it lies inside and then scale the value of t, - accordingly to compute the interpolant value. - - for numerical stability reasons we omit processing the final - interpolant in this loop since this last interpolant is always equal - to the last sample. - */ - for (int interpolantIndex = 0; interpolantIndex < (numInterpolants - 1); interpolantIndex++) { - double tGlobal = ((double) interpolantIndex) / (numInterpolants - 1.0); - double tInterval = tGlobal * (numSamples - 1.0); - int intervalIndex = (int) Math.floor(tInterval); - tInterval -= intervalIndex; - - double leftSample = samples[intervalIndex]; - double rightSample = samples[intervalIndex + 1]; - double interpolant = (1.0 - tInterval) * leftSample + tInterval * rightSample; - interpolants[interpolantIndex] = interpolant; - } - - // the final interpolant is always the final sample - interpolants[numInterpolants - 1] = samples[numSamples - 1]; - } - return interpolants; - } -} diff --git a/src/main/java/org/opensearch/ad/feature/FeatureManager.java b/src/main/java/org/opensearch/ad/feature/FeatureManager.java index cb86d2b86..07d5aef55 100644 --- a/src/main/java/org/opensearch/ad/feature/FeatureManager.java +++ b/src/main/java/org/opensearch/ad/feature/FeatureManager.java @@ -42,11 +42,11 @@ import org.opensearch.action.support.ThreadedActionListener; import org.opensearch.ad.CleanState; import org.opensearch.ad.common.exception.EndRunException; -import org.opensearch.ad.dataprocessor.Interpolator; import org.opensearch.ad.model.AnomalyDetector; import org.opensearch.ad.model.Entity; import org.opensearch.threadpool.ThreadPool; import org.opensearch.timeseries.constant.CommonMessages; +import org.opensearch.timeseries.dataprocessor.Imputer; /** * A facade managing feature data operations and buffers. @@ -59,7 +59,7 @@ public class FeatureManager implements CleanState { private final Map>>> detectorIdsToTimeShingles; private final SearchFeatureDao searchFeatureDao; - private final Interpolator interpolator; + private final Imputer imputer; private final Clock clock; private final int maxTrainSamples; @@ -78,7 +78,7 @@ public class FeatureManager implements CleanState { * Constructor with dependencies and configuration. * * @param searchFeatureDao DAO of features from search - * @param interpolator interpolator of samples + * @param imputer imputer of samples * @param clock clock for system time * @param maxTrainSamples max number of samples from search * @param maxSampleStride max stride between uninterpolated train samples @@ -94,7 +94,7 @@ public class FeatureManager implements CleanState { */ public FeatureManager( SearchFeatureDao searchFeatureDao, - Interpolator interpolator, + Imputer imputer, Clock clock, int maxTrainSamples, int maxSampleStride, @@ -109,7 +109,7 @@ public FeatureManager( String adThreadPoolName ) { this.searchFeatureDao = searchFeatureDao; - this.interpolator = interpolator; + this.imputer = imputer; this.clock = clock; this.maxTrainSamples = maxTrainSamples; this.maxSampleStride = maxSampleStride; @@ -592,8 +592,8 @@ void getSamplesForRanges( private List> getPreviewRanges(List> ranges, int stride, int shingleSize) { double[] rangeStarts = ranges.stream().mapToDouble(Entry::getKey).toArray(); double[] rangeEnds = ranges.stream().mapToDouble(Entry::getValue).toArray(); - double[] previewRangeStarts = interpolator.interpolate(new double[][] { rangeStarts }, stride * (ranges.size() - 1) + 1)[0]; - double[] previewRangeEnds = interpolator.interpolate(new double[][] { rangeEnds }, stride * (ranges.size() - 1) + 1)[0]; + double[] previewRangeStarts = imputer.impute(new double[][] { rangeStarts }, stride * (ranges.size() - 1) + 1)[0]; + double[] previewRangeEnds = imputer.impute(new double[][] { rangeEnds }, stride * (ranges.size() - 1) + 1)[0]; List> previewRanges = IntStream .range(shingleSize - 1, previewRangeStarts.length) .mapToObj(i -> new SimpleImmutableEntry<>((long) previewRangeStarts[i], (long) previewRangeEnds[i])) @@ -614,7 +614,7 @@ private Entry getPreviewFeatures(double[][] samples, int Entry unprocessedAndProcessed = Optional .of(samples) .map(m -> transpose(m)) - .map(m -> interpolator.interpolate(m, stride * (samples.length - 1) + 1)) + .map(m -> imputer.impute(m, stride * (samples.length - 1) + 1)) .map(m -> transpose(m)) .map(m -> new SimpleImmutableEntry<>(copyOfRange(m, shingleSize - 1, m.length), batchShingle(m, shingleSize))) .get(); diff --git a/src/main/java/org/opensearch/ad/feature/SearchFeatureDao.java b/src/main/java/org/opensearch/ad/feature/SearchFeatureDao.java index cdbab8c3b..8c73f3538 100644 --- a/src/main/java/org/opensearch/ad/feature/SearchFeatureDao.java +++ b/src/main/java/org/opensearch/ad/feature/SearchFeatureDao.java @@ -42,7 +42,6 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.ad.common.exception.AnomalyDetectionException; import org.opensearch.ad.constant.ADCommonName; -import org.opensearch.ad.dataprocessor.Interpolator; import org.opensearch.ad.model.AnomalyDetector; import org.opensearch.ad.model.Entity; import org.opensearch.ad.model.IntervalTimeConfiguration; @@ -72,6 +71,7 @@ import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.search.sort.FieldSortBuilder; import org.opensearch.search.sort.SortOrder; +import org.opensearch.timeseries.dataprocessor.Imputer; /** * DAO for features from search. @@ -86,7 +86,7 @@ public class SearchFeatureDao extends AbstractRetriever { // Dependencies private final Client client; private final NamedXContentRegistry xContent; - private final Interpolator interpolator; + private final Imputer imputer; private final SecurityClientUtil clientUtil; private volatile int maxEntitiesForPreview; private volatile int pageSize; @@ -98,7 +98,7 @@ public class SearchFeatureDao extends AbstractRetriever { public SearchFeatureDao( Client client, NamedXContentRegistry xContent, - Interpolator interpolator, + Imputer imputer, SecurityClientUtil clientUtil, Settings settings, ClusterService clusterService, @@ -110,7 +110,7 @@ public SearchFeatureDao( ) { this.client = client; this.xContent = xContent; - this.interpolator = interpolator; + this.imputer = imputer; this.clientUtil = clientUtil; this.maxEntitiesForPreview = maxEntitiesForPreview; @@ -130,7 +130,7 @@ public SearchFeatureDao( * * @param client ES client for queries * @param xContent ES XContentRegistry - * @param interpolator interpolator for missing values + * @param imputer imputer for missing values * @param clientUtil utility for ES client * @param settings ES settings * @param clusterService ES ClusterService @@ -140,7 +140,7 @@ public SearchFeatureDao( public SearchFeatureDao( Client client, NamedXContentRegistry xContent, - Interpolator interpolator, + Imputer imputer, SecurityClientUtil clientUtil, Settings settings, ClusterService clusterService, @@ -149,7 +149,7 @@ public SearchFeatureDao( this( client, xContent, - interpolator, + imputer, clientUtil, settings, clusterService, @@ -825,7 +825,7 @@ private Optional toMatrix(ArrayDeque sampledFeatures) { } private double[] getInterpolants(double[] previous, double[] next) { - return transpose(interpolator.interpolate(transpose(new double[][] { previous, next }), 3))[1]; + return transpose(imputer.impute(transpose(new double[][] { previous, next }), 3))[1]; } private double[][] transpose(double[][] matrix) { diff --git a/src/main/java/org/opensearch/ad/ml/EntityColdStarter.java b/src/main/java/org/opensearch/ad/ml/EntityColdStarter.java index 2c9095eb8..a7591752b 100644 --- a/src/main/java/org/opensearch/ad/ml/EntityColdStarter.java +++ b/src/main/java/org/opensearch/ad/ml/EntityColdStarter.java @@ -44,7 +44,6 @@ import org.opensearch.ad.caching.DoorKeeper; import org.opensearch.ad.common.exception.AnomalyDetectionException; import org.opensearch.ad.common.exception.EndRunException; -import org.opensearch.ad.dataprocessor.Interpolator; import org.opensearch.ad.feature.FeatureManager; import org.opensearch.ad.feature.SearchFeatureDao; import org.opensearch.ad.model.AnomalyDetector; @@ -57,6 +56,7 @@ import org.opensearch.ad.util.ExceptionUtil; import org.opensearch.common.settings.Settings; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.timeseries.dataprocessor.Imputer; import com.amazon.randomcutforest.config.Precision; import com.amazon.randomcutforest.parkservices.ThresholdedRandomCutForest; @@ -77,7 +77,7 @@ public class EntityColdStarter implements MaintenanceState, CleanState { private final double thresholdMinPvalue; private final int defaulStrideLength; private final int defaultNumberOfSamples; - private final Interpolator interpolator; + private final Imputer imputer; private final SearchFeatureDao searchFeatureDao; private Instant lastThrottledColdStartTime; private final FeatureManager featureManager; @@ -108,7 +108,7 @@ public class EntityColdStarter implements MaintenanceState, CleanState { * results are returned. * @param defaultSampleStride default sample distances measured in detector intervals. * @param defaultTrainSamples Default train samples to collect. - * @param interpolator Used to generate data points between samples. + * @param imputer Used to generate data points between samples. * @param searchFeatureDao Used to issue ES queries. * @param thresholdMinPvalue min P-value for thresholding * @param featureManager Used to create features for models. @@ -130,7 +130,7 @@ public EntityColdStarter( int numMinSamples, int defaultSampleStride, int defaultTrainSamples, - Interpolator interpolator, + Imputer imputer, SearchFeatureDao searchFeatureDao, double thresholdMinPvalue, FeatureManager featureManager, @@ -150,7 +150,7 @@ public EntityColdStarter( this.numMinSamples = numMinSamples; this.defaulStrideLength = defaultSampleStride; this.defaultNumberOfSamples = defaultTrainSamples; - this.interpolator = interpolator; + this.imputer = imputer; this.searchFeatureDao = searchFeatureDao; this.thresholdMinPvalue = thresholdMinPvalue; this.featureManager = featureManager; @@ -173,7 +173,7 @@ public EntityColdStarter( int numMinSamples, int maxSampleStride, int maxTrainSamples, - Interpolator interpolator, + Imputer imputer, SearchFeatureDao searchFeatureDao, double thresholdMinPvalue, FeatureManager featureManager, @@ -192,7 +192,7 @@ public EntityColdStarter( numMinSamples, maxSampleStride, maxTrainSamples, - interpolator, + imputer, searchFeatureDao, thresholdMinPvalue, featureManager, @@ -501,8 +501,8 @@ private void getFeatures( int numInterpolants = (i - lastSample.getLeft()) * stride + 1; double[][] points = featureManager .transpose( - interpolator - .interpolate( + imputer + .impute( featureManager.transpose(new double[][] { lastSample.getRight(), featuresOptional.get() }), numInterpolants ) diff --git a/src/main/java/org/opensearch/ad/model/ADTask.java b/src/main/java/org/opensearch/ad/model/ADTask.java index 9a9bcdbc5..283ff80de 100644 --- a/src/main/java/org/opensearch/ad/model/ADTask.java +++ b/src/main/java/org/opensearch/ad/model/ADTask.java @@ -17,7 +17,6 @@ import java.io.IOException; import java.time.Instant; -import org.opensearch.ad.annotation.Generated; import org.opensearch.ad.util.ParseUtils; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; @@ -26,6 +25,7 @@ import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.timeseries.annotation.Generated; import com.google.common.base.Objects; diff --git a/src/main/java/org/opensearch/ad/model/ADTaskProfile.java b/src/main/java/org/opensearch/ad/model/ADTaskProfile.java index 79c6d90c0..d95c6c579 100644 --- a/src/main/java/org/opensearch/ad/model/ADTaskProfile.java +++ b/src/main/java/org/opensearch/ad/model/ADTaskProfile.java @@ -19,13 +19,13 @@ import java.util.Objects; import org.opensearch.Version; -import org.opensearch.ad.annotation.Generated; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.io.stream.Writeable; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.timeseries.annotation.Generated; /** * One anomaly detection task means one detector starts to run until stopped. diff --git a/src/main/java/org/opensearch/ad/model/AnomalyDetector.java b/src/main/java/org/opensearch/ad/model/AnomalyDetector.java index ab6f8a74c..ae948f304 100644 --- a/src/main/java/org/opensearch/ad/model/AnomalyDetector.java +++ b/src/main/java/org/opensearch/ad/model/AnomalyDetector.java @@ -30,7 +30,6 @@ import java.util.stream.Collectors; import org.apache.logging.log4j.util.Strings; -import org.opensearch.ad.annotation.Generated; import org.opensearch.ad.common.exception.ADValidationException; import org.opensearch.ad.constant.ADCommonMessages; import org.opensearch.ad.constant.CommonValue; @@ -52,6 +51,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; +import org.opensearch.timeseries.annotation.Generated; import org.opensearch.timeseries.constant.CommonMessages; import org.opensearch.timeseries.constant.CommonName; diff --git a/src/main/java/org/opensearch/ad/model/AnomalyDetectorExecutionInput.java b/src/main/java/org/opensearch/ad/model/AnomalyDetectorExecutionInput.java index 6e636d00c..5e05378b4 100644 --- a/src/main/java/org/opensearch/ad/model/AnomalyDetectorExecutionInput.java +++ b/src/main/java/org/opensearch/ad/model/AnomalyDetectorExecutionInput.java @@ -16,12 +16,12 @@ import java.io.IOException; import java.time.Instant; -import org.opensearch.ad.annotation.Generated; import org.opensearch.ad.util.ParseUtils; import org.opensearch.common.Strings; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.timeseries.annotation.Generated; import com.google.common.base.Objects; diff --git a/src/main/java/org/opensearch/ad/model/AnomalyResult.java b/src/main/java/org/opensearch/ad/model/AnomalyResult.java index 8eea5b394..d14371f07 100644 --- a/src/main/java/org/opensearch/ad/model/AnomalyResult.java +++ b/src/main/java/org/opensearch/ad/model/AnomalyResult.java @@ -24,7 +24,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.opensearch.ad.annotation.Generated; import org.opensearch.ad.constant.CommonValue; import org.opensearch.ad.ml.ThresholdingResult; import org.opensearch.ad.util.ParseUtils; @@ -37,6 +36,7 @@ import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.timeseries.annotation.Generated; import org.opensearch.timeseries.constant.CommonName; import com.google.common.base.Objects; diff --git a/src/main/java/org/opensearch/ad/model/AnomalyResultBucket.java b/src/main/java/org/opensearch/ad/model/AnomalyResultBucket.java index 415e21481..8f91f34b5 100644 --- a/src/main/java/org/opensearch/ad/model/AnomalyResultBucket.java +++ b/src/main/java/org/opensearch/ad/model/AnomalyResultBucket.java @@ -15,7 +15,6 @@ import java.util.Map; import org.apache.commons.lang.builder.ToStringBuilder; -import org.opensearch.ad.annotation.Generated; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.io.stream.Writeable; @@ -23,6 +22,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.search.aggregations.bucket.composite.CompositeAggregation.Bucket; import org.opensearch.search.aggregations.metrics.InternalMax; +import org.opensearch.timeseries.annotation.Generated; import com.google.common.base.Objects; diff --git a/src/main/java/org/opensearch/ad/model/DetectionDateRange.java b/src/main/java/org/opensearch/ad/model/DetectionDateRange.java index 717e2cd62..856253379 100644 --- a/src/main/java/org/opensearch/ad/model/DetectionDateRange.java +++ b/src/main/java/org/opensearch/ad/model/DetectionDateRange.java @@ -17,7 +17,6 @@ import java.time.Instant; import org.apache.commons.lang.builder.ToStringBuilder; -import org.opensearch.ad.annotation.Generated; import org.opensearch.ad.util.ParseUtils; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; @@ -25,6 +24,7 @@ import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.timeseries.annotation.Generated; import com.google.common.base.Objects; diff --git a/src/main/java/org/opensearch/ad/model/DetectorInternalState.java b/src/main/java/org/opensearch/ad/model/DetectorInternalState.java index 0ab35cfbd..d8c8c47b1 100644 --- a/src/main/java/org/opensearch/ad/model/DetectorInternalState.java +++ b/src/main/java/org/opensearch/ad/model/DetectorInternalState.java @@ -16,13 +16,13 @@ import java.io.IOException; import java.time.Instant; -import org.opensearch.ad.annotation.Generated; import org.opensearch.ad.util.ParseUtils; import org.opensearch.core.ParseField; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.timeseries.annotation.Generated; import com.google.common.base.Objects; diff --git a/src/main/java/org/opensearch/ad/model/Entity.java b/src/main/java/org/opensearch/ad/model/Entity.java index d346d4034..f5e0085a3 100644 --- a/src/main/java/org/opensearch/ad/model/Entity.java +++ b/src/main/java/org/opensearch/ad/model/Entity.java @@ -25,7 +25,6 @@ import java.util.TreeMap; import org.apache.lucene.util.SetOnce; -import org.opensearch.ad.annotation.Generated; import org.opensearch.ad.settings.AnomalyDetectorSettings; import org.opensearch.common.Numbers; import org.opensearch.common.bytes.BytesReference; @@ -41,6 +40,7 @@ import org.opensearch.core.xcontent.XContentParser; import org.opensearch.core.xcontent.XContentParser.Token; import org.opensearch.index.query.TermQueryBuilder; +import org.opensearch.timeseries.annotation.Generated; import org.opensearch.timeseries.constant.CommonName; import com.google.common.base.Joiner; diff --git a/src/main/java/org/opensearch/ad/model/Feature.java b/src/main/java/org/opensearch/ad/model/Feature.java index 2b9248271..7ec86742a 100644 --- a/src/main/java/org/opensearch/ad/model/Feature.java +++ b/src/main/java/org/opensearch/ad/model/Feature.java @@ -16,7 +16,6 @@ import java.io.IOException; import org.apache.logging.log4j.util.Strings; -import org.opensearch.ad.annotation.Generated; import org.opensearch.ad.util.ParseUtils; import org.opensearch.common.UUIDs; import org.opensearch.common.io.stream.StreamInput; @@ -26,6 +25,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.search.aggregations.AggregationBuilder; +import org.opensearch.timeseries.annotation.Generated; import com.google.common.base.Objects; diff --git a/src/main/java/org/opensearch/ad/model/FeatureData.java b/src/main/java/org/opensearch/ad/model/FeatureData.java index 3b223bbdf..9df55d35d 100644 --- a/src/main/java/org/opensearch/ad/model/FeatureData.java +++ b/src/main/java/org/opensearch/ad/model/FeatureData.java @@ -15,11 +15,11 @@ import java.io.IOException; -import org.opensearch.ad.annotation.Generated; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.timeseries.annotation.Generated; import com.google.common.base.Objects; diff --git a/src/main/java/org/opensearch/ad/model/IntervalTimeConfiguration.java b/src/main/java/org/opensearch/ad/model/IntervalTimeConfiguration.java index e8e124587..cffb00b5f 100644 --- a/src/main/java/org/opensearch/ad/model/IntervalTimeConfiguration.java +++ b/src/main/java/org/opensearch/ad/model/IntervalTimeConfiguration.java @@ -17,11 +17,11 @@ import java.util.Locale; import java.util.Set; -import org.opensearch.ad.annotation.Generated; import org.opensearch.ad.constant.ADCommonMessages; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.timeseries.annotation.Generated; import org.opensearch.timeseries.constant.CommonMessages; import com.google.common.base.Objects; diff --git a/src/main/java/org/opensearch/ad/annotation/Generated.java b/src/main/java/org/opensearch/timeseries/annotation/Generated.java similarity index 94% rename from src/main/java/org/opensearch/ad/annotation/Generated.java rename to src/main/java/org/opensearch/timeseries/annotation/Generated.java index 08156483a..d3812c9ca 100644 --- a/src/main/java/org/opensearch/ad/annotation/Generated.java +++ b/src/main/java/org/opensearch/timeseries/annotation/Generated.java @@ -9,7 +9,7 @@ * GitHub history for details. */ -package org.opensearch.ad.annotation; +package org.opensearch.timeseries.annotation; import java.lang.annotation.ElementType; import java.lang.annotation.Retention; diff --git a/src/main/java/org/opensearch/timeseries/dataprocessor/FixedValueImputer.java b/src/main/java/org/opensearch/timeseries/dataprocessor/FixedValueImputer.java new file mode 100644 index 000000000..90804b1da --- /dev/null +++ b/src/main/java/org/opensearch/timeseries/dataprocessor/FixedValueImputer.java @@ -0,0 +1,53 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.timeseries.dataprocessor; + +import java.util.Arrays; + +/** + * fixing missing value (denoted using Double.NaN) using a fixed set of specified values. + * The 2nd parameter of interpolate is ignored as we infer the number of imputed values + * using the number of Double.NaN. + */ +public class FixedValueImputer extends Imputer { + private double[] fixedValue; + + public FixedValueImputer(double[] fixedValue) { + this.fixedValue = fixedValue; + } + + /** + * Given an array of samples, fill with given value. + * We will ignore the rest of samples beyond the 2nd element. + * + * @return an imputed array of size numImputed + */ + @Override + public double[][] impute(double[][] samples, int numImputed) { + int numFeatures = samples.length; + double[][] imputed = new double[numFeatures][numImputed]; + + for (int featureIndex = 0; featureIndex < numFeatures; featureIndex++) { + imputed[featureIndex] = singleFeatureInterpolate(samples[featureIndex], numImputed, fixedValue[featureIndex]); + } + return imputed; + } + + private double[] singleFeatureInterpolate(double[] samples, int numInterpolants, double defaultVal) { + return Arrays.stream(samples).map(d -> Double.isNaN(d) ? defaultVal : d).toArray(); + } + + @Override + protected double[] singleFeatureImpute(double[] samples, int numInterpolants) { + throw new UnsupportedOperationException("The operation is not supported"); + } +} diff --git a/src/main/java/org/opensearch/timeseries/dataprocessor/ImputationMethod.java b/src/main/java/org/opensearch/timeseries/dataprocessor/ImputationMethod.java new file mode 100644 index 000000000..90494862c --- /dev/null +++ b/src/main/java/org/opensearch/timeseries/dataprocessor/ImputationMethod.java @@ -0,0 +1,25 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.timeseries.dataprocessor; + +public enum ImputationMethod { + /** + * This method replaces all missing values with 0's. It's a simple approach, but it may introduce bias if the data is not centered around zero. + */ + ZERO, + /** + * This method replaces missing values with a predefined set of values. The values are the same for each input dimension, and they need to be specified by the user. + */ + FIXED_VALUES, + /** + * This method replaces missing values with the last known value in the respective input dimension. It's a commonly used method for time series data, where temporal continuity is expected. + */ + PREVIOUS, + /** + * This method estimates missing values by interpolating linearly between known values in the respective input dimension. This method assumes that the data follows a linear trend. + */ + LINEAR +} diff --git a/src/main/java/org/opensearch/timeseries/dataprocessor/ImputationOption.java b/src/main/java/org/opensearch/timeseries/dataprocessor/ImputationOption.java new file mode 100644 index 000000000..9943171e5 --- /dev/null +++ b/src/main/java/org/opensearch/timeseries/dataprocessor/ImputationOption.java @@ -0,0 +1,128 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.timeseries.dataprocessor; + +import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.Optional; + +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; + +public class ImputationOption implements Writeable, ToXContent { + // field name in toXContent + public static final String METHOD_FIELD = "method"; + public static final String DEFAULT_FILL_FIELD = "defaultFill"; + + private final ImputationMethod method; + private final Optional defaultFill; + + public ImputationOption(ImputationMethod method, Optional defaultFill) { + this.method = method; + this.defaultFill = defaultFill; + } + + public ImputationOption(ImputationMethod method) { + this(method, Optional.empty()); + } + + public ImputationOption(StreamInput in) throws IOException { + this.method = in.readEnum(ImputationMethod.class); + if (in.readBoolean()) { + this.defaultFill = Optional.of(in.readDoubleArray()); + } else { + this.defaultFill = Optional.empty(); + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeEnum(method); + if (defaultFill.isEmpty()) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + out.writeDoubleArray(defaultFill.get()); + } + } + + public static ImputationOption parse(XContentParser parser) throws IOException { + ImputationMethod method = ImputationMethod.ZERO; + List defaultFill = new ArrayList<>(); + + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = parser.currentName(); + parser.nextToken(); + switch (fieldName) { + case METHOD_FIELD: + method = ImputationMethod.valueOf(parser.text().toUpperCase(Locale.ROOT)); + break; + case DEFAULT_FILL_FIELD: + ensureExpectedToken(XContentParser.Token.START_ARRAY, parser.currentToken(), parser); + while (parser.nextToken() != XContentParser.Token.END_ARRAY) { + defaultFill.add(parser.doubleValue()); + } + break; + default: + break; + } + } + return new ImputationOption(method, Optional.of(defaultFill.stream().mapToDouble(Double::doubleValue).toArray())); + } + + public XContentBuilder toXContent(XContentBuilder builder) throws IOException { + return toXContent(builder, ToXContent.EMPTY_PARAMS); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + XContentBuilder xContentBuilder = builder.startObject(); + + xContentBuilder.field(METHOD_FIELD, method); + + if (!defaultFill.isEmpty()) { + builder.array(DEFAULT_FILL_FIELD, defaultFill.get()); + } + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (o == this) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + ImputationOption other = (ImputationOption) o; + return method == other.method + && (defaultFill.isEmpty() ? other.defaultFill.isEmpty() : Arrays.equals(defaultFill.get(), other.defaultFill.get())); + } + + @Override + public int hashCode() { + return Objects.hash(method, (defaultFill.isEmpty() ? 0 : Arrays.hashCode(defaultFill.get()))); + } + + public ImputationMethod getMethod() { + return method; + } + + public Optional getDefaultFill() { + return defaultFill; + } +} diff --git a/src/main/java/org/opensearch/timeseries/dataprocessor/Imputer.java b/src/main/java/org/opensearch/timeseries/dataprocessor/Imputer.java new file mode 100644 index 000000000..400aae44e --- /dev/null +++ b/src/main/java/org/opensearch/timeseries/dataprocessor/Imputer.java @@ -0,0 +1,54 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.timeseries.dataprocessor; + +/* + * An object for imputing feature vectors. + * + * In certain situations, due to time and compute cost, we are only allowed to + * query a sparse sample of data points / feature vectors from a cluster. + * However, we need a large sample of feature vectors in order to train our + * anomaly detection algorithms. An Imputer approximates the data points + * between a given, ordered list of samples. + */ +public abstract class Imputer { + + /** + * Imputes the given sample feature vectors. + * + * Computes a list `numImputed` feature vectors using the ordered list + * of `numSamples` input sample vectors where each sample vector has size + * `numFeatures`. + * + * + * @param samples A `numFeatures x numSamples` list of feature vectors. + * @param numImputed The desired number of imputed vectors. + * @return A `numFeatures x numImputed` list of feature vectors. + */ + public double[][] impute(double[][] samples, int numImputed) { + int numFeatures = samples.length; + double[][] interpolants = new double[numFeatures][numImputed]; + + for (int featureIndex = 0; featureIndex < numFeatures; featureIndex++) { + interpolants[featureIndex] = singleFeatureImpute(samples[featureIndex], numImputed); + } + return interpolants; + } + + /** + * compute per-feature impute value + * @param samples input array + * @param numImputed number of elements in the return array + * @return input array with missing values imputed + */ + protected abstract double[] singleFeatureImpute(double[] samples, int numImputed); +} diff --git a/src/main/java/org/opensearch/timeseries/dataprocessor/LinearUniformImputer.java b/src/main/java/org/opensearch/timeseries/dataprocessor/LinearUniformImputer.java new file mode 100644 index 000000000..e397cb1af --- /dev/null +++ b/src/main/java/org/opensearch/timeseries/dataprocessor/LinearUniformImputer.java @@ -0,0 +1,88 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.timeseries.dataprocessor; + +import java.util.Arrays; + +import com.google.common.math.DoubleMath; + +/** + * A piecewise linear imputer with uniformly spaced points. + * + * The LinearUniformImputer constructs a piecewise linear imputation on + * the input list of sample feature vectors. That is, between every consecutive + * pair of points we construct a linear imputation. The linear imputation + * is computed on a per-feature basis. + * + */ +public class LinearUniformImputer extends Imputer { + // if true, impute integral/floating-point results: when all samples are integral, + // the results are integral. Else, the results are floating points. + private boolean integerSensitive; + + public LinearUniformImputer(boolean integerSensitive) { + this.integerSensitive = integerSensitive; + } + + /* + * Piecewise linearly impute the given sample of one-dimensional + * features. + * + * Computes a list `numImputed` features using the ordered list of + * `numSamples` input one-dimensional samples. The imputed features are + * computing using a piecewise linear imputation. + * + * @param samples A `numSamples` sized list of sample features. + * @param numImputed The desired number of imputed features. + * @return A `numImputed` sized array of imputed features. + */ + @Override + public double[] singleFeatureImpute(double[] samples, int numImputed) { + int numSamples = samples.length; + double[] imputedValues = new double[numImputed]; + + if (numSamples == 0) { + imputedValues = new double[0]; + } else if (numSamples == 1) { + Arrays.fill(imputedValues, samples[0]); + } else { + /* assume the piecewise linear imputation between the samples is a + parameterized curve f(t) for t in [0, 1]. Each pair of samples + determines a interval [t_i, t_(i+1)]. For each imputed value we + determine which interval it lies inside and then scale the value of t, + accordingly to compute the imputed value. + + for numerical stability reasons we omit processing the final + imputed value in this loop since this last imputed value is always equal + to the last sample. + */ + for (int imputedIndex = 0; imputedIndex < (numImputed - 1); imputedIndex++) { + double tGlobal = (imputedIndex) / (numImputed - 1.0); + double tInterval = tGlobal * (numSamples - 1.0); + int intervalIndex = (int) Math.floor(tInterval); + tInterval -= intervalIndex; + + double leftSample = samples[intervalIndex]; + double rightSample = samples[intervalIndex + 1]; + double imputed = (1.0 - tInterval) * leftSample + tInterval * rightSample; + imputedValues[imputedIndex] = imputed; + } + + // the final imputed value is always the final sample + imputedValues[numImputed - 1] = samples[numSamples - 1]; + } + if (integerSensitive && Arrays.stream(samples).allMatch(DoubleMath::isMathematicalInteger)) { + imputedValues = Arrays.stream(imputedValues).map(Math::rint).toArray(); + } + return imputedValues; + } +} diff --git a/src/main/java/org/opensearch/timeseries/dataprocessor/PreviousValueImputer.java b/src/main/java/org/opensearch/timeseries/dataprocessor/PreviousValueImputer.java new file mode 100644 index 000000000..39f7f94f1 --- /dev/null +++ b/src/main/java/org/opensearch/timeseries/dataprocessor/PreviousValueImputer.java @@ -0,0 +1,48 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.timeseries.dataprocessor; + +/** + * Given an array of samples, fill missing values (represented using Double.NaN) + * with previous value. + * The return array may be smaller than the input array as we remove leading missing + * values after interpolation. If the first sample is Double.NaN + * as there is no last known value to fill in. + * The 2nd parameter of interpolate is ignored as we infer the number of imputed values + * using the number of Double.NaN. + * + */ +public class PreviousValueImputer extends Imputer { + + @Override + protected double[] singleFeatureImpute(double[] samples, int numInterpolants) { + int numSamples = samples.length; + double[] interpolants = new double[numSamples]; + + if (numSamples > 0) { + System.arraycopy(samples, 0, interpolants, 0, samples.length); + if (numSamples > 1) { + double lastKnownValue = Double.NaN; + for (int interpolantIndex = 0; interpolantIndex < numSamples; interpolantIndex++) { + if (Double.isNaN(interpolants[interpolantIndex])) { + if (!Double.isNaN(lastKnownValue)) { + interpolants[interpolantIndex] = lastKnownValue; + } + } else { + lastKnownValue = interpolants[interpolantIndex]; + } + } + } + } + return interpolants; + } +} diff --git a/src/main/java/org/opensearch/timeseries/dataprocessor/ZeroImputer.java b/src/main/java/org/opensearch/timeseries/dataprocessor/ZeroImputer.java new file mode 100644 index 000000000..dcc959b7b --- /dev/null +++ b/src/main/java/org/opensearch/timeseries/dataprocessor/ZeroImputer.java @@ -0,0 +1,27 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.timeseries.dataprocessor; + +import java.util.Arrays; + +/** + * fixing missing value (denoted using Double.NaN) using 0. + * The 2nd parameter of impute is ignored as we infer the number + * of imputed values using the number of Double.NaN. + */ +public class ZeroImputer extends Imputer { + + @Override + public double[] singleFeatureImpute(double[] samples, int numInterpolants) { + return Arrays.stream(samples).map(d -> Double.isNaN(d) ? 0.0 : d).toArray(); + } +} diff --git a/src/main/java/org/opensearch/timeseries/util/DataUtil.java b/src/main/java/org/opensearch/timeseries/util/DataUtil.java new file mode 100644 index 000000000..eacd85ee4 --- /dev/null +++ b/src/main/java/org/opensearch/timeseries/util/DataUtil.java @@ -0,0 +1,54 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.timeseries.util; + +import java.util.Arrays; + +public class DataUtil { + /** + * Removes leading rows in a 2D array that contain Double.NaN values. + * + * This method iterates over the rows of the provided 2D array. If a row is found + * where all elements are not Double.NaN, it removes this row and all rows before it + * from the array. The modified array, which may be smaller than the original, is then returned. + * + * Note: If all rows contain at least one Double.NaN, the method will return an empty array. + * + * @param arr The 2D array from which leading rows containing Double.NaN are to be removed. + * @return A possibly smaller 2D array with leading rows containing Double.NaN removed. + */ + public static double[][] ltrim(double[][] arr) { + int numRows = arr.length; + if (numRows == 0) { + return new double[0][0]; + } + + int numCols = arr[0].length; + int startIndex = numRows; // Initialized to numRows + for (int i = 0; i < numRows; i++) { + boolean hasNaN = false; + for (int j = 0; j < numCols; j++) { + if (Double.isNaN(arr[i][j])) { + hasNaN = true; + break; + } + } + if (!hasNaN) { + startIndex = i; + break; // Stop the loop as soon as a row without NaN is found + } + } + + return Arrays.copyOfRange(arr, startIndex, arr.length); + } + +} diff --git a/src/test/java/org/opensearch/ad/dataprocessor/SingleFeatureLinearUniformInterpolatorTests.java b/src/test/java/org/opensearch/ad/dataprocessor/SingleFeatureLinearUniformInterpolatorTests.java deleted file mode 100644 index e668b51a0..000000000 --- a/src/test/java/org/opensearch/ad/dataprocessor/SingleFeatureLinearUniformInterpolatorTests.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.ad.dataprocessor; - -import static org.junit.Assert.assertArrayEquals; - -import java.util.Arrays; -import java.util.Collection; - -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameters; - -@RunWith(Parameterized.class) -public class SingleFeatureLinearUniformInterpolatorTests { - - @Parameters - public static Collection data() { - double[] singleComponent = { -1.0, 2.0 }; - double[] multiComponent = { 0.0, 1.0, -1. }; - double oneThird = 1.0 / 3.0; - - return Arrays - .asList( - new Object[][] { - { new double[0], 1, new double[0] }, - { new double[] { 1 }, 2, new double[] { 1, 1 } }, - { singleComponent, 2, singleComponent }, - { singleComponent, 3, new double[] { -1.0, 0.5, 2.0 } }, - { singleComponent, 4, new double[] { -1.0, 0.0, 1.0, 2.0 } }, - { multiComponent, 3, multiComponent }, - { multiComponent, 4, new double[] { 0.0, 2 * oneThird, oneThird, -1.0 } }, - { multiComponent, 5, new double[] { 0.0, 0.5, 1.0, 0.0, -1.0 } }, - { multiComponent, 6, new double[] { 0.0, 0.4, 0.8, 0.6, -0.2, -1.0 } } } - ); - } - - private double[] input; - private int numInterpolants; - private double[] expected; - private SingleFeatureLinearUniformInterpolator interpolator; - - public SingleFeatureLinearUniformInterpolatorTests(double[] input, int numInterpolants, double[] expected) { - this.input = input; - this.numInterpolants = numInterpolants; - this.expected = expected; - } - - @Before - public void setUp() { - this.interpolator = new SingleFeatureLinearUniformInterpolator(); - } - - @Test - public void testInterpolation() { - double[] actual = interpolator.interpolate(input, numInterpolants); - double delta = 1e-8; - assertArrayEquals(expected, actual, delta); - } -} diff --git a/src/test/java/org/opensearch/ad/feature/FeatureManagerTests.java b/src/test/java/org/opensearch/ad/feature/FeatureManagerTests.java index 64ca51936..445669153 100644 --- a/src/test/java/org/opensearch/ad/feature/FeatureManagerTests.java +++ b/src/test/java/org/opensearch/ad/feature/FeatureManagerTests.java @@ -58,14 +58,13 @@ import org.opensearch.action.ActionListener; import org.opensearch.ad.AnomalyDetectorPlugin; import org.opensearch.ad.common.exception.EndRunException; -import org.opensearch.ad.dataprocessor.Interpolator; -import org.opensearch.ad.dataprocessor.LinearUniformInterpolator; -import org.opensearch.ad.dataprocessor.SingleFeatureLinearUniformInterpolator; import org.opensearch.ad.model.AnomalyDetector; import org.opensearch.ad.model.Entity; import org.opensearch.ad.model.IntervalTimeConfiguration; import org.opensearch.ad.util.ArrayEqMatcher; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.timeseries.dataprocessor.Imputer; +import org.opensearch.timeseries.dataprocessor.LinearUniformImputer; @RunWith(JUnitParamsRunner.class) @SuppressWarnings("unchecked") @@ -91,7 +90,7 @@ public class FeatureManagerTests { private SearchFeatureDao searchFeatureDao; @Mock - private Interpolator interpolator; + private Imputer imputer; @Mock private Clock clock; @@ -125,7 +124,7 @@ public void setup() { intervalInMilliseconds = detectorIntervalTimeConfig.toDuration().toMillis(); when(detector.getDetectorIntervalInMilliseconds()).thenReturn(intervalInMilliseconds); - Interpolator interpolator = new LinearUniformInterpolator(new SingleFeatureLinearUniformInterpolator()); + Imputer imputer = new LinearUniformImputer(false); ExecutorService executorService = mock(ExecutorService.class); @@ -139,7 +138,7 @@ public void setup() { this.featureManager = spy( new FeatureManager( searchFeatureDao, - interpolator, + imputer, clock, maxTrainSamples, maxSampleStride, @@ -215,7 +214,7 @@ public void getColdStartData_returnExpectedToListener( featureManager = spy( new FeatureManager( searchFeatureDao, - interpolator, + imputer, clock, maxTrainSamples, maxSampleStride, @@ -449,11 +448,10 @@ private void getPreviewFeaturesTemplate(List> samplesResults, return null; }).when(searchFeatureDao).getFeatureSamplesForPeriods(eq(detector), eq(sampleRanges), any()); - when(interpolator.interpolate(argThat(new ArrayEqMatcher<>(new double[][] { { 1, 3 } })), eq(3))) - .thenReturn(new double[][] { { 1, 2, 3 } }); - when(interpolator.interpolate(argThat(new ArrayEqMatcher<>(new double[][] { { 0, 120000 } })), eq(3))) + when(imputer.impute(argThat(new ArrayEqMatcher<>(new double[][] { { 1, 3 } })), eq(3))).thenReturn(new double[][] { { 1, 2, 3 } }); + when(imputer.impute(argThat(new ArrayEqMatcher<>(new double[][] { { 0, 120000 } })), eq(3))) .thenReturn(new double[][] { { 0, 60000, 120000 } }); - when(interpolator.interpolate(argThat(new ArrayEqMatcher<>(new double[][] { { 60000, 180000 } })), eq(3))) + when(imputer.impute(argThat(new ArrayEqMatcher<>(new double[][] { { 60000, 180000 } })), eq(3))) .thenReturn(new double[][] { { 60000, 120000, 180000 } }); ActionListener listener = mock(ActionListener.class); diff --git a/src/test/java/org/opensearch/ad/feature/NoPowermockSearchFeatureDaoTests.java b/src/test/java/org/opensearch/ad/feature/NoPowermockSearchFeatureDaoTests.java index 67e73cdb5..60fb5e273 100644 --- a/src/test/java/org/opensearch/ad/feature/NoPowermockSearchFeatureDaoTests.java +++ b/src/test/java/org/opensearch/ad/feature/NoPowermockSearchFeatureDaoTests.java @@ -58,8 +58,6 @@ import org.opensearch.ad.AbstractADTest; import org.opensearch.ad.NodeStateManager; import org.opensearch.ad.TestHelpers; -import org.opensearch.ad.dataprocessor.LinearUniformInterpolator; -import org.opensearch.ad.dataprocessor.SingleFeatureLinearUniformInterpolator; import org.opensearch.ad.model.AnomalyDetector; import org.opensearch.ad.model.Entity; import org.opensearch.ad.model.Feature; @@ -100,6 +98,8 @@ import org.opensearch.search.aggregations.metrics.InternalMax; import org.opensearch.search.aggregations.metrics.SumAggregationBuilder; import org.opensearch.search.internal.InternalSearchResponse; +import org.opensearch.timeseries.dataprocessor.Imputer; +import org.opensearch.timeseries.dataprocessor.LinearUniformImputer; import com.carrotsearch.hppc.BitMixer; import com.google.common.collect.ImmutableList; @@ -115,7 +115,7 @@ public class NoPowermockSearchFeatureDaoTests extends AbstractADTest { private AnomalyDetector detector; private Client client; private SearchFeatureDao searchFeatureDao; - private LinearUniformInterpolator interpolator; + private Imputer imputer; private SecurityClientUtil clientUtil; private Settings settings; private ClusterService clusterService; @@ -154,7 +154,7 @@ public void setUp() throws Exception { client = mock(Client.class); when(client.threadPool()).thenReturn(threadPool); - interpolator = new LinearUniformInterpolator(new SingleFeatureLinearUniformInterpolator()); + imputer = new LinearUniformImputer(false); settings = Settings.EMPTY; ClusterSettings clusterSettings = new ClusterSettings( @@ -178,7 +178,7 @@ public void setUp() throws Exception { searchFeatureDao = new SearchFeatureDao( client, xContentRegistry(), // Important. Without this, ParseUtils cannot parse anything - interpolator, + imputer, clientUtil, settings, clusterService, @@ -365,7 +365,7 @@ public void testGetHighestCountEntitiesExhaustedPages() throws InterruptedExcept searchFeatureDao = new SearchFeatureDao( client, xContentRegistry(), - interpolator, + imputer, clientUtil, settings, clusterService, @@ -411,7 +411,7 @@ public void testGetHighestCountEntitiesNotEnoughTime() throws InterruptedExcepti searchFeatureDao = new SearchFeatureDao( client, xContentRegistry(), - interpolator, + imputer, clientUtil, settings, clusterService, diff --git a/src/test/java/org/opensearch/ad/feature/SearchFeatureDaoParamTests.java b/src/test/java/org/opensearch/ad/feature/SearchFeatureDaoParamTests.java index a6380b8cf..74b1d030b 100644 --- a/src/test/java/org/opensearch/ad/feature/SearchFeatureDaoParamTests.java +++ b/src/test/java/org/opensearch/ad/feature/SearchFeatureDaoParamTests.java @@ -55,9 +55,6 @@ import org.opensearch.ad.AnomalyDetectorPlugin; import org.opensearch.ad.NodeStateManager; import org.opensearch.ad.constant.ADCommonName; -import org.opensearch.ad.dataprocessor.Interpolator; -import org.opensearch.ad.dataprocessor.LinearUniformInterpolator; -import org.opensearch.ad.dataprocessor.SingleFeatureLinearUniformInterpolator; import org.opensearch.ad.model.AnomalyDetector; import org.opensearch.ad.model.IntervalTimeConfiguration; import org.opensearch.ad.settings.AnomalyDetectorSettings; @@ -82,6 +79,8 @@ import org.opensearch.search.aggregations.metrics.Percentile; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.timeseries.dataprocessor.Imputer; +import org.opensearch.timeseries.dataprocessor.LinearUniformImputer; import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; @@ -150,7 +149,7 @@ public class SearchFeatureDaoParamTests { private MultiSearchRequest multiSearchRequest; private IntervalTimeConfiguration detectionInterval; private String detectorId; - private Interpolator interpolator; + private Imputer imputer; private Settings settings; @Before @@ -158,7 +157,7 @@ public void setup() throws Exception { MockitoAnnotations.initMocks(this); PowerMockito.mockStatic(ParseUtils.class); - interpolator = new LinearUniformInterpolator(new SingleFeatureLinearUniformInterpolator()); + imputer = new LinearUniformImputer(false); ExecutorService executorService = mock(ExecutorService.class); when(threadPool.executor(AnomalyDetectorPlugin.AD_THREAD_POOL_NAME)).thenReturn(executorService); @@ -179,7 +178,7 @@ public void setup() throws Exception { }).when(nodeStateManager).getAnomalyDetector(any(String.class), any(ActionListener.class)); clientUtil = new SecurityClientUtil(nodeStateManager, settings); searchFeatureDao = spy( - new SearchFeatureDao(client, xContent, interpolator, clientUtil, settings, null, AnomalyDetectorSettings.NUM_SAMPLES_PER_TREE) + new SearchFeatureDao(client, xContent, imputer, clientUtil, settings, null, AnomalyDetectorSettings.NUM_SAMPLES_PER_TREE) ); detectionInterval = new IntervalTimeConfiguration(1, ChronoUnit.MINUTES); diff --git a/src/test/java/org/opensearch/ad/feature/SearchFeatureDaoTests.java b/src/test/java/org/opensearch/ad/feature/SearchFeatureDaoTests.java index 25541cf3b..407ccfce5 100644 --- a/src/test/java/org/opensearch/ad/feature/SearchFeatureDaoTests.java +++ b/src/test/java/org/opensearch/ad/feature/SearchFeatureDaoTests.java @@ -60,9 +60,6 @@ import org.opensearch.ad.AnomalyDetectorPlugin; import org.opensearch.ad.NodeStateManager; import org.opensearch.ad.constant.ADCommonName; -import org.opensearch.ad.dataprocessor.Interpolator; -import org.opensearch.ad.dataprocessor.LinearUniformInterpolator; -import org.opensearch.ad.dataprocessor.SingleFeatureLinearUniformInterpolator; import org.opensearch.ad.model.AnomalyDetector; import org.opensearch.ad.model.Entity; import org.opensearch.ad.model.IntervalTimeConfiguration; @@ -98,6 +95,8 @@ import org.opensearch.search.aggregations.metrics.Percentile; import org.opensearch.search.builder.SearchSourceBuilder; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.timeseries.dataprocessor.Imputer; +import org.opensearch.timeseries.dataprocessor.LinearUniformImputer; import org.powermock.api.mockito.PowerMockito; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; @@ -153,7 +152,7 @@ public class SearchFeatureDaoTests { private Map aggsMap; private IntervalTimeConfiguration detectionInterval; private String detectorId; - private Interpolator interpolator; + private Imputer imputer; private Settings settings; @Before @@ -161,7 +160,7 @@ public void setup() throws Exception { MockitoAnnotations.initMocks(this); PowerMockito.mockStatic(ParseUtils.class); - interpolator = new LinearUniformInterpolator(new SingleFeatureLinearUniformInterpolator()); + imputer = new LinearUniformImputer(false); ExecutorService executorService = mock(ExecutorService.class); when(threadPool.executor(AnomalyDetectorPlugin.AD_THREAD_POOL_NAME)).thenReturn(executorService); @@ -182,7 +181,7 @@ public void setup() throws Exception { }).when(nodeStateManager).getAnomalyDetector(any(String.class), any(ActionListener.class)); clientUtil = new SecurityClientUtil(nodeStateManager, settings); searchFeatureDao = spy( - new SearchFeatureDao(client, xContent, interpolator, clientUtil, settings, null, AnomalyDetectorSettings.NUM_SAMPLES_PER_TREE) + new SearchFeatureDao(client, xContent, imputer, clientUtil, settings, null, AnomalyDetectorSettings.NUM_SAMPLES_PER_TREE) ); detectionInterval = new IntervalTimeConfiguration(1, ChronoUnit.MINUTES); diff --git a/src/test/java/org/opensearch/ad/ml/AbstractCosineDataTest.java b/src/test/java/org/opensearch/ad/ml/AbstractCosineDataTest.java index 2107d4e59..0e4952600 100644 --- a/src/test/java/org/opensearch/ad/ml/AbstractCosineDataTest.java +++ b/src/test/java/org/opensearch/ad/ml/AbstractCosineDataTest.java @@ -38,10 +38,6 @@ import org.opensearch.ad.MemoryTracker; import org.opensearch.ad.NodeStateManager; import org.opensearch.ad.TestHelpers; -import org.opensearch.ad.dataprocessor.IntegerSensitiveSingleFeatureLinearUniformInterpolator; -import org.opensearch.ad.dataprocessor.Interpolator; -import org.opensearch.ad.dataprocessor.LinearUniformInterpolator; -import org.opensearch.ad.dataprocessor.SingleFeatureLinearUniformInterpolator; import org.opensearch.ad.feature.FeatureManager; import org.opensearch.ad.feature.SearchFeatureDao; import org.opensearch.ad.model.AnomalyDetector; @@ -61,6 +57,8 @@ import org.opensearch.test.OpenSearchTestCase; import org.opensearch.threadpool.ThreadPool; import org.opensearch.timeseries.constant.CommonName; +import org.opensearch.timeseries.dataprocessor.Imputer; +import org.opensearch.timeseries.dataprocessor.LinearUniformImputer; import com.google.common.collect.ImmutableList; @@ -75,7 +73,7 @@ public class AbstractCosineDataTest extends AbstractADTest { EntityColdStarter entityColdStarter; NodeStateManager stateManager; SearchFeatureDao searchFeatureDao; - Interpolator interpolator; + Imputer imputer; CheckpointDao checkpoint; FeatureManager featureManager; Settings settings; @@ -152,16 +150,14 @@ public void setUp() throws Exception { clusterService ); - SingleFeatureLinearUniformInterpolator singleFeatureLinearUniformInterpolator = - new IntegerSensitiveSingleFeatureLinearUniformInterpolator(); - interpolator = new LinearUniformInterpolator(singleFeatureLinearUniformInterpolator); + imputer = new LinearUniformImputer(true); searchFeatureDao = mock(SearchFeatureDao.class); checkpoint = mock(CheckpointDao.class); featureManager = new FeatureManager( searchFeatureDao, - interpolator, + imputer, clock, AnomalyDetectorSettings.MAX_TRAIN_SAMPLE, AnomalyDetectorSettings.MAX_SAMPLE_STRIDE, @@ -189,7 +185,7 @@ public void setUp() throws Exception { numMinSamples, AnomalyDetectorSettings.MAX_SAMPLE_STRIDE, AnomalyDetectorSettings.MAX_TRAIN_SAMPLE, - interpolator, + imputer, searchFeatureDao, AnomalyDetectorSettings.THRESHOLD_MIN_PVALUE, featureManager, diff --git a/src/test/java/org/opensearch/ad/ml/EntityColdStarterTests.java b/src/test/java/org/opensearch/ad/ml/EntityColdStarterTests.java index a6d5d34c1..dc77931f8 100644 --- a/src/test/java/org/opensearch/ad/ml/EntityColdStarterTests.java +++ b/src/test/java/org/opensearch/ad/ml/EntityColdStarterTests.java @@ -166,9 +166,9 @@ public void testColdStart() throws InterruptedException, IOException { // for function interpolate: // 1st parameter is a matrix of size numFeatures * numSamples // 2nd parameter is the number of interpolants including two samples - double[][] interval1 = interpolator.interpolate(new double[][] { new double[] { sample1[0], sample2[0] } }, 61); + double[][] interval1 = imputer.impute(new double[][] { new double[] { sample1[0], sample2[0] } }, 61); expectedColdStartData.addAll(convertToFeatures(interval1, 60)); - double[][] interval2 = interpolator.interpolate(new double[][] { new double[] { sample2[0], sample3[0] } }, 61); + double[][] interval2 = imputer.impute(new double[][] { new double[] { sample2[0], sample3[0] } }, 61); expectedColdStartData.addAll(convertToFeatures(interval2, 61)); assertEquals(121, expectedColdStartData.size()); @@ -302,11 +302,11 @@ public void testTwoSegmentsWithSingleSample() throws InterruptedException, IOExc // for function interpolate: // 1st parameter is a matrix of size numFeatures * numSamples // 2nd parameter is the number of interpolants including two samples - double[][] interval1 = interpolator.interpolate(new double[][] { new double[] { sample1[0], sample2[0] } }, 61); + double[][] interval1 = imputer.impute(new double[][] { new double[] { sample1[0], sample2[0] } }, 61); expectedColdStartData.addAll(convertToFeatures(interval1, 60)); - double[][] interval2 = interpolator.interpolate(new double[][] { new double[] { sample2[0], sample3[0] } }, 61); + double[][] interval2 = imputer.impute(new double[][] { new double[] { sample2[0], sample3[0] } }, 61); expectedColdStartData.addAll(convertToFeatures(interval2, 60)); - double[][] interval3 = interpolator.interpolate(new double[][] { new double[] { sample3[0], sample5[0] } }, 121); + double[][] interval3 = imputer.impute(new double[][] { new double[] { sample3[0], sample5[0] } }, 121); expectedColdStartData.addAll(convertToFeatures(interval3, 121)); assertTrue("size: " + model.getSamples().size(), model.getSamples().isEmpty()); assertEquals(241, expectedColdStartData.size()); @@ -358,13 +358,13 @@ public void testTwoSegments() throws InterruptedException, IOException { // for function interpolate: // 1st parameter is a matrix of size numFeatures * numSamples // 2nd parameter is the number of interpolants including two samples - double[][] interval1 = interpolator.interpolate(new double[][] { new double[] { sample1[0], sample2[0] } }, 61); + double[][] interval1 = imputer.impute(new double[][] { new double[] { sample1[0], sample2[0] } }, 61); expectedColdStartData.addAll(convertToFeatures(interval1, 60)); - double[][] interval2 = interpolator.interpolate(new double[][] { new double[] { sample2[0], sample3[0] } }, 61); + double[][] interval2 = imputer.impute(new double[][] { new double[] { sample2[0], sample3[0] } }, 61); expectedColdStartData.addAll(convertToFeatures(interval2, 60)); - double[][] interval3 = interpolator.interpolate(new double[][] { new double[] { sample3[0], sample5[0] } }, 121); + double[][] interval3 = imputer.impute(new double[][] { new double[] { sample3[0], sample5[0] } }, 121); expectedColdStartData.addAll(convertToFeatures(interval3, 120)); - double[][] interval4 = interpolator.interpolate(new double[][] { new double[] { sample5[0], sample6[0] } }, 61); + double[][] interval4 = imputer.impute(new double[][] { new double[] { sample5[0], sample6[0] } }, 61); expectedColdStartData.addAll(convertToFeatures(interval4, 61)); assertEquals(301, expectedColdStartData.size()); assertTrue("size: " + model.getSamples().size(), model.getSamples().isEmpty()); @@ -701,7 +701,7 @@ public void testAccuracyOneMinuteIntervalNoInterpolation() throws Exception { numMinSamples, AnomalyDetectorSettings.MAX_SAMPLE_STRIDE, AnomalyDetectorSettings.MAX_TRAIN_SAMPLE, - interpolator, + imputer, searchFeatureDao, AnomalyDetectorSettings.THRESHOLD_MIN_PVALUE, featureManager, diff --git a/src/test/java/org/opensearch/ad/ml/HCADModelPerfTests.java b/src/test/java/org/opensearch/ad/ml/HCADModelPerfTests.java index c51bc11b5..359d47e5f 100644 --- a/src/test/java/org/opensearch/ad/ml/HCADModelPerfTests.java +++ b/src/test/java/org/opensearch/ad/ml/HCADModelPerfTests.java @@ -113,7 +113,7 @@ private void averageAccuracyTemplate( featureManager = new FeatureManager( searchFeatureDao, - interpolator, + imputer, clock, AnomalyDetectorSettings.MAX_TRAIN_SAMPLE, AnomalyDetectorSettings.MAX_SAMPLE_STRIDE, @@ -138,7 +138,7 @@ private void averageAccuracyTemplate( numMinSamples, AnomalyDetectorSettings.MAX_SAMPLE_STRIDE, AnomalyDetectorSettings.MAX_TRAIN_SAMPLE, - interpolator, + imputer, searchFeatureDao, AnomalyDetectorSettings.THRESHOLD_MIN_PVALUE, featureManager, diff --git a/src/test/java/org/opensearch/ad/ml/ModelManagerTests.java b/src/test/java/org/opensearch/ad/ml/ModelManagerTests.java index 1b4797eb6..8fbb7b75f 100644 --- a/src/test/java/org/opensearch/ad/ml/ModelManagerTests.java +++ b/src/test/java/org/opensearch/ad/ml/ModelManagerTests.java @@ -62,9 +62,6 @@ import org.opensearch.ad.caching.EntityCache; import org.opensearch.ad.common.exception.LimitExceededException; import org.opensearch.ad.common.exception.ResourceNotFoundException; -import org.opensearch.ad.dataprocessor.IntegerSensitiveSingleFeatureLinearUniformInterpolator; -import org.opensearch.ad.dataprocessor.LinearUniformInterpolator; -import org.opensearch.ad.dataprocessor.SingleFeatureLinearUniformInterpolator; import org.opensearch.ad.feature.FeatureManager; import org.opensearch.ad.feature.SearchFeatureDao; import org.opensearch.ad.ml.ModelManager.ModelType; @@ -80,6 +77,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.monitor.jvm.JvmService; import org.opensearch.threadpool.ThreadPool; +import org.opensearch.timeseries.dataprocessor.LinearUniformImputer; import org.powermock.modules.junit4.PowerMockRunner; import org.powermock.modules.junit4.PowerMockRunnerDelegate; @@ -914,9 +912,7 @@ public void getNullState() { public void getEmptyStateFullSamples() { SearchFeatureDao searchFeatureDao = mock(SearchFeatureDao.class); - SingleFeatureLinearUniformInterpolator singleFeatureLinearUniformInterpolator = - new IntegerSensitiveSingleFeatureLinearUniformInterpolator(); - LinearUniformInterpolator interpolator = new LinearUniformInterpolator(singleFeatureLinearUniformInterpolator); + LinearUniformImputer interpolator = new LinearUniformImputer(true); NodeStateManager stateManager = mock(NodeStateManager.class); featureManager = new FeatureManager( diff --git a/src/test/java/org/opensearch/ad/rest/AnomalyDetectorRestApiIT.java b/src/test/java/org/opensearch/ad/rest/AnomalyDetectorRestApiIT.java index b7605ce90..3e439f213 100644 --- a/src/test/java/org/opensearch/ad/rest/AnomalyDetectorRestApiIT.java +++ b/src/test/java/org/opensearch/ad/rest/AnomalyDetectorRestApiIT.java @@ -1673,7 +1673,7 @@ public void testSearchTopAnomalyResultsOnNonExistentResultIndex() throws IOExcep // Delete any existing result index if (indexExistsWithAdminClient(ADCommonName.ANOMALY_RESULT_INDEX_ALIAS)) { - deleteIndexWithAdminClient(ADCommonName.ANOMALY_RESULT_INDEX_ALIAS); + deleteIndexWithAdminClient(ADCommonName.ANOMALY_RESULT_INDEX_ALIAS + "*"); } Response response = searchTopAnomalyResults( detector.getDetectorId(), @@ -1710,12 +1710,8 @@ public void testSearchTopAnomalyResultsOnEmptyResultIndex() throws IOException { client() ); - // Clear any existing result index, create an empty one - // Use the DELETE API to delete the index or indices associated with the alias first - // then Use the DELETE API to remove the alias. if (indexExistsWithAdminClient(ADCommonName.ANOMALY_RESULT_INDEX_ALIAS)) { deleteIndexWithAdminClient(ADCommonName.ANOMALY_RESULT_INDEX_ALIAS + "*"); - deleteIndexWithAdminClient(ADCommonName.ANOMALY_RESULT_INDEX_ALIAS); } TestHelpers.createEmptyAnomalyResultIndex(adminClient()); Response response = searchTopAnomalyResults( diff --git a/src/test/java/org/opensearch/ad/rest/SecureADRestIT.java b/src/test/java/org/opensearch/ad/rest/SecureADRestIT.java index 0a3ec8c76..a96ba20f1 100644 --- a/src/test/java/org/opensearch/ad/rest/SecureADRestIT.java +++ b/src/test/java/org/opensearch/ad/rest/SecureADRestIT.java @@ -17,6 +17,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Map; +import java.util.Random; import org.apache.hc.core5.http.HttpHeaders; import org.apache.hc.core5.http.HttpHost; @@ -60,49 +61,74 @@ public class SecureADRestIT extends AnomalyDetectorRestTestCase { private String indexAllAccessRole = "index_all_access"; private String indexSearchAccessRole = "index_all_search"; + /** + * Create an unguessable password. Simple password are weak due to https://tinyurl.com/383em9zk + * @return a random password. + */ + public static String generatePassword() { + String characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + + Random rng = new Random(); + + char[] password = new char[10]; + for (int i = 0; i < 10; i++) { + password[i] = characters.charAt(rng.nextInt(characters.length())); + } + + return new String(password); + } + @Before public void setupSecureTests() throws IOException { if (!isHttps()) throw new IllegalArgumentException("Secure Tests are running but HTTPS is not set"); createIndexRole(indexAllAccessRole, "*"); createSearchRole(indexSearchAccessRole, "*"); - createUser(aliceUser, aliceUser, new ArrayList<>(Arrays.asList("odfe"))); - aliceClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), aliceUser, aliceUser) + String alicePassword = generatePassword(); + createUser(aliceUser, alicePassword, new ArrayList<>(Arrays.asList("odfe"))); + aliceClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), aliceUser, alicePassword) .setSocketTimeout(60000) .build(); - createUser(bobUser, bobUser, new ArrayList<>(Arrays.asList("odfe"))); - bobClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), bobUser, bobUser) + String bobPassword = generatePassword(); + createUser(bobUser, bobPassword, new ArrayList<>(Arrays.asList("odfe"))); + bobClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), bobUser, bobPassword) .setSocketTimeout(60000) .build(); - createUser(catUser, catUser, new ArrayList<>(Arrays.asList("aes"))); - catClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), catUser, catUser) + String catPassword = generatePassword(); + createUser(catUser, catPassword, new ArrayList<>(Arrays.asList("aes"))); + catClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), catUser, catPassword) .setSocketTimeout(60000) .build(); - createUser(dogUser, dogUser, new ArrayList<>(Arrays.asList())); - dogClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), dogUser, dogUser) + String dogPassword = generatePassword(); + createUser(dogUser, dogPassword, new ArrayList<>(Arrays.asList())); + dogClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), dogUser, dogPassword) .setSocketTimeout(60000) .build(); - createUser(elkUser, elkUser, new ArrayList<>(Arrays.asList("odfe"))); - elkClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), elkUser, elkUser) + String elkPassword = generatePassword(); + createUser(elkUser, elkPassword, new ArrayList<>(Arrays.asList("odfe"))); + elkClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), elkUser, elkPassword) .setSocketTimeout(60000) .build(); - createUser(fishUser, fishUser, new ArrayList<>(Arrays.asList("odfe", "aes"))); - fishClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), fishUser, fishUser) + String fishPassword = generatePassword(); + createUser(fishUser, fishPassword, new ArrayList<>(Arrays.asList("odfe", "aes"))); + fishClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), fishUser, fishPassword) .setSocketTimeout(60000) .build(); - createUser(goatUser, goatUser, new ArrayList<>(Arrays.asList("opensearch"))); - goatClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), goatUser, goatUser) + String goatPassword = generatePassword(); + createUser(goatUser, goatPassword, new ArrayList<>(Arrays.asList("opensearch"))); + goatClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), goatUser, goatPassword) .setSocketTimeout(60000) .build(); - createUser(lionUser, lionUser, new ArrayList<>(Arrays.asList("opensearch"))); - lionClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), lionUser, lionUser) + String lionPassword = generatePassword(); + createUser(lionUser, lionPassword, new ArrayList<>(Arrays.asList("opensearch"))); + lionClient = new SecureRestClientBuilder(getClusterHosts().toArray(new HttpHost[0]), isHttps(), lionUser, lionPassword) .setSocketTimeout(60000) .build(); diff --git a/src/test/java/org/opensearch/timeseries/dataprocessor/FixedValueImputerTests.java b/src/test/java/org/opensearch/timeseries/dataprocessor/FixedValueImputerTests.java new file mode 100644 index 000000000..8c88489a2 --- /dev/null +++ b/src/test/java/org/opensearch/timeseries/dataprocessor/FixedValueImputerTests.java @@ -0,0 +1,40 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.timeseries.dataprocessor; + +import static org.junit.Assert.assertArrayEquals; + +import org.junit.Test; + +public class FixedValueImputerTests { + + @Test + public void testImpute() { + // Initialize the FixedValueImputer with some fixed values + double[] fixedValues = { 2.0, 3.0 }; + FixedValueImputer imputer = new FixedValueImputer(fixedValues); + + // Create a sample array with some missing values (Double.NaN) + double[][] samples = { { 1.0, Double.NaN, 3.0 }, { Double.NaN, 2.0, 3.0 } }; + + // Call the impute method + double[][] imputed = imputer.impute(samples, 3); + + // Check the results + double[][] expected = { { 1.0, 2.0, 3.0 }, { 3.0, 2.0, 3.0 } }; + double delta = 0.0001; + + for (int i = 0; i < expected.length; i++) { + assertArrayEquals("The arrays are not equal", expected[i], imputed[i], delta); + } + } +} diff --git a/src/test/java/org/opensearch/timeseries/dataprocessor/ImputationOptionTests.java b/src/test/java/org/opensearch/timeseries/dataprocessor/ImputationOptionTests.java new file mode 100644 index 000000000..a0df85ead --- /dev/null +++ b/src/test/java/org/opensearch/timeseries/dataprocessor/ImputationOptionTests.java @@ -0,0 +1,124 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.timeseries.dataprocessor; + +import java.io.IOException; +import java.util.Optional; + +import org.opensearch.common.bytes.BytesReference; +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.io.stream.StreamInput; +import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.xcontent.DeprecationHandler; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.test.OpenSearchTestCase; + +public class ImputationOptionTests extends OpenSearchTestCase { + + public void testStreamInputAndOutput() throws IOException { + // Prepare the data to be read by the StreamInput object. + ImputationMethod method = ImputationMethod.PREVIOUS; + double[] defaultFill = { 1.0, 2.0, 3.0 }; + + ImputationOption option = new ImputationOption(method, Optional.of(defaultFill)); + + // Write the ImputationOption to the StreamOutput. + BytesStreamOutput out = new BytesStreamOutput(); + option.writeTo(out); + + StreamInput in = out.bytes().streamInput(); + + // Create an ImputationOption using the mocked StreamInput. + ImputationOption inOption = new ImputationOption(in); + + // Check that the created ImputationOption has the correct values. + assertEquals(method, inOption.getMethod()); + assertArrayEquals(defaultFill, inOption.getDefaultFill().get(), 1e-6); + } + + public void testToXContent() throws IOException { + double[] defaultFill = { 1.0, 2.0, 3.0 }; + ImputationOption imputationOption = new ImputationOption(ImputationMethod.FIXED_VALUES, Optional.of(defaultFill)); + + String xContent = "{" + "\"method\":\"FIXED_VALUES\"," + "\"defaultFill\":[1.0,2.0,3.0]" + "}"; + + XContentBuilder builder = imputationOption.toXContent(JsonXContent.contentBuilder(), ToXContent.EMPTY_PARAMS); + String actualJson = BytesReference.bytes(builder).utf8ToString(); + + assertEquals(xContent, actualJson); + } + + public void testParse() throws IOException { + String xContent = "{" + "\"method\":\"FIXED_VALUES\"," + "\"defaultFill\":[1.0,2.0,3.0]" + "}"; + + double[] defaultFill = { 1.0, 2.0, 3.0 }; + ImputationOption imputationOption = new ImputationOption(ImputationMethod.FIXED_VALUES, Optional.of(defaultFill)); + + try ( + XContentParser parser = JsonXContent.jsonXContent + .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, xContent) + ) { + // advance to first token + XContentParser.Token token = parser.nextToken(); + if (token != XContentParser.Token.START_OBJECT) { + throw new IOException("Expected data to start with an Object"); + } + + ImputationOption parsedOption = ImputationOption.parse(parser); + + assertEquals(imputationOption.getMethod(), parsedOption.getMethod()); + assertTrue(imputationOption.getDefaultFill().isPresent()); + assertTrue(parsedOption.getDefaultFill().isPresent()); + assertEquals(imputationOption.getDefaultFill().get().length, parsedOption.getDefaultFill().get().length); + for (int i = 0; i < imputationOption.getDefaultFill().get().length; i++) { + assertEquals(imputationOption.getDefaultFill().get()[i], parsedOption.getDefaultFill().get()[i], 0); + } + } + } + + public void testEqualsAndHashCode() { + double[] defaultFill1 = { 1.0, 2.0, 3.0 }; + double[] defaultFill2 = { 4.0, 5.0, 6.0 }; + + ImputationOption option1 = new ImputationOption(ImputationMethod.FIXED_VALUES, Optional.of(defaultFill1)); + ImputationOption option2 = new ImputationOption(ImputationMethod.FIXED_VALUES, Optional.of(defaultFill1)); + ImputationOption option3 = new ImputationOption(ImputationMethod.LINEAR, Optional.of(defaultFill2)); + + // Test reflexivity + assertTrue(option1.equals(option1)); + + // Test symmetry + assertTrue(option1.equals(option2)); + assertTrue(option2.equals(option1)); + + // Test transitivity + ImputationOption option2Clone = new ImputationOption(ImputationMethod.FIXED_VALUES, Optional.of(defaultFill1)); + assertTrue(option1.equals(option2)); + assertTrue(option2.equals(option2Clone)); + assertTrue(option1.equals(option2Clone)); + + // Test consistency: ultiple invocations of a.equals(b) consistently return true or consistently return false. + assertTrue(option1.equals(option2)); + assertTrue(option1.equals(option2)); + + // Test non-nullity + assertFalse(option1.equals(null)); + + // Test hashCode consistency + assertEquals(option1.hashCode(), option1.hashCode()); + + // Test hashCode equality + assertTrue(option1.equals(option2)); + assertEquals(option1.hashCode(), option2.hashCode()); + + // Test inequality + assertFalse(option1.equals(option3)); + assertNotEquals(option1.hashCode(), option3.hashCode()); + } +} diff --git a/src/test/java/org/opensearch/timeseries/dataprocessor/IntegerSensitiveLinearUniformImputerTests.java b/src/test/java/org/opensearch/timeseries/dataprocessor/IntegerSensitiveLinearUniformImputerTests.java new file mode 100644 index 000000000..12da46895 --- /dev/null +++ b/src/test/java/org/opensearch/timeseries/dataprocessor/IntegerSensitiveLinearUniformImputerTests.java @@ -0,0 +1,78 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.timeseries.dataprocessor; + +import static org.junit.Assert.assertArrayEquals; + +import java.util.Arrays; +import java.util.Collection; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +/** + * Compared to MultiFeatureLinearUniformImputerTests, outputs are different and + * integerSensitive is enabled + * + */ +@RunWith(Parameterized.class) +public class IntegerSensitiveLinearUniformImputerTests { + + @Parameters + public static Collection data() { + double[][] singleComponent = { { -1.0, 2.0 }, { 1.0, 1.0 } }; + double[][] multiComponent = { { 0.0, 1.0, -1.0 }, { 1.0, 1.0, 1.0 } }; + + return Arrays + .asList( + new Object[][] { + // after integer sensitive rint rounding + { singleComponent, 2, singleComponent }, + { singleComponent, 3, new double[][] { { -1.0, 0, 2.0 }, { 1.0, 1.0, 1.0 } } }, + { singleComponent, 4, new double[][] { { -1.0, 0.0, 1.0, 2.0 }, { 1.0, 1.0, 1.0, 1.0 } } }, + { multiComponent, 3, multiComponent }, + { multiComponent, 4, new double[][] { { 0.0, 1.0, 0.0, -1.0 }, { 1.0, 1.0, 1.0, 1.0 } } }, + { multiComponent, 5, new double[][] { { 0.0, 0.0, 1.0, 0.0, -1.0 }, { 1.0, 1.0, 1.0, 1.0, 1.0 } } }, + { multiComponent, 6, new double[][] { { 0.0, 0.0, 1.0, 1.0, -0.0, -1.0 }, { 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 } } }, } + ); + } + + private double[][] input; + private int numInterpolants; + private double[][] expected; + private Imputer imputer; + + public IntegerSensitiveLinearUniformImputerTests(double[][] input, int numInterpolants, double[][] expected) { + this.input = input; + this.numInterpolants = numInterpolants; + this.expected = expected; + } + + @Before + public void setUp() { + this.imputer = new LinearUniformImputer(true); + } + + @Test + public void testImputation() { + double[][] actual = imputer.impute(input, numInterpolants); + double delta = 1e-8; + int numFeatures = expected.length; + + for (int i = 0; i < numFeatures; i++) { + assertArrayEquals(expected[i], actual[i], delta); + } + } +} diff --git a/src/test/java/org/opensearch/ad/dataprocessor/LinearUniformInterpolatorTests.java b/src/test/java/org/opensearch/timeseries/dataprocessor/MultiFeatureLinearUniformImputerTests.java similarity index 81% rename from src/test/java/org/opensearch/ad/dataprocessor/LinearUniformInterpolatorTests.java rename to src/test/java/org/opensearch/timeseries/dataprocessor/MultiFeatureLinearUniformImputerTests.java index 2d39566ca..3656be278 100644 --- a/src/test/java/org/opensearch/ad/dataprocessor/LinearUniformInterpolatorTests.java +++ b/src/test/java/org/opensearch/timeseries/dataprocessor/MultiFeatureLinearUniformImputerTests.java @@ -9,7 +9,7 @@ * GitHub history for details. */ -package org.opensearch.ad.dataprocessor; +package org.opensearch.timeseries.dataprocessor; import static org.junit.Assert.assertArrayEquals; @@ -23,7 +23,7 @@ import org.junit.runners.Parameterized.Parameters; @RunWith(Parameterized.class) -public class LinearUniformInterpolatorTests { +public class MultiFeatureLinearUniformImputerTests { @Parameters public static Collection data() { @@ -34,6 +34,7 @@ public static Collection data() { return Arrays .asList( new Object[][] { + // no integer sensitive rint rounding at the end of singleFeatureImpute. { singleComponent, 2, singleComponent }, { singleComponent, 3, new double[][] { { -1.0, 0.5, 2.0 }, { 1.0, 1.0, 1.0 } } }, { singleComponent, 4, new double[][] { { -1.0, 0.0, 1.0, 2.0 }, { 1.0, 1.0, 1.0, 1.0 } } }, @@ -47,9 +48,9 @@ public static Collection data() { private double[][] input; private int numInterpolants; private double[][] expected; - private LinearUniformInterpolator interpolator; + private Imputer imputer; - public LinearUniformInterpolatorTests(double[][] input, int numInterpolants, double[][] expected) { + public MultiFeatureLinearUniformImputerTests(double[][] input, int numInterpolants, double[][] expected) { this.input = input; this.numInterpolants = numInterpolants; this.expected = expected; @@ -57,12 +58,12 @@ public LinearUniformInterpolatorTests(double[][] input, int numInterpolants, dou @Before public void setUp() { - this.interpolator = new LinearUniformInterpolator(new SingleFeatureLinearUniformInterpolator()); + this.imputer = new LinearUniformImputer(false); } @Test - public void testInterpolation() { - double[][] actual = interpolator.interpolate(input, numInterpolants); + public void testImputation() { + double[][] actual = imputer.impute(input, numInterpolants); double delta = 1e-8; int numFeatures = expected.length; diff --git a/src/test/java/org/opensearch/timeseries/dataprocessor/PreviousValueImputerTests.java b/src/test/java/org/opensearch/timeseries/dataprocessor/PreviousValueImputerTests.java new file mode 100644 index 000000000..ff0cfbba2 --- /dev/null +++ b/src/test/java/org/opensearch/timeseries/dataprocessor/PreviousValueImputerTests.java @@ -0,0 +1,28 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.timeseries.dataprocessor; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.jupiter.api.Test; + +public class PreviousValueImputerTests { + @Test + void testSingleFeatureImpute() { + PreviousValueImputer imputer = new PreviousValueImputer(); + + double[] samples = { 1.0, Double.NaN, 3.0, Double.NaN, 5.0 }; + double[] expected = { 1.0, 1.0, 3.0, 3.0, 5.0 }; + + assertArrayEquals(expected, imputer.singleFeatureImpute(samples, 0), "Imputation failed"); + + // The second test checks whether the method removes leading Double.NaN values from the array + samples = new double[] { Double.NaN, 2.0, Double.NaN, 4.0 }; + expected = new double[] { Double.NaN, 2.0, 2.0, 4.0 }; + + assertArrayEquals(expected, imputer.singleFeatureImpute(samples, 0), "Imputation failed with leading NaN"); + } +} diff --git a/src/test/java/org/opensearch/ad/dataprocessor/IntegerSensitiveSingleFeatureLinearUniformInterpolatorTests.java b/src/test/java/org/opensearch/timeseries/dataprocessor/SingleFeatureLinearUniformImputerTests.java similarity index 54% rename from src/test/java/org/opensearch/ad/dataprocessor/IntegerSensitiveSingleFeatureLinearUniformInterpolatorTests.java rename to src/test/java/org/opensearch/timeseries/dataprocessor/SingleFeatureLinearUniformImputerTests.java index 045c05297..17aae0422 100644 --- a/src/test/java/org/opensearch/ad/dataprocessor/IntegerSensitiveSingleFeatureLinearUniformInterpolatorTests.java +++ b/src/test/java/org/opensearch/timeseries/dataprocessor/SingleFeatureLinearUniformImputerTests.java @@ -9,7 +9,7 @@ * GitHub history for details. */ -package org.opensearch.ad.dataprocessor; +package org.opensearch.timeseries.dataprocessor; import static org.junit.Assert.assertTrue; @@ -23,25 +23,28 @@ import org.junit.runner.RunWith; @RunWith(JUnitParamsRunner.class) -public class IntegerSensitiveSingleFeatureLinearUniformInterpolatorTests { +public class SingleFeatureLinearUniformImputerTests { - private IntegerSensitiveSingleFeatureLinearUniformInterpolator interpolator; + private Imputer imputer; @Before public void setup() { - interpolator = new IntegerSensitiveSingleFeatureLinearUniformInterpolator(); + imputer = new LinearUniformImputer(false); } - private Object[] interpolateData() { + private Object[] imputeData() { return new Object[] { new Object[] { new double[] { 25.25, 25.75 }, 3, new double[] { 25.25, 25.5, 25.75 } }, new Object[] { new double[] { 25, 75 }, 3, new double[] { 25, 50, 75 } }, - new Object[] { new double[] { 25, 75.5 }, 3, new double[] { 25, 50.25, 75.5 } }, }; + new Object[] { new double[] { 25, 75.5 }, 3, new double[] { 25, 50.25, 75.5 } }, + new Object[] { new double[] { 25.25, 25.75 }, 3, new double[] { 25.25, 25.5, 25.75 } }, + new Object[] { new double[] { 25, 75 }, 3, new double[] { 25, 50, 75 } }, + new Object[] { new double[] { 25, 75.5 }, 3, new double[] { 25, 50.25, 75.5 } } }; } @Test - @Parameters(method = "interpolateData") - public void interpolate_returnExpected(double[] samples, int num, double[] expected) { - assertTrue(Arrays.equals(expected, interpolator.interpolate(samples, num))); + @Parameters(method = "imputeData") + public void impute_returnExpected(double[] samples, int num, double[] expected) { + assertTrue(Arrays.equals(expected, imputer.singleFeatureImpute(samples, num))); } } diff --git a/src/test/java/org/opensearch/timeseries/dataprocessor/ZeroImputerTests.java b/src/test/java/org/opensearch/timeseries/dataprocessor/ZeroImputerTests.java new file mode 100644 index 000000000..17f79cd0c --- /dev/null +++ b/src/test/java/org/opensearch/timeseries/dataprocessor/ZeroImputerTests.java @@ -0,0 +1,45 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.timeseries.dataprocessor; + +import static org.junit.Assert.assertArrayEquals; + +import junitparams.JUnitParamsRunner; +import junitparams.Parameters; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(JUnitParamsRunner.class) +public class ZeroImputerTests { + + private Imputer imputer; + + @Before + public void setup() { + imputer = new ZeroImputer(); + } + + private Object[] imputeData() { + return new Object[] { + new Object[] { new double[] { 25.25, Double.NaN, 25.75 }, 3, new double[] { 25.25, 0, 25.75 } }, + new Object[] { new double[] { Double.NaN, 25, 75 }, 3, new double[] { 0, 25, 75 } }, + new Object[] { new double[] { 25, 75.5, Double.NaN }, 3, new double[] { 25, 75.5, 0 } }, }; + } + + @Test + @Parameters(method = "imputeData") + public void impute_returnExpected(double[] samples, int num, double[] expected) { + assertArrayEquals("The arrays are not equal", expected, imputer.singleFeatureImpute(samples, num), 0.001); + } +} diff --git a/src/test/java/org/opensearch/timeseries/util/LTrimTests.java b/src/test/java/org/opensearch/timeseries/util/LTrimTests.java new file mode 100644 index 000000000..24ed663e8 --- /dev/null +++ b/src/test/java/org/opensearch/timeseries/util/LTrimTests.java @@ -0,0 +1,49 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.timeseries.util; + +import org.opensearch.test.OpenSearchTestCase; + +public class LTrimTests extends OpenSearchTestCase { + + public void testLtrimEmptyArray() { + + double[][] input = {}; + double[][] expectedOutput = {}; + + assertArrayEquals(expectedOutput, DataUtil.ltrim(input)); + } + + public void testLtrimAllNaN() { + + double[][] input = { { Double.NaN, Double.NaN }, { Double.NaN, Double.NaN }, { Double.NaN, Double.NaN } }; + double[][] expectedOutput = {}; + + assertArrayEquals(expectedOutput, DataUtil.ltrim(input)); + } + + public void testLtrimSomeNaN() { + + double[][] input = { { Double.NaN, Double.NaN }, { 1.0, 2.0 }, { 3.0, 4.0 } }; + double[][] expectedOutput = { { 1.0, 2.0 }, { 3.0, 4.0 } }; + + assertArrayEquals(expectedOutput, DataUtil.ltrim(input)); + } + + public void testLtrimNoNaN() { + + double[][] input = { { 1.0, 2.0 }, { 3.0, 4.0 }, { 5.0, 6.0 } }; + double[][] expectedOutput = { { 1.0, 2.0 }, { 3.0, 4.0 }, { 5.0, 6.0 } }; + + assertArrayEquals(expectedOutput, DataUtil.ltrim(input)); + } +} From 5059bfded58ebdcc430415a0f1cab9780c710d02 Mon Sep 17 00:00:00 2001 From: Kaituo Li Date: Mon, 22 May 2023 15:32:25 -0700 Subject: [PATCH 2/2] Various changes The commit makes all new files use the shortened header text and addresses compiler/test issues due to core refactoring Signed-off-by: Kaituo Li --- build.gradle | 1 + .../opensearch/ad/caching/PriorityCache.java | 2 +- .../NotSerializedADExceptionName.java | 22 +++++++++---------- .../model/AnomalyDetectorExecutionInput.java | 2 +- .../CheckPointMaintainRequestAdapter.java | 2 +- .../ad/ratelimit/CheckpointWriteWorker.java | 2 +- .../ad/rest/RestGetAnomalyDetectorAction.java | 2 +- .../RestPreviewAnomalyDetectorAction.java | 2 +- .../rest/RestStatsAnomalyDetectorAction.java | 2 +- .../ADBatchAnomalyResultRequest.java | 2 +- .../ad/transport/ADCancelTaskRequest.java | 2 +- .../ad/transport/ADTaskProfileRequest.java | 2 +- .../ad/transport/AnomalyResultRequest.java | 2 +- .../DeleteAnomalyDetectorRequest.java | 2 +- .../ad/transport/DeleteModelRequest.java | 2 +- .../ad/transport/EntityProfileRequest.java | 2 +- .../ad/transport/EntityResultRequest.java | 2 +- .../GetAnomalyDetectorTransportAction.java | 2 +- .../ad/transport/RCFPollingRequest.java | 2 +- .../ad/transport/RCFResultRequest.java | 2 +- ...SearchTopAnomalyResultTransportAction.java | 2 +- .../ad/transport/StopDetectorRequest.java | 2 +- .../ad/transport/ThresholdResultRequest.java | 2 +- .../ad/util/ADSafeSecurityInjector.java | 2 +- .../dataprocessor/FixedValueImputer.java | 8 +------ .../timeseries/dataprocessor/Imputer.java | 8 +------ .../dataprocessor/LinearUniformImputer.java | 8 +------ .../dataprocessor/PreviousValueImputer.java | 8 +------ .../timeseries/dataprocessor/ZeroImputer.java | 8 +------ .../opensearch/timeseries/util/DataUtil.java | 8 +------ .../ad/e2e/AbstractSyntheticDataTest.java | 5 +++-- .../ad/util/ExceptionUtilsTests.java | 2 +- .../dataprocessor/FixedValueImputerTests.java | 8 +------ ...gerSensitiveLinearUniformImputerTests.java | 8 +------ .../dataprocessor/ZeroImputerTests.java | 8 +------ .../timeseries/util/LTrimTests.java | 8 +------ 36 files changed, 48 insertions(+), 106 deletions(-) diff --git a/build.gradle b/build.gradle index 99085787a..5bb23bb78 100644 --- a/build.gradle +++ b/build.gradle @@ -163,6 +163,7 @@ dependencies { testImplementation 'org.junit.jupiter:junit-jupiter-api:5.7.2' testImplementation 'org.junit.jupiter:junit-jupiter-params:5.7.2' testImplementation 'org.junit.jupiter:junit-jupiter-engine:5.7.2' + testImplementation "org.opensearch:opensearch-core:${opensearch_version}" testRuntimeOnly 'org.junit.vintage:junit-vintage-engine:5.7.2' testCompileOnly 'junit:junit:4.13.2' } diff --git a/src/main/java/org/opensearch/ad/caching/PriorityCache.java b/src/main/java/org/opensearch/ad/caching/PriorityCache.java index 31c183ca8..1130a57c5 100644 --- a/src/main/java/org/opensearch/ad/caching/PriorityCache.java +++ b/src/main/java/org/opensearch/ad/caching/PriorityCache.java @@ -57,10 +57,10 @@ import org.opensearch.ad.settings.EnabledSetting; import org.opensearch.ad.util.DateUtils; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.Strings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.common.Strings; import org.opensearch.threadpool.ThreadPool; import org.opensearch.timeseries.constant.CommonMessages; diff --git a/src/main/java/org/opensearch/ad/common/exception/NotSerializedADExceptionName.java b/src/main/java/org/opensearch/ad/common/exception/NotSerializedADExceptionName.java index 0631d29d3..31668cb5e 100644 --- a/src/main/java/org/opensearch/ad/common/exception/NotSerializedADExceptionName.java +++ b/src/main/java/org/opensearch/ad/common/exception/NotSerializedADExceptionName.java @@ -16,7 +16,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.opensearch.OpenSearchException; +import org.opensearch.BaseExceptionsHelper; import org.opensearch.common.io.stream.NotSerializableExceptionWrapper; /** @@ -29,16 +29,16 @@ */ public enum NotSerializedADExceptionName { - RESOURCE_NOT_FOUND_EXCEPTION_NAME_UNDERSCORE(OpenSearchException.getExceptionName(new ResourceNotFoundException("", ""))), - LIMIT_EXCEEDED_EXCEPTION_NAME_UNDERSCORE(OpenSearchException.getExceptionName(new LimitExceededException("", "", false))), - END_RUN_EXCEPTION_NAME_UNDERSCORE(OpenSearchException.getExceptionName(new EndRunException("", "", false))), - ANOMALY_DETECTION_EXCEPTION_NAME_UNDERSCORE(OpenSearchException.getExceptionName(new AnomalyDetectionException("", ""))), - INTERNAL_FAILURE_NAME_UNDERSCORE(OpenSearchException.getExceptionName(new InternalFailure("", ""))), - CLIENT_EXCEPTION_NAME_UNDERSCORE(OpenSearchException.getExceptionName(new ClientException("", ""))), - CANCELLATION_EXCEPTION_NAME_UNDERSCORE(OpenSearchException.getExceptionName(new ADTaskCancelledException("", ""))), - DUPLICATE_TASK_EXCEPTION_NAME_UNDERSCORE(OpenSearchException.getExceptionName(new DuplicateTaskException(""))), - AD_VERSION_EXCEPTION_NAME_UNDERSCORE(OpenSearchException.getExceptionName(new ADVersionException(""))), - AD_VALIDATION_EXCEPTION_NAME_UNDERSCORE(OpenSearchException.getExceptionName(new ADValidationException("", null, null))); + RESOURCE_NOT_FOUND_EXCEPTION_NAME_UNDERSCORE(BaseExceptionsHelper.getExceptionName(new ResourceNotFoundException("", ""))), + LIMIT_EXCEEDED_EXCEPTION_NAME_UNDERSCORE(BaseExceptionsHelper.getExceptionName(new LimitExceededException("", "", false))), + END_RUN_EXCEPTION_NAME_UNDERSCORE(BaseExceptionsHelper.getExceptionName(new EndRunException("", "", false))), + ANOMALY_DETECTION_EXCEPTION_NAME_UNDERSCORE(BaseExceptionsHelper.getExceptionName(new AnomalyDetectionException("", ""))), + INTERNAL_FAILURE_NAME_UNDERSCORE(BaseExceptionsHelper.getExceptionName(new InternalFailure("", ""))), + CLIENT_EXCEPTION_NAME_UNDERSCORE(BaseExceptionsHelper.getExceptionName(new ClientException("", ""))), + CANCELLATION_EXCEPTION_NAME_UNDERSCORE(BaseExceptionsHelper.getExceptionName(new ADTaskCancelledException("", ""))), + DUPLICATE_TASK_EXCEPTION_NAME_UNDERSCORE(BaseExceptionsHelper.getExceptionName(new DuplicateTaskException(""))), + AD_VERSION_EXCEPTION_NAME_UNDERSCORE(BaseExceptionsHelper.getExceptionName(new ADVersionException(""))), + AD_VALIDATION_EXCEPTION_NAME_UNDERSCORE(BaseExceptionsHelper.getExceptionName(new ADValidationException("", null, null))); private static final Logger LOG = LogManager.getLogger(NotSerializedADExceptionName.class); private final String name; diff --git a/src/main/java/org/opensearch/ad/model/AnomalyDetectorExecutionInput.java b/src/main/java/org/opensearch/ad/model/AnomalyDetectorExecutionInput.java index 5e05378b4..b98fdbb58 100644 --- a/src/main/java/org/opensearch/ad/model/AnomalyDetectorExecutionInput.java +++ b/src/main/java/org/opensearch/ad/model/AnomalyDetectorExecutionInput.java @@ -17,7 +17,7 @@ import java.time.Instant; import org.opensearch.ad.util.ParseUtils; -import org.opensearch.common.Strings; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; diff --git a/src/main/java/org/opensearch/ad/ratelimit/CheckPointMaintainRequestAdapter.java b/src/main/java/org/opensearch/ad/ratelimit/CheckPointMaintainRequestAdapter.java index 8723e3d35..072855069 100644 --- a/src/main/java/org/opensearch/ad/ratelimit/CheckPointMaintainRequestAdapter.java +++ b/src/main/java/org/opensearch/ad/ratelimit/CheckPointMaintainRequestAdapter.java @@ -27,10 +27,10 @@ import org.opensearch.ad.ml.ModelState; import org.opensearch.ad.util.DateUtils; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.Strings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.common.Strings; public class CheckPointMaintainRequestAdapter { private static final Logger LOG = LogManager.getLogger(CheckPointMaintainRequestAdapter.class); diff --git a/src/main/java/org/opensearch/ad/ratelimit/CheckpointWriteWorker.java b/src/main/java/org/opensearch/ad/ratelimit/CheckpointWriteWorker.java index 56caddd54..fc7fc0a07 100644 --- a/src/main/java/org/opensearch/ad/ratelimit/CheckpointWriteWorker.java +++ b/src/main/java/org/opensearch/ad/ratelimit/CheckpointWriteWorker.java @@ -39,9 +39,9 @@ import org.opensearch.ad.model.AnomalyDetector; import org.opensearch.ad.util.ExceptionUtil; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.Strings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; +import org.opensearch.core.common.Strings; import org.opensearch.threadpool.ThreadPool; public class CheckpointWriteWorker extends BatchWorker { diff --git a/src/main/java/org/opensearch/ad/rest/RestGetAnomalyDetectorAction.java b/src/main/java/org/opensearch/ad/rest/RestGetAnomalyDetectorAction.java index 46cb52843..330a3a68e 100644 --- a/src/main/java/org/opensearch/ad/rest/RestGetAnomalyDetectorAction.java +++ b/src/main/java/org/opensearch/ad/rest/RestGetAnomalyDetectorAction.java @@ -30,7 +30,7 @@ import org.opensearch.ad.transport.GetAnomalyDetectorAction; import org.opensearch.ad.transport.GetAnomalyDetectorRequest; import org.opensearch.client.node.NodeClient; -import org.opensearch.common.Strings; +import org.opensearch.core.common.Strings; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.RestActions; diff --git a/src/main/java/org/opensearch/ad/rest/RestPreviewAnomalyDetectorAction.java b/src/main/java/org/opensearch/ad/rest/RestPreviewAnomalyDetectorAction.java index e300689d4..abae558cc 100644 --- a/src/main/java/org/opensearch/ad/rest/RestPreviewAnomalyDetectorAction.java +++ b/src/main/java/org/opensearch/ad/rest/RestPreviewAnomalyDetectorAction.java @@ -28,7 +28,7 @@ import org.opensearch.ad.transport.PreviewAnomalyDetectorAction; import org.opensearch.ad.transport.PreviewAnomalyDetectorRequest; import org.opensearch.ad.util.RestHandlerUtils; -import org.opensearch.common.Strings; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.BytesRestResponse; diff --git a/src/main/java/org/opensearch/ad/rest/RestStatsAnomalyDetectorAction.java b/src/main/java/org/opensearch/ad/rest/RestStatsAnomalyDetectorAction.java index 5fa9bf686..ae0886604 100644 --- a/src/main/java/org/opensearch/ad/rest/RestStatsAnomalyDetectorAction.java +++ b/src/main/java/org/opensearch/ad/rest/RestStatsAnomalyDetectorAction.java @@ -29,7 +29,7 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.Strings; +import org.opensearch.core.common.Strings; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.RestToXContentListener; diff --git a/src/main/java/org/opensearch/ad/transport/ADBatchAnomalyResultRequest.java b/src/main/java/org/opensearch/ad/transport/ADBatchAnomalyResultRequest.java index 53cdb4a76..7d2d86f90 100644 --- a/src/main/java/org/opensearch/ad/transport/ADBatchAnomalyResultRequest.java +++ b/src/main/java/org/opensearch/ad/transport/ADBatchAnomalyResultRequest.java @@ -19,9 +19,9 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.ad.model.ADTask; import org.opensearch.ad.model.AnomalyDetector; -import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.Strings; public class ADBatchAnomalyResultRequest extends ActionRequest { private ADTask adTask; diff --git a/src/main/java/org/opensearch/ad/transport/ADCancelTaskRequest.java b/src/main/java/org/opensearch/ad/transport/ADCancelTaskRequest.java index 4b0d02250..f289e6fc1 100644 --- a/src/main/java/org/opensearch/ad/transport/ADCancelTaskRequest.java +++ b/src/main/java/org/opensearch/ad/transport/ADCancelTaskRequest.java @@ -19,9 +19,9 @@ import org.opensearch.action.support.nodes.BaseNodesRequest; import org.opensearch.ad.constant.ADCommonMessages; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.Strings; public class ADCancelTaskRequest extends BaseNodesRequest { diff --git a/src/main/java/org/opensearch/ad/transport/ADTaskProfileRequest.java b/src/main/java/org/opensearch/ad/transport/ADTaskProfileRequest.java index 4fe5e5bec..c2c05ff05 100644 --- a/src/main/java/org/opensearch/ad/transport/ADTaskProfileRequest.java +++ b/src/main/java/org/opensearch/ad/transport/ADTaskProfileRequest.java @@ -19,9 +19,9 @@ import org.opensearch.action.support.nodes.BaseNodesRequest; import org.opensearch.ad.constant.ADCommonMessages; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.Strings; public class ADTaskProfileRequest extends BaseNodesRequest { diff --git a/src/main/java/org/opensearch/ad/transport/AnomalyResultRequest.java b/src/main/java/org/opensearch/ad/transport/AnomalyResultRequest.java index bb9b3f110..b9efc10b3 100644 --- a/src/main/java/org/opensearch/ad/transport/AnomalyResultRequest.java +++ b/src/main/java/org/opensearch/ad/transport/AnomalyResultRequest.java @@ -22,11 +22,11 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.ad.constant.ADCommonMessages; import org.opensearch.ad.constant.ADCommonName; -import org.opensearch.common.Strings; import org.opensearch.common.io.stream.InputStreamStreamInput; import org.opensearch.common.io.stream.OutputStreamStreamOutput; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.timeseries.constant.CommonMessages; diff --git a/src/main/java/org/opensearch/ad/transport/DeleteAnomalyDetectorRequest.java b/src/main/java/org/opensearch/ad/transport/DeleteAnomalyDetectorRequest.java index 624dbc6c0..22686616b 100644 --- a/src/main/java/org/opensearch/ad/transport/DeleteAnomalyDetectorRequest.java +++ b/src/main/java/org/opensearch/ad/transport/DeleteAnomalyDetectorRequest.java @@ -18,9 +18,9 @@ import org.opensearch.action.ActionRequest; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.ad.constant.ADCommonMessages; -import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.Strings; public class DeleteAnomalyDetectorRequest extends ActionRequest { diff --git a/src/main/java/org/opensearch/ad/transport/DeleteModelRequest.java b/src/main/java/org/opensearch/ad/transport/DeleteModelRequest.java index 080df95d6..2710fca8b 100644 --- a/src/main/java/org/opensearch/ad/transport/DeleteModelRequest.java +++ b/src/main/java/org/opensearch/ad/transport/DeleteModelRequest.java @@ -20,9 +20,9 @@ import org.opensearch.ad.constant.ADCommonMessages; import org.opensearch.ad.constant.ADCommonName; import org.opensearch.cluster.node.DiscoveryNode; -import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/src/main/java/org/opensearch/ad/transport/EntityProfileRequest.java b/src/main/java/org/opensearch/ad/transport/EntityProfileRequest.java index a4950334a..76addebcf 100644 --- a/src/main/java/org/opensearch/ad/transport/EntityProfileRequest.java +++ b/src/main/java/org/opensearch/ad/transport/EntityProfileRequest.java @@ -23,9 +23,9 @@ import org.opensearch.ad.constant.ADCommonName; import org.opensearch.ad.model.Entity; import org.opensearch.ad.model.EntityProfileName; -import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/src/main/java/org/opensearch/ad/transport/EntityResultRequest.java b/src/main/java/org/opensearch/ad/transport/EntityResultRequest.java index beb9acd62..bafab7f80 100644 --- a/src/main/java/org/opensearch/ad/transport/EntityResultRequest.java +++ b/src/main/java/org/opensearch/ad/transport/EntityResultRequest.java @@ -24,9 +24,9 @@ import org.opensearch.ad.constant.ADCommonMessages; import org.opensearch.ad.constant.ADCommonName; import org.opensearch.ad.model.Entity; -import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.timeseries.constant.CommonMessages; diff --git a/src/main/java/org/opensearch/ad/transport/GetAnomalyDetectorTransportAction.java b/src/main/java/org/opensearch/ad/transport/GetAnomalyDetectorTransportAction.java index d870a02d7..3eda10a94 100644 --- a/src/main/java/org/opensearch/ad/transport/GetAnomalyDetectorTransportAction.java +++ b/src/main/java/org/opensearch/ad/transport/GetAnomalyDetectorTransportAction.java @@ -60,11 +60,11 @@ import org.opensearch.client.Client; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.CheckedConsumer; -import org.opensearch.common.Strings; import org.opensearch.common.inject.Inject; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.commons.authuser.User; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.rest.RestStatus; diff --git a/src/main/java/org/opensearch/ad/transport/RCFPollingRequest.java b/src/main/java/org/opensearch/ad/transport/RCFPollingRequest.java index 339980f40..a0e2c1e49 100644 --- a/src/main/java/org/opensearch/ad/transport/RCFPollingRequest.java +++ b/src/main/java/org/opensearch/ad/transport/RCFPollingRequest.java @@ -19,9 +19,9 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.ad.constant.ADCommonMessages; import org.opensearch.ad.constant.ADCommonName; -import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/src/main/java/org/opensearch/ad/transport/RCFResultRequest.java b/src/main/java/org/opensearch/ad/transport/RCFResultRequest.java index 114fb6fca..d06f1eae8 100644 --- a/src/main/java/org/opensearch/ad/transport/RCFResultRequest.java +++ b/src/main/java/org/opensearch/ad/transport/RCFResultRequest.java @@ -19,9 +19,9 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.ad.constant.ADCommonMessages; import org.opensearch.ad.constant.ADCommonName; -import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.timeseries.constant.CommonName; diff --git a/src/main/java/org/opensearch/ad/transport/SearchTopAnomalyResultTransportAction.java b/src/main/java/org/opensearch/ad/transport/SearchTopAnomalyResultTransportAction.java index a56458337..f9a837dbd 100644 --- a/src/main/java/org/opensearch/ad/transport/SearchTopAnomalyResultTransportAction.java +++ b/src/main/java/org/opensearch/ad/transport/SearchTopAnomalyResultTransportAction.java @@ -39,8 +39,8 @@ import org.opensearch.ad.model.AnomalyResultBucket; import org.opensearch.ad.transport.handler.ADSearchHandler; import org.opensearch.client.Client; -import org.opensearch.common.Strings; import org.opensearch.common.inject.Inject; +import org.opensearch.core.common.Strings; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.ExistsQueryBuilder; import org.opensearch.index.query.QueryBuilder; diff --git a/src/main/java/org/opensearch/ad/transport/StopDetectorRequest.java b/src/main/java/org/opensearch/ad/transport/StopDetectorRequest.java index 7bfc2c973..787bb851a 100644 --- a/src/main/java/org/opensearch/ad/transport/StopDetectorRequest.java +++ b/src/main/java/org/opensearch/ad/transport/StopDetectorRequest.java @@ -21,11 +21,11 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.ad.constant.ADCommonMessages; import org.opensearch.ad.constant.ADCommonName; -import org.opensearch.common.Strings; import org.opensearch.common.io.stream.InputStreamStreamInput; import org.opensearch.common.io.stream.OutputStreamStreamOutput; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; diff --git a/src/main/java/org/opensearch/ad/transport/ThresholdResultRequest.java b/src/main/java/org/opensearch/ad/transport/ThresholdResultRequest.java index a71e9116f..72751bf9a 100644 --- a/src/main/java/org/opensearch/ad/transport/ThresholdResultRequest.java +++ b/src/main/java/org/opensearch/ad/transport/ThresholdResultRequest.java @@ -19,9 +19,9 @@ import org.opensearch.action.ActionRequestValidationException; import org.opensearch.ad.constant.ADCommonMessages; import org.opensearch.ad.constant.ADCommonName; -import org.opensearch.common.Strings; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.timeseries.constant.CommonName; diff --git a/src/main/java/org/opensearch/ad/util/ADSafeSecurityInjector.java b/src/main/java/org/opensearch/ad/util/ADSafeSecurityInjector.java index 8fcd11ce0..b38455d36 100644 --- a/src/main/java/org/opensearch/ad/util/ADSafeSecurityInjector.java +++ b/src/main/java/org/opensearch/ad/util/ADSafeSecurityInjector.java @@ -19,10 +19,10 @@ import org.opensearch.ad.NodeStateManager; import org.opensearch.ad.common.exception.EndRunException; import org.opensearch.ad.model.AnomalyDetector; -import org.opensearch.common.Strings; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.commons.authuser.User; +import org.opensearch.core.common.Strings; public class ADSafeSecurityInjector extends SafeSecurityInjector { private static final Logger LOG = LogManager.getLogger(ADSafeSecurityInjector.class); diff --git a/src/main/java/org/opensearch/timeseries/dataprocessor/FixedValueImputer.java b/src/main/java/org/opensearch/timeseries/dataprocessor/FixedValueImputer.java index 90804b1da..9b8f6bf21 100644 --- a/src/main/java/org/opensearch/timeseries/dataprocessor/FixedValueImputer.java +++ b/src/main/java/org/opensearch/timeseries/dataprocessor/FixedValueImputer.java @@ -1,12 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ package org.opensearch.timeseries.dataprocessor; diff --git a/src/main/java/org/opensearch/timeseries/dataprocessor/Imputer.java b/src/main/java/org/opensearch/timeseries/dataprocessor/Imputer.java index 400aae44e..4e885421c 100644 --- a/src/main/java/org/opensearch/timeseries/dataprocessor/Imputer.java +++ b/src/main/java/org/opensearch/timeseries/dataprocessor/Imputer.java @@ -1,12 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ package org.opensearch.timeseries.dataprocessor; diff --git a/src/main/java/org/opensearch/timeseries/dataprocessor/LinearUniformImputer.java b/src/main/java/org/opensearch/timeseries/dataprocessor/LinearUniformImputer.java index e397cb1af..2fa3fd651 100644 --- a/src/main/java/org/opensearch/timeseries/dataprocessor/LinearUniformImputer.java +++ b/src/main/java/org/opensearch/timeseries/dataprocessor/LinearUniformImputer.java @@ -1,12 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ package org.opensearch.timeseries.dataprocessor; diff --git a/src/main/java/org/opensearch/timeseries/dataprocessor/PreviousValueImputer.java b/src/main/java/org/opensearch/timeseries/dataprocessor/PreviousValueImputer.java index 39f7f94f1..e91c90814 100644 --- a/src/main/java/org/opensearch/timeseries/dataprocessor/PreviousValueImputer.java +++ b/src/main/java/org/opensearch/timeseries/dataprocessor/PreviousValueImputer.java @@ -1,12 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ package org.opensearch.timeseries.dataprocessor; diff --git a/src/main/java/org/opensearch/timeseries/dataprocessor/ZeroImputer.java b/src/main/java/org/opensearch/timeseries/dataprocessor/ZeroImputer.java index dcc959b7b..1d0656de1 100644 --- a/src/main/java/org/opensearch/timeseries/dataprocessor/ZeroImputer.java +++ b/src/main/java/org/opensearch/timeseries/dataprocessor/ZeroImputer.java @@ -1,12 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ package org.opensearch.timeseries.dataprocessor; diff --git a/src/main/java/org/opensearch/timeseries/util/DataUtil.java b/src/main/java/org/opensearch/timeseries/util/DataUtil.java index eacd85ee4..4f417e4f7 100644 --- a/src/main/java/org/opensearch/timeseries/util/DataUtil.java +++ b/src/main/java/org/opensearch/timeseries/util/DataUtil.java @@ -1,12 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ package org.opensearch.timeseries.util; diff --git a/src/test/java/org/opensearch/ad/e2e/AbstractSyntheticDataTest.java b/src/test/java/org/opensearch/ad/e2e/AbstractSyntheticDataTest.java index c0a7676fa..02bf8dbd4 100644 --- a/src/test/java/org/opensearch/ad/e2e/AbstractSyntheticDataTest.java +++ b/src/test/java/org/opensearch/ad/e2e/AbstractSyntheticDataTest.java @@ -35,8 +35,8 @@ import org.opensearch.client.Response; import org.opensearch.client.RestClient; import org.opensearch.client.WarningsHandler; -import org.opensearch.common.Strings; import org.opensearch.common.xcontent.json.JsonXContent; +import org.opensearch.core.common.Strings; import org.opensearch.core.xcontent.XContentBuilder; import com.google.common.collect.ImmutableList; @@ -66,7 +66,7 @@ protected void disableResourceNotFoundFaultTolerence() throws IOException { settingCommand.endObject(); settingCommand.endObject(); Request request = new Request("PUT", "/_cluster/settings"); - request.setJsonEntity(Strings.toString(settingCommand)); + request.setJsonEntity(org.opensearch.common.Strings.toString(settingCommand)); adminClient().performRequest(request); } @@ -192,6 +192,7 @@ protected String createDetector( return detectorId; } + @Override protected void waitAllSyncheticDataIngested(int expectedSize, String datasetName, RestClient client) throws Exception { int maxWaitCycles = 3; do { diff --git a/src/test/java/org/opensearch/ad/util/ExceptionUtilsTests.java b/src/test/java/org/opensearch/ad/util/ExceptionUtilsTests.java index 94af37770..00c9d1a2d 100644 --- a/src/test/java/org/opensearch/ad/util/ExceptionUtilsTests.java +++ b/src/test/java/org/opensearch/ad/util/ExceptionUtilsTests.java @@ -56,7 +56,7 @@ public void testCountInStats() { public void testGetErrorMessage() { assertEquals("test", ExceptionUtil.getErrorMessage(new AnomalyDetectionException("test"))); assertEquals("test", ExceptionUtil.getErrorMessage(new IllegalArgumentException("test"))); - assertEquals("org.opensearch.OpenSearchException: test", ExceptionUtil.getErrorMessage(new OpenSearchException("test"))); + assertEquals("OpenSearchException[test]", ExceptionUtil.getErrorMessage(new OpenSearchException("test"))); assertTrue( ExceptionUtil .getErrorMessage(new RuntimeException("test")) diff --git a/src/test/java/org/opensearch/timeseries/dataprocessor/FixedValueImputerTests.java b/src/test/java/org/opensearch/timeseries/dataprocessor/FixedValueImputerTests.java index 8c88489a2..81b9b5bfb 100644 --- a/src/test/java/org/opensearch/timeseries/dataprocessor/FixedValueImputerTests.java +++ b/src/test/java/org/opensearch/timeseries/dataprocessor/FixedValueImputerTests.java @@ -1,12 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ package org.opensearch.timeseries.dataprocessor; diff --git a/src/test/java/org/opensearch/timeseries/dataprocessor/IntegerSensitiveLinearUniformImputerTests.java b/src/test/java/org/opensearch/timeseries/dataprocessor/IntegerSensitiveLinearUniformImputerTests.java index 12da46895..03e8b6cb3 100644 --- a/src/test/java/org/opensearch/timeseries/dataprocessor/IntegerSensitiveLinearUniformImputerTests.java +++ b/src/test/java/org/opensearch/timeseries/dataprocessor/IntegerSensitiveLinearUniformImputerTests.java @@ -1,12 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ package org.opensearch.timeseries.dataprocessor; diff --git a/src/test/java/org/opensearch/timeseries/dataprocessor/ZeroImputerTests.java b/src/test/java/org/opensearch/timeseries/dataprocessor/ZeroImputerTests.java index 17f79cd0c..a13189db2 100644 --- a/src/test/java/org/opensearch/timeseries/dataprocessor/ZeroImputerTests.java +++ b/src/test/java/org/opensearch/timeseries/dataprocessor/ZeroImputerTests.java @@ -1,12 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ package org.opensearch.timeseries.dataprocessor; diff --git a/src/test/java/org/opensearch/timeseries/util/LTrimTests.java b/src/test/java/org/opensearch/timeseries/util/LTrimTests.java index 24ed663e8..384982828 100644 --- a/src/test/java/org/opensearch/timeseries/util/LTrimTests.java +++ b/src/test/java/org/opensearch/timeseries/util/LTrimTests.java @@ -1,12 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - * - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. */ package org.opensearch.timeseries.util;