diff --git a/dqops/src/main/java/com/dqops/checks/AbstractCheckSpec.java b/dqops/src/main/java/com/dqops/checks/AbstractCheckSpec.java index 0cc53b62bf..d70a136d11 100644 --- a/dqops/src/main/java/com/dqops/checks/AbstractCheckSpec.java +++ b/dqops/src/main/java/com/dqops/checks/AbstractCheckSpec.java @@ -48,6 +48,7 @@ import lombok.ToString; import tech.tablesaw.api.IntColumn; +import java.time.Instant; import java.util.Objects; /** @@ -121,6 +122,13 @@ public abstract class AbstractCheckSpec targetCheckCloned = defaultCheck.deepClone(); targetCheckCloned.setDefaultCheck(true); + targetCheckCloned.setPolicyLastModified(policyLastModified); FieldInfo targetCategoryCheckFieldInfo = targetCategoryChildMap.getReflectionClassInfo().getFieldByYamlName(defaultChecksEntry.getChildName()); targetCategoryCheckFieldInfo.setFieldValue(targetCheckCloned, targetCategoryContainer); } @@ -244,6 +248,7 @@ public void copyChecksToContainer(AbstractRootChecksContainerSpec targetChecksCo CustomCheckSpec clonedTargetCustomCheck = (CustomCheckSpec)defaultCategoryCustomCheckKeyValue.getValue().deepClone(); clonedTargetCustomCheck.setDefaultCheck(true); + clonedTargetCustomCheck.setPolicyLastModified(policyLastModified); targetCategoryCustomChecks.put(customCheckName, clonedTargetCustomCheck); } } @@ -266,6 +271,7 @@ public void copyChecksToContainer(AbstractRootChecksContainerSpec targetChecksCo CustomCheckSpec clonedTargetCustomCheck = (CustomCheckSpec)defaultCustomCheckKeyValue.getValue().deepClone(); clonedTargetCustomCheck.setDefaultCheck(true); + clonedTargetCustomCheck.setPolicyLastModified(policyLastModified); targetCustomChecks.put(customCheckName, clonedTargetCustomCheck); } } diff --git a/dqops/src/main/java/com/dqops/checks/defaults/DefaultObservabilityConfigurationServiceImpl.java b/dqops/src/main/java/com/dqops/checks/defaults/DefaultObservabilityConfigurationServiceImpl.java index 6c866701c0..37ea498f7f 100644 --- a/dqops/src/main/java/com/dqops/checks/defaults/DefaultObservabilityConfigurationServiceImpl.java +++ b/dqops/src/main/java/com/dqops/checks/defaults/DefaultObservabilityConfigurationServiceImpl.java @@ -96,7 +96,7 @@ public void applyDefaultChecksOnTableOnly(ConnectionSpec connectionSpec, continue; } - defaultChecksPattern.applyOnTable(targetTableSpec, providerDialectSettings); + defaultChecksPattern.applyOnTable(targetTableSpec, providerDialectSettings, tableQualityPolicyWrapper.getLastModified()); } } @@ -129,7 +129,7 @@ public void applyDefaultChecksOnColumn(ConnectionSpec connectionSpec, continue; } - defaultChecksPattern.applyOnColumn(targetTableSpec, targetColumnSpec, providerDialectSettings); + defaultChecksPattern.applyOnColumn(targetTableSpec, targetColumnSpec, providerDialectSettings, columnQualityPolicyWrapper.getLastModified()); } } } diff --git a/dqops/src/main/java/com/dqops/execution/checks/TableCheckExecutionServiceImpl.java b/dqops/src/main/java/com/dqops/execution/checks/TableCheckExecutionServiceImpl.java index 8f0517a510..fca262665a 100644 --- a/dqops/src/main/java/com/dqops/execution/checks/TableCheckExecutionServiceImpl.java +++ b/dqops/src/main/java/com/dqops/execution/checks/TableCheckExecutionServiceImpl.java @@ -90,6 +90,7 @@ import tech.tablesaw.api.*; import tech.tablesaw.columns.Column; +import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneId; @@ -249,13 +250,13 @@ public CheckExecutionSummary executeChecksOnTable(ExecutionContext executionCont .collect(Collectors.toList()); executeSingleTableChecks(executionContext, userHome, userTimeWindowFilters, progressListener, dummySensorExecution, executionTarget, jobCancellationToken, checkExecutionSummary, singleTableChecks, tableSpec, outputSensorReadoutsSnapshot, allNormalizedSensorResultsTable, checkResultsSnapshot, - allRuleEvaluationResultsTable, allErrorsTable, executionStatistics, checksForErrorSampling, collectErrorSamples); + allRuleEvaluationResultsTable, allErrorsTable, executionStatistics, checksForErrorSampling, collectErrorSamples, targetTable.getLastModified()); List> tableComparisonChecks = checks.stream().filter(c -> c.isTableComparisonCheck()) .collect(Collectors.toList()); executeTableComparisonChecks(executionContext, userHome, userTimeWindowFilters, progressListener, dummySensorExecution, executionTarget, jobCancellationToken, checkExecutionSummary, tableComparisonChecks, tableSpec, outputSensorReadoutsSnapshot, allNormalizedSensorResultsTable, checkResultsSnapshot, - allRuleEvaluationResultsTable, allErrorsTable, executionStatistics); + allRuleEvaluationResultsTable, allErrorsTable, executionStatistics, targetTable.getLastModified()); if (outputSensorReadoutsSnapshot.getTableDataChanges().hasChanges() && !dummySensorExecution) { allNormalizedSensorResultsTable.removeColumns(severityColumnTemporary); // removed, it was temporary @@ -326,14 +327,15 @@ public void executeSingleTableChecks( Table allErrorsTable, TableChecksExecutionStatistics executionStatistics, List> checksNotPassedForErrorCollection, - Boolean collectErrorSamples) { + Boolean collectErrorSamples, + Instant tableYamlLastModified) { if (checks.isEmpty()) { return; } List allPreparedSensors = this.prepareSensors( checks, executionContext, userHome, tableSpec, userTimeWindowFilters, progressListener, - allErrorsTable, checkExecutionSummary, executionStatistics, jobCancellationToken); + allErrorsTable, checkExecutionSummary, executionStatistics, jobCancellationToken, tableYamlLastModified); GroupedSensorsCollection groupedSensorsCollection = new GroupedSensorsCollection(this.dqoSensorLimitsConfigurationProperties.getMaxMergedQueries()); groupedSensorsCollection.addAllPreparedSensors(allPreparedSensors); @@ -487,7 +489,8 @@ public void executeTableComparisonChecks( CheckResultsSnapshot checkResultsSnapshot, Table allRuleEvaluationResultsTable, Table allErrorsTable, - TableChecksExecutionStatistics executionStatistics) { + TableChecksExecutionStatistics executionStatistics, + Instant tableYamlLastModified) { if (checks.isEmpty()) { return; } @@ -498,7 +501,7 @@ public void executeTableComparisonChecks( List allPreparedSensorsOnComparedTable = this.prepareSensors( checks, executionContext, userHome, tableSpec, userTimeWindowFilters, progressListener, - allErrorsTable, checkExecutionSummary, executionStatistics, jobCancellationToken); + allErrorsTable, checkExecutionSummary, executionStatistics, jobCancellationToken, tableYamlLastModified); GroupedSensorsCollection groupedSensorsCollectionOnComparedTable = new GroupedSensorsCollection(this.dqoSensorLimitsConfigurationProperties.getMaxMergedQueries()); groupedSensorsCollectionOnComparedTable.addAllPreparedSensors(allPreparedSensorsOnComparedTable); @@ -509,7 +512,7 @@ public void executeTableComparisonChecks( List allPreparedSensorsOnReferenceTables = this.prepareComparisonSensorsOnReferenceTable( checks, executionContext, userHome, userTimeWindowFilters, progressListener, - checkExecutionSummary, executionStatistics, jobCancellationToken); + checkExecutionSummary, executionStatistics, jobCancellationToken, tableYamlLastModified); GroupedSensorsCollection groupedSensorsCollectionOnReferenceTables = new GroupedSensorsCollection(this.dqoSensorLimitsConfigurationProperties.getMaxMergedQueries()); groupedSensorsCollectionOnReferenceTables.addAllPreparedSensors(allPreparedSensorsOnReferenceTables); @@ -634,6 +637,7 @@ public void executeTableComparisonChecks( * @param checkExecutionSummary Check execution summary where results are added. * @param executionStatistics Execution statistics - counts of checks and errors. * @param jobCancellationToken Job cancellation token - to cancel the preparation by the user. + * @param tableYamlLastModified The timestamp when the table YAML was last modified. * @return List of prepared sensors. */ public List prepareSensors(Collection> checks, @@ -645,7 +649,8 @@ public List prepareSensors(Collection sensorPrepareResults = new ArrayList<>(); int sensorResultId = 0; @@ -657,7 +662,7 @@ public List prepareSensors(Collection prepareSensors(Collection prepareComparisonSensorsOnReferenceTable(Collection> checks, @@ -756,7 +762,8 @@ public List prepareComparisonSensorsOnReferenceTable(Collec CheckExecutionProgressListener progressListener, CheckExecutionSummary checkExecutionSummary, TableChecksExecutionStatistics executionStatistics, - JobCancellationToken jobCancellationToken) { + JobCancellationToken jobCancellationToken, + Instant tableYamlLastModified) { List sensorPrepareResults = new ArrayList<>(); int sensorResultId = 0; @@ -766,7 +773,7 @@ public List prepareComparisonSensorsOnReferenceTable(Collec } try { - SensorExecutionRunParameters sensorRunParameters = createSensorRunParametersToReferenceTable(userHome, checkSpec, userTimeWindowFilters); + SensorExecutionRunParameters sensorRunParameters = createSensorRunParametersToReferenceTable(userHome, checkSpec, userTimeWindowFilters, tableYamlLastModified); if (!sensorRunParameters.isSuccess()) { this.userErrorLogger.logCheck(sensorRunParameters.toString() + " failed to capture the initial configuration, error: " + (sensorRunParameters.getSensorConfigurationException() != null ? @@ -950,12 +957,14 @@ public List executeSensors(GroupedSensorsCollection group * @param tableSpec Target table to query. * @param checkSpec Check specification. * @param userTimeWindowFilters Optional user time window filters, to run the checks for a given time period. + * @param tableYamlLastModified The timestamp when the table YAML was last modified. * @return Sensor run parameters. */ public SensorExecutionRunParameters createSensorRunParameters(UserHome userHome, TableSpec tableSpec, AbstractCheckSpec checkSpec, - TimeWindowFilterParameters userTimeWindowFilters) { + TimeWindowFilterParameters userTimeWindowFilters, + Instant tableYamlLastModified) { try { HierarchyId checkHierarchyId = checkSpec.getHierarchyId(); ConnectionWrapper connectionWrapper = userHome.findConnectionFor(checkHierarchyId); @@ -1027,7 +1036,7 @@ public SensorExecutionRunParameters createSensorRunParameters(UserHome userHome, SensorExecutionRunParameters sensorRunParameters = this.sensorExecutionRunParametersFactory.createSensorParameters( userHome, connectionSpec, tableSpec, columnSpec, checkSpec, customCheckDefinitionSpec, checkType, dataGroupingConfigurationForComparison, - tableComparisonConfigurationSpec, timeSeriesConfigurationSpec, userTimeWindowFilters, dialectSettings); + tableComparisonConfigurationSpec, timeSeriesConfigurationSpec, userTimeWindowFilters, dialectSettings, tableYamlLastModified); sensorRunParameters.appendAdditionalFilter(extraComparisonFilter); return sensorRunParameters; } @@ -1044,11 +1053,13 @@ public SensorExecutionRunParameters createSensorRunParameters(UserHome userHome, * @param userHome User home with the metadata. * @param checkSpec Table comparison check specification on the compared table. * @param userTimeWindowFilters Optional user time window filters, to run the checks for a given time period. + * @param tableYamlLastModified The timestamp when the table YAML as last modified. * @return Sensor run parameters. */ public SensorExecutionRunParameters createSensorRunParametersToReferenceTable(UserHome userHome, AbstractCheckSpec checkSpec, - TimeWindowFilterParameters userTimeWindowFilters) { + TimeWindowFilterParameters userTimeWindowFilters, + Instant tableYamlLastModified) { try { HierarchyId checkHierarchyId = checkSpec.getHierarchyId(); TableWrapper comparedTableWrapper = userHome.findTableFor(checkHierarchyId); @@ -1145,7 +1156,7 @@ public SensorExecutionRunParameters createSensorRunParametersToReferenceTable(Us SensorExecutionRunParameters sensorRunParameters = this.sensorExecutionRunParametersFactory.createSensorParameters( userHome, referencedConnectionSpec, referencedTableSpec, referencedColumnSpec, checkSpec, customCheckDefinitionSpec, checkType, referencedTableGroupingConfiguration, - tableComparisonConfigurationSpec, timeSeriesConfigurationSpec, timeWindowConfigurationFromComparedTable, referencedDialectSettings); + tableComparisonConfigurationSpec, timeSeriesConfigurationSpec, timeWindowConfigurationFromComparedTable, referencedDialectSettings, tableYamlLastModified); sensorRunParameters.setTableComparisonConfiguration(tableComparisonConfigurationSpec); sensorRunParameters.setReferenceColumnName(referencedColumnName); sensorRunParameters.appendAdditionalFilter(tableComparisonConfigurationSpec.getReferenceTableFilter()); diff --git a/dqops/src/main/java/com/dqops/execution/checks/ruleeval/RuleEvaluationServiceImpl.java b/dqops/src/main/java/com/dqops/execution/checks/ruleeval/RuleEvaluationServiceImpl.java index 888cd3c979..a28311f4a1 100644 --- a/dqops/src/main/java/com/dqops/execution/checks/ruleeval/RuleEvaluationServiceImpl.java +++ b/dqops/src/main/java/com/dqops/execution/checks/ruleeval/RuleEvaluationServiceImpl.java @@ -176,6 +176,17 @@ public RuleEvaluationResult evaluateRules(ExecutionContext executionContext, LocalDateTime timePeriodLocal = timePeriodColumn.get(allSensorResultsRowIndex); HistoricDataPoint[] previousDataPoints = null; // combined data points from current readouts and historic sensor readouts + if (previousDataPointTimeSeriesCollectorOld != null && timePeriodLocal != null && sensorRunParameters.getCheckConfiguredAt() != null) { + HistoricResultPreviousRun previousResult = previousDataPointTimeSeriesCollectorOld.getPreviousResult(timePeriodLocal); + if (previousResult != null && Objects.equals(previousResult.getLastActualValue(), actualValue) && + (expectedValueFromSensor == null || Objects.equals(previousResult.getLastExpectedValue(), expectedValueFromSensor)) && + sensorRunParameters.getCheckConfiguredAt().isBefore(previousResult.getExecutedAt())) { + // no data changes, we can ignore calculating the rule again + continue; + } + } + + if (customSeverity == null) { if (historicDataPointGrouping == HistoricDataPointsGrouping.last_n_readouts) { // these checks do not have real time periods, we just take the last data points, also we don't want the current sensor results diff --git a/dqops/src/main/java/com/dqops/execution/rules/HistoricDataPointTimeSeriesCollector.java b/dqops/src/main/java/com/dqops/execution/rules/HistoricDataPointTimeSeriesCollector.java index abae391cb6..c6566723b8 100644 --- a/dqops/src/main/java/com/dqops/execution/rules/HistoricDataPointTimeSeriesCollector.java +++ b/dqops/src/main/java/com/dqops/execution/rules/HistoricDataPointTimeSeriesCollector.java @@ -38,6 +38,7 @@ public class HistoricDataPointTimeSeriesCollector { private final InstantColumn timePeriodUtcColumn; private final DoubleColumn actualValueColumn; private final DoubleColumn expectedValueColumn; + private final InstantColumn executedAtColumn; private final TimePeriodGradient gradient; private final ZoneId timeZoneId; private LongIndex timePeriodIndex; @@ -53,10 +54,11 @@ public HistoricDataPointTimeSeriesCollector(Table timeSeriesData, TimePeriodGradient gradient, ZoneId timeZoneId) { this.timeSeriesData = timeSeriesData; - this.timePeriodColumn = (DateTimeColumn) timeSeriesData.column(SensorReadoutsColumnNames.TIME_PERIOD_COLUMN_NAME); - this.timePeriodUtcColumn = (InstantColumn) timeSeriesData.column(SensorReadoutsColumnNames.TIME_PERIOD_UTC_COLUMN_NAME); - this.actualValueColumn = (DoubleColumn) timeSeriesData.column(SensorReadoutsColumnNames.ACTUAL_VALUE_COLUMN_NAME); - this.expectedValueColumn = (DoubleColumn) timeSeriesData.column(SensorReadoutsColumnNames.EXPECTED_VALUE_COLUMN_NAME); + this.timePeriodColumn = timeSeriesData.dateTimeColumn(SensorReadoutsColumnNames.TIME_PERIOD_COLUMN_NAME); + this.timePeriodUtcColumn = timeSeriesData.instantColumn(SensorReadoutsColumnNames.TIME_PERIOD_UTC_COLUMN_NAME); + this.actualValueColumn = timeSeriesData.doubleColumn(SensorReadoutsColumnNames.ACTUAL_VALUE_COLUMN_NAME); + this.expectedValueColumn = timeSeriesData.doubleColumn(SensorReadoutsColumnNames.EXPECTED_VALUE_COLUMN_NAME); + this.executedAtColumn = timeSeriesData.instantColumn(SensorReadoutsColumnNames.EXECUTED_AT_COLUMN_NAME); this.gradient = gradient; this.timeZoneId = timeZoneId; } @@ -148,4 +150,33 @@ public HistoricDataPoint[] getHistoricContinuousResultsBefore(LocalDateTime read return historicDataPoints; } + + /** + * Looks up the previous result for the same time period. + * @param readoutTimestamp The time period to find. + * @return Previous result with its execution time, or null when no value was found. + */ + public HistoricResultPreviousRun getPreviousResult(LocalDateTime readoutTimestamp) { + if (this.timePeriodIndex == null) { + this.timePeriodIndex = new LongIndex(this.timePeriodColumn); + } + + Selection rowSelection = this.timePeriodIndex.get(readoutTimestamp); + if (rowSelection.isEmpty()) { + return null; + } + + int[] rowIndexes = rowSelection.toArray(); + int rowIndex = rowIndexes[0]; + + if (this.executedAtColumn.isMissing(rowIndex)) { + return null; + } + + Double previousActualValue = this.actualValueColumn.get(rowIndex); + Double previousExpectedValue = this.expectedValueColumn.get(rowIndex); + Instant previousExecutedAt = this.executedAtColumn.get(rowIndex); + + return new HistoricResultPreviousRun(previousActualValue, previousExpectedValue, previousExecutedAt); + } } diff --git a/dqops/src/main/java/com/dqops/execution/rules/HistoricResultPreviousRun.java b/dqops/src/main/java/com/dqops/execution/rules/HistoricResultPreviousRun.java new file mode 100644 index 0000000000..2158ae4cfb --- /dev/null +++ b/dqops/src/main/java/com/dqops/execution/rules/HistoricResultPreviousRun.java @@ -0,0 +1,66 @@ +/* + * Copyright © 2021 DQOps (support@dqops.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.dqops.execution.rules; + +import java.time.Instant; + +/** + * A wrapper object that contains the previous sensor readout for the current time period if this check was run again and + * we want to detect if the sensor value has not changed, to avoid running the rule again, which would generate additional alerts. + * It stores also the sensor's execution timestamp. + */ +public final class HistoricResultPreviousRun { + private final Double lastActualValue; + private final Double lastExpectedValue; + private final Instant executedAt; + + /** + * Creates a pair of the last result and its execution timestamp. + * @param lastActualValue The last sensor value. + * @param lastExpectedValue The last expected value (for data comparisons). + * @param executedAt When it was captured. + */ + public HistoricResultPreviousRun(Double lastActualValue, Double lastExpectedValue, Instant executedAt) { + this.lastActualValue = lastActualValue; + this.lastExpectedValue = lastExpectedValue; + this.executedAt = executedAt; + } + + /** + * Returns the last sensor value that was captured. + * @return Last sensor value. + */ + public Double getLastActualValue() { + return lastActualValue; + } + + /** + * Returns the last expected value that was captured. + * @return Last expected value. + */ + public Double getLastExpectedValue() { + return lastExpectedValue; + } + + /** + * Returns the timestamp when the sensor was executed for the last time. + * @return Last execution timestamp. + */ + public Instant getExecutedAt() { + return executedAt; + } +} diff --git a/dqops/src/main/java/com/dqops/execution/sensors/SensorExecutionRunParameters.java b/dqops/src/main/java/com/dqops/execution/sensors/SensorExecutionRunParameters.java index 3b91318d1e..a0ac63c471 100644 --- a/dqops/src/main/java/com/dqops/execution/sensors/SensorExecutionRunParameters.java +++ b/dqops/src/main/java/com/dqops/execution/sensors/SensorExecutionRunParameters.java @@ -83,6 +83,7 @@ public class SensorExecutionRunParameters { private int rowCountLimit = 1000; private boolean failOnSensorReadoutLimitExceeded = true; private ErrorSamplingRenderParameters errorSamplingRenderParameters; + private Instant checkConfiguredAt; /** @@ -116,6 +117,7 @@ public SensorExecutionRunParameters(AbstractCheckSpec check, Throwable * @param rowCountLimit Sets the row count limit. * @param failOnSensorReadoutLimitExceeded Fail when the row count limit is exceeded. * @param errorSamplingRenderParameters Optional parameters for error sampling. When present (not null), an error sampling template is used to capture error samples. + * @param checkConfiguredAt The timestamp when this check was probably configured for the last time. */ public SensorExecutionRunParameters( ConnectionSpec connection, @@ -135,7 +137,8 @@ public SensorExecutionRunParameters( CheckSearchFilters checkSearchFilter, int rowCountLimit, boolean failOnSensorReadoutLimitExceeded, - ErrorSamplingRenderParameters errorSamplingRenderParameters) { + ErrorSamplingRenderParameters errorSamplingRenderParameters, + Instant checkConfiguredAt) { this.success = true; this.connection = connection; this.table = table; @@ -158,6 +161,7 @@ public SensorExecutionRunParameters( if (timeWindowFilter != null && !Strings.isNullOrEmpty(timeWindowFilter.getWhereFilter())) { this.additionalFilters.add(timeWindowFilter.getWhereFilter()); } + this.checkConfiguredAt = checkConfiguredAt; } /** @@ -556,6 +560,23 @@ public void setErrorSamplingRenderParameters(ErrorSamplingRenderParameters error this.errorSamplingRenderParameters = errorSamplingRenderParameters; } + /** + * Returns the timestamp when this check was probably configured for the last time. It is the file modification timestamp + * of the table YAML file or data quality policy file, depending on the source of configuring this check. + * @return Check configuration timestamp. + */ + public Instant getCheckConfiguredAt() { + return checkConfiguredAt; + } + + /** + * Sets the timestamp when the check was configured. + * @param checkConfiguredAt Check configured at timestamp. + */ + public void setCheckConfiguredAt(Instant checkConfiguredAt) { + this.checkConfiguredAt = checkConfiguredAt; + } + /** * Returns the time series gradient that is used for time partitioning the analyzed table. * @return Time period gradient. diff --git a/dqops/src/main/java/com/dqops/execution/sensors/SensorExecutionRunParametersFactory.java b/dqops/src/main/java/com/dqops/execution/sensors/SensorExecutionRunParametersFactory.java index bb07136e10..f55157e3f0 100644 --- a/dqops/src/main/java/com/dqops/execution/sensors/SensorExecutionRunParametersFactory.java +++ b/dqops/src/main/java/com/dqops/execution/sensors/SensorExecutionRunParametersFactory.java @@ -31,6 +31,8 @@ import com.dqops.metadata.userhome.UserHome; import com.dqops.statistics.AbstractStatisticsCollectorSpec; +import java.time.Instant; + /** * Factory for {@link SensorExecutionRunParameters} objects. Expands all parameters in the form ${ENV_VAR} or ${sm://secret-name} */ @@ -51,6 +53,7 @@ public interface SensorExecutionRunParametersFactory { * @param userTimeWindowFilters Optional user provided time window filters to analyze a time range of data or recent months/days. * When not provided, the defaults are copied from the table's incremental time window configuration for a matching partition time scale. * @param dialectSettings Dialect settings. + * @param tableYamlLastModified The timestamp when the table YAML was last modified. * @return Sensor execution run parameters. */ SensorExecutionRunParameters createSensorParameters(UserHome userHome, @@ -64,7 +67,8 @@ SensorExecutionRunParameters createSensorParameters(UserHome userHome, TableComparisonConfigurationSpec tableComparisonConfigurationSpec, TimeSeriesConfigurationSpec timeSeriesConfigurationSpec, TimeWindowFilterParameters userTimeWindowFilters, - ProviderDialectSettings dialectSettings); + ProviderDialectSettings dialectSettings, + Instant tableYamlLastModified); /** * Creates a sensor parameters object for a statistics collector. The sensor parameter object contains cloned, truncated and expanded (parameter expansion) diff --git a/dqops/src/main/java/com/dqops/execution/sensors/SensorExecutionRunParametersFactoryImpl.java b/dqops/src/main/java/com/dqops/execution/sensors/SensorExecutionRunParametersFactoryImpl.java index 93c8469601..33e8ef778c 100644 --- a/dqops/src/main/java/com/dqops/execution/sensors/SensorExecutionRunParametersFactoryImpl.java +++ b/dqops/src/main/java/com/dqops/execution/sensors/SensorExecutionRunParametersFactoryImpl.java @@ -52,6 +52,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import java.time.Instant; import java.util.List; import java.util.Optional; @@ -95,6 +96,7 @@ public SensorExecutionRunParametersFactoryImpl(SecretValueProvider secretValuePr * @param userTimeWindowFilters Optional user provided time window filters to analyze a time range of data or recent months/days. * When not provided, the defaults are copied from the table's incremental time window configuration for a matching partition time scale. * @param dialectSettings Dialect settings. + * @param tableYamlLastModified The timestamp when the table YAML was last modified. * @return Sensor execution run parameters. */ @Override @@ -109,7 +111,8 @@ public SensorExecutionRunParameters createSensorParameters(UserHome userHome, TableComparisonConfigurationSpec tableComparisonConfigurationSpec, TimeSeriesConfigurationSpec timeSeriesConfigurationSpec, TimeWindowFilterParameters userTimeWindowFilters, - ProviderDialectSettings dialectSettings) { + ProviderDialectSettings dialectSettings, + Instant tableYamlLastModified) { SecretValueLookupContext secretValueLookupContext = new SecretValueLookupContext(userHome); ConnectionSpec expandedConnection = connection.expandAndTrim(this.secretValueProvider, secretValueLookupContext); @@ -151,12 +154,23 @@ public SensorExecutionRunParameters createSensorParameters(UserHome userHome, exactCheckSearchFilters.setCheckName(check.getCheckName()); exactCheckSearchFilters.setSensorName(effectiveSensorRuleNames.getSensorName()); + Instant checkConfiguredAt = tableYamlLastModified; + if (check.getPolicyLastModified() != null) { + if (tableYamlLastModified == null) { + checkConfiguredAt = check.getPolicyLastModified(); + } else { + if (check.getPolicyLastModified().isAfter(tableYamlLastModified)) { + checkConfiguredAt = check.getPolicyLastModified(); + } + } + } + int rowCountLimit = checkType == CheckType.partitioned ? this.sensorLimitsConfigurationProperties.getSensorReadoutLimitPartitioned() : this.sensorLimitsConfigurationProperties.getSensorReadoutLimit(); return new SensorExecutionRunParameters(expandedConnection, expandedTable, expandedColumn, check, null, effectiveSensorRuleNames, checkType, timeSeries, timeWindowFilterParameters, dataGroupingConfiguration, tableComparisonConfigurationSpec, null, sensorParameters, dialectSettings, exactCheckSearchFilters, - rowCountLimit, this.sensorLimitsConfigurationProperties.isFailOnSensorReadoutLimitExceeded(), null); + rowCountLimit, this.sensorLimitsConfigurationProperties.isFailOnSensorReadoutLimitExceeded(), null, checkConfiguredAt); } /** @@ -212,7 +226,7 @@ public SensorExecutionRunParameters createStatisticsSensorParameters(DqoHome dqo dataGroupingConfigurationSpec, null, null, sensorParameters, dialectSettings, null, this.sensorLimitsConfigurationProperties.getSensorReadoutLimit(), false, // statistics is opportunistic, we do not fail, we just collect something for data groups - null); + null, null); } /** @@ -328,7 +342,7 @@ public SensorExecutionRunParameters createErrorSamplerSensorParameters(DqoHome d return new SensorExecutionRunParameters(expandedConnection, expandedTable, expandedColumn, check, null, effectiveSensorRuleNames, checkType, timeSeries, timeWindowFilterParameters, dataGroupingConfiguration, null, null, sensorParameters, dialectSettings, exactCheckSearchFilters, - rowCountLimit, this.sensorLimitsConfigurationProperties.isFailOnSensorReadoutLimitExceeded(), errorSamplingRenderParameters); + rowCountLimit, this.sensorLimitsConfigurationProperties.isFailOnSensorReadoutLimitExceeded(), errorSamplingRenderParameters, null); } /** diff --git a/dqops/src/main/java/com/dqops/metadata/policies/column/ColumnQualityPolicySpec.java b/dqops/src/main/java/com/dqops/metadata/policies/column/ColumnQualityPolicySpec.java index 911217fa3a..a7e3f7af21 100644 --- a/dqops/src/main/java/com/dqops/metadata/policies/column/ColumnQualityPolicySpec.java +++ b/dqops/src/main/java/com/dqops/metadata/policies/column/ColumnQualityPolicySpec.java @@ -47,6 +47,7 @@ import com.google.common.base.Strings; import lombok.EqualsAndHashCode; +import java.time.Instant; import java.util.Objects; /** @@ -301,26 +302,30 @@ public void setPolicyName(String policyName) { * @param tableSpec Parent table specification. * @param targetColumn Target column. * @param dialectSettings Dialect settings, to decide if the checks are applicable. + * @param policyLastModified The timestamp when the policy file was last modified. */ - public void applyOnColumn(TableSpec tableSpec, ColumnSpec targetColumn, ProviderDialectSettings dialectSettings) { + public void applyOnColumn(TableSpec tableSpec, + ColumnSpec targetColumn, + ProviderDialectSettings dialectSettings, + Instant policyLastModified) { DataTypeCategory dataTypeCategory = dialectSettings.detectColumnType(targetColumn.getTypeSnapshot()); if (this.profilingChecks != null && !this.profilingChecks.isDefault()) { AbstractRootChecksContainerSpec tableProfilingContainer = targetColumn.getColumnCheckRootContainer(CheckType.profiling, null, true); - this.profilingChecks.copyChecksToContainer(tableProfilingContainer, tableSpec, dataTypeCategory, dialectSettings); + this.profilingChecks.copyChecksToContainer(tableProfilingContainer, tableSpec, dataTypeCategory, dialectSettings, policyLastModified); } if (this.monitoringChecks != null) { AbstractRootChecksContainerSpec defaultChecksDaily = this.monitoringChecks.getDaily(); if (defaultChecksDaily != null && !defaultChecksDaily.isDefault()) { AbstractRootChecksContainerSpec targetContainer = targetColumn.getColumnCheckRootContainer(CheckType.monitoring, CheckTimeScale.daily, true); - defaultChecksDaily.copyChecksToContainer(targetContainer, tableSpec, dataTypeCategory, dialectSettings); + defaultChecksDaily.copyChecksToContainer(targetContainer, tableSpec, dataTypeCategory, dialectSettings, policyLastModified); } AbstractRootChecksContainerSpec defaultChecksMonthly = this.monitoringChecks.getMonthly(); if (defaultChecksMonthly != null && !defaultChecksMonthly.isDefault()) { AbstractRootChecksContainerSpec targetContainer = targetColumn.getColumnCheckRootContainer(CheckType.monitoring, CheckTimeScale.monthly, true); - defaultChecksMonthly.copyChecksToContainer(targetContainer, tableSpec, dataTypeCategory, dialectSettings); + defaultChecksMonthly.copyChecksToContainer(targetContainer, tableSpec, dataTypeCategory, dialectSettings, policyLastModified); } } @@ -329,13 +334,13 @@ public void applyOnColumn(TableSpec tableSpec, ColumnSpec targetColumn, Provider AbstractRootChecksContainerSpec defaultChecksDaily = this.partitionedChecks.getDaily(); if (defaultChecksDaily != null && !defaultChecksDaily.isDefault()) { AbstractRootChecksContainerSpec targetContainer = targetColumn.getColumnCheckRootContainer(CheckType.partitioned, CheckTimeScale.daily, true); - defaultChecksDaily.copyChecksToContainer(targetContainer, tableSpec, dataTypeCategory, dialectSettings); + defaultChecksDaily.copyChecksToContainer(targetContainer, tableSpec, dataTypeCategory, dialectSettings, policyLastModified); } AbstractRootChecksContainerSpec defaultChecksMonthly = this.partitionedChecks.getMonthly(); if (defaultChecksMonthly != null && !defaultChecksMonthly.isDefault()) { AbstractRootChecksContainerSpec targetContainer = targetColumn.getColumnCheckRootContainer(CheckType.partitioned, CheckTimeScale.monthly, true); - defaultChecksMonthly.copyChecksToContainer(targetContainer, tableSpec, dataTypeCategory, dialectSettings); + defaultChecksMonthly.copyChecksToContainer(targetContainer, tableSpec, dataTypeCategory, dialectSettings, policyLastModified); } } } diff --git a/dqops/src/main/java/com/dqops/metadata/policies/table/TableQualityPolicySpec.java b/dqops/src/main/java/com/dqops/metadata/policies/table/TableQualityPolicySpec.java index ffaccaa6d9..f819eefafc 100644 --- a/dqops/src/main/java/com/dqops/metadata/policies/table/TableQualityPolicySpec.java +++ b/dqops/src/main/java/com/dqops/metadata/policies/table/TableQualityPolicySpec.java @@ -40,6 +40,7 @@ import com.google.common.base.Strings; import lombok.EqualsAndHashCode; +import java.time.Instant; import java.util.Objects; @@ -294,24 +295,27 @@ public void setPolicyName(String policyName) { * Applies the default checks on a target table. * @param targetTable Target table. * @param dialectSettings Dialect settings, to decide if the checks are applicable. + * @param policyLastModified The timestamp when the policy YAML file was last modified. */ - public void applyOnTable(TableSpec targetTable, ProviderDialectSettings dialectSettings) { + public void applyOnTable(TableSpec targetTable, + ProviderDialectSettings dialectSettings, + Instant policyLastModified) { if (this.profilingChecks != null && !this.profilingChecks.isDefault()) { AbstractRootChecksContainerSpec tableProfilingContainer = targetTable.getTableCheckRootContainer(CheckType.profiling, null, true); - this.profilingChecks.copyChecksToContainer(tableProfilingContainer, targetTable, null, dialectSettings); + this.profilingChecks.copyChecksToContainer(tableProfilingContainer, targetTable, null, dialectSettings, policyLastModified); } if (this.monitoringChecks != null) { AbstractRootChecksContainerSpec defaultChecksDaily = this.monitoringChecks.getDaily(); if (defaultChecksDaily != null && !defaultChecksDaily.isDefault()) { AbstractRootChecksContainerSpec targetContainer = targetTable.getTableCheckRootContainer(CheckType.monitoring, CheckTimeScale.daily, true); - defaultChecksDaily.copyChecksToContainer(targetContainer, targetTable, null, dialectSettings); + defaultChecksDaily.copyChecksToContainer(targetContainer, targetTable, null, dialectSettings, policyLastModified); } AbstractRootChecksContainerSpec defaultChecksMonthly = this.monitoringChecks.getMonthly(); if (defaultChecksMonthly != null && !defaultChecksMonthly.isDefault()) { AbstractRootChecksContainerSpec targetContainer = targetTable.getTableCheckRootContainer(CheckType.monitoring, CheckTimeScale.monthly, true); - defaultChecksMonthly.copyChecksToContainer(targetContainer, targetTable, null, dialectSettings); + defaultChecksMonthly.copyChecksToContainer(targetContainer, targetTable, null, dialectSettings, policyLastModified); } } @@ -320,13 +324,13 @@ public void applyOnTable(TableSpec targetTable, ProviderDialectSettings dialectS AbstractRootChecksContainerSpec defaultChecksDaily = this.partitionedChecks.getDaily(); if (defaultChecksDaily != null && !defaultChecksDaily.isDefault()) { AbstractRootChecksContainerSpec targetContainer = targetTable.getTableCheckRootContainer(CheckType.partitioned, CheckTimeScale.daily, true); - defaultChecksDaily.copyChecksToContainer(targetContainer, targetTable,null, dialectSettings); + defaultChecksDaily.copyChecksToContainer(targetContainer, targetTable,null, dialectSettings, policyLastModified); } AbstractRootChecksContainerSpec defaultChecksMonthly = this.partitionedChecks.getMonthly(); if (defaultChecksMonthly != null && !defaultChecksMonthly.isDefault()) { AbstractRootChecksContainerSpec targetContainer = targetTable.getTableCheckRootContainer(CheckType.partitioned, CheckTimeScale.monthly, true); - defaultChecksMonthly.copyChecksToContainer(targetContainer, targetTable, null, dialectSettings); + defaultChecksMonthly.copyChecksToContainer(targetContainer, targetTable, null, dialectSettings, policyLastModified); } } } diff --git a/dqops/src/main/java/com/dqops/utils/docs/checks/CheckDocumentationModelFactoryImpl.java b/dqops/src/main/java/com/dqops/utils/docs/checks/CheckDocumentationModelFactoryImpl.java index 41094873cd..951e477a98 100644 --- a/dqops/src/main/java/com/dqops/utils/docs/checks/CheckDocumentationModelFactoryImpl.java +++ b/dqops/src/main/java/com/dqops/utils/docs/checks/CheckDocumentationModelFactoryImpl.java @@ -712,6 +712,7 @@ protected List generateProviderSampl new CheckSearchFilters(), 1000, true, + null, null ); diff --git a/dqops/src/main/resources/static/swagger-api/dqops-api-swagger-2.json b/dqops/src/main/resources/static/swagger-api/dqops-api-swagger-2.json index d865ad2672..7afbfd24b5 100644 --- a/dqops/src/main/resources/static/swagger-api/dqops-api-swagger-2.json +++ b/dqops/src/main/resources/static/swagger-api/dqops-api-swagger-2.json @@ -37835,7 +37835,7 @@ "provider_type" : { "type" : "string", "description" : "Database provider type (required). Accepts: bigquery, snowflake, etc.", - "enum" : [ "bigquery", "databricks", "mysql", "oracle", "postgresql", "duckdb", "presto", "redshift", "snowflake", "spark", "sqlserver", "trino", "hana", "db2", "mariadb", "clickhouse", "questdb", "teradata" ] + "enum" : [ "bigquery", "clickhouse", "databricks", "db2", "duckdb", "hana", "mariadb", "mysql", "oracle", "postgresql", "presto", "questdb", "redshift", "snowflake", "spark", "sqlserver", "teradata", "trino" ] }, "bigquery" : { "description" : "BigQuery connection parameters. Specify parameters in the bigquery section.", @@ -37969,7 +37969,7 @@ "provider_type" : { "type" : "string", "description" : "Database provider type (required).", - "enum" : [ "bigquery", "databricks", "mysql", "oracle", "postgresql", "duckdb", "presto", "redshift", "snowflake", "spark", "sqlserver", "trino", "hana", "db2", "mariadb", "clickhouse", "questdb", "teradata" ] + "enum" : [ "bigquery", "clickhouse", "databricks", "db2", "duckdb", "hana", "mariadb", "mysql", "oracle", "postgresql", "presto", "questdb", "redshift", "snowflake", "spark", "sqlserver", "teradata", "trino" ] }, "bigquery" : { "description" : "BigQuery connection parameters. Specify parameters in the bigquery section.", @@ -41148,7 +41148,7 @@ "provider_type" : { "type" : "string", "description" : "Provider type.", - "enum" : [ "bigquery", "databricks", "mysql", "oracle", "postgresql", "duckdb", "presto", "redshift", "snowflake", "spark", "sqlserver", "trino", "hana", "db2", "mariadb", "clickhouse", "questdb", "teradata" ] + "enum" : [ "bigquery", "clickhouse", "databricks", "db2", "duckdb", "hana", "mariadb", "mysql", "oracle", "postgresql", "presto", "questdb", "redshift", "snowflake", "spark", "sqlserver", "teradata", "trino" ] }, "custom" : { "type" : "boolean", @@ -41171,7 +41171,7 @@ "providerType" : { "type" : "string", "description" : "Provider type.", - "enum" : [ "bigquery", "databricks", "mysql", "oracle", "postgresql", "duckdb", "presto", "redshift", "snowflake", "spark", "sqlserver", "trino", "hana", "db2", "mariadb", "clickhouse", "questdb", "teradata" ] + "enum" : [ "bigquery", "clickhouse", "databricks", "db2", "duckdb", "hana", "mariadb", "mysql", "oracle", "postgresql", "presto", "questdb", "redshift", "snowflake", "spark", "sqlserver", "teradata", "trino" ] }, "providerSensorDefinitionSpec" : { "description" : "Provider specific sensor definition specification", diff --git a/dqops/src/main/resources/static/swagger-api/dqops-api-swagger-2.yaml b/dqops/src/main/resources/static/swagger-api/dqops-api-swagger-2.yaml index 215ec0d35a..3926149cf4 100644 --- a/dqops/src/main/resources/static/swagger-api/dqops-api-swagger-2.yaml +++ b/dqops/src/main/resources/static/swagger-api/dqops-api-swagger-2.yaml @@ -36525,23 +36525,23 @@ definitions: \ etc." enum: - "bigquery" + - "clickhouse" - "databricks" + - "db2" + - "duckdb" + - "hana" + - "mariadb" - "mysql" - "oracle" - "postgresql" - - "duckdb" - "presto" + - "questdb" - "redshift" - "snowflake" - "spark" - "sqlserver" - - "trino" - - "hana" - - "db2" - - "mariadb" - - "clickhouse" - - "questdb" - "teradata" + - "trino" bigquery: description: "BigQuery connection parameters. Specify parameters in the bigquery\ \ section." @@ -36662,23 +36662,23 @@ definitions: description: "Database provider type (required)." enum: - "bigquery" + - "clickhouse" - "databricks" + - "db2" + - "duckdb" + - "hana" + - "mariadb" - "mysql" - "oracle" - "postgresql" - - "duckdb" - "presto" + - "questdb" - "redshift" - "snowflake" - "spark" - "sqlserver" - - "trino" - - "hana" - - "db2" - - "mariadb" - - "clickhouse" - - "questdb" - "teradata" + - "trino" bigquery: description: "BigQuery connection parameters. Specify parameters in the bigquery\ \ section." @@ -39736,23 +39736,23 @@ definitions: description: "Provider type." enum: - "bigquery" + - "clickhouse" - "databricks" + - "db2" + - "duckdb" + - "hana" + - "mariadb" - "mysql" - "oracle" - "postgresql" - - "duckdb" - "presto" + - "questdb" - "redshift" - "snowflake" - "spark" - "sqlserver" - - "trino" - - "hana" - - "db2" - - "mariadb" - - "clickhouse" - - "questdb" - "teradata" + - "trino" custom: type: "boolean" description: "This connection specific template is a custom sensor template\ @@ -39774,23 +39774,23 @@ definitions: description: "Provider type." enum: - "bigquery" + - "clickhouse" - "databricks" + - "db2" + - "duckdb" + - "hana" + - "mariadb" - "mysql" - "oracle" - "postgresql" - - "duckdb" - "presto" + - "questdb" - "redshift" - "snowflake" - "spark" - "sqlserver" - - "trino" - - "hana" - - "db2" - - "mariadb" - - "clickhouse" - - "questdb" - "teradata" + - "trino" providerSensorDefinitionSpec: description: "Provider specific sensor definition specification" $ref: "#/definitions/ProviderSensorDefinitionSpec" diff --git a/dqops/src/test/java/com/dqops/data/readouts/normalization/SensorReadoutsNormalizationServiceImplTests.java b/dqops/src/test/java/com/dqops/data/readouts/normalization/SensorReadoutsNormalizationServiceImplTests.java index cf3794fa92..24f61562d2 100644 --- a/dqops/src/test/java/com/dqops/data/readouts/normalization/SensorReadoutsNormalizationServiceImplTests.java +++ b/dqops/src/test/java/com/dqops/data/readouts/normalization/SensorReadoutsNormalizationServiceImplTests.java @@ -92,6 +92,7 @@ void setUp() { null, 1000, true, + null, null); sensorExecutionResult = new SensorExecutionResult(this.sensorExecutionRunParameters, this.table); } diff --git a/dqops/src/test/java/com/dqops/execution/checks/ruleeval/RuleEvaluationServiceImplTests.java b/dqops/src/test/java/com/dqops/execution/checks/ruleeval/RuleEvaluationServiceImplTests.java index 4a32f69625..6451c7e7f5 100644 --- a/dqops/src/test/java/com/dqops/execution/checks/ruleeval/RuleEvaluationServiceImplTests.java +++ b/dqops/src/test/java/com/dqops/execution/checks/ruleeval/RuleEvaluationServiceImplTests.java @@ -108,6 +108,7 @@ void setUp() { null, 1000, true, + null, null); progressListener = new CheckExecutionProgressListenerStub(); sensorExecutionResult = new SensorExecutionResult(this.sensorExecutionRunParameters, this.table); diff --git a/dqops/src/test/java/com/dqops/execution/sensors/SensorExecutionRunParametersObjectMother.java b/dqops/src/test/java/com/dqops/execution/sensors/SensorExecutionRunParametersObjectMother.java index 7aa2d83aeb..5f21e951eb 100644 --- a/dqops/src/test/java/com/dqops/execution/sensors/SensorExecutionRunParametersObjectMother.java +++ b/dqops/src/test/java/com/dqops/execution/sensors/SensorExecutionRunParametersObjectMother.java @@ -57,7 +57,7 @@ public static SensorExecutionRunParametersFactory getFactory() { public static SensorExecutionRunParameters createEmptyBigQuery() { return new SensorExecutionRunParameters(BigQueryConnectionSpecObjectMother.create(), null, null, null, null, null, null, null, null, null, - null, null, null, null, null, 1000, false, null); + null, null, null, null, null, 1000, false, null, null); } /** @@ -82,7 +82,7 @@ public static SensorExecutionRunParameters createForTableAndCheck( SensorExecutionRunParameters sensorExecutionRunParameters = factory.createSensorParameters( userHome, connectionWrapper.getSpec(), tableWrapper.getSpec(), null, checkSpec, null, checkType, null, - null, timeSeriesConfigurationSpec, null, dialectSettings); + null, timeSeriesConfigurationSpec, null, dialectSettings, null); return sensorExecutionRunParameters; } @@ -103,7 +103,7 @@ public static SensorExecutionRunParameters createForTableForProfilingCheck( SensorExecutionRunParameters sensorExecutionRunParameters = factory.createSensorParameters(null, connectionSpec, tableSpec, null, checkSpec, null, CheckType.profiling, null, null, - timeSeriesConfigurationSpec, null, dialectSettings); + timeSeriesConfigurationSpec, null, dialectSettings, null); return sensorExecutionRunParameters; } @@ -127,7 +127,7 @@ public static SensorExecutionRunParameters createForTableForMonitoringCheck( SensorExecutionRunParameters sensorExecutionRunParameters = factory.createSensorParameters(null, connectionSpec, tableSpec, null, checkSpec, null, CheckType.monitoring, null, null, - timeSeriesConfigurationSpec, null, dialectSettings); + timeSeriesConfigurationSpec, null, dialectSettings, null); return sensorExecutionRunParameters; } @@ -163,7 +163,7 @@ else if (timeScale == CheckTimeScale.monthly) { SensorExecutionRunParameters sensorExecutionRunParameters = factory.createSensorParameters(null, connectionSpec, tableSpec, null, checkSpec, null, CheckType.partitioned, null, null, - timeSeriesConfigurationSpec, userTimeWindowFilters, dialectSettings); + timeSeriesConfigurationSpec, userTimeWindowFilters, dialectSettings, null); return sensorExecutionRunParameters; } @@ -218,7 +218,7 @@ public static SensorExecutionRunParameters createForTableColumnAndCheck( SensorExecutionRunParameters sensorExecutionRunParameters = factory.createSensorParameters(userHome, connectionSpec, tableSpec, columnSpec, checkSpec, null, checkType, null, null, - timeSeriesConfigurationSpec, null, dialectSettings); + timeSeriesConfigurationSpec, null, dialectSettings, null); return sensorExecutionRunParameters; } @@ -266,7 +266,7 @@ public static SensorExecutionRunParameters createForTableColumnForProfilingCheck SensorExecutionRunParameters sensorExecutionRunParameters = factory.createSensorParameters(null, connectionSpec, tableSpec, columnSpec, checkSpec, null, CheckType.profiling, null, null, - timeSeriesConfigurationSpec, null, dialectSettings); + timeSeriesConfigurationSpec, null, dialectSettings, null); return sensorExecutionRunParameters; } @@ -292,7 +292,7 @@ public static SensorExecutionRunParameters createForTableColumnForMonitoringChec SensorExecutionRunParameters sensorExecutionRunParameters = factory.createSensorParameters(null, connectionSpec, tableSpec, columnSpec, checkSpec, null, CheckType.monitoring, null, null, - timeSeriesConfigurationSpec, null, dialectSettings); + timeSeriesConfigurationSpec, null, dialectSettings, null); return sensorExecutionRunParameters; } @@ -331,7 +331,7 @@ else if (checkTimeScale == CheckTimeScale.monthly) { SensorExecutionRunParameters sensorExecutionRunParameters = factory.createSensorParameters(null, connectionSpec, tableSpec, columnSpec, checkSpec, null, CheckType.partitioned, null, null, - timeSeriesConfigurationSpec, userTimeWindowFilters, dialectSettings); + timeSeriesConfigurationSpec, userTimeWindowFilters, dialectSettings, null); return sensorExecutionRunParameters; } diff --git a/dqops/src/test/java/com/dqops/metadata/policies/column/ColumnQualityPolicySpecTests.java b/dqops/src/test/java/com/dqops/metadata/policies/column/ColumnQualityPolicySpecTests.java index 3359b5f98e..dab9784081 100644 --- a/dqops/src/test/java/com/dqops/metadata/policies/column/ColumnQualityPolicySpecTests.java +++ b/dqops/src/test/java/com/dqops/metadata/policies/column/ColumnQualityPolicySpecTests.java @@ -64,7 +64,7 @@ public void applyOnColumn_whenProfilingChecksSetOnColumn_thenConfiguresThemOnCol ColumnNullsProfilingChecksSpec nullsDefaults = new ColumnNullsProfilingChecksSpec(); columnDefaults.setNulls(nullsDefaults); nullsDefaults.setProfileNullsCount(new ColumnNullsCountCheckSpec()); - this.sut.applyOnColumn(this.targetTable, targetColumn, this.dialectSettings); + this.sut.applyOnColumn(this.targetTable, targetColumn, this.dialectSettings, null); Assertions.assertNotNull(targetColumn.getProfilingChecks().getNulls().getProfileNullsCount()); } @@ -76,7 +76,7 @@ public void applyOnColumn_whenDailyMonitoringChecksSetOnColumn_thenConfiguresThe ColumnNullsDailyMonitoringChecksSpec nullsDefaults = new ColumnNullsDailyMonitoringChecksSpec(); columnDefaults.setNulls(nullsDefaults); nullsDefaults.setDailyNullsCount(new ColumnNullsCountCheckSpec()); - this.sut.applyOnColumn(this.targetTable, targetColumn, this.dialectSettings); + this.sut.applyOnColumn(this.targetTable, targetColumn, this.dialectSettings, null); Assertions.assertNotNull(targetColumn.getMonitoringChecks().getDaily().getNulls().getDailyNullsCount()); } @@ -88,7 +88,7 @@ public void applyOnColumn_whenMonthlyMonitoringChecksSetOnColumn_thenConfiguresT ColumnNullsMonthlyMonitoringChecksSpec nullsDefaults = new ColumnNullsMonthlyMonitoringChecksSpec(); columnDefaults.setNulls(nullsDefaults); nullsDefaults.setMonthlyNullsCount(new ColumnNullsCountCheckSpec()); - this.sut.applyOnColumn(this.targetTable, targetColumn, this.dialectSettings); + this.sut.applyOnColumn(this.targetTable, targetColumn, this.dialectSettings, null); Assertions.assertNotNull(targetColumn.getMonitoringChecks().getMonthly().getNulls().getMonthlyNullsCount()); } @@ -102,7 +102,7 @@ public void applyOnColumn_whenProfilingChecksSetOnColumnAndTypeRequiresTextAndTy ColumnTextProfilingChecksSpec textDefault = new ColumnTextProfilingChecksSpec(); columnDefaults.setText(textDefault); textDefault.setProfileTextLengthAboveMaxLength(new ColumnTextLengthAboveMaxLengthCheckSpec()); - this.sut.applyOnColumn(this.targetTable, targetColumn, this.dialectSettings); + this.sut.applyOnColumn(this.targetTable, targetColumn, this.dialectSettings, null); Assertions.assertNotNull(targetColumn.getProfilingChecks().getText().getProfileTextLengthAboveMaxLength()); } @@ -116,7 +116,7 @@ public void applyOnColumn_whenProfilingChecksSetOnColumnAndTypeRequiresTextButTy ColumnTextProfilingChecksSpec textDefault = new ColumnTextProfilingChecksSpec(); columnDefaults.setText(textDefault); textDefault.setProfileTextLengthAboveMaxLength(new ColumnTextLengthAboveMaxLengthCheckSpec()); - this.sut.applyOnColumn(this.targetTable, targetColumn, this.dialectSettings); + this.sut.applyOnColumn(this.targetTable, targetColumn, this.dialectSettings, null); Assertions.assertNull(targetColumn.getProfilingChecks().getText()); } @@ -130,7 +130,7 @@ public void applyOnColumn_whenProfilingChecksSetOnColumnAndTypeRequiresTextButTy ColumnTextProfilingChecksSpec textDefault = new ColumnTextProfilingChecksSpec(); columnDefaults.setText(textDefault); textDefault.setProfileTextLengthAboveMaxLength(new ColumnTextLengthAboveMaxLengthCheckSpec()); - this.sut.applyOnColumn(this.targetTable, targetColumn, this.dialectSettings); + this.sut.applyOnColumn(this.targetTable, targetColumn, this.dialectSettings, null); Assertions.assertNull(targetColumn.getProfilingChecks().getText()); } @@ -145,7 +145,7 @@ public void applyOnColumn_whenDailyPartitionChecksSetOnColumnAndTableHasPartitio ColumnTextDailyPartitionedChecksSpec textDefault = new ColumnTextDailyPartitionedChecksSpec(); columnDefaults.setText(textDefault); textDefault.setDailyPartitionTextMaxLength(new ColumnTextMaxLengthCheckSpec()); - this.sut.applyOnColumn(this.targetTable, targetColumn, this.dialectSettings); + this.sut.applyOnColumn(this.targetTable, targetColumn, this.dialectSettings, null); Assertions.assertNotNull(targetColumn.getPartitionedChecks().getDaily().getText().getDailyPartitionTextMaxLength()); } @@ -159,7 +159,7 @@ public void applyOnColumn_whenDailyPartitionChecksSetOnColumnAndTableNotConfigur ColumnTextDailyPartitionedChecksSpec textDefault = new ColumnTextDailyPartitionedChecksSpec(); columnDefaults.setText(textDefault); textDefault.setDailyPartitionTextMaxLength(new ColumnTextMaxLengthCheckSpec()); - this.sut.applyOnColumn(this.targetTable, targetColumn, this.dialectSettings); + this.sut.applyOnColumn(this.targetTable, targetColumn, this.dialectSettings, null); Assertions.assertNull(targetColumn.getPartitionedChecks()); } diff --git a/dqops/src/test/java/com/dqops/metadata/policies/table/TableQualityPolicySpecTests.java b/dqops/src/test/java/com/dqops/metadata/policies/table/TableQualityPolicySpecTests.java index 7c303b0856..b004789ec3 100644 --- a/dqops/src/test/java/com/dqops/metadata/policies/table/TableQualityPolicySpecTests.java +++ b/dqops/src/test/java/com/dqops/metadata/policies/table/TableQualityPolicySpecTests.java @@ -55,7 +55,7 @@ void setUp() { @Test public void applyOnTable_whenNoDefaultChecks_thenDoesNothing() { - this.sut.applyOnTable(targetTable, this.dialectSettings); + this.sut.applyOnTable(targetTable, this.dialectSettings, null); } @Test @@ -65,7 +65,7 @@ public void applyOnTable_whenProfilingChecksSet_thenConfiguresThemOnTable() { TableVolumeProfilingChecksSpec volumeDefaults = new TableVolumeProfilingChecksSpec(); tableDefaults.setVolume(volumeDefaults); volumeDefaults.setProfileRowCountAnomaly(new TableRowCountAnomalyDifferencingCheckSpec()); - this.sut.applyOnTable(targetTable, this.dialectSettings); + this.sut.applyOnTable(targetTable, this.dialectSettings, null); Assertions.assertNotNull(targetTable.getProfilingChecks().getVolume().getProfileRowCountAnomaly()); } @@ -77,7 +77,7 @@ public void applyOnTable_whenDailyMonitoringChecksSet_thenConfiguresThemOnTable( TableVolumeDailyMonitoringChecksSpec volumeDefaults = new TableVolumeDailyMonitoringChecksSpec(); tableDefaults.setVolume(volumeDefaults); volumeDefaults.setDailyRowCountAnomaly(new TableRowCountAnomalyDifferencingCheckSpec()); - this.sut.applyOnTable(targetTable, this.dialectSettings); + this.sut.applyOnTable(targetTable, this.dialectSettings, null); Assertions.assertNotNull(targetTable.getMonitoringChecks().getDaily().getVolume().getDailyRowCountAnomaly()); } @@ -89,7 +89,7 @@ public void applyOnTable_whenMonthlyMonitoringChecksSet_thenConfiguresThemOnTabl TableVolumeMonthlyMonitoringChecksSpec volumeDefaults = new TableVolumeMonthlyMonitoringChecksSpec(); tableDefaults.setVolume(volumeDefaults); volumeDefaults.setMonthlyRowCount(new TableRowCountCheckSpec()); - this.sut.applyOnTable(targetTable, this.dialectSettings); + this.sut.applyOnTable(targetTable, this.dialectSettings, null); Assertions.assertNotNull(targetTable.getMonitoringChecks().getMonthly().getVolume().getMonthlyRowCount()); }