diff options
Diffstat (limited to '.venv/lib/python3.12/site-packages/azure/ai/ml/_schema/monitoring/thresholds.py')
| -rw-r--r-- | .venv/lib/python3.12/site-packages/azure/ai/ml/_schema/monitoring/thresholds.py | 211 |
1 files changed, 211 insertions, 0 deletions
diff --git a/.venv/lib/python3.12/site-packages/azure/ai/ml/_schema/monitoring/thresholds.py b/.venv/lib/python3.12/site-packages/azure/ai/ml/_schema/monitoring/thresholds.py new file mode 100644 index 00000000..b7970fca --- /dev/null +++ b/.venv/lib/python3.12/site-packages/azure/ai/ml/_schema/monitoring/thresholds.py @@ -0,0 +1,211 @@ +# --------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# --------------------------------------------------------- + +# pylint: disable=unused-argument, name-too-long + +from marshmallow import fields, post_load + +from azure.ai.ml.constants._monitoring import MonitorFeatureType +from azure.ai.ml._schema.core.fields import StringTransformedEnum, NestedField +from azure.ai.ml._schema.core.schema import PatchedSchemaMeta + + +class MetricThresholdSchema(metaclass=PatchedSchemaMeta): + threshold = fields.Number() + + +class NumericalDriftMetricsSchema(metaclass=PatchedSchemaMeta): + jensen_shannon_distance = fields.Number() + normalized_wasserstein_distance = fields.Number() + population_stability_index = fields.Number() + two_sample_kolmogorov_smirnov_test = fields.Number() + + @post_load + def make(self, data, **kwargs): + from azure.ai.ml.entities._monitoring.thresholds import NumericalDriftMetrics + + return NumericalDriftMetrics(**data) + + +class CategoricalDriftMetricsSchema(metaclass=PatchedSchemaMeta): + jensen_shannon_distance = fields.Number() + population_stability_index = fields.Number() + pearsons_chi_squared_test = fields.Number() + + @post_load + def make(self, data, **kwargs): + from azure.ai.ml.entities._monitoring.thresholds import CategoricalDriftMetrics + + return CategoricalDriftMetrics(**data) + + +class DataDriftMetricThresholdSchema(MetricThresholdSchema): + data_type = StringTransformedEnum(allowed_values=[MonitorFeatureType.NUMERICAL, MonitorFeatureType.CATEGORICAL]) + + numerical = NestedField(NumericalDriftMetricsSchema) + categorical = NestedField(CategoricalDriftMetricsSchema) + + @post_load + def make(self, data, **kwargs): + from azure.ai.ml.entities._monitoring.thresholds import DataDriftMetricThreshold + + return DataDriftMetricThreshold(**data) + + +class DataQualityMetricsNumericalSchema(metaclass=PatchedSchemaMeta): + null_value_rate = fields.Number() + data_type_error_rate = fields.Number() + out_of_bounds_rate = fields.Number() + + @post_load + def make(self, data, **kwargs): + from azure.ai.ml.entities._monitoring.thresholds import DataQualityMetricsNumerical + + return DataQualityMetricsNumerical(**data) + + +class DataQualityMetricsCategoricalSchema(metaclass=PatchedSchemaMeta): + null_value_rate = fields.Number() + data_type_error_rate = fields.Number() + out_of_bounds_rate = fields.Number() + + @post_load + def make(self, data, **kwargs): + from azure.ai.ml.entities._monitoring.thresholds import DataQualityMetricsCategorical + + return DataQualityMetricsCategorical(**data) + + +class DataQualityMetricThresholdSchema(MetricThresholdSchema): + data_type = StringTransformedEnum(allowed_values=[MonitorFeatureType.NUMERICAL, MonitorFeatureType.CATEGORICAL]) + numerical = NestedField(DataQualityMetricsNumericalSchema) + categorical = NestedField(DataQualityMetricsCategoricalSchema) + + @post_load + def make(self, data, **kwargs): + from azure.ai.ml.entities._monitoring.thresholds import DataQualityMetricThreshold + + return DataQualityMetricThreshold(**data) + + +class PredictionDriftMetricThresholdSchema(MetricThresholdSchema): + data_type = StringTransformedEnum(allowed_values=[MonitorFeatureType.NUMERICAL, MonitorFeatureType.CATEGORICAL]) + numerical = NestedField(NumericalDriftMetricsSchema) + categorical = NestedField(CategoricalDriftMetricsSchema) + + @post_load + def make(self, data, **kwargs): + from azure.ai.ml.entities._monitoring.thresholds import PredictionDriftMetricThreshold + + return PredictionDriftMetricThreshold(**data) + + +# pylint: disable-next=name-too-long +class FeatureAttributionDriftMetricThresholdSchema(MetricThresholdSchema): + normalized_discounted_cumulative_gain = fields.Number() + + @post_load + def make(self, data, **kwargs): + from azure.ai.ml.entities._monitoring.thresholds import FeatureAttributionDriftMetricThreshold + + return FeatureAttributionDriftMetricThreshold(**data) + + +class ModelPerformanceClassificationThresholdsSchema(metaclass=PatchedSchemaMeta): + accuracy = fields.Number() + precision = fields.Number() + recall = fields.Number() + + @post_load + def make(self, data, **kwargs): + from azure.ai.ml.entities._monitoring.thresholds import ModelPerformanceClassificationThresholds + + return ModelPerformanceClassificationThresholds(**data) + + +class ModelPerformanceRegressionThresholdsSchema(metaclass=PatchedSchemaMeta): + mae = fields.Number() + mse = fields.Number() + rmse = fields.Number() + + @post_load + def make(self, data, **kwargs): + from azure.ai.ml.entities._monitoring.thresholds import ModelPerformanceRegressionThresholds + + return ModelPerformanceRegressionThresholds(**data) + + +class ModelPerformanceMetricThresholdSchema(MetricThresholdSchema): + classification = NestedField(ModelPerformanceClassificationThresholdsSchema) + regression = NestedField(ModelPerformanceRegressionThresholdsSchema) + + @post_load + def make(self, data, **kwargs): + from azure.ai.ml.entities._monitoring.thresholds import ModelPerformanceMetricThreshold + + return ModelPerformanceMetricThreshold(**data) + + +class CustomMonitoringMetricThresholdSchema(MetricThresholdSchema): + metric_name = fields.Str() + + @post_load + def make(self, data, **kwargs): + from azure.ai.ml.entities._monitoring.thresholds import CustomMonitoringMetricThreshold + + return CustomMonitoringMetricThreshold(**data) + + +class GenerationSafetyQualityMetricThresholdSchema(metaclass=PatchedSchemaMeta): # pylint: disable=name-too-long + groundedness = fields.Dict( + keys=StringTransformedEnum( + allowed_values=["aggregated_groundedness_pass_rate", "acceptable_groundedness_score_per_instance"] + ), + values=fields.Number(), + ) + relevance = fields.Dict( + keys=StringTransformedEnum( + allowed_values=["aggregated_relevance_pass_rate", "acceptable_relevance_score_per_instance"] + ), + values=fields.Number(), + ) + coherence = fields.Dict( + keys=StringTransformedEnum( + allowed_values=["aggregated_coherence_pass_rate", "acceptable_coherence_score_per_instance"] + ), + values=fields.Number(), + ) + fluency = fields.Dict( + keys=StringTransformedEnum( + allowed_values=["aggregated_fluency_pass_rate", "acceptable_fluency_score_per_instance"] + ), + values=fields.Number(), + ) + similarity = fields.Dict( + keys=StringTransformedEnum( + allowed_values=["aggregated_similarity_pass_rate", "acceptable_similarity_score_per_instance"] + ), + values=fields.Number(), + ) + + @post_load + def make(self, data, **kwargs): + from azure.ai.ml.entities._monitoring.thresholds import GenerationSafetyQualityMonitoringMetricThreshold + + return GenerationSafetyQualityMonitoringMetricThreshold(**data) + + +class GenerationTokenStatisticsMonitorMetricThresholdSchema( + metaclass=PatchedSchemaMeta +): # pylint: disable=name-too-long + totaltoken = fields.Dict( + keys=StringTransformedEnum(allowed_values=["total_token_count", "total_token_count_per_group"]), + values=fields.Number(), + ) + + @post_load + def make(self, data, **kwargs): + from azure.ai.ml.entities._monitoring.thresholds import GenerationTokenStatisticsMonitorMetricThreshold + + return GenerationTokenStatisticsMonitorMetricThreshold(**data) |