Experimenting with Greedy Sampling

samples/Sentry.Samples.AspNetCore.Mvc/Program.cs

@@ -25,11 +25,23 @@
         // Example: Disabling support to compressed responses:
         options.DecompressionMethods = DecompressionMethods.None;
 
-        options.MaxQueueItems = 100;
+        options.MaxQueueItems = 30;
         options.ShutdownTimeout = TimeSpan.FromSeconds(5);
 
+        options.Debug = true;
+        options.DiagnosticLevel = SentryLevel.Debug;
+
         options.TracesSampleRate = 1.0; // For production you may want to lower this to stay inside your quota
 
+        var sampler = new DynamicSampler(options);
+        options.TracesSampler = sampler.SampleRate;
+
+        options.ExperimentalMetrics = new ExperimentalMetricsOptions()
+        {
+            EnableCodeLocations = false,
+            CaptureSystemDiagnosticsMeters = SentryMeters.All
+        };
+
         // Configures the root scope
         options.ConfigureScope(s => s.SetTag("Always sent", "this tag"));
     });

src/Sentry/DynamicSampler.cs

@@ -0,0 +1,186 @@
+#if NET8_0_OR_GREATER
+using System.Diagnostics.Metrics;
+using Sentry.Extensibility;
+
+namespace Sentry;
+
+/// <summary>
+/// Custom sampler that adjusts the sample rate based on the number of discarded envelopes and open queue slots.
+/// </summary>
+public class DynamicSampler : IDisposable
+{
+    private readonly SentryOptions _options;
+    private readonly int _envelopesDiscardedThreshold;
+    private readonly int _openQueueSlotsThreshold;
+
+    private readonly CancellationTokenSource _shutdownSource;
+
+    private double _sampleRate = 1.0;
+    private readonly ReaderWriterLockSlim _sampleLock = new();
+
+    private int _discardedEnvelopes = 0;
+
+    private List<int> _openQueueSlotObservations = new();
+    private readonly object _openQueueSlotObservationsLock = new();
+
+    private volatile bool _disposed;
+    private readonly MeterListener _meterListener = new();
+
+    private Task AdjustRateTask { get; }
+
+    /// <summary>
+    /// Creates a dynamic sampler
+    /// </summary>
+    /// <param name="options">A <see cref="SentryOptions"/> instance</param>
+    /// <param name="envelopesDiscardedThreshold">If the number of discarded envelopes in a given period is above this number, the sample rate will be adjusted down</param>
+    /// <param name="openQueueSlotsThreshold">If the average number of open queue slots is below this number in a given sample period, the sample rate will be adjusted up</param>
+    /// <param name="shutdownSource">A cancellation token for the sample update task</param>
+    public DynamicSampler(SentryOptions options, int envelopesDiscardedThreshold = 1,
+        int openQueueSlotsThreshold = 10, CancellationTokenSource? shutdownSource = null)
+    {
+        _options = options;
+        _envelopesDiscardedThreshold = envelopesDiscardedThreshold;
+        _openQueueSlotsThreshold = openQueueSlotsThreshold;
+        _shutdownSource = shutdownSource ?? new CancellationTokenSource();
+        _meterListener.InstrumentPublished = (instrument, listener) =>
+        {
+            if (instrument.Meter.Name is "Sentry.Internal.Backgroundworker")
+            {
+                listener.EnableMeasurementEvents(instrument);
+            }
+        };
+
+        _meterListener.SetMeasurementEventCallback<int>(OnMeasurementRecorded);
+
+        // Start the meterListener, enabling InstrumentPublished callbacks.
+        _meterListener.Start();
+
+        AdjustRateTask = Task.Run(AdjustRateAsync);
+    }
+
+    private void OnMeasurementRecorded(Instrument instrument, int measurement, ReadOnlySpan<KeyValuePair<string, object?>> tags, object? state)
+    {
+        switch (instrument.Name)
+        {
+            case "sentry.open_queue_slots":
+                lock (_openQueueSlotObservationsLock)
+                {
+                    _openQueueSlotObservations.Add(measurement);
+                }
+                break;
+            case "sentry.envelopes_discarded":
+                Interlocked.Increment(ref _discardedEnvelopes);
+                break;
+        }
+    }
+
+    private async Task AdjustRateAsync()
+    {
+        while (!_shutdownSource.Token.IsCancellationRequested)
+        {
+            await Task.Delay(TimeSpan.FromSeconds(1), _shutdownSource.Token).ConfigureAwait(false);
+
+            // Check if the rate should be adjusted down (e.g. if we're discarding too many envelopes)
+            var discardedEnvelopes = Interlocked.Exchange(ref _discardedEnvelopes, 0);
+            if (discardedEnvelopes > _envelopesDiscardedThreshold)
+            {
+                _sampleLock.EnterWriteLock();
+                try
+                {
+                    // Sample half as many events
+                    _sampleRate /= 2;
+                    _options.LogDebug("Sample rate lowered to {0}", _sampleRate);
+                }
+                finally
+                {
+                    _sampleLock.ExitWriteLock();
+                }
+                continue;
+            }
+
+            // Check if the rate should be adjusted up (e.g. if we're seeing lots of open queue slots)
+            double? observedMean = null;
+            lock (_openQueueSlotObservations)
+            {
+                if (_openQueueSlotObservations.Any())
+                {
+                    observedMean = _openQueueSlotObservations.Average();
+                    _openQueueSlotObservations = new();
+                }
+            }
+            if (observedMean is { } openQueueSlots && openQueueSlots > _openQueueSlotsThreshold)
+            {
+                _sampleLock.EnterUpgradeableReadLock();
+                try
+                {
+                    if (_sampleRate < 1.0)
+                    {
+                        _sampleLock.EnterWriteLock();
+                        try
+                        {
+                            // Sample twice as many events
+                            _sampleRate = Math.Min(1, _sampleRate * 2);
+                            _options.LogDebug("Sample rate increased to {0}", _sampleRate);
+                        }
+                        finally
+                        {
+                            _sampleLock.ExitWriteLock();
+                        }
+                    }
+                }
+                finally
+                {
+                    _sampleLock.ExitUpgradeableReadLock();
+                }
+            }
+        }
+    }
+
+    /// <summary>
+    /// Returns the current sample rate
+    /// </summary>
+    /// <param name="context">The <see cref="TransactionSamplingContext"/></param>
+    /// <returns></returns>
+    public double? SampleRate(TransactionSamplingContext context)
+    {
+        _sampleLock.EnterReadLock();
+        try
+        {
+            return _sampleRate;
+        }
+        finally
+        {
+            _sampleLock.ExitReadLock();
+        }
+    }
+
+    /// <inheritdoc cref="IDisposable.Dispose"/>
+    public void Dispose()
+    {
+        if (_disposed)
+        {
+            return;
+        }
+
+        _disposed = true;
+        try
+        {
+            _shutdownSource.Cancel();
+            AdjustRateTask.Wait();
+        }
+        catch (OperationCanceledException)
+        {
+            _options.LogDebug("Stopping the background worker due to a cancellation.");
+        }
+        catch (Exception exception)
+        {
+            _options.LogError(exception, "Stopping the background worker threw an exception.");
+        }
+        finally
+        {
+            _shutdownSource.Dispose();
+            _sampleLock.Dispose();
+        }
+    }
+}
+#endif

src/Sentry/Internal/BackgroundWorker.cs

@@ -1,3 +1,6 @@
+#if NET8_0_OR_GREATER
+using System.Diagnostics.Metrics;
+#endif
 using Sentry.Extensibility;
 using Sentry.Internal.Extensions;
 using Sentry.Internal.Http;
@@ -7,6 +10,13 @@ namespace Sentry.Internal;
 
 internal class BackgroundWorker : IBackgroundWorker, IDisposable
 {
+#if NET8_0_OR_GREATER
+    private const string MeterName = "Sentry.Internal.Backgroundworker";
+    private readonly Histogram<int> _openQueueSlots;
+    private readonly Histogram<int> _envelopesInQueue;
+    private readonly Counter<int> _envelopesDiscarded;
+#endif
+
     private readonly ITransport _transport;
     private readonly SentryOptions _options;
     private readonly ConcurrentQueue<Envelope> _queue;
@@ -26,11 +36,21 @@ internal class BackgroundWorker : IBackgroundWorker, IDisposable
     public BackgroundWorker(
         ITransport transport,
         SentryOptions options,
+#if NET8_0_OR_GREATER
+        IMeterFactory? meterFactory = null,
+#endif
         CancellationTokenSource? shutdownSource = null,
-        ConcurrentQueue<Envelope>? queue = null)
+        ConcurrentQueue<Envelope>? queue = null
+        )
     {
         _transport = transport;
         _options = options;
+#if NET8_0_OR_GREATER
+        var meter = (meterFactory ?? SentryMeterFactory.Instance).Create(new MeterOptions(MeterName));
+        _openQueueSlots = meter.CreateHistogram<int>($"sentry.open_queue_slots");
+        _envelopesInQueue = meter.CreateHistogram<int>($"sentry.envelopes_in_queue");
+        _envelopesDiscarded = meter.CreateCounter<int>($"sentry.envelopes_discarded");
+#endif
         _queue = queue ?? new ConcurrentQueue<Envelope>();
         _maxItems = options.MaxQueueItems;
         _shutdownSource = shutdownSource ?? new CancellationTokenSource();
@@ -67,13 +87,20 @@ public bool EnqueueEnvelope(Envelope envelope, bool process)
         if (Interlocked.Increment(ref _currentItems) > _maxItems)
         {
             Interlocked.Decrement(ref _currentItems);
+#if NET8_0_OR_GREATER
+            _envelopesDiscarded.Add(1);
+#endif
             _options.ClientReportRecorder.RecordDiscardedEvents(DiscardReason.QueueOverflow, envelope);
             _options.LogInfo("Discarding envelope {0} because the queue is full.", eventId);
             return false;
         }
 
         _options.LogDebug("Enqueuing envelope {0}", eventId);
         _queue.Enqueue(envelope);
+#if NET8_0_OR_GREATER
+        _envelopesInQueue.Record(_currentItems);
+        _openQueueSlots.Record(_maxItems - _currentItems);
+#endif
 
         if (process)
         {
@@ -176,6 +203,10 @@ private async Task DoWorkAsync()
                         _options.LogDebug("De-queueing event {0}", eventId);
                         _queue.TryDequeue(out _);
                         Interlocked.Decrement(ref _currentItems);
+#if NET8_0_OR_GREATER
+                        _envelopesInQueue.Record(_currentItems);
+                        _openQueueSlots.Record(_maxItems - _currentItems);
+#endif
                         OnFlushObjectReceived?.Invoke(envelope, EventArgs.Empty);
                     }
                 }

src/Sentry/Internal/SentryMeterFactory.cs

@@ -0,0 +1,76 @@
+#if NET8_0_OR_GREATER
+using System.Diagnostics.Metrics;
+
+namespace Sentry.Internal;
+
+/// <summary>
+/// This is a simplified MeterFactory that only caches meters and does not support scoping. It also assumes there will
+/// be no dimensionality in the meters. The main reason we have this is that that we can't rely on Dependency injection
+/// so we can't get a DefaultMeterFactory from DI.
+/// </summary>
+internal class SentryMeterFactory : IMeterFactory
+{
+    private static readonly Lazy<SentryMeterFactory> LazyInstance = new();
+    public static SentryMeterFactory Instance => LazyInstance.Value;
+
+    private readonly Dictionary<string, Meter> _cachedMeters = new();
+    private bool _disposed;
+
+    public Meter Create(MeterOptions options)
+    {
+        if (options is null)
+        {
+            throw new ArgumentNullException(nameof(options));
+        }
+
+        if (options.Scope is not null)
+        {
+            throw new ArgumentException("The SentryMeterFactory does not support scopes");
+        }
+
+        if (options.Tags != null && options.Tags.Any())
+        {
+            throw new ArgumentException("The SentryMeterFactory does not support tags");
+        }
+
+        Debug.Assert(options.Name is not null);
+
+        lock (_cachedMeters)
+        {
+            if (_disposed)
+            {
+                throw new ObjectDisposedException(nameof(SentryMeterFactory));
+            }
+
+            if (_cachedMeters.TryGetValue(options.Name, out var meter))
+            {
+                return meter;
+            }
+
+            meter = new Meter(options.Name, options.Version, options.Tags, scope: this);
+            _cachedMeters.Add(options.Name, meter);
+            return meter;
+        }
+    }
+
+    public void Dispose()
+    {
+        lock (_cachedMeters)
+        {
+            if (_disposed)
+            {
+                return;
+            }
+
+            _disposed = true;
+
+            foreach (var meter in _cachedMeters.Values)
+            {
+                meter.Dispose();
+            }
+
+            _cachedMeters.Clear();
+        }
+    }
+}
+#endif

src/Sentry/SentryMeters.cs

@@ -0,0 +1,33 @@
+namespace Sentry;
+
+/// <summary>
+/// Sentry metrics that can be added to
+/// <see cref="ExperimentalMetricsOptions.CaptureSystemDiagnosticsMeters"/>
+/// </summary>
+public static partial class SentryMeters
+{
+    private const string BackgroundWorkerPattern = @"^Sentry\.Internal\.Backgroundworker$";
+
+    /// <summary>
+    /// Matches the built in <see cref="System.Net.Http"/> metrics
+    /// </summary>
+#if NET8_0_OR_GREATER
+    public static readonly SubstringOrRegexPattern BackgroundWorker = SystemNetHttpRegex();
+
+    [GeneratedRegex(BackgroundWorkerPattern, RegexOptions.Compiled)]
+    private static partial Regex SystemNetHttpRegex();
+#else
+    public static readonly SubstringOrRegexPattern BackgroundWorker = new Regex(BackgroundWorkerPattern, RegexOptions.Compiled);
+#endif
+
+    private static readonly Lazy<IList<SubstringOrRegexPattern>> LazyAll = new(() => new List<SubstringOrRegexPattern>
+    {
+        BackgroundWorker,
+    });
+
+    /// <summary>
+    /// Matches all built in metrics
+    /// </summary>
+    /// <returns></returns>
+    public static IList<SubstringOrRegexPattern> All => LazyAll.Value;
+}