Fix zero live statistics caused by chart trimming (#9520)

* Keep daily sample of statistics charts

* Aggregate OHLC when trimming the daily sample of statistics charts

Author: JosueNina

Engine/Results/LiveTradingResultHandler.cs

@@ -336,21 +336,7 @@ private void Update()
                     if (utcNow > _nextChartTrimming)
                     {
                         Log.Debug("LiveTradingResultHandler.Update(): Trimming charts");
-                        var timeLimitUtc = utcNow.AddDays(-2);
-                        lock (ChartLock)
-                        {
-                            foreach (var chart in Charts)
-                            {
-                                foreach (var series in chart.Value.Series)
-                                {
-                                    // trim data that's older than 2 days
-                                    series.Value.Values =
-                                        (from v in series.Value.Values
-                                         where v.Time > timeLimitUtc
-                                         select v).ToList();
-                                }
-                            }
-                        }
+                        TrimCharts(utcNow);
                         _nextChartTrimming = DateTime.UtcNow.AddMinutes(10);
                         Log.Debug("LiveTradingResultHandler.Update(): Finished trimming charts");
                     }
@@ -382,6 +368,85 @@ protected virtual void SetNextStatusUpdate()
             _nextStatusUpdate = DateTime.UtcNow.AddMinutes(10);
         }
 
+        /// <summary>
+        /// Trims old points from each chart series. The statistics series (equity, return and benchmark) keep
+        /// full resolution for the last 2 days and a daily sample for up to 2 years. Every other series keeps
+        /// only the last 2 days.
+        /// </summary>
+        protected virtual void TrimCharts(DateTime utcNow)
+        {
+            var fullResolutionLimit = utcNow.AddDays(-2);
+            var dailySampleLimit = utcNow.AddDays(-730);
+
+            lock (ChartLock)
+            {
+                foreach (var chart in Charts)
+                {
+                    foreach (var series in chart.Value.Series)
+                    {
+                        var isStatisticsSeries =
+                            (chart.Key == StrategyEquityKey && (series.Key == EquityKey || series.Key == ReturnKey)) ||
+                            (chart.Key == BenchmarkKey && series.Key == BenchmarkKey);
+
+                        if (isStatisticsSeries)
+                        {
+                            series.Value.Values = TrimToDailySample(series.Value.Values, fullResolutionLimit, dailySampleLimit);
+                        }
+                        else
+                        {
+                            series.Value.Values = series.Value.Values
+                                .Where(point => point.Time > fullResolutionLimit)
+                                .ToList();
+                        }
+                    }
+                }
+            }
+        }
+
+        /// <summary>
+        /// Keeps all points within the full resolution limit, then one aggregated point per day down to the daily sample limit, and drops the rest
+        /// </summary>
+        private static List<ISeriesPoint> TrimToDailySample(List<ISeriesPoint> values, DateTime fullResolutionLimit, DateTime dailySampleLimit)
+        {
+            var dailySamples = values
+                .Where(point => point.Time > dailySampleLimit && point.Time <= fullResolutionLimit)
+                .GroupBy(point => point.Time.Date)
+                .Select(AggregateDailySample);
+
+            var fullResolution = values.Where(point => point.Time > fullResolutionLimit);
+
+            return dailySamples.Concat(fullResolution).ToList();
+        }
+
+        /// <summary>
+        /// Aggregates a single day's points into one, keeping the full OHLC for candlestick series
+        /// </summary>
+        private static ISeriesPoint AggregateDailySample(IEnumerable<ISeriesPoint> dayPoints)
+        {
+            ISeriesPoint last = null;
+            Candlestick aggregated = null;
+            foreach (var point in dayPoints)
+            {
+                last = point;
+                if (point is Candlestick candlestick)
+                {
+                    aggregated ??= new Candlestick();
+                    aggregated.Update(candlestick.Open);
+                    aggregated.Update(candlestick.High);
+                    aggregated.Update(candlestick.Low);
+                    aggregated.Update(candlestick.Close);
+                }
+            }
+
+            if (aggregated == null)
+            {
+                return last;
+            }
+
+            aggregated.Time = last.Time;
+            return aggregated;
+        }
+
         /// <summary>
         /// Stores the order events
         /// </summary>

Tests/Engine/Results/LiveTradingResultHandlerTests.cs

@@ -150,7 +150,7 @@ public void DailySampleValueBasedOnMarketHour(bool extendedMarketHoursEnabled)
             using var messagging = new QuantConnect.Messaging.Messaging();
             var referenceDate = new DateTime(2020, 11, 25);
             var resultHandler = new LiveTradingResultHandler();
-            resultHandler.Initialize(new (new LiveNodePacket(), messagging, api, new BacktestingTransactionHandler(), null));
+            resultHandler.Initialize(new(new LiveNodePacket(), messagging, api, new BacktestingTransactionHandler(), null));
 
             try
             {
@@ -222,6 +222,102 @@ public void MessagesArePrefixedWithAlgorithmTime()
             Assert.That(messages, Has.All.StartsWith(algorithmTimePrefix));
         }
 
+        [Test]
+        public void TrimChartsKeepsDailySampleOfStatisticsSeries()
+        {
+            var handler = new TestableLiveTradingResultHandler();
+            var utcNow = new DateTime(2020, 11, 25, 12, 0, 0, DateTimeKind.Utc);
+
+            var benchmarkChart = new Chart(BaseResultsHandler.BenchmarkKey);
+            benchmarkChart.Series.Add(BaseResultsHandler.BenchmarkKey, new Series(BaseResultsHandler.BenchmarkKey));
+            handler.Charts[BaseResultsHandler.BenchmarkKey] = benchmarkChart;
+
+            var customChart = new Chart("MyCustomChart");
+            customChart.Series.Add("MyMetric", new Series("MyMetric"));
+            handler.Charts["MyCustomChart"] = customChart;
+
+            var returnSeries = handler.Charts[BaseResultsHandler.StrategyEquityKey].Series[BaseResultsHandler.ReturnKey];
+            var equitySeries = handler.Charts[BaseResultsHandler.StrategyEquityKey].Series[BaseResultsHandler.EquityKey];
+            var benchmarkSeries = benchmarkChart.Series[BaseResultsHandler.BenchmarkKey];
+            var customSeries = customChart.Series["MyMetric"];
+
+            // Return and Benchmark: one point per day, going beyond 2 years
+            for (var i = 800; i >= 1; i--)
+            {
+                var t = utcNow.AddDays(-i);
+                returnSeries.Values.Add(new ChartPoint(t, i));
+                benchmarkSeries.Values.Add(new ChartPoint(t, i));
+            }
+
+            // Equity: several points per day for older days, with varying OHLC so the high and low come from intraday candles
+            foreach (var day in new[] { 5, 4, 3 })
+            {
+                var date = utcNow.AddDays(-day).Date;
+                equitySeries.Values.Add(new Candlestick(date.AddHours(10), 100, 105, 98, 101));
+                equitySeries.Values.Add(new Candlestick(date.AddHours(14), 101, 120, 99, 102));
+                equitySeries.Values.Add(new Candlestick(date.AddHours(16), 102, 106, 85, 103));
+            }
+            // Two recent points within the 2 day window
+            equitySeries.Values.Add(new Candlestick(utcNow.AddHours(-5), 200, 210, 195, 205));
+            equitySeries.Values.Add(new Candlestick(utcNow.AddHours(-1), 205, 215, 200, 211));
+
+            // Custom chart: not a statistics series, so no daily sample
+            for (var i = 5; i >= 1; i--)
+            {
+                customSeries.Values.Add(new ChartPoint(utcNow.AddDays(-i), i));
+            }
+
+            handler.PublicTrimCharts(utcNow);
+
+            // Return and Benchmark keep one point per day, up to 2 years
+            var dailyStatsCutoff = utcNow.AddDays(-730);
+            Assert.IsTrue(returnSeries.Values.All(v => v.Time > dailyStatsCutoff));
+            Assert.IsTrue(benchmarkSeries.Values.All(v => v.Time > dailyStatsCutoff));
+            Assert.AreEqual(729, returnSeries.Values.Count);
+            Assert.AreEqual(729, benchmarkSeries.Values.Count);
+
+            // Equity keeps all recent points and one aggregated candlestick per day for older ones
+            Assert.AreEqual(5, equitySeries.Values.Count);
+            foreach (var day in new[] { 5, 4, 3 })
+            {
+                var date = utcNow.AddDays(-day).Date;
+                var samplesForDay = equitySeries.Values.Where(v => v.Time.Date == date).Cast<Candlestick>().ToList();
+                Assert.AreEqual(1, samplesForDay.Count);
+                // The whole day OHLC is aggregated, not just the last candle
+                var candle = samplesForDay[0];
+                Assert.AreEqual(100, candle.Open);
+                Assert.AreEqual(120, candle.High);
+                Assert.AreEqual(85, candle.Low);
+                Assert.AreEqual(103, candle.Close);
+            }
+
+            // Recent points are kept at full resolution
+            var recent = equitySeries.Values.Where(v => v.Time > utcNow.AddDays(-2)).Cast<Candlestick>().ToList();
+            Assert.AreEqual(2, recent.Count);
+            Assert.AreEqual(205, recent[0].Close);
+            Assert.AreEqual(211, recent[1].Close);
+
+            // Custom chart keeps only the last 2 days
+            var defaultCutoff = utcNow.AddDays(-2);
+            Assert.IsTrue(customSeries.Values.All(v => v.Time > defaultCutoff));
+            Assert.AreEqual(1, customSeries.Values.Count);
+
+            // Trimming runs repeatedly in production, so a second pass must leave the already trimmed series unchanged
+            var equitySnapshot = equitySeries.Values.Cast<Candlestick>()
+                .Select(v => (v.Time, v.Open, v.High, v.Low, v.Close)).ToList();
+            handler.PublicTrimCharts(utcNow);
+            Assert.AreEqual(729, returnSeries.Values.Count);
+            Assert.AreEqual(729, benchmarkSeries.Values.Count);
+            Assert.AreEqual(1, customSeries.Values.Count);
+            CollectionAssert.AreEqual(equitySnapshot, equitySeries.Values.Cast<Candlestick>()
+                .Select(v => (v.Time, v.Open, v.High, v.Low, v.Close)).ToList());
+        }
+
+        private class TestableLiveTradingResultHandler : LiveTradingResultHandler
+        {
+            public void PublicTrimCharts(DateTime utcNow) => TrimCharts(utcNow);
+        }
+
         private class TestDataFeed : IDataFeed
         {
             public bool IsActive { get; }