Fix chart trimming breaking live trading statistics

Engine/Results/BaseResultsHandler.cs

@@ -986,7 +986,7 @@ protected Dictionary<string, string> GetAlgorithmState(DateTime? endTime = null)
         /// <summary>
         /// Will generate the statistics results and update the provided runtime statistics
         /// </summary>
-        protected StatisticsResults GenerateStatisticsResults(Dictionary<string, Chart> charts,
+        protected virtual StatisticsResults GenerateStatisticsResults(Dictionary<string, Chart> charts,
             SortedDictionary<DateTime, decimal> profitLoss = null, CapacityEstimate estimatedStrategyCapacity = null)
         {
             var statisticsResults = new StatisticsResults();

Engine/Results/LiveTradingResultHandler.cs

@@ -79,6 +79,8 @@ public class LiveTradingResultHandler : BaseResultsHandler, IResultHandler
         private bool _userExchangeIsOpen;
         private DateTime _lastChartSampleLogicCheck;
         private readonly Dictionary<string, SecurityExchangeHours> _exchangeHours;
+        protected readonly SortedDictionary<DateTime, decimal> _dailyEquityClose = new();
+        protected readonly object _dailyEquityCloseLock = new();
 
 
         /// <summary>
@@ -336,21 +338,7 @@ private void Update()
                     if (utcNow > _nextChartTrimming)
                     {
                         Log.Debug("LiveTradingResultHandler.Update(): Trimming charts");
-                        var timeLimitUtc = utcNow.AddDays(-2);
-                        lock (ChartLock)
-                        {
-                            foreach (var chart in Charts)
-                            {
-                                foreach (var series in chart.Value.Series)
-                                {
-                                    // trim data that's older than 2 days
-                                    series.Value.Values =
-                                        (from v in series.Value.Values
-                                         where v.Time > timeLimitUtc
-                                         select v).ToList();
-                                }
-                            }
-                        }
+                        TrimCharts(utcNow);
                         _nextChartTrimming = DateTime.UtcNow.AddMinutes(10);
                         Log.Debug("LiveTradingResultHandler.Update(): Finished trimming charts");
                     }
@@ -382,6 +370,86 @@ protected virtual void SetNextStatusUpdate()
             _nextStatusUpdate = DateTime.UtcNow.AddMinutes(10);
         }
 
+        /// <summary>
+        /// Removes chart series points older than their retention window: 10 days for performance charts, 2 days for all others.
+        /// </summary>
+        public override void Sample(DateTime time)
+        {
+            base.Sample(time);
+            if (!Algorithm.IsWarmingUp)
+            {
+                lock (_dailyEquityCloseLock)
+                {
+                    _dailyEquityClose[time.Date] = GetPortfolioValue();
+                }
+            }
+        }
+
+        protected override StatisticsResults GenerateStatisticsResults(Dictionary<string, Chart> charts,
+            SortedDictionary<DateTime, decimal> profitLoss = null, CapacityEstimate estimatedStrategyCapacity = null)
+        {
+            List<KeyValuePair<DateTime, decimal>> historicalCloses;
+            lock (_dailyEquityCloseLock)
+            {
+                historicalCloses = _dailyEquityClose.ToList();
+            }
+
+            if (historicalCloses.Count > 0 &&
+                charts.TryGetValue(StrategyEquityKey, out var strategyEquity) &&
+                strategyEquity.Series.TryGetValue(EquityKey, out var equitySeries))
+            {
+                var existingDates = new HashSet<DateTime>(equitySeries.Values.Select(v => v.Time.Date));
+                var toInject = historicalCloses
+                    .Where(kvp => !existingDates.Contains(kvp.Key))
+                    .Select(kvp => (ISeriesPoint)new Candlestick(kvp.Key, kvp.Value, kvp.Value, kvp.Value, kvp.Value))
+                    .ToList();
+
+                if (toInject.Count > 0)
+                {
+                    var mergedEquity = (CandlestickSeries)equitySeries.Clone();
+                    mergedEquity.Values.InsertRange(0, toInject);
+
+                    var mergedChart = new Chart(StrategyEquityKey);
+                    foreach (var kvp in strategyEquity.Series)
+                    {
+                        mergedChart.Series[kvp.Key] = kvp.Key == EquityKey ? mergedEquity : kvp.Value;
+                    }
+
+                    charts = new Dictionary<string, Chart>(charts) { [StrategyEquityKey] = mergedChart };
+                }
+            }
+
+            return base.GenerateStatisticsResults(charts, profitLoss, estimatedStrategyCapacity);
+        }
+
+        /// <summary>
+        /// Removes chart series points older than their retention window: 2 years for daily statistics series
+        /// (Return, Benchmark), 2 days for all others.
+        /// </summary>
+        protected virtual void TrimCharts(DateTime utcNow)
+        {
+            var defaultLimit = utcNow.AddDays(-2);
+            var dailyStatsLimit = utcNow.AddDays(-730);
+
+            lock (ChartLock)
+            {
+                foreach (var chart in Charts)
+                {
+                    foreach (var series in chart.Value.Series)
+                    {
+                        var isDailyStatsSeries =
+                            (chart.Key == StrategyEquityKey && series.Key == ReturnKey) ||
+                            (chart.Key == BenchmarkKey && series.Key == BenchmarkKey);
+
+                        var timeLimitUtc = isDailyStatsSeries ? dailyStatsLimit : defaultLimit;
+                        series.Value.Values = series.Value.Values
+                            .Where(v => v.Time > timeLimitUtc)
+                            .ToList();
+                    }
+                }
+            }
+        }
+
         /// <summary>
         /// Stores the order events
         /// </summary>

Tests/Engine/Results/LiveTradingResultHandlerTests.cs

@@ -149,7 +149,7 @@ public void DailySampleValueBasedOnMarketHour(bool extendedMarketHoursEnabled)
             using var messagging = new QuantConnect.Messaging.Messaging();
             var referenceDate = new DateTime(2020, 11, 25);
             var resultHandler = new LiveTradingResultHandler();
-            resultHandler.Initialize(new (new LiveNodePacket(), messagging, api, new BacktestingTransactionHandler(), null));
+            resultHandler.Initialize(new(new LiveNodePacket(), messagging, api, new BacktestingTransactionHandler(), null));
 
             try
             {
@@ -190,6 +190,62 @@ public void DailySampleValueBasedOnMarketHour(bool extendedMarketHoursEnabled)
             }
         }
 
+        [Test]
+        public void TrimChartsUsesLongerWindowForPerformanceCharts()
+        {
+            var handler = new TestableLiveTradingResultHandler();
+            var utcNow = new DateTime(2020, 11, 25, 12, 0, 0, DateTimeKind.Utc);
+
+            var benchmarkChart = new Chart(BaseResultsHandler.BenchmarkKey);
+            benchmarkChart.Series.Add(BaseResultsHandler.BenchmarkKey, new Series(BaseResultsHandler.BenchmarkKey));
+            handler.Charts[BaseResultsHandler.BenchmarkKey] = benchmarkChart;
+
+            // Add a custom user chart to verify it still uses the 2 day window 
+            var customChart = new Chart("MyCustomChart");
+            customChart.Series.Add("MyMetric", new Series("MyMetric"));
+            handler.Charts["MyCustomChart"] = customChart;
+
+            var returnSeries = handler.Charts[BaseResultsHandler.StrategyEquityKey].Series[BaseResultsHandler.ReturnKey];
+            var equitySeries = handler.Charts[BaseResultsHandler.StrategyEquityKey].Series[BaseResultsHandler.EquityKey];
+            var benchmarkSeries = benchmarkChart.Series[BaseResultsHandler.BenchmarkKey];
+            var customSeries = customChart.Series["MyMetric"];
+
+            // performance charts: 15 daily samples covering well beyond both trim windows
+            for (var i = 15; i >= 1; i--)
+            {
+                var t = utcNow.AddDays(-i);
+                returnSeries.Values.Add(new ChartPoint(t, i));
+                benchmarkSeries.Values.Add(new ChartPoint(t, i));
+                equitySeries.Values.Add(new Candlestick(t, 100, 110, 90, 105));
+            }
+
+            // custom chart: 5 samples to verify it uses the 2-day window
+            for (var i = 5; i >= 1; i--)
+            {
+                customSeries.Values.Add(new ChartPoint(utcNow.AddDays(-i), i));
+            }
+
+            handler.PublicTrimCharts(utcNow);
+
+            // performance charts keep 10 day window
+            var performanceChartsCutoff = utcNow.AddDays(-10);
+            Assert.IsTrue(returnSeries.Values.All(v => v.Time > performanceChartsCutoff));
+            Assert.IsTrue(benchmarkSeries.Values.All(v => v.Time > performanceChartsCutoff));
+            Assert.IsTrue(equitySeries.Values.All(v => v.Time > performanceChartsCutoff));
+            Assert.AreEqual(9, returnSeries.Values.Count);
+            Assert.AreEqual(9, benchmarkSeries.Values.Count);
+
+            // other charts keep 2 day window
+            var otherChartsCutoff = utcNow.AddDays(-2);
+            Assert.IsTrue(customSeries.Values.All(v => v.Time > otherChartsCutoff));
+            Assert.AreEqual(1, customSeries.Values.Count);
+        }
+
+        private class TestableLiveTradingResultHandler : LiveTradingResultHandler
+        {
+            public void PublicTrimCharts(DateTime utcNow) => TrimCharts(utcNow);
+        }
+
         private class TestDataFeed : IDataFeed
         {
             public bool IsActive { get; }