diff --git a/Engine/Results/BaseResultsHandler.cs b/Engine/Results/BaseResultsHandler.cs index 407f55bf1826..dce5cfa65182 100644 --- a/Engine/Results/BaseResultsHandler.cs +++ b/Engine/Results/BaseResultsHandler.cs @@ -986,7 +986,7 @@ protected Dictionary GetAlgorithmState(DateTime? endTime = null) /// /// Will generate the statistics results and update the provided runtime statistics /// - protected StatisticsResults GenerateStatisticsResults(Dictionary charts, + protected virtual StatisticsResults GenerateStatisticsResults(Dictionary charts, SortedDictionary profitLoss = null, CapacityEstimate estimatedStrategyCapacity = null) { var statisticsResults = new StatisticsResults(); diff --git a/Engine/Results/LiveTradingResultHandler.cs b/Engine/Results/LiveTradingResultHandler.cs index 954c95cd3411..442aa09c64c4 100644 --- a/Engine/Results/LiveTradingResultHandler.cs +++ b/Engine/Results/LiveTradingResultHandler.cs @@ -79,6 +79,8 @@ public class LiveTradingResultHandler : BaseResultsHandler, IResultHandler private bool _userExchangeIsOpen; private DateTime _lastChartSampleLogicCheck; private readonly Dictionary _exchangeHours; + protected readonly SortedDictionary _dailyEquityClose = new(); + protected readonly object _dailyEquityCloseLock = new(); /// @@ -336,21 +338,7 @@ private void Update() if (utcNow > _nextChartTrimming) { Log.Debug("LiveTradingResultHandler.Update(): Trimming charts"); - var timeLimitUtc = utcNow.AddDays(-2); - lock (ChartLock) - { - foreach (var chart in Charts) - { - foreach (var series in chart.Value.Series) - { - // trim data that's older than 2 days - series.Value.Values = - (from v in series.Value.Values - where v.Time > timeLimitUtc - select v).ToList(); - } - } - } + TrimCharts(utcNow); _nextChartTrimming = DateTime.UtcNow.AddMinutes(10); Log.Debug("LiveTradingResultHandler.Update(): Finished trimming charts"); } @@ -382,6 +370,86 @@ protected virtual void SetNextStatusUpdate() _nextStatusUpdate = DateTime.UtcNow.AddMinutes(10); } + /// + /// Removes chart series points older than their retention window: 10 days for performance charts, 2 days for all others. + /// + public override void Sample(DateTime time) + { + base.Sample(time); + if (!Algorithm.IsWarmingUp) + { + lock (_dailyEquityCloseLock) + { + _dailyEquityClose[time.Date] = GetPortfolioValue(); + } + } + } + + protected override StatisticsResults GenerateStatisticsResults(Dictionary charts, + SortedDictionary profitLoss = null, CapacityEstimate estimatedStrategyCapacity = null) + { + List> historicalCloses; + lock (_dailyEquityCloseLock) + { + historicalCloses = _dailyEquityClose.ToList(); + } + + if (historicalCloses.Count > 0 && + charts.TryGetValue(StrategyEquityKey, out var strategyEquity) && + strategyEquity.Series.TryGetValue(EquityKey, out var equitySeries)) + { + var existingDates = new HashSet(equitySeries.Values.Select(v => v.Time.Date)); + var toInject = historicalCloses + .Where(kvp => !existingDates.Contains(kvp.Key)) + .Select(kvp => (ISeriesPoint)new Candlestick(kvp.Key, kvp.Value, kvp.Value, kvp.Value, kvp.Value)) + .ToList(); + + if (toInject.Count > 0) + { + var mergedEquity = (CandlestickSeries)equitySeries.Clone(); + mergedEquity.Values.InsertRange(0, toInject); + + var mergedChart = new Chart(StrategyEquityKey); + foreach (var kvp in strategyEquity.Series) + { + mergedChart.Series[kvp.Key] = kvp.Key == EquityKey ? mergedEquity : kvp.Value; + } + + charts = new Dictionary(charts) { [StrategyEquityKey] = mergedChart }; + } + } + + return base.GenerateStatisticsResults(charts, profitLoss, estimatedStrategyCapacity); + } + + /// + /// Removes chart series points older than their retention window: 2 years for daily statistics series + /// (Return, Benchmark), 2 days for all others. + /// + protected virtual void TrimCharts(DateTime utcNow) + { + var defaultLimit = utcNow.AddDays(-2); + var dailyStatsLimit = utcNow.AddDays(-730); + + lock (ChartLock) + { + foreach (var chart in Charts) + { + foreach (var series in chart.Value.Series) + { + var isDailyStatsSeries = + (chart.Key == StrategyEquityKey && series.Key == ReturnKey) || + (chart.Key == BenchmarkKey && series.Key == BenchmarkKey); + + var timeLimitUtc = isDailyStatsSeries ? dailyStatsLimit : defaultLimit; + series.Value.Values = series.Value.Values + .Where(v => v.Time > timeLimitUtc) + .ToList(); + } + } + } + } + /// /// Stores the order events /// diff --git a/Tests/Engine/Results/LiveTradingResultHandlerTests.cs b/Tests/Engine/Results/LiveTradingResultHandlerTests.cs index ba3da3aac2d1..08f1c1bb6248 100644 --- a/Tests/Engine/Results/LiveTradingResultHandlerTests.cs +++ b/Tests/Engine/Results/LiveTradingResultHandlerTests.cs @@ -149,7 +149,7 @@ public void DailySampleValueBasedOnMarketHour(bool extendedMarketHoursEnabled) using var messagging = new QuantConnect.Messaging.Messaging(); var referenceDate = new DateTime(2020, 11, 25); var resultHandler = new LiveTradingResultHandler(); - resultHandler.Initialize(new (new LiveNodePacket(), messagging, api, new BacktestingTransactionHandler(), null)); + resultHandler.Initialize(new(new LiveNodePacket(), messagging, api, new BacktestingTransactionHandler(), null)); try { @@ -190,6 +190,62 @@ public void DailySampleValueBasedOnMarketHour(bool extendedMarketHoursEnabled) } } + [Test] + public void TrimChartsUsesLongerWindowForPerformanceCharts() + { + var handler = new TestableLiveTradingResultHandler(); + var utcNow = new DateTime(2020, 11, 25, 12, 0, 0, DateTimeKind.Utc); + + var benchmarkChart = new Chart(BaseResultsHandler.BenchmarkKey); + benchmarkChart.Series.Add(BaseResultsHandler.BenchmarkKey, new Series(BaseResultsHandler.BenchmarkKey)); + handler.Charts[BaseResultsHandler.BenchmarkKey] = benchmarkChart; + + // Add a custom user chart to verify it still uses the 2 day window + var customChart = new Chart("MyCustomChart"); + customChart.Series.Add("MyMetric", new Series("MyMetric")); + handler.Charts["MyCustomChart"] = customChart; + + var returnSeries = handler.Charts[BaseResultsHandler.StrategyEquityKey].Series[BaseResultsHandler.ReturnKey]; + var equitySeries = handler.Charts[BaseResultsHandler.StrategyEquityKey].Series[BaseResultsHandler.EquityKey]; + var benchmarkSeries = benchmarkChart.Series[BaseResultsHandler.BenchmarkKey]; + var customSeries = customChart.Series["MyMetric"]; + + // performance charts: 15 daily samples covering well beyond both trim windows + for (var i = 15; i >= 1; i--) + { + var t = utcNow.AddDays(-i); + returnSeries.Values.Add(new ChartPoint(t, i)); + benchmarkSeries.Values.Add(new ChartPoint(t, i)); + equitySeries.Values.Add(new Candlestick(t, 100, 110, 90, 105)); + } + + // custom chart: 5 samples to verify it uses the 2-day window + for (var i = 5; i >= 1; i--) + { + customSeries.Values.Add(new ChartPoint(utcNow.AddDays(-i), i)); + } + + handler.PublicTrimCharts(utcNow); + + // performance charts keep 10 day window + var performanceChartsCutoff = utcNow.AddDays(-10); + Assert.IsTrue(returnSeries.Values.All(v => v.Time > performanceChartsCutoff)); + Assert.IsTrue(benchmarkSeries.Values.All(v => v.Time > performanceChartsCutoff)); + Assert.IsTrue(equitySeries.Values.All(v => v.Time > performanceChartsCutoff)); + Assert.AreEqual(9, returnSeries.Values.Count); + Assert.AreEqual(9, benchmarkSeries.Values.Count); + + // other charts keep 2 day window + var otherChartsCutoff = utcNow.AddDays(-2); + Assert.IsTrue(customSeries.Values.All(v => v.Time > otherChartsCutoff)); + Assert.AreEqual(1, customSeries.Values.Count); + } + + private class TestableLiveTradingResultHandler : LiveTradingResultHandler + { + public void PublicTrimCharts(DateTime utcNow) => TrimCharts(utcNow); + } + private class TestDataFeed : IDataFeed { public bool IsActive { get; }