using ZymonicGateway.Contracts; using ZymonicGateway.Options; using ZymonicServices; using System.Text.Json.Nodes; namespace ZymonicGateway.Services; public class ZymonicGatewayQueueRunner : BackgroundService { private readonly IServiceProvider _services; private readonly ZymonicGatewayPluginCatalog _pluginCatalog; private readonly IZymonicGatewayDebugRequestDecider _debugRequestDecider; private readonly ZymonicGatewayOptions _options; private readonly ILogger _logger; public ZymonicGatewayQueueRunner( IServiceProvider services, ZymonicGatewayPluginCatalog pluginCatalog, IZymonicGatewayDebugRequestDecider debugRequestDecider, Microsoft.Extensions.Options.IOptions options, ILogger logger) { _services = services; _pluginCatalog = pluginCatalog; _debugRequestDecider = debugRequestDecider; _options = options.Value; _logger = logger; } protected override async Task ExecuteAsync(CancellationToken stoppingToken) { if (!_options.QueueRunner.Enabled) { _logger.LogInformation("ZymonicGateway queue runner is disabled."); return; } var handlers = _pluginCatalog.Plugins.SelectMany(plugin => plugin.QueueHandlers).ToArray(); if (handlers.Length == 0) { _logger.LogWarning("ZymonicGateway queue runner has no plugin queue handlers."); } var workers = Enumerable.Range(0, Math.Max(1, _options.QueueRunner.MaxParallelRequests)) .Select(workerId => RunWorkerAsync(workerId, handlers, stoppingToken)); await Task.WhenAll(workers); } private async Task RunWorkerAsync(int workerId, IReadOnlyList handlers, CancellationToken ct) { using var timer = new PeriodicTimer(TimeSpan.FromSeconds(Math.Max(1, _options.QueueRunner.PollIntervalSeconds))); while (!ct.IsCancellationRequested) { try { await ProcessOneAsync(workerId, handlers, ct); } catch (OperationCanceledException) when (ct.IsCancellationRequested) { return; } catch (Exception ex) { _logger.LogError(ex, "ZymonicGateway queue worker {WorkerId} failed unexpectedly.", workerId); } await timer.WaitForNextTickAsync(ct); } } private async Task ProcessOneAsync(int workerId, IReadOnlyList handlers, CancellationToken ct) { using var scope = _services.CreateScope(); var queueStore = scope.ServiceProvider.GetRequiredService(); var request = await queueStore.ClaimNextQueuedRequestAsync( _options.QueueRunner.MaxAttempts, TimeSpan.FromSeconds(Math.Max(0, _options.QueueRunner.RetryDelaySeconds)), ct); if (request is null) { return; } var work = new ZymonicGatewayQueuedWork { Services = scope.ServiceProvider, RequestId = request.RequestId, QueuedAtUtc = request.QueuedAtUtc, AttemptCount = request.AttemptCount, Source = request.Source, Route = request.Route, EffectiveUser = request.EffectiveUser, AuthMethod = request.AuthMethod, RequestPayloadJson = request.RequestPayloadJson, RequestMetadataJson = request.RequestMetadataJson }; work.IsManualRetry = IsManualRetry(request.RequestMetadataJson); work.DebugMode = ManualRetryDebugMode(request.RequestMetadataJson) ?? _debugRequestDecider.ShouldDebug(work); _logger.LogInformation( "Queued request {RequestId} debug decision: {DebugMode}. Source={Source}, Route={Route}, EffectiveUser={EffectiveUser}, AuthMethod={AuthMethod}", work.RequestId, work.DebugMode, work.Source ?? "", work.Route ?? "", work.EffectiveUser ?? "", work.AuthMethod ?? ""); var handler = handlers.FirstOrDefault(handler => handler.CanHandle(work)); if (handler is null) { await queueStore.FailRequestAsync(request.RequestId, $"No queue handler found for {request.Source}:{request.Route}", _options.QueueRunner.MaxAttempts, ct); return; } try { _logger.LogInformation("Worker {WorkerId} processing queued request {RequestId} with handler {Handler}.", workerId, request.RequestId, handler.Name); await handler.HandleAsync(work, ct); await queueStore.CompleteRequestAsync(request.RequestId, ct); } catch (Exception ex) { _logger.LogError(ex, "Queued request {RequestId} failed on attempt {AttemptCount}.", request.RequestId, request.AttemptCount); var maxAttempts = ex is IZymonicPermanentFailure ? request.AttemptCount : _options.QueueRunner.MaxAttempts; await queueStore.FailRequestAsync(request.RequestId, ex.Message, maxAttempts, ct); } } private static bool IsManualRetry(string? metadataJson) { if (string.IsNullOrWhiteSpace(metadataJson)) { return false; } try { var metadata = JsonNode.Parse(metadataJson) as JsonObject; return metadata?["manualRetry"]?.GetValue() ?? false; } catch { return false; } } private static bool? ManualRetryDebugMode(string? metadataJson) { if (string.IsNullOrWhiteSpace(metadataJson)) { return null; } try { var metadata = JsonNode.Parse(metadataJson) as JsonObject; return metadata?["manualRetryDebugMode"]?.GetValue(); } catch { return null; } } }