From 554bb9e88927766b75694589f0eb146659b618e6 Mon Sep 17 00:00:00 2001 From: Waldek Mastykarz Date: Fri, 24 Jan 2025 08:12:22 +0100 Subject: [PATCH] Adds UrlDiscoveryPlugin and preset (#969) --- dev-proxy-plugins/Reporters/JsonReporter.cs | 163 +++++++++++------- .../Reporters/MarkdownReporter.cs | 40 ++++- .../Reporters/PlainTextReporter.cs | 33 +++- .../RequestLogs/ExecutionSummaryPlugin.cs | 1 + .../RequestLogs/UrlDiscoveryPlugin.cs | 46 +++++ dev-proxy/presets/urls-to-watch.json | 19 ++ 6 files changed, 236 insertions(+), 66 deletions(-) create mode 100644 dev-proxy-plugins/RequestLogs/UrlDiscoveryPlugin.cs create mode 100644 dev-proxy/presets/urls-to-watch.json diff --git a/dev-proxy-plugins/Reporters/JsonReporter.cs b/dev-proxy-plugins/Reporters/JsonReporter.cs index 5497e7f4..11cd3304 100644 --- a/dev-proxy-plugins/Reporters/JsonReporter.cs +++ b/dev-proxy-plugins/Reporters/JsonReporter.cs @@ -1,65 +1,100 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System.Text.Json; -using Microsoft.DevProxy.Abstractions; -using Microsoft.DevProxy.Plugins.RequestLogs; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; - -namespace Microsoft.DevProxy.Plugins.Reporters; - -public class JsonReporter(IPluginEvents pluginEvents, IProxyContext context, ILogger logger, ISet urlsToWatch, IConfigurationSection? configSection = null) : BaseReporter(pluginEvents, context, logger, urlsToWatch, configSection) -{ - public override string Name => nameof(JsonReporter); - public override string FileExtension => ".json"; - - private readonly Dictionary> _transformers = new() - { - { typeof(ExecutionSummaryPluginReportByUrl), TransformExecutionSummary }, - { typeof(ExecutionSummaryPluginReportByMessageType), TransformExecutionSummary }, - }; - - protected override string GetReport(KeyValuePair report) - { - Logger.LogDebug("Serializing report {reportKey}...", report.Key); - - var reportData = report.Value; - var reportType = reportData.GetType(); - - if (_transformers.TryGetValue(reportType, out var transform)) - { - Logger.LogDebug("Transforming {reportType} using {transform}...", reportType.Name, transform.Method.Name); - reportData = transform(reportData); - } - else - { - Logger.LogDebug("No transformer found for {reportType}", reportType.Name); - } - - if (reportData is string strVal) - { - Logger.LogDebug("{reportKey} is a string. Checking if it's JSON...", report.Key); - - try - { - JsonSerializer.Deserialize(strVal); - Logger.LogDebug("{reportKey} is already JSON, ignore", report.Key); - // already JSON, ignore - return strVal; - } - catch - { - Logger.LogDebug("{reportKey} is not JSON, serializing...", report.Key); - } - } - - return JsonSerializer.Serialize(reportData, ProxyUtils.JsonSerializerOptions); - } - - private static object TransformExecutionSummary(object report) - { - var executionSummaryReport = (ExecutionSummaryPluginReportBase)report; - return executionSummaryReport.Data; - } +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text; +using System.Text.Json; +using Microsoft.DevProxy.Abstractions; +using Microsoft.DevProxy.Plugins.RequestLogs; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DevProxy.Plugins.Reporters; + +public class JsonReporter(IPluginEvents pluginEvents, IProxyContext context, ILogger logger, ISet urlsToWatch, IConfigurationSection? configSection = null) : BaseReporter(pluginEvents, context, logger, urlsToWatch, configSection) +{ + public override string Name => nameof(JsonReporter); + private string _fileExtension = ".json"; + public override string FileExtension => _fileExtension; + + private readonly Dictionary> _transformers = new() + { + { typeof(ExecutionSummaryPluginReportByUrl), TransformExecutionSummary }, + { typeof(ExecutionSummaryPluginReportByMessageType), TransformExecutionSummary }, + { typeof(UrlDiscoveryPluginReport), TransformUrlDiscoveryReport } + }; + + protected override string GetReport(KeyValuePair report) + { + Logger.LogDebug("Serializing report {reportKey}...", report.Key); + + var reportData = report.Value; + var reportType = reportData.GetType(); + _fileExtension = reportType.Name == nameof(UrlDiscoveryPluginReport) ? ".jsonc" : ".json"; + + if (_transformers.TryGetValue(reportType, out var transform)) + { + Logger.LogDebug("Transforming {reportType} using {transform}...", reportType.Name, transform.Method.Name); + reportData = transform(reportData); + } + else + { + Logger.LogDebug("No transformer found for {reportType}", reportType.Name); + } + + if (reportData is string strVal) + { + Logger.LogDebug("{reportKey} is a string. Checking if it's JSON...", report.Key); + + try + { + JsonSerializer.Deserialize(strVal, ProxyUtils.JsonSerializerOptions); + Logger.LogDebug("{reportKey} is already JSON, ignore", report.Key); + // already JSON, ignore + return strVal; + } + catch + { + Logger.LogDebug("{reportKey} is not JSON, serializing...", report.Key); + } + } + + return JsonSerializer.Serialize(reportData, ProxyUtils.JsonSerializerOptions); + } + + private static object TransformExecutionSummary(object report) + { + var executionSummaryReport = (ExecutionSummaryPluginReportBase)report; + return executionSummaryReport.Data; + } + + private static object TransformUrlDiscoveryReport(object report) + { + var urlDiscoveryPluginReport = (UrlDiscoveryPluginReport)report; + + var sb = new StringBuilder(); + sb.AppendLine("{"); + sb.AppendLine(" // Wildcards"); + sb.AppendLine(" // "); + sb.AppendLine(" // You can use wildcards to catch multiple URLs with the same pattern."); + sb.AppendLine(" // For example, you can use the following URL pattern to catch all API requests to"); + sb.AppendLine(" // JSON Placeholder API:"); + sb.AppendLine(" // "); + sb.AppendLine(" // https://jsonplaceholder.typicode.com/*"); + sb.AppendLine(" // "); + sb.AppendLine(" // Excluding URLs"); + sb.AppendLine(" // "); + sb.AppendLine(" // You can exclude URLs with ! to prevent them from being intercepted."); + sb.AppendLine(" // For example, you can exclude the URL https://jsonplaceholder.typicode.com/authors"); + sb.AppendLine(" // by using the following URL pattern:"); + sb.AppendLine(" // "); + sb.AppendLine(" // !https://jsonplaceholder.typicode.com/authors"); + sb.AppendLine(" // https://jsonplaceholder.typicode.com/*"); + sb.AppendLine(" \"urlsToWatch\": ["); + sb.AppendJoin($",{Environment.NewLine}", urlDiscoveryPluginReport.Data.Select(u => $" \"{u}\"")); + sb.AppendLine(""); + sb.AppendLine(" ]"); + sb.AppendLine("}"); + + return sb.ToString(); + } } \ No newline at end of file diff --git a/dev-proxy-plugins/Reporters/MarkdownReporter.cs b/dev-proxy-plugins/Reporters/MarkdownReporter.cs index 5dac6ecd..a44e9be9 100644 --- a/dev-proxy-plugins/Reporters/MarkdownReporter.cs +++ b/dev-proxy-plugins/Reporters/MarkdownReporter.cs @@ -24,7 +24,8 @@ public class MarkdownReporter(IPluginEvents pluginEvents, IProxyContext context, { typeof(HttpFileGeneratorPlugin), TransformHttpFileGeneratorReport }, { typeof(GraphMinimalPermissionsGuidancePluginReport), TransformMinimalPermissionsGuidanceReport }, { typeof(GraphMinimalPermissionsPluginReport), TransformMinimalPermissionsReport }, - { typeof(OpenApiSpecGeneratorPluginReport), TransformOpenApiSpecGeneratorReport } + { typeof(OpenApiSpecGeneratorPluginReport), TransformOpenApiSpecGeneratorReport }, + { typeof(UrlDiscoveryPluginReport), TransformUrlDiscoveryReport } }; private const string _requestsInterceptedMessage = "Requests intercepted"; @@ -483,6 +484,43 @@ void transformPermissionsInfo(GraphMinimalPermissionsInfo permissionsInfo, strin return sb.ToString(); } + private static string? TransformUrlDiscoveryReport(object report) + { + var urlDiscoveryPluginReport = (UrlDiscoveryPluginReport)report; + + var sb = new StringBuilder(); + sb.AppendLine("## Wildcards"); + sb.AppendLine(""); + sb.AppendLine("You can use wildcards to catch multiple URLs with the same pattern."); + sb.AppendLine("For example, you can use the following URL pattern to catch all API requests to"); + sb.AppendLine("JSON Placeholder API:"); + sb.AppendLine(""); + sb.AppendLine("```text"); + sb.AppendLine("https://jsonplaceholder.typicode.com/*"); + sb.AppendLine("```"); + sb.AppendLine(""); + sb.AppendLine("## Excluding URLs"); + sb.AppendLine(""); + sb.AppendLine("You can exclude URLs with ! to prevent them from being intercepted."); + sb.AppendLine("For example, you can exclude the URL `https://jsonplaceholder.typicode.com/authors`"); + sb.AppendLine("by using the following URL pattern:"); + sb.AppendLine(""); + sb.AppendLine("```text"); + sb.AppendLine("!https://jsonplaceholder.typicode.com/authors"); + sb.AppendLine("https://jsonplaceholder.typicode.com/*"); + sb.AppendLine("```"); + sb.AppendLine(""); + sb.AppendLine("Intercepted URLs:"); + sb.AppendLine(); + sb.AppendLine("```text"); + + sb.AppendJoin(Environment.NewLine, urlDiscoveryPluginReport.Data); + + sb.AppendLine(""); + sb.AppendLine("```"); + return sb.ToString(); + } + private static string? TransformHttpFileGeneratorReport(object report) { var httpFileGeneratorReport = (HttpFileGeneratorPluginReport)report; diff --git a/dev-proxy-plugins/Reporters/PlainTextReporter.cs b/dev-proxy-plugins/Reporters/PlainTextReporter.cs index 352e3f0d..eb9c4dff 100644 --- a/dev-proxy-plugins/Reporters/PlainTextReporter.cs +++ b/dev-proxy-plugins/Reporters/PlainTextReporter.cs @@ -24,7 +24,8 @@ public class PlainTextReporter(IPluginEvents pluginEvents, IProxyContext context { typeof(HttpFileGeneratorPluginReport), TransformHttpFileGeneratorReport }, { typeof(GraphMinimalPermissionsGuidancePluginReport), TransformMinimalPermissionsGuidanceReport }, { typeof(GraphMinimalPermissionsPluginReport), TransformMinimalPermissionsReport }, - { typeof(OpenApiSpecGeneratorPluginReport), TransformOpenApiSpecGeneratorReport } + { typeof(OpenApiSpecGeneratorPluginReport), TransformOpenApiSpecGeneratorReport }, + { typeof(UrlDiscoveryPluginReport), TransformUrlDiscoveryReport } }; private const string _requestsInterceptedMessage = "Requests intercepted"; @@ -75,6 +76,36 @@ public class PlainTextReporter(IPluginEvents pluginEvents, IProxyContext context return sb.ToString(); } + private static string? TransformUrlDiscoveryReport(object report) + { + var urlDiscoveryPluginReport = (UrlDiscoveryPluginReport)report; + + var sb = new StringBuilder(); + sb.AppendLine("Wildcards"); + sb.AppendLine(""); + sb.AppendLine("You can use wildcards to catch multiple URLs with the same pattern."); + sb.AppendLine("For example, you can use the following URL pattern to catch all API requests to"); + sb.AppendLine("JSON Placeholder API:"); + sb.AppendLine(""); + sb.AppendLine("https://jsonplaceholder.typicode.com/*"); + sb.AppendLine(""); + sb.AppendLine("Excluding URLs"); + sb.AppendLine(""); + sb.AppendLine("You can exclude URLs with ! to prevent them from being intercepted."); + sb.AppendLine("For example, you can exclude the URL https://jsonplaceholder.typicode.com/authors"); + sb.AppendLine("by using the following URL pattern:"); + sb.AppendLine(""); + sb.AppendLine("!https://jsonplaceholder.typicode.com/authors"); + sb.AppendLine("https://jsonplaceholder.typicode.com/*"); + sb.AppendLine(""); + sb.AppendLine("Intercepted URLs:"); + sb.AppendLine(); + + sb.AppendJoin(Environment.NewLine, urlDiscoveryPluginReport.Data); + + return sb.ToString(); + } + private static string? TransformExecutionSummaryByMessageType(object report) { var executionSummaryReport = (ExecutionSummaryPluginReportByMessageType)report; diff --git a/dev-proxy-plugins/RequestLogs/ExecutionSummaryPlugin.cs b/dev-proxy-plugins/RequestLogs/ExecutionSummaryPlugin.cs index aa1cfc94..4810e6fc 100644 --- a/dev-proxy-plugins/RequestLogs/ExecutionSummaryPlugin.cs +++ b/dev-proxy-plugins/RequestLogs/ExecutionSummaryPlugin.cs @@ -79,6 +79,7 @@ private Task AfterRecordingStopAsync(object? sender, RecordingArgs e) { if (!e.RequestLogs.Any()) { + Logger.LogRequest("No messages recorded", MessageType.Skipped); return Task.CompletedTask; } diff --git a/dev-proxy-plugins/RequestLogs/UrlDiscoveryPlugin.cs b/dev-proxy-plugins/RequestLogs/UrlDiscoveryPlugin.cs new file mode 100644 index 00000000..69684241 --- /dev/null +++ b/dev-proxy-plugins/RequestLogs/UrlDiscoveryPlugin.cs @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.Extensions.Configuration; +using Microsoft.DevProxy.Abstractions; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DevProxy.Plugins.RequestLogs; + +public class UrlDiscoveryPluginReport +{ + public required List Data { get; init; } +} + +public class UrlDiscoveryPlugin(IPluginEvents pluginEvents, IProxyContext context, ILogger logger, ISet urlsToWatch, IConfigurationSection? configSection = null) : BaseReportingPlugin(pluginEvents, context, logger, urlsToWatch, configSection) +{ + public override string Name => nameof(UrlDiscoveryPlugin); + private readonly ExecutionSummaryPluginConfiguration _configuration = new(); + + public override async Task RegisterAsync() + { + await base.RegisterAsync(); + + ConfigSection?.Bind(_configuration); + + PluginEvents.AfterRecordingStop += AfterRecordingStopAsync; + } + + private Task AfterRecordingStopAsync(object? sender, RecordingArgs e) + { + if (!e.RequestLogs.Any()) + { + Logger.LogRequest("No messages recorded", MessageType.Skipped); + return Task.CompletedTask; + } + + UrlDiscoveryPluginReport report = new() + { + Data = [.. e.RequestLogs.Select(log => log.Context?.Session.HttpClient.Request.RequestUri.ToString()).Distinct().Order()] + }; + + StoreReport(report, e); + + return Task.CompletedTask; + } +} diff --git a/dev-proxy/presets/urls-to-watch.json b/dev-proxy/presets/urls-to-watch.json new file mode 100644 index 00000000..b2ab2a7c --- /dev/null +++ b/dev-proxy/presets/urls-to-watch.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://raw.githubusercontent.com/microsoft/dev-proxy/main/schemas/v0.24.0/rc.schema.json", + "plugins": [ + { + "name": "UrlDiscoveryPlugin", + "enabled": true, + "pluginPath": "~appFolder/plugins/dev-proxy-plugins.dll" + }, + { + "name": "PlainTextReporter", + "enabled": true, + "pluginPath": "~appFolder/plugins/dev-proxy-plugins.dll" + } + ], + "urlsToWatch": [ + "https://*/*" + ], + "record": true +} \ No newline at end of file