From 233e78266e45c715505e4dddf9c98eceb5375097 Mon Sep 17 00:00:00 2001 From: Paul Warwick Date: Wed, 9 Sep 2020 13:51:54 +0100 Subject: [PATCH] baseline-1-resourceapi-Issue#60 --- .../Invictus.Testing.ResourceApi.csproj | 31 ++ .../ResourceApiAuthentication.cs | 137 +++++ .../ResourceApiProvider.cs | 488 ++++++++++++++++++ src/Invictus.Testing.sln | 6 + 4 files changed, 662 insertions(+) create mode 100644 src/Invictus.Testing.ResourceApi/Invictus.Testing.ResourceApi.csproj create mode 100644 src/Invictus.Testing.ResourceApi/ResourceApiAuthentication.cs create mode 100644 src/Invictus.Testing.ResourceApi/ResourceApiProvider.cs diff --git a/src/Invictus.Testing.ResourceApi/Invictus.Testing.ResourceApi.csproj b/src/Invictus.Testing.ResourceApi/Invictus.Testing.ResourceApi.csproj new file mode 100644 index 0000000..e62a221 --- /dev/null +++ b/src/Invictus.Testing.ResourceApi/Invictus.Testing.ResourceApi.csproj @@ -0,0 +1,31 @@ + + + + netstandard2.0 + Codit + Codit + Git + Azure;Resource Api;Testing + Provides capabilities for easily testing Azure resources via Api. + Copyright (c) Codit + https://github.com/invictus-integration/testing-framework/blob/master/LICENSE + https://github.com/invictus-integration/testing-framework + https://github.com/invictus-integration/testing-framework + https://raw.githubusercontent.com/invictus-integration/testing-framework/master/docs/images/invictus-small.png + true + true + + + + + + + + + + + + + + + diff --git a/src/Invictus.Testing.ResourceApi/ResourceApiAuthentication.cs b/src/Invictus.Testing.ResourceApi/ResourceApiAuthentication.cs new file mode 100644 index 0000000..0b33b13 --- /dev/null +++ b/src/Invictus.Testing.ResourceApi/ResourceApiAuthentication.cs @@ -0,0 +1,137 @@ +using GuardNet; +using System.Net.Http; +using System.Collections.Generic; +using System.Globalization; +using Newtonsoft.Json.Linq; +using System.Threading.Tasks; +using System; + +using ISecretProvider = Arcus.Security.Core.ISecretProvider; +using Microsoft.IdentityModel.Clients.ActiveDirectory; + +namespace Codit.Testing.ResourceApi +{ + /// + /// Authentication representation to authenticate with resources running on Azure. + /// + public class ResourceApiAuthentication + { + private readonly Func> _authenticateAsync; + private ResourceApiAuthentication(Func> authenticateAsync) + { + Guard.NotNull(authenticateAsync, nameof(authenticateAsync)); + + _authenticateAsync = authenticateAsync; + } + + /// + /// Uses the service principal to authenticate with Azure. + /// + /// The ID where the resources are located on Azure. + /// The ID that identifies the subscription on Azure. + /// The ID of the client or application that has access to the logic apps running on Azure. + /// The secret of the client or application that has access to the logic apps running on Azure. + /// The provider to get the client secret; using the . + public static ResourceApiAuthentication UsingServicePrincipal(string tenantId, string subscriptionId, string clientId, string clientSecretKey, ISecretProvider secretProvider) + { + Guard.NotNullOrWhitespace(tenantId, nameof(tenantId)); + Guard.NotNullOrWhitespace(subscriptionId, nameof(subscriptionId)); + Guard.NotNullOrWhitespace(clientId, nameof(clientId)); + Guard.NotNullOrWhitespace(clientSecretKey, nameof(clientSecretKey)); + Guard.NotNull(secretProvider, nameof(secretProvider)); + + return new ResourceApiAuthentication(async () => + { + string clientSecret = await secretProvider.GetRawSecretAsync(clientSecretKey); + var managementClient = await AuthenticateResourceManagerAsync(subscriptionId, tenantId, clientId, clientSecret); + return managementClient; + }); + } + + /// + /// Uses the service principal to authenticate with Azure. + /// + /// The ID where the resources are located on Azure. + /// The ID that identifies the subscription on Azure. + /// The ID of the client or application that has access to the logic apps running on Azure. + /// The secret of the client or application that has access to the logic apps running on Azure. + public static ResourceApiAuthentication UsingServicePrincipal(string tenantId, string subscriptionId, string clientId, string clientSecret) + { + Guard.NotNullOrWhitespace(tenantId, nameof(tenantId)); + Guard.NotNullOrWhitespace(subscriptionId, nameof(subscriptionId)); + Guard.NotNullOrWhitespace(clientId, nameof(clientId)); + Guard.NotNullOrWhitespace(clientSecret, nameof(clientSecret)); + + return new ResourceApiAuthentication( + () => AuthenticateResourceManagerAsync(subscriptionId, tenantId, clientId, clientSecret)); + } + + /// + /// Uses the service principal to authenticate with Azure. + /// + /// The ID where the resources are located on Azure. + /// The ID that identifies the subscription on Azure. + /// The ID of the client or application that has access to the logic apps running on Azure. + /// The secret of the client or application that has access to the logic apps running on Azure. + /// The resource string for Auth context. + /// The authUri context. + public static ResourceApiAuthentication UsingServicePrincipal(string tenantId, string subscriptionId, string clientId, string clientSecret, string resource, string authUri) + { + Guard.NotNullOrWhitespace(tenantId, nameof(tenantId)); + Guard.NotNullOrWhitespace(clientId, nameof(clientId)); + Guard.NotNullOrWhitespace(clientSecret, nameof(clientSecret)); + Guard.NotNullOrWhitespace(subscriptionId, nameof(authUri)); + Guard.NotNullOrWhitespace(subscriptionId, nameof(resource)); + + string authority = string.Format(CultureInfo.InvariantCulture, authUri, tenantId); + + return new ResourceApiAuthentication( + () => AccessTokenUmt(clientId, clientSecret, resource, authority)); + } +/// + /// Authenticate with Azure with the previously chosen authentication mechanism. + /// + /// + /// The management client to interact with logic app resources running on Azure. + /// + public async Task AuthenticateAsync() + { + return await _authenticateAsync(); + } + + private static Task AccessTokenUmt(string clientId, string clientSecret, string adAppId, string authContext) + { + Task token = Task.Factory.StartNew(() => + { + var clientCredential = new ClientCredential(clientId, clientSecret); + AuthenticationContext context = new AuthenticationContext(authContext, false); + AuthenticationResult authenticationResult = context.AcquireTokenAsync(adAppId, clientCredential).Result; + + return authenticationResult.AccessToken; + }); + return token; + } + + private static async Task AuthenticateResourceManagerAsync(string subscriptionId, string tenantId, string clientId, string clientSecret) + { + string baseAddress = string.Format(CultureInfo.InvariantCulture, "https://login.microsoftonline.com/{0}/oauth2/token", tenantId); + string resource = "https://management.azure.com/"; + string grant_type = "client_credentials"; + + var form = new Dictionary + { + {"grant_type", grant_type}, + {"client_id", clientId}, + {"client_secret", clientSecret}, + {"resource", resource}, + }; + + var httpClient = new System.Net.Http.HttpClient(); + HttpResponseMessage tokenResponse = await httpClient.PostAsync(baseAddress, new FormUrlEncodedContent(form)); + var jsonContent = await tokenResponse.Content.ReadAsStringAsync(); + dynamic data = JObject.Parse(jsonContent); + var token = ((Newtonsoft.Json.Linq.JValue)((Newtonsoft.Json.Linq.JProperty)((Newtonsoft.Json.Linq.JContainer)data).Last).Value).Value; + return token.ToString(); + } + } +} diff --git a/src/Invictus.Testing.ResourceApi/ResourceApiProvider.cs b/src/Invictus.Testing.ResourceApi/ResourceApiProvider.cs new file mode 100644 index 0000000..c2afe8f --- /dev/null +++ b/src/Invictus.Testing.ResourceApi/ResourceApiProvider.cs @@ -0,0 +1,488 @@ +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Web; +using GuardNet; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Newtonsoft.Json.Linq; +using Polly; +using Polly.Retry; +using Polly.Timeout; + +namespace Codit.Testing.ResourceApi +{ /// + /// Component to provide access in a reliable manner on api testable resources running in Azure. + /// + public class ResourceApiProvider + { + private readonly ILogger _logger; + private DateTimeOffset _startTime = DateTimeOffset.UtcNow; + private readonly TimeSpan _retryInterval = TimeSpan.FromSeconds(1); + private bool _hasBodyFilter; + private bool _hasUrlFilter; + private bool _hasHeaders; + private bool _hasActionValues; + private bool _hasActionParameterValues; + private bool _skipBearerToken = false; + private readonly string _definitionName; + + private readonly ResourceApiAuthentication _authentication; + + private string _baseUrlPattern, _urlAction, _urlFilter, _bodyFilter; + private TimeSpan _timeout = TimeSpan.FromSeconds(90); + + private IDictionary _baseUrlValues = new Dictionary(); + private IDictionary _urlFilterValues = new Dictionary(); + private IDictionary _bodyFilterValues = new Dictionary(); + private IDictionary _headerValues = new Dictionary(); + private IDictionary _actionValues = new Dictionary(); + private IDictionary _actionParameterValues = new Dictionary(); + + //private static System.Net.Http.HttpClient _httpClient = new System.Net.Http.HttpClient(); + + private ResourceApiProvider( + string definitionName, + ResourceApiAuthentication authentication, + ILogger logger) + { + Guard.NotNull(definitionName, nameof(definitionName)); + Guard.NotNull(authentication, nameof(authentication)); + Guard.NotNull(logger, nameof(logger)); + + _definitionName = definitionName; + _authentication = authentication; + _logger = logger; + } + + /// + /// Creates a new instance of the class. + /// + /// The context for the Api connection with Azure. + /// The authentication mechanism to authenticate with Azure. + public static ResourceApiProvider LocatedAt( + string definitionName, + ResourceApiAuthentication authentication) + { + Guard.NotNull(definitionName, nameof(definitionName)); + Guard.NotNull(authentication, nameof(authentication)); + + return LocatedAt(definitionName, authentication, NullLogger.Instance); + } + + /// + /// Creates a new instance of the class. + /// + /// The context for the Api connection with Azure. + /// The authentication mechanism to authenticate with Azure. + /// The instance to write diagnostic trace messages while interacting with the provider. + public static ResourceApiProvider LocatedAt( + string definitionName, + ResourceApiAuthentication authentication, + ILogger logger) + { + Guard.NotNull(definitionName, nameof(definitionName)); + Guard.NotNull(authentication, nameof(authentication)); + + logger = logger ?? NullLogger.Instance; + return new ResourceApiProvider(definitionName, authentication, logger); + } + + /// + /// Sets the values to build the Resource API Url. + /// + /// the dictionary of values to build the Resource API Url. + public ResourceApiProvider WithBaseUrlValues(Dictionary baseUrlValues) + { + Guard.NotNull(baseUrlValues, nameof(baseUrlValues)); + + _baseUrlValues = baseUrlValues; + return this; + } + + /// + /// Sets the values to build the Url Filter. + /// + /// the dictionary of values to build the Resource API Url Filter. + public ResourceApiProvider WithUrlFilterValues(Dictionary urlFilterValues) + { + _urlFilterValues = urlFilterValues; + return this; + } + + /// + /// Sets the values to build the Body Filter. + /// + /// the dictionary of values to build the Resource API Body Filter. + public ResourceApiProvider WithBodyFilterValues(Dictionary bodyFilterValues) + { + Guard.NotNull(bodyFilterValues, nameof(bodyFilterValues)); + + _bodyFilterValues = bodyFilterValues; + return this; + } + + /// + /// Sets the values to build the API Header. + /// + /// the dictionary of values to build the Resource API Url. + public ResourceApiProvider WithHeaderValues(Dictionary headerValues) + { + Guard.NotNull(headerValues, nameof(headerValues)); + + _headerValues = headerValues; + _hasHeaders = true; + return this; + } + + /// + /// Sets the values for URL with Action value replacement tags. + /// + /// the tag replacement to build the Resource API Url. + public ResourceApiProvider WithUrlActionValues(Dictionary actionValues) + { + Guard.NotNull(actionValues, nameof(actionValues)); + + _actionValues = actionValues; + _hasActionValues = true; + return this; + } + + /// + /// Sets the values for build a composite URL query. + /// + /// the query elements. + public ResourceApiProvider WithUrlActionQueryValues(Dictionary actionParameterValues) + { + Guard.NotNull(actionParameterValues, nameof(actionParameterValues)); + + _actionParameterValues = actionParameterValues; + _hasActionParameterValues = true; + return this; + } + + /// + /// Sets the time period in which the retrieval of the logic app runs should succeed. + /// + /// The period to retrieve logic app runs. + public ResourceApiProvider WithTimeout(TimeSpan timeout) + { + _timeout = timeout; + return this; + } + + /// + /// Sets the Authentication to skip Bearer Token. + /// + /// The flag to skip Bearer Token Auth. + public ResourceApiProvider WithNoBearerTokenAuthentication(bool skipBearerToken) + { + _skipBearerToken = skipBearerToken; + return this; + } + + /// + /// Sets URL replacement string + /// + /// The URL replacement string. + public ResourceApiProvider WithBaseUrlPattern(string baseUrlPattern) + { + Guard.NotNullOrEmpty(baseUrlPattern, nameof(baseUrlPattern)); + _baseUrlPattern = baseUrlPattern; + return this; + } + + /// + /// Sets the urlAction string. + /// + /// The urlAction string. + public ResourceApiProvider WithUrlAction(string urlAction) + { + _urlAction = urlAction; + return this; + } + + /// + /// Sets the urlFilter string. + /// + /// The urlFilter string. + public ResourceApiProvider WithUrlFilter(string urlFilter) + { + Guard.NotNullOrEmpty(urlFilter, nameof(urlFilter)); + _urlFilter = urlFilter; + _hasUrlFilter = true; + return this; + } + + /// + /// Sets the bodyFilter Pattern. + /// + /// The bodyFilter Pattern. + public ResourceApiProvider WithBodyFilter(string bodyFilter) + { + Guard.NotNull(bodyFilter, nameof(bodyFilter)); + + _bodyFilter = bodyFilter; + _hasBodyFilter = true; + return this; + } + + /// + /// Runs the current url request. + /// + public async Task RunAsync() + { + var cancellationTokenSource = new CancellationTokenSource(); + cancellationTokenSource.CancelAfter(_timeout); + + string requestResult = string.Empty; + + while (!cancellationTokenSource.Token.IsCancellationRequested) + { + try + { + requestResult = await PostRequestAsync(); + await Task.Delay(TimeSpan.FromSeconds(1), cancellationTokenSource.Token); + } + catch (Exception exception) + { + _logger.LogError(exception, _definitionName + "=Polling for Resource Api request was faulted: {Message}", exception.Message); + } + } + + return requestResult; + } + + /// + /// Starts polling for a series of Resource Api response objects corresponding to the previously set filtering criteria. + /// + public async Task> PollForResponsesAsync() + { + var cancellationTokenSource = new CancellationTokenSource(); + cancellationTokenSource.CancelAfter(_timeout); + + IEnumerable responses = Enumerable.Empty(); + + while (!cancellationTokenSource.Token.IsCancellationRequested) + { + try + { + responses = await GetResponsesAsync(); + await Task.Delay(TimeSpan.FromSeconds(1), cancellationTokenSource.Token); + } + catch (Exception exception) + { + _logger.LogError(exception, _definitionName + "=Polling for Resource Api responses was faulted: {Message}", exception.Message); + } + } + + return responses ?? Enumerable.Empty(); + } + + /// + /// Starts polling for a corresponding to the previously set filtering criteria. + /// + /// The minimum amount of responses to retrieve. + public async Task> PollForResponsesAsync(int minimumNumberOfItems) + { + Guard.NotLessThanOrEqualTo(minimumNumberOfItems, 0, nameof(minimumNumberOfItems)); + + string amount = minimumNumberOfItems == 1 ? "any" : minimumNumberOfItems.ToString(); + + RetryPolicy> retryPolicy = + Policy.HandleResult>(currentResponses => + { + int count = currentResponses.Count(); + bool isStillPending = count < minimumNumberOfItems; + + _logger.LogTrace(_definitionName + "=Polling for {Amount} Resource Api responses, whilst got now {Current} ", amount, count); + return isStillPending; + }).Or(ex => + { + _logger.LogError(ex, _definitionName + "=Polling for Resource Api responses was faulted: {Message}", ex.Message); + return true; + }) + .WaitAndRetryForeverAsync(index => + { + _logger.LogTrace(_definitionName + "=Could not retrieve Resource Api responses in time, wait 1s and try again..."); + return _retryInterval; + }); + + PolicyResult> result = + await Policy.TimeoutAsync(_timeout) + .WrapAsync(retryPolicy) + .ExecuteAndCaptureAsync(GetResponsesAsync); + + if (result.Outcome == OutcomeType.Failure) + { + if (result.FinalException is null + || result.FinalException.GetType() == typeof(TimeoutRejectedException)) + { + string filterType = _hasBodyFilter + ? $"{Environment.NewLine} with body filter" + : $"{Environment.NewLine} with url filter"; + + throw new TimeoutException( + $"Could not in the given timeout span ({_timeout:g}) retrieve {amount} Resource Api responses " + + $"{Environment.NewLine} with StartTime >= {_startTime.UtcDateTime:O}" + + filterType); + } + + throw result.FinalException; + } + + _logger.LogTrace(_definitionName + "=Polling finished successful with {Count} Resource Api responses", result.Result.Count()); + return result.Result; + } + + /// + /// Start polling for a single Resource Api response object. + /// + public async Task PollForSingleResponseAsync() + { + IEnumerable responses = await PollForResponsesAsync(minimumNumberOfItems: 1); + return responses.FirstOrDefault(); + } + + private async Task PostRequestAsync() + { + string urlFilter = string.Empty; + StringContent httpFilterBody = null; + + string token = await _authentication.AuthenticateAsync(); + + Guard.NotNullOrEmpty(_baseUrlPattern, nameof(_baseUrlPattern)); + Guard.NotNullOrEmpty(_urlAction, nameof(_urlAction)); + Guard.NotNull(_baseUrlValues, nameof(_baseUrlValues)); + var url = BuildPattern(_baseUrlPattern, _baseUrlValues); + url += _urlAction; + + if (_hasBodyFilter) + { + Guard.NotNull(_bodyFilterValues, nameof(_bodyFilterValues)); + httpFilterBody = new StringContent(BuildPattern(_bodyFilter, _bodyFilterValues), + Encoding.UTF8, + "application/json"); + } + else + { + if (_hasUrlFilter) + { + Guard.NotNull(_urlFilterValues, nameof(_urlFilterValues)); + urlFilter = BuildPattern(_urlFilter, _urlFilterValues); + url += urlFilter; + } + } + + //_headerValues + + System.Net.Http.HttpClient httpClient = new System.Net.Http.HttpClient(); + httpClient.DefaultRequestHeaders.Add("Authorization", ("Bearer " + token)); + + using (HttpResponseMessage response = await httpClient.PostAsync(url, httpFilterBody)) + { + return await response.Content.ReadAsStringAsync(); + } + } + + private async Task> GetResponsesAsync() + { + string urlFilter = string.Empty; + StringContent httpFilterBody = null; + + string token = await _authentication.AuthenticateAsync(); + + Guard.NotNullOrEmpty(_baseUrlPattern, nameof(_baseUrlPattern)); + Guard.NotNullOrEmpty(_urlAction, nameof(_urlAction)); + Guard.NotNull(_baseUrlValues, nameof(_baseUrlValues)); + var url = BuildPattern(_baseUrlPattern, _baseUrlValues); + url += _urlAction; + + if (_hasBodyFilter) + { + Guard.NotNull(_bodyFilterValues, nameof(_bodyFilterValues)); + httpFilterBody = new StringContent(BuildPattern(_bodyFilter, _bodyFilterValues), + Encoding.UTF8, + "application/json"); + } + else + { + if (_hasUrlFilter) + { + Guard.NotNull(_urlFilterValues, nameof(_urlFilterValues)); + urlFilter = BuildPattern(_urlFilter, _urlFilterValues); + url += urlFilter; + } + } + + System.Net.Http.HttpClient httpClient = new System.Net.Http.HttpClient(); + if (!_skipBearerToken) + { + httpClient.DefaultRequestHeaders.Add("Authorization", ("Bearer " + token)); + } + + if (_hasHeaders) + { + foreach (var header in _headerValues) + { + httpClient.DefaultRequestHeaders.Add(header.Key, header.Value); + } + } + + if(_hasActionValues) + { + foreach (var value in _actionValues) + { + url = url.Replace(value.Key, value.Value); + } + } + + if (_hasActionParameterValues) + { + string parameterString = string.Empty; + foreach (var keyPair in _actionParameterValues) + { + parameterString += ((parameterString == string.Empty) ? "" : "&") + + string.Format( + "{0}={1}", + HttpUtility.UrlEncode(keyPair.Key), + HttpUtility.UrlEncode(keyPair.Value)); + }; + // + url += "?" + parameterString; + } + + HttpResponseMessage response = await httpClient.PostAsync(url, httpFilterBody); + + var responseContent = await response.Content.ReadAsStringAsync(); + + JObject valueResponses = JObject.Parse(responseContent); + + _logger.LogTrace(_definitionName + "=Query returned {ValueResponsesCount} values", valueResponses.Count); + + var returnResponses = new Collection(); + foreach (var valueResponse in valueResponses) + { + returnResponses.Add(valueResponse); + } + + _logger.LogTrace(_definitionName + "=Query resulted in {ResponseCount} Resource Api responses", returnResponses.Count); + return returnResponses.AsEnumerable(); + } + + private string BuildPattern(string pattern, IDictionary values) + { + foreach (var value in values) + { + pattern = pattern.Replace(value.Key, value.Value); + } + return pattern; + } + + } +} diff --git a/src/Invictus.Testing.sln b/src/Invictus.Testing.sln index c6f0543..be73dd5 100644 --- a/src/Invictus.Testing.sln +++ b/src/Invictus.Testing.sln @@ -9,6 +9,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{65BD93B8 EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Invictus.Testing.Tests.Integration", "Invictus.Testing.Tests.Integration\Invictus.Testing.Tests.Integration.csproj", "{621838F3-EF62-46C4-9517-77C2FF919A29}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Invictus.Testing.ResourceApi", "Invictus.Testing.ResourceApi\Invictus.Testing.ResourceApi.csproj", "{CB8C7BED-3F6C-4DDF-BDF4-77404B25E366}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -23,6 +25,10 @@ Global {621838F3-EF62-46C4-9517-77C2FF919A29}.Debug|Any CPU.Build.0 = Debug|Any CPU {621838F3-EF62-46C4-9517-77C2FF919A29}.Release|Any CPU.ActiveCfg = Release|Any CPU {621838F3-EF62-46C4-9517-77C2FF919A29}.Release|Any CPU.Build.0 = Release|Any CPU + {CB8C7BED-3F6C-4DDF-BDF4-77404B25E366}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CB8C7BED-3F6C-4DDF-BDF4-77404B25E366}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CB8C7BED-3F6C-4DDF-BDF4-77404B25E366}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CB8C7BED-3F6C-4DDF-BDF4-77404B25E366}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE