using Azure.ResourceManager; using Azure.ResourceManager.AppContainers; using JetBrains.Annotations; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Proto.Cluster.AzureContainerApps.Stores.ResourceTags; using Proto.Utils; namespace Proto.Cluster.AzureContainerApps; /// /// A cluster provider that uses Azure Container Apps to host the cluster. /// [PublicAPI] public class AzureContainerAppsProvider : IClusterProvider { private readonly IArmClientProvider _armClientProvider; private readonly IClusterMemberStore _clusterMemberStore; private readonly IOptions _options; private readonly ILogger _logger; private readonly string? _containerAppName; private readonly string _revisionName; private readonly string _replicaName; private readonly string _advertisedHost; private string _memberId = null!; private string _address = null!; private Cluster _cluster = null!; private string _clusterName = null!; private string[] _kinds = null!; private int _port; private ArmClient _client = null!; /// /// Use this constructor to create a new instance. /// /// An to create instances. /// The store to use for storing member information. /// The options for this provider. /// The logger to use. public AzureContainerAppsProvider( IArmClientProvider armClientProvider, IClusterMemberStore clusterMemberStore, IOptions options, ILogger logger) { _armClientProvider = armClientProvider; _clusterMemberStore = clusterMemberStore; _options = options; _logger = logger; _containerAppName = Environment.GetEnvironmentVariable("CONTAINER_APP_NAME") ?? throw new Exception("No app name provided"); _revisionName = Environment.GetEnvironmentVariable("CONTAINER_APP_REVISION") ?? throw new Exception("No app revision provided"); _replicaName = Environment.GetEnvironmentVariable("HOSTNAME") ?? throw new Exception("No replica name provided"); _advertisedHost = ConfigUtils.FindSmallestIpAddress().ToString(); } /// public async Task StartMemberAsync(Cluster cluster) { var clusterName = cluster.Config.ClusterName; var (host, port) = cluster.System.GetAddress(); var kinds = cluster.GetClusterKinds(); _cluster = cluster; _clusterName = clusterName; _memberId = cluster.System.Id; _port = port; _kinds = kinds; _address = $"{host}:{port}"; _client = await _armClientProvider.CreateClientAsync(); //await CleanupStoreAsync(cluster); await RegisterMemberAsync().ConfigureAwait(false); StartClusterMonitor(); } /// public Task StartClientAsync(Cluster cluster) { var clusterName = cluster.Config.ClusterName; var (_, port) = cluster.System.GetAddress(); _cluster = cluster; _clusterName = clusterName; _memberId = cluster.System.Id; _port = port; _kinds = Array.Empty(); StartClusterMonitor(); return Task.CompletedTask; } /// public async Task ShutdownAsync(bool graceful) => await DeregisterMemberAsync().ConfigureAwait(false); private async Task CleanupStoreAsync(Cluster cluster) { await _clusterMemberStore.ClearAsync(cluster.Config.ClusterName); } private async Task RegisterMemberAsync() { await Retry.Try(RegisterMemberInternal, retryCount: Retry.Forever, onError: OnError, onFailed: OnFailed).ConfigureAwait(false); void OnError(int attempt, Exception exception) => _logger.LogWarning(exception, "Failed to register service"); void OnFailed(Exception exception) => _logger.LogError(exception, "Failed to register service"); } private async Task RegisterMemberInternal() { var subscriptionId = _options.Value.SubscriptionId; var resourceGroupName = _options.Value.ResourceGroupName; var resourceGroup = await _client.GetResourceGroupByNameAsync(resourceGroupName, subscriptionId).ConfigureAwait(false); var containerApp = await resourceGroup.GetContainerAppAsync(_containerAppName).ConfigureAwait(false); var revision = await containerApp.Value.GetContainerAppRevisionAsync(_revisionName).ConfigureAwait(false); if ((revision.Value.Data.TrafficWeight ?? 0) == 0) return; var member = new Member { Id = _memberId, Host = _advertisedHost, Port = _port, }; _logger.LogInformation( "[Cluster][AzureContainerAppsProvider] Registering service {ReplicaName} on {IpAddress}", _replicaName, _address); member.Kinds.AddRange(_kinds); await _clusterMemberStore.RegisterAsync(_clusterName, member).ConfigureAwait(false); } private void StartClusterMonitor() { var pollInterval = _options.Value.PollInterval; var storeName = _clusterMemberStore.GetType().Name; _ = SafeTask.Run(async () => { while (!_cluster.System.Shutdown.IsCancellationRequested) { _logger.LogInformation("Looking for members in {Store}", storeName); try { var members = (await _clusterMemberStore.ListAsync().ConfigureAwait(false)).ToArray(); if (members.Any()) { _logger.LogInformation("Got members {Members}", members.Length); _cluster.MemberList.UpdateClusterTopology(members); } else { _logger.LogWarning("Failed to get members from {Store}", storeName); } } catch (Exception x) { _logger.LogError(x, "Failed to get members from {Store}", storeName); } await Task.Delay(pollInterval).ConfigureAwait(false); } } ); } private async Task DeregisterMemberAsync() { await Retry.Try(DeregisterMemberInner, onError: OnError, onFailed: OnFailed).ConfigureAwait(false); void OnError(int attempt, Exception exception) => _logger.LogWarning(exception, "Failed to deregister service"); void OnFailed(Exception exception) => _logger.LogError(exception, "Failed to deregister service"); } private async Task DeregisterMemberInner() { _logger.LogInformation("[Cluster][AzureContainerAppsProvider] Unregistering member {ReplicaName} on {IpAddress}", _replicaName, _address); await _clusterMemberStore.UnregisterAsync(_memberId).ConfigureAwait(false); } }