How to dynamically generate prometheus configuration

[paulomorgado]

I'm using ASPIRE_ENABLE_CONTAINER_TUNNEL with podman in order for containers to invoke the host.

To use a Prometheus container to scrape metrics from resources, I'm using this:

const string producerMetricsPath = "WebRTCProducer/touchpoint/metrics";
const string signalingMetricsPath = "WebRTCSignaling/0/metrics";
IResourceBuilder<ProjectResource>? producerResourceBuilder = null;
IResourceBuilder<ProjectResource>? signalingResourceBuilder = null;
foreach (var resource in builder.Resources)
{
	if (resource is ProjectResource projectResource)
	{
		switch (resource.Name)
		{
			case "WebRTC-Producer":
				producerResourceBuilder = builder.CreateResourceBuilder(projectResource);
				producerResourceBuilder.WithHttpEndpoint(name: "prometheus", env: "PROMETHEUS_PORT");
				producerResourceBuilder.WithEnvironment(ctx =>
				{
					ctx.EnvironmentVariables["Prometheus__MetricsServer__Enabled"] = "true";
					var prometheusEndpoint = producerResourceBuilder.GetEndpoint("prometheus");
					ctx.EnvironmentVariables["Prometheus__MetricsServer__Url"] = ReferenceExpression.Create($"http://localhost:{prometheusEndpoint.Property(EndpointProperty.TargetPort)}/{producerMetricsPath}");
				});

				prometheus.WaitFor(producerResourceBuilder);

				break;

			case "WebRTC-Signaling":
				signalingResourceBuilder = builder.CreateResourceBuilder(projectResource);
				signalingResourceBuilder.WithHttpEndpoint(name: "prometheus", env: "PROMETHEUS_PORT");
				signalingResourceBuilder.WithEnvironment(ctx =>
				{
					ctx.EnvironmentVariables["Prometheus__MetricsServer__Enabled"] = "true";
					var prometheusEndpoint = signalingResourceBuilder.GetEndpoint("prometheus");
					ctx.EnvironmentVariables["Prometheus__MetricsServer__Url"] = ReferenceExpression.Create($"http://localhost:{prometheusEndpoint.Property(EndpointProperty.TargetPort)}/{signalingMetricsPath}");
				});

				prometheus.WaitFor(signalingResourceBuilder);

				break;
		}
	}
}

if (producerResourceBuilder is { } || signalingResourceBuilder is { })
{
	prometheus
		.WithBindMount("Configuration/Prometheus/rules.yml", "/etc/prometheus/rules.yml")
		;

	if (producerResourceBuilder is { })
	{
		prometheus.WithReference(producerResourceBuilder.GetEndpoint("prometheus"));
	}

	if (signalingResourceBuilder is { })
	{
		prometheus.WithReference(signalingResourceBuilder.GetEndpoint("prometheus"));
	}

	// Write to Configuration/Prometheus/prometheus.yml
	prometheus.WithContainerFiles(
		"/etc/prometheus",
		async (ctx, ct) =>
		{
			var prometheusConfigBuilder = new StringBuilder()
				.AppendLine("""
				global:
				  scrape_interval: 5s

				rule_files:
				  - "rules.yml"

				scrape_configs:
				""");

			if (producerResourceBuilder is { })
			{
				var producerEndpoint = producerResourceBuilder.GetEndpoint("prometheus");
				prometheusConfigBuilder
					.AppendLine($"""
						  - job_name: 'producer'
							metrics_path: '/{producerMetricsPath}'
							static_configs:
							  - targets:
								- 'aspire.dev.internal:{producerEndpoint.Port}'
						""");
			}

			if (signalingResourceBuilder is { })
			{
				var signalingEndpoint = signalingResourceBuilder.GetEndpoint("prometheus");
				prometheusConfigBuilder
					.AppendLine($"""
						  - job_name: 'signaling'
							metrics_path: '/{signalingMetricsPath}'
							static_configs:
							  - targets: 
								- 'aspire.dev.internal:{signalingEndpoint.Port}'
						""");
			}

			return [
				new ContainerFile
				{
					Name = "prometheus.yml",
					Contents = prometheusConfigBuilder.ToString(),
					Mode = UnixFileMode.UserRead | UnixFileMode.GroupRead | UnixFileMode.OtherRead // 0644
				}
			];
		});
}

However, the prometheus.yml file does not have neither the target port nor the proxied, like the environment variables created by WithReference.

What am I doing wrong?