Using OpenTelemetry Java SDK, a new Span is created when requests reach a new service
Overview
This issue occurs in the following circumstances:
- An organization creates a Distributed Application in the vFunction Server UI
- The organization has some Services that use the OpenTelemetry Auto-Instrumentation Java Agent
- The organization has other Services that use the OpenTelemetry Java SDK. For example:
import io.opentelemetry.api.OpenTelemetry;
import io.opentelemetry.api.common.AttributeKey;
import io.opentelemetry.api.trace.Tracer;
import io.opentelemetry.exporter.otlp.http.trace.OtlpHttpSpanExporter;
import io.opentelemetry.sdk.OpenTelemetrySdk;
import io.opentelemetry.sdk.resources.Resource;
import io.opentelemetry.sdk.trace.SdkTracerProvider;
import io.opentelemetry.sdk.trace.export.BatchSpanProcessor;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.env.Environment;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManagerFactory;
import javax.net.ssl.X509TrustManager;
import java.io.InputStream;
import java.security.KeyStore;
import java.security.SecureRandom;
import static io.opentelemetry.semconv.ServiceAttributes.SERVICE_NAME;
/**
* This configuration provides a custom {@link OtlpHttpSpanExporter} bean.
* The Spring Boot OpenTelemetry auto-configuration will automatically detect and use this bean
* instead of creating a default one. This allows for advanced customization, such as adding
* a custom SSLContext for client certificate authentication, without disabling the rest of the
* auto-configuration (like Web MVC tracing).
*/
@Configuration
public class OtelExporterConfig {
private final Environment env;
public OtelExporterConfig(final Environment env) {
this.env = env;
}
@Bean(name= "openTelemetryBean")
public OpenTelemetry openTelemetry() {
try {
System.out.println("OpenTelemetry bean is being created");
// Retrieve the password from an environment variable
String trustStorePassword = env.getProperty("TRUSTSTORE_PASSWORD");
if (trustStorePassword == null || trustStorePassword.isEmpty()) {
throw new IllegalArgumentException("Environment variable TRUSTSTORE_PASSWORD is not set or empty");
}
InputStream trustStoreStream = getClass().getClassLoader()
.getResourceAsStream("TRUSTSTORE_REMOVED");
if (trustStoreStream == null) {
throw new IllegalArgumentException("Truststore file not found in the specified path");
}
KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
trustStore.load(trustStoreStream, trustStorePassword.toCharArray());
TrustManagerFactory tmf = TrustManagerFactory.getInstance(
TrustManagerFactory.getDefaultAlgorithm());
tmf.init(trustStore);
SSLContext sslContext = SSLContext.getInstance("TLS");
sslContext.init(null, tmf.getTrustManagers(), new SecureRandom());
OtlpHttpSpanExporter spanExporter= OtlpHttpSpanExporter.builder()
.setEndpoint(env.getProperty("OTEL_EXPORTER_OTLP_ENDPOINT"))
.addHeader("X-VF-APP", "UUID_REMOVED")
.addHeader("Content-Type", "application/x-protobuf")
.setSslContext(sslContext, (X509TrustManager) tmf.getTrustManagers()[0])
.build();
// Define the service.name resource
Resource resource = Resource.getDefault()
.merge(Resource.create(io.opentelemetry.api.common.Attributes.of(
SERVICE_NAME, "SERVICE_NAME_REMOVED")));
SdkTracerProvider tracerProvider = SdkTracerProvider.builder()
.addSpanProcessor(BatchSpanProcessor.builder(spanExporter).build())
.setResource(resource)
.build();
return OpenTelemetrySdk.builder()
.setTracerProvider(tracerProvider)
.buildAndRegisterGlobal();
} catch (Exception e) {
throw new RuntimeException("Failed to create OTLP exporter with disabled SSL verification", e);
}
}
@Bean
public Tracer tracer(@Qualifier("openTelemetryBean") OpenTelemetry openTelemetry) {
return openTelemetry.getTracer("customersetup");
}
}
- The organization starts Learning in the vFunction Server UI
- Unexpectedly, flows that should pass through multiple hops that originate in the Auto-Instrumented Services and pass to the SDK Services are displayed as two separate flows
- Unexpectedly, after turning on Debugging to an OpenTelemetry Collector to see the Trace ID, Span ID and Parent Span IDs for all incoming requests, the SDK Services show that flows have a new Trace ID and new Parent Span ID rather than maintaining the same values created in Auto-Instrumentation Services. For example:
#### Span from Auto-Instrumentation Service
Span #0
Trace ID : 424161da377f35acc60e34c3041237ee
Parent ID :
ID : 24310128516586e1
Name : GET /app/Main.jsp
Kind : Server
Start time : 2025-12-04 23:01:27.76 +0000 UTC
End time : 2025-12-04 23:01:28.284331223 +0000 UTC
Status code : Unset
Status message :
Attributes:
-> url.query: Str(selectedURL=/CustomerView&CustCd=E022)
### Multi-hop Flow from Auto-Instrumentation Service to SDK Service
### Contains new Trace ID with no Parent ID
Span #0
Trace ID : 99e9c1c8b507c86bce513a3f573ad62b
Parent ID :
ID : db151886ae4ce46d
Name : GET /app/customer/getCustBaseCustCd
Kind : Server
Start time : 2025-12-04 23:01:27.791683745 +0000 UTC
End time : 2025-12-04 23:01:27.798291884 +0000 UTC
Status code : Unset
Status message :
Attributes:
-> url.query: Str(custCd=E022)
Solution
Modify the configuration of the SDK Service to ensure that Parent Spans are continued through multi-hop flows rather than having new, independent spans being created. For example:
import io.opentelemetry.exporter.otlp.http.logs.OtlpHttpLogRecordExporterBuilder;
import io.opentelemetry.exporter.otlp.http.metrics.OtlpHttpMetricExporterBuilder;
import io.opentelemetry.exporter.otlp.http.trace.OtlpHttpSpanExporter;
import io.opentelemetry.exporter.otlp.http.trace.OtlpHttpSpanExporterBuilder;
import io.opentelemetry.exporter.otlp.http.metrics.OtlpHttpMetricExporter;
import io.opentelemetry.exporter.otlp.http.logs.OtlpHttpLogRecordExporter;
import io.opentelemetry.sdk.autoconfigure.spi.AutoConfigurationCustomizerProvider;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.env.Environment;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManagerFactory;
import javax.net.ssl.X509TrustManager;
import java.io.InputStream;
import java.security.KeyStore;
import java.security.SecureRandom;
@Configuration
public class CustomOtel {
private final Environment env;
public CustomOtel(final Environment env) {
this.env = env;
}
@Bean
public AutoConfigurationCustomizerProvider otelCustomizer() {
try {
// Retrieve the password from an environment variable
String trustStorePassword = env.getProperty("TRUSTSTORE_PASSWORD");
if (trustStorePassword == null || trustStorePassword.isEmpty()) {
throw new IllegalArgumentException("Environment variable TRUSTSTORE_PASSWORD is not set or empty");
}
InputStream trustStoreStream;
trustStoreStream = getClass().getClassLoader().getResourceAsStream("REMOVED_TRUSTSTORE");
if (trustStoreStream == null) {
throw new IllegalArgumentException("Truststore file not found at classpath 'REMOVED_TRUSTSTORE'");
}
KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
trustStore.load(trustStoreStream, trustStorePassword.toCharArray());
TrustManagerFactory tmf = TrustManagerFactory.getInstance(
TrustManagerFactory.getDefaultAlgorithm());
tmf.init(trustStore);
SSLContext sslContext = SSLContext.getInstance("TLS");
sslContext.init(null, tmf.getTrustManagers(), new SecureRandom());
return p ->
p.addSpanExporterCustomizer((exporter, config) -> {
if (exporter instanceof OtlpHttpSpanExporter) {
OtlpHttpSpanExporterBuilder builder = ((OtlpHttpSpanExporter) exporter).toBuilder();
String endpoint = env.getProperty("OTEL_EXPORTER_OTLP_ENDPOINT");
builder.setEndpoint(endpoint);
String xVfApp = env.getProperty("X_VF_APP", "REMOVED_VF_UUID");
builder.addHeader("X-VF-APP", xVfApp);
builder.setSslContext(sslContext, (X509TrustManager) tmf.getTrustManagers()[0]);
return builder.build();
}
return exporter;
})
.addMetricExporterCustomizer((exporter, config) -> {
if (exporter instanceof OtlpHttpMetricExporter) {
OtlpHttpMetricExporterBuilder builder = ((OtlpHttpMetricExporter) exporter).toBuilder();
String endpoint = env.getProperty("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", env.getProperty("OTEL_EXPORTER_OTLP_ENDPOINT"));
if (endpoint != null && !endpoint.isBlank()) {
builder.setEndpoint(endpoint);
}
builder.setSslContext(sslContext, (X509TrustManager) tmf.getTrustManagers()[0]);
return builder.build();
}
return exporter;
})
.addLogRecordExporterCustomizer((exporter, config) -> {
if (exporter instanceof OtlpHttpLogRecordExporter) {
OtlpHttpLogRecordExporterBuilder builder = ((OtlpHttpLogRecordExporter) exporter).toBuilder();
String endpoint = env.getProperty("OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", env.getProperty("OTEL_EXPORTER_OTLP_ENDPOINT"));
if (endpoint != null && !endpoint.isBlank()) {
builder.setEndpoint(endpoint);
}
builder.setSslContext(sslContext, (X509TrustManager) tmf.getTrustManagers()[0]);
return builder.build();
}
return exporter;
});
} catch (Exception e) {
throw new RuntimeException("Failed to create OTLP exporter with disabled SSL verification", e);
}
}
private static boolean hasAny(io.opentelemetry.sdk.autoconfigure.spi.ConfigProperties config, String... keys) {
for (String k : keys) {
String v = config.getString(k);
if (v != null && !v.isBlank()) return true;
}
return false;
}
}