diff --git a/java-datastore/google-cloud-datastore/pom.xml b/java-datastore/google-cloud-datastore/pom.xml index 2138b34a687a..6239803f65ae 100644 --- a/java-datastore/google-cloud-datastore/pom.xml +++ b/java-datastore/google-cloud-datastore/pom.xml @@ -79,6 +79,10 @@ com.google.protobuf protobuf-java + + com.google.protobuf + protobuf-java-util + com.google.api gax @@ -196,12 +200,20 @@ io.opentelemetry opentelemetry-sdk - test io.opentelemetry opentelemetry-sdk-common - test + + + com.google.cloud + google-cloud-monitoring + 3.91.0 + + + com.google.api.grpc + proto-google-cloud-monitoring-v3 + 3.91.0 io.opentelemetry @@ -216,7 +228,6 @@ io.opentelemetry opentelemetry-sdk-metrics - test io.opentelemetry diff --git a/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java b/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java index c7b938e5edcb..700b5e4faf4a 100644 --- a/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java +++ b/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreImpl.java @@ -47,6 +47,7 @@ import com.google.cloud.ServiceOptions; import com.google.cloud.datastore.execution.AggregationQueryExecutor; import com.google.cloud.datastore.spi.v1.DatastoreRpc; +import com.google.cloud.datastore.telemetry.BuiltInDatastoreMetricsProvider; import com.google.cloud.datastore.telemetry.DatastoreMetricsRecorder; import com.google.cloud.datastore.telemetry.TelemetryConstants; import com.google.cloud.datastore.telemetry.TelemetryUtils; @@ -69,7 +70,9 @@ import com.google.datastore.v1.TransactionOptions; import com.google.protobuf.ByteString; import io.grpc.Status; +import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.OpenTelemetrySdk; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -98,7 +101,9 @@ final class DatastoreImpl extends BaseService implements Datas private final com.google.cloud.datastore.telemetry.TraceUtil otelTraceUtil = getOptions().getTraceUtil(); - private final DatastoreMetricsRecorder metricsRecorder = getOptions().getMetricsRecorder(); + private final DatastoreMetricsRecorder metricsRecorder; + private final OpenTelemetry builtInOpenTelemetry; + private final ReadOptionProtoPreparer readOptionProtoPreparer; private final AggregationQueryExecutor aggregationQueryExecutor; @@ -107,6 +112,8 @@ final class DatastoreImpl extends BaseService implements Datas this.datastoreRpc = options.getDatastoreRpcV1(); retrySettings = MoreObjects.firstNonNull(options.getRetrySettings(), ServiceOptions.getNoRetrySettings()); + builtInOpenTelemetry = BuiltInDatastoreMetricsProvider.INSTANCE.createOpenTelemetry(options); + metricsRecorder = DatastoreMetricsRecorder.getInstance(options, builtInOpenTelemetry); readOptionProtoPreparer = new ReadOptionProtoPreparer(); aggregationQueryExecutor = @@ -162,6 +169,16 @@ public T call() throws DatastoreException { } } + /** + * Closes the Datastore client and releases all resources. + * + *

This method closes the underlying RPC channel and then closes the {@link + * com.google.cloud.datastore.telemetry.DatastoreMetricsRecorder}. For clients using the built-in + * Cloud Monitoring exporter, closing the recorder flushes any buffered metrics and shuts down the + * private {@link io.opentelemetry.sdk.OpenTelemetrySdk} instance. For clients using a + * user-provided {@link io.opentelemetry.api.OpenTelemetry} instance, the recorder close is a + * no-op since the user owns that instance's lifecycle. + */ @Override public void close() throws Exception { try { @@ -169,6 +186,14 @@ public void close() throws Exception { } catch (Exception e) { logger.log(Level.WARNING, "Failed to close channels", e); } + // Shut down the built-in OTel SDK as we manage its lifecycle + if (builtInOpenTelemetry instanceof OpenTelemetrySdk) { + try { + ((OpenTelemetrySdk) builtInOpenTelemetry).close(); + } catch (Exception e) { + logger.log(Level.WARNING, "Failed to close built-in OpenTelemetry SDK instance.", e); + } + } } @Override @@ -242,7 +267,9 @@ public T call() throws DatastoreException { private void recordAttempt(String status) { Map attributes = TelemetryUtils.buildMetricAttributes( - TelemetryConstants.METHOD_TRANSACTION_COMMIT, status); + TelemetryConstants.METHOD_TRANSACTION_COMMIT, + status, + datastore.getOptions().getDatabaseId()); metricsRecorder.recordTransactionAttemptCount(1, attributes); } } @@ -280,7 +307,8 @@ public T runInTransaction( } finally { long latencyMs = stopwatch.elapsed(TimeUnit.MILLISECONDS); Map attributes = - TelemetryUtils.buildMetricAttributes(TelemetryConstants.METHOD_TRANSACTION_RUN, status); + TelemetryUtils.buildMetricAttributes( + TelemetryConstants.METHOD_TRANSACTION_RUN, status, getOptions().getDatabaseId()); metricsRecorder.recordTransactionLatency(latencyMs, attributes); span.end(); } @@ -789,7 +817,8 @@ private T runWithObservability( DatastoreOptions options = getOptions(); Callable attemptCallable = - TelemetryUtils.attemptMetricsCallable(callable, metricsRecorder, methodName); + TelemetryUtils.attemptMetricsCallable( + callable, metricsRecorder, methodName, options.getDatabaseId()); try (TraceUtil.Scope ignored = span.makeCurrent()) { return RetryHelper.runWithRetries( attemptCallable, retrySettings, exceptionHandler, options.getClock()); @@ -799,7 +828,11 @@ private T runWithObservability( throw DatastoreException.translateAndThrow(e); } finally { TelemetryUtils.recordOperationMetrics( - metricsRecorder, operationStopwatch, methodName, operationStatus); + metricsRecorder, + operationStopwatch, + methodName, + operationStatus, + options.getDatabaseId()); span.end(); } } diff --git a/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreOpenTelemetryOptions.java b/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreOpenTelemetryOptions.java index 50444353751b..12c55011fd17 100644 --- a/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreOpenTelemetryOptions.java +++ b/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreOpenTelemetryOptions.java @@ -16,25 +16,39 @@ package com.google.cloud.datastore; +import com.google.api.core.BetaApi; import io.opentelemetry.api.OpenTelemetry; import javax.annotation.Nonnull; import javax.annotation.Nullable; +/** + * Represents the options that are used to configure the use of OpenTelemetry for telemetry + * collection in the Datastore SDK. + */ public class DatastoreOpenTelemetryOptions { private final boolean tracingEnabled; private final boolean metricsEnabled; + private final boolean exportBuiltinMetricsToGoogleCloudMonitoring; private final @Nullable OpenTelemetry openTelemetry; DatastoreOpenTelemetryOptions(Builder builder) { this.tracingEnabled = builder.tracingEnabled; this.metricsEnabled = builder.metricsEnabled; + this.exportBuiltinMetricsToGoogleCloudMonitoring = + builder.exportBuiltinMetricsToGoogleCloudMonitoring; this.openTelemetry = builder.openTelemetry; } /** - * Returns whether either tracing or metrics are enabled. Telemetry is disabled by default. + * Returns whether either tracing or custom metrics (via a user-provided {@link OpenTelemetry} + * instance) are enabled. + * + *

Note: This method does not reflect the state of built-in metrics export to + * Google Cloud Monitoring, which is controlled separately by {@link + * #isExportBuiltinMetricsToGoogleCloudMonitoring()} and is {@code false} by default. To check + * whether any telemetry is active, also consult that flag. * - * @return {@code true} if either tracing or metrics are enabled, {@code false} otherwise. + * @return {@code true} if tracing or custom OTel metrics are enabled, {@code false} otherwise. */ public boolean isEnabled() { return tracingEnabled || metricsEnabled; @@ -50,7 +64,7 @@ public boolean isTracingEnabled() { } /** - * Returns whether metrics are enabled. + * Returns whether metrics are enabled for the custom (user-provided) OpenTelemetry backend. * * @return {@code true} if metrics are enabled, {@code false} otherwise. */ @@ -58,16 +72,46 @@ public boolean isMetricsEnabled() { return metricsEnabled; } + /** + * Returns whether built-in metrics should be exported to Google Cloud Monitoring. + * + *

When enabled, client-side metrics are automatically exported to Google Cloud Monitoring + * using the Cloud Monitoring API. This is independent of the custom OpenTelemetry backend + * configured via {@link #getOpenTelemetry()}. + * + * @return {@code true} if built-in metrics export to Cloud Monitoring is enabled, {@code false} + * otherwise. + */ + @BetaApi + public boolean isExportBuiltinMetricsToGoogleCloudMonitoring() { + return exportBuiltinMetricsToGoogleCloudMonitoring; + } + + /** + * Returns the custom {@link OpenTelemetry} instance, if one was provided. + * + * @return the custom {@link OpenTelemetry} instance, or {@code null} if none was provided. + */ @Nullable public OpenTelemetry getOpenTelemetry() { return openTelemetry; } + /** + * Returns a new {@link Builder} initialized with the values from this options instance. + * + * @return a new {@link Builder}. + */ @Nonnull public DatastoreOpenTelemetryOptions.Builder toBuilder() { return new DatastoreOpenTelemetryOptions.Builder(this); } + /** + * Returns a new default {@link Builder}. + * + * @return a new {@link Builder}. + */ @Nonnull public static DatastoreOpenTelemetryOptions.Builder newBuilder() { return new DatastoreOpenTelemetryOptions.Builder(); @@ -77,18 +121,23 @@ public static class Builder { private boolean tracingEnabled; private boolean metricsEnabled; + private boolean exportBuiltinMetricsToGoogleCloudMonitoring; @Nullable private OpenTelemetry openTelemetry; private Builder() { tracingEnabled = false; metricsEnabled = false; + // TODO(b/405457573): This is disabled by default until the Firestore namespace is deployed + exportBuiltinMetricsToGoogleCloudMonitoring = false; openTelemetry = null; } private Builder(DatastoreOpenTelemetryOptions options) { this.tracingEnabled = options.tracingEnabled; this.metricsEnabled = options.metricsEnabled; + this.exportBuiltinMetricsToGoogleCloudMonitoring = + options.exportBuiltinMetricsToGoogleCloudMonitoring; this.openTelemetry = options.openTelemetry; } @@ -96,6 +145,7 @@ private Builder(DatastoreOpenTelemetryOptions options) { * Sets whether tracing should be enabled. * * @param enabled Whether tracing should be enabled. + * @return this builder instance. */ @Nonnull public DatastoreOpenTelemetryOptions.Builder setTracingEnabled(boolean enabled) { @@ -104,10 +154,10 @@ public DatastoreOpenTelemetryOptions.Builder setTracingEnabled(boolean enabled) } /** - * Sets whether metrics should be enabled. + * Sets whether metrics should be enabled for the custom (user-provided) OpenTelemetry backend. * * @param enabled Whether metrics should be enabled. - * @return the builder instance. + * @return this builder instance. */ @Nonnull DatastoreOpenTelemetryOptions.Builder setMetricsEnabled(boolean enabled) { @@ -115,12 +165,30 @@ DatastoreOpenTelemetryOptions.Builder setMetricsEnabled(boolean enabled) { return this; } + /** + * Sets whether built-in metrics should be exported to Google Cloud Monitoring. + * + *

When enabled, client-side metrics are automatically exported to Google Cloud Monitoring + * using the Cloud Monitoring API. This can be disabled to prevent metrics from being sent to + * Cloud Monitoring while still allowing metrics to flow to a custom OpenTelemetry backend. + * + * @param exportBuiltinMetrics Whether built-in metrics should be exported to Cloud Monitoring. + * @return this builder instance. + */ + @BetaApi + public DatastoreOpenTelemetryOptions.Builder setExportBuiltinMetricsToGoogleCloudMonitoring( + boolean exportBuiltinMetrics) { + this.exportBuiltinMetricsToGoogleCloudMonitoring = exportBuiltinMetrics; + return this; + } + /** * Sets the {@link OpenTelemetry} to use with this Datastore instance. If telemetry collection - * is enabled, but an `OpenTelemetry` is not provided, the Datastore SDK will attempt to use the - * `GlobalOpenTelemetry`. + * is enabled, but an {@code OpenTelemetry} is not provided, the Datastore SDK will attempt to + * use the {@code GlobalOpenTelemetry}. * * @param openTelemetry The OpenTelemetry that should be used by this Datastore instance. + * @return this builder instance. */ @Nonnull public DatastoreOpenTelemetryOptions.Builder setOpenTelemetry( @@ -129,6 +197,11 @@ public DatastoreOpenTelemetryOptions.Builder setOpenTelemetry( return this; } + /** + * Builds a new {@link DatastoreOpenTelemetryOptions} instance from this builder. + * + * @return a new {@link DatastoreOpenTelemetryOptions}. + */ @Nonnull public DatastoreOpenTelemetryOptions build() { return new DatastoreOpenTelemetryOptions(this); diff --git a/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreOptions.java b/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreOptions.java index c1453b22bfde..38aef8f0e56d 100644 --- a/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreOptions.java +++ b/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/DatastoreOptions.java @@ -31,7 +31,6 @@ import com.google.cloud.datastore.spi.v1.DatastoreRpc; import com.google.cloud.datastore.spi.v1.GrpcDatastoreRpc; import com.google.cloud.datastore.spi.v1.HttpDatastoreRpc; -import com.google.cloud.datastore.telemetry.DatastoreMetricsRecorder; import com.google.cloud.datastore.v1.DatastoreSettings; import com.google.cloud.grpc.GrpcTransportOptions; import com.google.cloud.http.HttpTransportOptions; @@ -82,7 +81,6 @@ public class DatastoreOptions extends ServiceOptions { private String namespace; @@ -240,7 +233,6 @@ private DatastoreOptions(Builder builder) { ? builder.openTelemetryOptions : DatastoreOpenTelemetryOptions.newBuilder().build(); this.traceUtil = com.google.cloud.datastore.telemetry.TraceUtil.getInstance(this); - this.metricsRecorder = DatastoreMetricsRecorder.getInstance(this); namespace = MoreObjects.firstNonNull(builder.namespace, defaultNamespace()); databaseId = MoreObjects.firstNonNull(builder.databaseId, DEFAULT_DATABASE_ID); diff --git a/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/RetryAndTraceDatastoreRpcDecorator.java b/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/RetryAndTraceDatastoreRpcDecorator.java index 515e431af431..97bd5ccad395 100644 --- a/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/RetryAndTraceDatastoreRpcDecorator.java +++ b/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/RetryAndTraceDatastoreRpcDecorator.java @@ -24,8 +24,8 @@ import com.google.cloud.RetryHelper; import com.google.cloud.RetryHelper.RetryHelperException; import com.google.cloud.datastore.spi.v1.DatastoreRpc; +import com.google.cloud.datastore.telemetry.CompositeDatastoreMetricsRecorder; import com.google.cloud.datastore.telemetry.DatastoreMetricsRecorder; -import com.google.cloud.datastore.telemetry.NoOpDatastoreMetricsRecorder; import com.google.cloud.datastore.telemetry.TelemetryConstants; import com.google.cloud.datastore.telemetry.TelemetryUtils; import com.google.cloud.datastore.telemetry.TraceUtil; @@ -48,7 +48,9 @@ import com.google.datastore.v1.RunAggregationQueryResponse; import com.google.datastore.v1.RunQueryRequest; import com.google.datastore.v1.RunQueryResponse; +import java.util.ArrayList; import java.util.concurrent.Callable; +import javax.annotation.Nonnull; /** * An implementation of {@link DatastoreRpc} which acts as a Decorator and decorates the underlying @@ -58,7 +60,7 @@ public class RetryAndTraceDatastoreRpcDecorator implements DatastoreRpc { private final DatastoreRpc datastoreRpc; - private final com.google.cloud.datastore.telemetry.TraceUtil otelTraceUtil; + private final TraceUtil otelTraceUtil; private final RetrySettings retrySettings; private final DatastoreOptions datastoreOptions; private final DatastoreMetricsRecorder metricsRecorder; @@ -73,7 +75,7 @@ public RetryAndTraceDatastoreRpcDecorator( this.retrySettings = retrySettings; this.datastoreOptions = datastoreOptions; this.otelTraceUtil = otelTraceUtil; - this.metricsRecorder = new NoOpDatastoreMetricsRecorder(); + this.metricsRecorder = new CompositeDatastoreMetricsRecorder(new ArrayList<>()); } private RetryAndTraceDatastoreRpcDecorator(Builder builder) { @@ -94,10 +96,8 @@ public static class Builder { private RetrySettings retrySettings; private DatastoreOptions datastoreOptions; - // Defaults configured for this class - private DatastoreMetricsRecorder metricsRecorder = new NoOpDatastoreMetricsRecorder(); - - private Builder() {} + private DatastoreMetricsRecorder metricsRecorder = + new CompositeDatastoreMetricsRecorder(new ArrayList<>()); public Builder setDatastoreRpc(DatastoreRpc datastoreRpc) { this.datastoreRpc = datastoreRpc; @@ -119,8 +119,8 @@ public Builder setDatastoreOptions(DatastoreOptions datastoreOptions) { return this; } - public Builder setMetricsRecorder(DatastoreMetricsRecorder metricsRecorder) { - Preconditions.checkNotNull(metricsRecorder, "metricsRecorder can not be null"); + @InternalApi + public Builder setMetricsRecorder(@Nonnull DatastoreMetricsRecorder metricsRecorder) { this.metricsRecorder = metricsRecorder; return this; } @@ -130,6 +130,7 @@ public RetryAndTraceDatastoreRpcDecorator build() { Preconditions.checkNotNull(otelTraceUtil, "otelTraceUtil is required"); Preconditions.checkNotNull(retrySettings, "retrySettings is required"); Preconditions.checkNotNull(datastoreOptions, "datastoreOptions is required"); + Preconditions.checkNotNull(metricsRecorder, "metricsRecorder is required"); return new RetryAndTraceDatastoreRpcDecorator(this); } } @@ -176,9 +177,8 @@ public RunAggregationQueryResponse runAggregationQuery(RunAggregationQueryReques boolean isTransactional = readOptions.hasTransaction() || readOptions.hasNewTransaction(); String spanName = (isTransactional - ? com.google.cloud.datastore.telemetry.TraceUtil - .SPAN_NAME_TRANSACTION_RUN_AGGREGATION_QUERY - : com.google.cloud.datastore.telemetry.TraceUtil.SPAN_NAME_RUN_AGGREGATION_QUERY); + ? TraceUtil.SPAN_NAME_TRANSACTION_RUN_AGGREGATION_QUERY + : TraceUtil.SPAN_NAME_RUN_AGGREGATION_QUERY); return invokeRpc( () -> datastoreRpc.runAggregationQuery(request), spanName, @@ -205,7 +205,8 @@ O invokeRpc(Callable block, String startSpan, String methodName) { String operationStatus = StatusCode.Code.UNKNOWN.toString(); try (TraceUtil.Scope ignored = span.makeCurrent()) { Callable callable = - TelemetryUtils.attemptMetricsCallable(block, metricsRecorder, methodName); + TelemetryUtils.attemptMetricsCallable( + block, metricsRecorder, methodName, datastoreOptions.getDatabaseId()); O result = RetryHelper.runWithRetries( callable, this.retrySettings, EXCEPTION_HANDLER, this.datastoreOptions.getClock()); @@ -217,7 +218,11 @@ O invokeRpc(Callable block, String startSpan, String methodName) { throw DatastoreException.translateAndThrow(e); } finally { TelemetryUtils.recordOperationMetrics( - metricsRecorder, stopwatch, methodName, operationStatus); + metricsRecorder, + stopwatch, + methodName, + operationStatus, + datastoreOptions.getDatabaseId()); span.end(); } } diff --git a/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/BuiltInDatastoreMetricsProvider.java b/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/BuiltInDatastoreMetricsProvider.java new file mode 100644 index 000000000000..a538e1318135 --- /dev/null +++ b/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/BuiltInDatastoreMetricsProvider.java @@ -0,0 +1,200 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.datastore.telemetry; + +import com.google.api.core.InternalApi; +import com.google.auth.Credentials; +import com.google.cloud.NoCredentials; +import com.google.cloud.datastore.DatastoreOptions; +import com.google.common.base.Preconditions; +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hashing; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; +import io.opentelemetry.sdk.resources.Resource; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * Provides a built-in {@link OpenTelemetry} instance for Datastore client-side metrics. + * + *

This class is responsible for configuring a private {@link OpenTelemetrySdk} that exports + * metrics to Google Cloud Monitoring using a {@link DatastoreCloudMonitoringExporter}. + * + *

The implementation follows the pattern used in other Google Cloud client libraries, providing + * automated environment detection and resource attribute configuration for the {@link + * TelemetryConstants#DATASTORE_RESOURCE_TYPE} monitored resource. + */ +@InternalApi +public class BuiltInDatastoreMetricsProvider { + + public static final BuiltInDatastoreMetricsProvider INSTANCE = + new BuiltInDatastoreMetricsProvider(); + + private static final Logger logger = + Logger.getLogger(BuiltInDatastoreMetricsProvider.class.getName()); + + // volatile ensures that writes from one thread (first call to detectClientLocation) are + // immediately visible to all other threads sharing this singleton INSTANCE. Without it, a + // thread could read a stale null and re-enter the initialization branch. + private static volatile String location; + private static final String DEFAULT_LOCATION = "global"; + + private BuiltInDatastoreMetricsProvider() {} + + static Map buildClientAttributes() { + Map attrs = new HashMap<>(); + attrs.put( + TelemetryConstants.CLIENT_UID_KEY.getKey(), hashClientUId(UUID.randomUUID().toString())); + attrs.put(TelemetryConstants.SERVICE_KEY.getKey(), TelemetryConstants.SERVICE_VALUE); + return attrs; + } + + /** + * Generates a 6-digit zero-padded all lower case hexadecimal representation of hash of the + * accounting group. The hash utilizes the 10 most significant bits of the value returned by + * `Hashing.goodFastHash(64).hashBytes()`, so effectively the returned values are uniformly + * distributed in the range [000000, 0003ff]. + * + *

The primary purpose of this function is to generate a hash value for the `client_uid` metric + * field. The range of values is chosen to be small enough to keep the cardinality under control. + * + *

Note: If at later time the range needs to be increased, it can be done by increasing the + * value of `kPrefixLength` to up to 24 bits without changing the format of the returned value. + * + * @return Returns a 6-digit zero-padded all lower case hexadecimal representation of hash of the + * accounting group. + */ + private static String hashClientUId(String uuid) { + if (uuid == null) { + return "000000"; + } + + HashFunction hashFunction = Hashing.goodFastHash(64); + long hash = hashFunction.hashBytes(uuid.getBytes()).asLong(); + // Don't change this value without reading above comment + int kPrefixLength = 10; + long shiftedValue = hash >>> (64 - kPrefixLength); + return String.format("%06x", shiftedValue); + } + + /** + * Creates a new {@link OpenTelemetry} instance for a single Datastore client's built-in metrics. + * + *

Each call returns a dedicated {@link OpenTelemetrySdk} wrapping an {@link SdkMeterProvider} + * configured with the provided project's monitored resource attributes and a {@link + * DatastoreCloudMonitoringExporter}. No global or shared state is modified. + * + *

Lifecycle: The returned instance is owned by the caller. It should be closed by + * calling {@link io.opentelemetry.sdk.OpenTelemetrySdk#close()} (or via {@link + * OpenTelemetryDatastoreMetricsRecorder#close()}) when the associated Datastore client is closed. + * + *

No caching is performed here; callers are responsible for holding the returned instance for + * the lifetime of their Datastore client. + * + * @param options Datastore Client Options + * @return a new {@link OpenTelemetry} instance, or {@code null} if it could not be created. + */ + @Nullable + public OpenTelemetry createOpenTelemetry(@Nonnull DatastoreOptions options) { + Credentials credentials = + Preconditions.checkNotNull( + options.getCredentials(), "Credentials cannot be null for built in metrics"); + String projectId = options.getProjectId(); + String databaseId = options.getDatabaseId(); + + // No need to send metrics when using an emulator + String emulatorHost = + System.getProperty( + DatastoreOptions.LOCAL_HOST_ENV_VAR, + System.getenv(DatastoreOptions.LOCAL_HOST_ENV_VAR)); + boolean emulatorEnabled = emulatorHost != null && !emulatorHost.isEmpty(); + + if (emulatorEnabled) { + logger.log(Level.FINE, "Emulator detected in Datastore. Metrics are not being recorded."); + return OpenTelemetry.noop(); + } + if (credentials instanceof NoCredentials) { + logger.log(Level.WARNING, "Built-in metrics exporting is disabled when using NoCredentials."); + return OpenTelemetry.noop(); + } + + SdkMeterProviderBuilder sdkMeterProviderBuilder = SdkMeterProvider.builder(); + Map clientAttributes = buildClientAttributes(); + DatastoreCloudMonitoringExporter exporter = + DatastoreCloudMonitoringExporter.create( + projectId, databaseId, credentials, clientAttributes); + if (exporter == null) { + logger.log( + Level.WARNING, + "Built-in metrics exporting is disabled as the exporter could not be created."); + return OpenTelemetry.noop(); + } + + // Register Datastore-specific views and the PeriodicMetricReader. + DatastoreBuiltInMetricsView.registerBuiltinMetrics(exporter, sdkMeterProviderBuilder); + // Configure the monitored resource attributes for this specific client. + sdkMeterProviderBuilder.setResource( + Resource.create(createResourceAttributes(projectId, databaseId))); + SdkMeterProvider sdkMeterProvider = sdkMeterProviderBuilder.build(); + return OpenTelemetrySdk.builder().setMeterProvider(sdkMeterProvider).build(); + } + + /** + * Detects the client's GCP location (region). + * + *

To avoid dependencies on external resource detection libraries, this implementation + * currently defaults to "global". + * + * @return the detected location, or "global" if detection fails. + */ + String detectClientLocation() { + if (location == null) { + location = DEFAULT_LOCATION; + } + return location; + } + + /** + * Creates resource attributes for the {@link TelemetryConstants#DATASTORE_RESOURCE_TYPE} + * monitored resource. + * + *

These attributes are attached to the OTel {@link Resource} and used by the exporter to + * populate the resource labels in Cloud Monitoring. + * + * @param projectId the GCP project ID. + * @param databaseId the Datastore database ID. + * @return the resource attributes. + */ + Attributes createResourceAttributes(String projectId, String databaseId) { + AttributesBuilder attributesBuilder = + Attributes.builder() + .put(TelemetryConstants.PROJECT_ID_KEY, projectId) + .put(TelemetryConstants.DATABASE_ID_KEY, databaseId) + .put(TelemetryConstants.LOCATION_ID_KEY, detectClientLocation()); + return attributesBuilder.build(); + } +} diff --git a/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/CompositeDatastoreMetricsRecorder.java b/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/CompositeDatastoreMetricsRecorder.java new file mode 100644 index 000000000000..fe49312cc116 --- /dev/null +++ b/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/CompositeDatastoreMetricsRecorder.java @@ -0,0 +1,116 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.datastore.telemetry; + +import com.google.api.core.InternalApi; +import com.google.common.annotations.VisibleForTesting; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * A {@link DatastoreMetricsRecorder} implementation that fans out recording calls to multiple + * underlying recorders. + * + *

This allows simultaneous recording to both built-in (Cloud Monitoring) and custom + * (user-provided) OpenTelemetry backends. + */ +@InternalApi +public class CompositeDatastoreMetricsRecorder implements DatastoreMetricsRecorder { + + private static final Logger logger = + Logger.getLogger(CompositeDatastoreMetricsRecorder.class.getName()); + + private final List recorders; + + public CompositeDatastoreMetricsRecorder(List recorders) { + this.recorders = recorders; + } + + @VisibleForTesting + List getMetricRecorders() { + return recorders; + } + + /** {@inheritDoc} */ + @Override + public void recordTransactionLatency(double latencyMs, Map attributes) { + for (DatastoreMetricsRecorder recorder : recorders) { + recorder.recordTransactionLatency(latencyMs, attributes); + } + } + + /** {@inheritDoc} */ + @Override + public void recordTransactionAttemptCount(long count, Map attributes) { + for (DatastoreMetricsRecorder recorder : recorders) { + recorder.recordTransactionAttemptCount(count, attributes); + } + } + + /** {@inheritDoc} */ + @Override + public void recordAttemptLatency(double latencyMs, Map attributes) { + for (DatastoreMetricsRecorder recorder : recorders) { + recorder.recordAttemptLatency(latencyMs, attributes); + } + } + + /** {@inheritDoc} */ + @Override + public void recordAttemptCount(long count, Map attributes) { + for (DatastoreMetricsRecorder recorder : recorders) { + recorder.recordAttemptCount(count, attributes); + } + } + + /** {@inheritDoc} */ + @Override + public void recordOperationLatency(double latencyMs, Map attributes) { + for (DatastoreMetricsRecorder recorder : recorders) { + recorder.recordOperationLatency(latencyMs, attributes); + } + } + + /** {@inheritDoc} */ + @Override + public void recordOperationCount(long count, Map attributes) { + for (DatastoreMetricsRecorder recorder : recorders) { + recorder.recordOperationCount(count, attributes); + } + } + + /** + * Closes all underlying recorders. + * + *

Each recorder's own {@link DatastoreMetricsRecorder#close()} semantics apply: recorders that + * own their {@link io.opentelemetry.api.OpenTelemetry} instance (built-in path) will flush and + * shut down; recorders backed by a user-provided instance will no-op. All recorders are closed + * even if one throws an exception. + */ + @Override + public void close() { + for (int i = recorders.size() - 1; i >= 0; i--) { + try { + recorders.get(i).close(); + } catch (Exception e) { + logger.log(Level.WARNING, "Failed to close metrics recorder", e); + } + } + } +} diff --git a/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DatastoreBuiltInMetricsView.java b/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DatastoreBuiltInMetricsView.java new file mode 100644 index 000000000000..206ad81ef350 --- /dev/null +++ b/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DatastoreBuiltInMetricsView.java @@ -0,0 +1,164 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.datastore.telemetry; + +import com.google.api.gax.tracing.OpenTelemetryMetricsRecorder; +import com.google.common.collect.ImmutableList; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.sdk.metrics.Aggregation; +import io.opentelemetry.sdk.metrics.InstrumentSelector; +import io.opentelemetry.sdk.metrics.InstrumentType; +import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; +import io.opentelemetry.sdk.metrics.View; +import io.opentelemetry.sdk.metrics.export.MetricExporter; +import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Configures OpenTelemetry Views for Datastore built-in metrics. + * + *

Views are a critical "post-processing" layer in this implementation. They are used here for + * three primary reasons: + * + *

    + *
  1. Metric Mapping (GAX Alignment): GAX records generic metrics like {@code + * operation_latency}. We use Views to catch these metrics and prepend the official {@link + * TelemetryConstants#METRIC_PREFIX}, ensuring all RPC metrics follow the Firestore domain + * naming convention. + *
  2. Latency Precision: Datastore operations vary from sub-millisecond lookups to + * multi-second transactional commits. Default OTel histogram buckets do not handle this well. + * Define explicit {@link DatastoreBuiltInMetricsView#BUCKET_BOUNDARIES} to ensure that + * latency heatmaps in Cloud Monitoring are more readable. + *
  3. Cloud Monitoring Schema Alignment: The Cloud Monitoring API is strict about labels. + * Exporting unexpected attributes can cause the entire export to fail. Views allow us to + * strictly filter attributes down to the {@link TelemetryConstants#COMMON_ATTRIBUTES}. + *
+ */ +class DatastoreBuiltInMetricsView { + + // Standard bucket boundaries for latency metrics in milliseconds. + private static final List BUCKET_BOUNDARIES = + ImmutableList.of( + 0.0, 0.5, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, + 16.0, 17.0, 18.0, 19.0, 20.0, 25.0, 30.0, 40.0, 50.0, 65.0, 80.0, 100.0, 130.0, 160.0, + 200.0, 250.0, 300.0, 400.0, 500.0, 650.0, 800.0, 1000.0, 2000.0, 5000.0, 10000.0, 20000.0, + 50000.0, 100000.0, 200000.0, 400000.0, 800000.0, 1600000.0, 3200000.0); + + private static final Aggregation AGGREGATION_WITH_MILLIS_HISTOGRAM = + Aggregation.explicitBucketHistogram(BUCKET_BOUNDARIES); + + private DatastoreBuiltInMetricsView() {} + + /** + * Registers Datastore built-in metrics and views on the provided {@link SdkMeterProviderBuilder}. + * + *

This method acts as the central configuration point for the metrics pipeline, stitching + * together the recording (SDK), the processing (Views), and the delivery (Exporter). + * + *

Views are scoped to the SdkMeterProvider they are registered on. This method is called + * exclusively from BuiltInDatastoreMetricsProvider, which constructs a private SdkMeterProvider + * for the built-in Cloud Monitoring export path. A user-provided OpenTelemetry instance has its + * own independent MeterProvider, so these views never affect user-configured backends. + * + * @param metricExporter the exporter to use for metrics. + * @param builder the builder to register views and the {@link PeriodicMetricReader} on. + */ + static void registerBuiltinMetrics( + MetricExporter metricExporter, SdkMeterProviderBuilder builder) { + registerGaxViews(builder); + registerDatastoreViews(builder); + // Metrics are collected in-memory and flushed periodically to avoid impacting the + // performance of critical-path Datastore operations. + builder.registerMetricReader(PeriodicMetricReader.create(metricExporter)); + } + + /** + * Configures views for metrics generated by the GAX library. + * + *

GAX provides standardized RPC metrics but uses a generic namespace. We apply these Views to + * "claim" those metrics for Datastore, ensuring they appear in the official Cloud Monitoring + * dashboard with the correct names and the necessary millisecond-level precision. + */ + private static void registerGaxViews(SdkMeterProviderBuilder builder) { + for (String metricName : TelemetryConstants.GAX_METRICS) { + Aggregation aggregation = Aggregation.defaultAggregation(); + + InstrumentType type = InstrumentType.COUNTER; + String unit = "1"; + + // Latency metrics use histograms with specific millisecond buckets. + if (TelemetryConstants.GAX_HISTOGRAM_METRICS.contains(metricName)) { + aggregation = AGGREGATION_WITH_MILLIS_HISTOGRAM; + type = InstrumentType.HISTOGRAM; + unit = "ms"; + } + + // Select metrics from the GAX meter scope. + InstrumentSelector selector = + InstrumentSelector.builder() + .setMeterName(OpenTelemetryMetricsRecorder.GAX_METER_NAME) + .setName(metricName) + .setType(type) + .setUnit(unit) + .build(); + + // Only allow common attributes to be attached to the metrics. + Set attributesFilter = + TelemetryConstants.COMMON_ATTRIBUTES.stream() + .map(AttributeKey::getKey) + .collect(Collectors.toSet()); + + String renamedMetricName = + TelemetryConstants.GAX_METRIC_NAME_MAP.getOrDefault(metricName, metricName); + + // Rename the metric to use the Datastore prefix for Cloud Monitoring. + View view = + View.builder() + .setName(TelemetryConstants.METRIC_PREFIX + "/" + renamedMetricName) + .setAggregation(aggregation) + .setAttributeFilter(attributesFilter) + .build(); + + builder.registerView(selector, view); + } + } + + /** + * Configures views for metrics generated specifically by the Datastore SDK. + * + *

For Datastore-native metrics (like transaction latency), we use Views primarily to enforce a + * strict attribute allowlist. This ensures that only approved labels (e.g., service, method, + * status) are exported, keeping the Cloud Monitoring time series clean and within quota limits. + */ + private static void registerDatastoreViews(SdkMeterProviderBuilder builder) { + // Select all metrics from the Datastore meter scope. + InstrumentSelector selector = + InstrumentSelector.builder().setMeterName(TelemetryConstants.DATASTORE_METER_NAME).build(); + + // Filter to allow only common attributes. + Set attributesFilter = + TelemetryConstants.COMMON_ATTRIBUTES.stream() + .map(AttributeKey::getKey) + .collect(Collectors.toSet()); + + View view = View.builder().setAttributeFilter(attributesFilter).build(); + + builder.registerView(selector, view); + } +} diff --git a/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DatastoreCloudMonitoringExporter.java b/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DatastoreCloudMonitoringExporter.java new file mode 100644 index 000000000000..83f1961621eb --- /dev/null +++ b/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DatastoreCloudMonitoringExporter.java @@ -0,0 +1,346 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.datastore.telemetry; + +import com.google.api.core.ApiFuture; +import com.google.api.core.ApiFutureCallback; +import com.google.api.core.ApiFutures; +import com.google.api.gax.core.FixedCredentialsProvider; +import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider; +import com.google.api.gax.rpc.PermissionDeniedException; +import com.google.auth.Credentials; +import com.google.cloud.monitoring.v3.MetricServiceClient; +import com.google.cloud.monitoring.v3.MetricServiceSettings; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Iterables; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.monitoring.v3.CreateTimeSeriesRequest; +import com.google.monitoring.v3.ProjectName; +import com.google.monitoring.v3.TimeSeries; +import com.google.protobuf.Empty; +import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.metrics.InstrumentType; +import io.opentelemetry.sdk.metrics.data.AggregationTemporality; +import io.opentelemetry.sdk.metrics.data.MetricData; +import io.opentelemetry.sdk.metrics.export.MetricExporter; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * Datastore Cloud Monitoring OpenTelemetry Exporter. + * + *

The exporter will look for all Datastore-owned metrics under {@link + * TelemetryConstants#METRIC_PREFIX} instrumentation scope and upload it via the Google Cloud + * Monitoring API. + * + *

This implementation is a standalone exporter that does not depend on the {@code + * com.google.cloud.opentelemetry:exporter-metrics} library, to avoid external version management + * and ensure tight integration with Datastore requirements. + * + *

The implementation in this file is inspired from the original work done in the Spanner client + * library (SpannerCloudMonitoringExporter) to export metrics. The logic has been adapted for + * Datastore's use case. + */ +class DatastoreCloudMonitoringExporter implements MetricExporter { + + private static final Logger logger = + Logger.getLogger(DatastoreCloudMonitoringExporter.class.getName()); + + /** + * Wrapper class to hold a {@link MetricServiceClient} and its reference count. This is used to + * share the client across multiple exporter instances. + */ + static class CachedMetricsClient { + final MetricServiceClient client; + final AtomicInteger refCount = new AtomicInteger(0); + + CachedMetricsClient(MetricServiceClient client) { + this.client = client; + } + } + + /** + * Shared cache for {@link MetricServiceClient} instances, keyed by + * "projectId:databaseId:credentialsHashCode". Sharing a single gRPC channel across exporter + * instances that target the same project, database, and credentials avoids per-client channel + * overhead (threads, connections, memory). The credentials hash ensures that clients using + * different credentials get their own isolated channel. Reference counting is used to safely shut + * down the client when no longer needed. + */ + static final ConcurrentHashMap METRICS_CLIENT_CACHE = + new ConcurrentHashMap<>(); + + private final MetricServiceClient client; + private final Map clientAttributes; + private final String cacheKey; + + // This is the quota limit from Cloud Monitoring. More details in + // https://cloud.google.com/monitoring/quotas#custom_metrics_quotas. + private static final int EXPORT_BATCH_SIZE_LIMIT = 200; + + // Increase max metadata size to 32MB to avoid "Header size exceeded" errors + // when receiving large error payloads from Cloud Monitoring. + private static final int MAX_METADATA_SIZE = 32 * 1024 * 1024; + + // Flag to prevent log spam of any export failures + private final AtomicBoolean datastoreExportFailureLogged = new AtomicBoolean(false); + + // Flag to prevent double shutdown of this exporter instance + private final AtomicBoolean isExporterShutDown = new AtomicBoolean(false); + + private final String projectId; + + /** + * Creates a new instance of the exporter. + * + *

The gRPC channel is configured with a 32MB inbound metadata limit ({@link + * #MAX_METADATA_SIZE}) to prevent "Header size exceeded" errors when Cloud Monitoring returns + * large error payloads in gRPC trailers. The default gRPC limit is too small for some error + * responses and can mask the real failure reason. + * + *

{@code createTimeSeries} is used instead of {@code createServiceTimeSeries} because the + * Firestore namespace in Cloud Monitoring (b/405457573) has not yet been deployed as a service + * resource. Once the namespace is available, this should be migrated to {@code + * createServiceTimeSeries} for correct quota and resource attribution. + * + * @param projectId the GCP project ID where metrics will be exported. + * @param credentials the credentials used to authenticate with Cloud Monitoring. + * @return a new {@link DatastoreCloudMonitoringExporter} instance. + */ + @Nullable + static DatastoreCloudMonitoringExporter create( + String projectId, + String databaseId, + Credentials credentials, + Map clientAttributes) { + int credHash = credentials != null ? credentials.hashCode() : 0; + String key = projectId + ":" + databaseId + ":" + credHash; + + // Use compute to acquire or create the client atomically with reference counting. + // If creation fails, we log the error and return null so it's not added to the map. + CachedMetricsClient cachedMetricsClient = + METRICS_CLIENT_CACHE.compute( + key, + (k, v) -> { + if (v == null) { + try { + v = new CachedMetricsClient(createMetricServiceClient(credentials)); + } catch (IOException e) { + logger.log( + Level.WARNING, + "Failed to create MetricServiceClient for metrics export. Monitoring will be disabled.", + e); + return null; // Do not add to map + } + } + v.refCount.incrementAndGet(); + return v; + }); + + // If there is no client in the cache (creation failed), return null. + if (cachedMetricsClient == null) { + return null; + } + + return new DatastoreCloudMonitoringExporter( + key, projectId, cachedMetricsClient.client, clientAttributes); + } + + private static MetricServiceClient createMetricServiceClient(Credentials credentials) + throws IOException { + MetricServiceSettings.Builder settingsBuilder = MetricServiceSettings.newBuilder(); + + InstantiatingGrpcChannelProvider transportChannelProvider = + MetricServiceSettings.defaultGrpcTransportProviderBuilder() + .setMaxInboundMetadataSize(MAX_METADATA_SIZE) + .build(); + settingsBuilder.setTransportChannelProvider(transportChannelProvider); + + settingsBuilder.setCredentialsProvider(FixedCredentialsProvider.create(credentials)); + + settingsBuilder + .createTimeSeriesSettings() + .setSimpleTimeoutNoRetriesDuration(Duration.ofMinutes(1)); + + return MetricServiceClient.create(settingsBuilder.build()); + } + + @VisibleForTesting + DatastoreCloudMonitoringExporter( + String cacheKey, + String projectId, + MetricServiceClient client, + Map clientAttributes) { + this.cacheKey = cacheKey; + this.client = client; + this.projectId = projectId; + this.clientAttributes = clientAttributes; + } + + /** + * Exports the provided collection of {@link MetricData} to Cloud Monitoring. + * + * @param collection the collection of metric data to export. + * @return a {@link CompletableResultCode} indicating the result of the export operation. + */ + @Override + public CompletableResultCode export(@Nonnull Collection collection) { + if (isExporterShutDown.get()) { + logger.log(Level.WARNING, "Exporter is shut down"); + return CompletableResultCode.ofFailure(); + } + + // Skips exporting if there's none + if (collection.isEmpty()) { + return CompletableResultCode.ofSuccess(); + } + + List datastoreTimeSeries; + try { + // Convert OTel MetricData to Cloud Monitoring TimeSeries. + datastoreTimeSeries = + DatastoreCloudMonitoringExporterUtils.convertToDatastoreTimeSeries( + new ArrayList<>(collection), clientAttributes); + } catch (Throwable e) { + logger.log( + Level.WARNING, + "Failed to convert Datastore metric data to Cloud Monitoring timeseries.", + e); + return CompletableResultCode.ofFailure(); + } + + ProjectName projectName = ProjectName.of(projectId); + + // Perform the actual network call to Cloud Monitoring. + ApiFuture> futureList = exportTimeSeriesInBatch(projectName, datastoreTimeSeries); + + CompletableResultCode datastoreExportCode = new CompletableResultCode(); + ApiFutures.addCallback( + futureList, + new ApiFutureCallback>() { + @Override + public void onFailure(Throwable throwable) { + // Only log failure once to avoid log spam, then reset on success. + if (datastoreExportFailureLogged.compareAndSet(false, true)) { + String msg = "createTimeSeries request failed for Datastore metrics."; + + if (throwable instanceof PermissionDeniedException) { + msg += + String.format( + " Need monitoring metric writer permission on project=%s.", + projectName.getProject()); + } + logger.log(Level.WARNING, msg, throwable); + } + datastoreExportCode.fail(); + } + + @Override + public void onSuccess(List empty) { + // When an export succeeded reset the export failure flag to false so if there's a + // transient failure it'll be logged. + datastoreExportFailureLogged.set(false); + datastoreExportCode.succeed(); + } + }, + MoreExecutors.directExecutor()); + + return datastoreExportCode; + } + + /** Batches and sends the {@link TimeSeries} to Cloud Monitoring. */ + private ApiFuture> exportTimeSeriesInBatch( + ProjectName projectName, List timeSeries) { + List> batchResults = new ArrayList<>(); + + for (List batch : Iterables.partition(timeSeries, EXPORT_BATCH_SIZE_LIMIT)) { + CreateTimeSeriesRequest req = + CreateTimeSeriesRequest.newBuilder() + .setName(projectName.toString()) + .addAllTimeSeries(batch) + .build(); + batchResults.add(this.client.createTimeSeriesCallable().futureCall(req)); + } + + return ApiFutures.allAsList(batchResults); + } + + /** + * Best-effort flush of any pending exports. + * + *

This implementation is a no-op and always returns {@link CompletableResultCode#ofSuccess()}. + * Because exports are performed asynchronously via {@link ApiFuture} callbacks, this flush cannot + * guarantee that all concurrent in-flight network requests have completed by the time this method + * returns. For a stronger guarantee, callers should invoke {@code SdkMeterProvider.forceFlush()}, + * which coordinates across the {@link io.opentelemetry.sdk.metrics.export.MetricReader} and + * ensures a full collection cycle completes before returning. + */ + @Override + public CompletableResultCode flush() { + return CompletableResultCode.ofSuccess(); + } + + /** Shuts down the exporter and the underlying {@link MetricServiceClient}. */ + @Override + public CompletableResultCode shutdown() { + if (!isExporterShutDown.compareAndSet(false, true)) { + logger.log(Level.WARNING, "shutdown is called multiple times"); + return CompletableResultCode.ofSuccess(); + } + CompletableResultCode shutdownResult = new CompletableResultCode(); + try { + // Atomically decrement reference count and cleanup if zero. + METRICS_CLIENT_CACHE.compute( + cacheKey, + (k, v) -> { + if (v != null && v.refCount.decrementAndGet() == 0) { + v.client.shutdown(); + return null; // Remove from map to prevent leaks + } + + return v; + }); + shutdownResult.succeed(); + } catch (Throwable e) { + logger.log(Level.WARNING, "failed to shutdown the monitoring client", e); + shutdownResult.fail(); + } + return shutdownResult; + } + + /** + * Returns the {@link AggregationTemporality} for this exporter. + * + *

For Google Cloud Monitoring, we always use {@link AggregationTemporality#CUMULATIVE} to + * maintain a continuous count or sum over time. + */ + @Override + public AggregationTemporality getAggregationTemporality(@Nonnull InstrumentType instrumentType) { + return AggregationTemporality.CUMULATIVE; + } +} diff --git a/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DatastoreCloudMonitoringExporterUtils.java b/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DatastoreCloudMonitoringExporterUtils.java new file mode 100644 index 000000000000..949bdff3c293 --- /dev/null +++ b/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DatastoreCloudMonitoringExporterUtils.java @@ -0,0 +1,274 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.datastore.telemetry; + +import static com.google.api.MetricDescriptor.MetricKind.CUMULATIVE; +import static com.google.api.MetricDescriptor.MetricKind.GAUGE; +import static com.google.api.MetricDescriptor.MetricKind.UNRECOGNIZED; +import static com.google.api.MetricDescriptor.ValueType.DISTRIBUTION; +import static com.google.api.MetricDescriptor.ValueType.DOUBLE; +import static com.google.api.MetricDescriptor.ValueType.INT64; +import static com.google.cloud.datastore.telemetry.TelemetryConstants.DATASTORE_RESOURCE_TYPE; + +import com.google.api.Distribution; +import com.google.api.Distribution.BucketOptions; +import com.google.api.Distribution.BucketOptions.Explicit; +import com.google.api.Metric; +import com.google.api.MetricDescriptor.MetricKind; +import com.google.api.MetricDescriptor.ValueType; +import com.google.api.MonitoredResource; +import com.google.monitoring.v3.Point; +import com.google.monitoring.v3.TimeInterval; +import com.google.monitoring.v3.TimeSeries; +import com.google.monitoring.v3.TypedValue; +import com.google.protobuf.util.Timestamps; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.sdk.metrics.data.AggregationTemporality; +import io.opentelemetry.sdk.metrics.data.DoublePointData; +import io.opentelemetry.sdk.metrics.data.HistogramData; +import io.opentelemetry.sdk.metrics.data.HistogramPointData; +import io.opentelemetry.sdk.metrics.data.LongPointData; +import io.opentelemetry.sdk.metrics.data.MetricData; +import io.opentelemetry.sdk.metrics.data.MetricDataType; +import io.opentelemetry.sdk.metrics.data.PointData; +import io.opentelemetry.sdk.metrics.data.SumData; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Utility class for converting OpenTelemetry metrics to Google Cloud Monitoring format. + * + *

This class contains the logic to map OpenTelemetry {@link MetricData} and {@link PointData} to + * Cloud Monitoring {@link TimeSeries}, including resource label mapping and attribute conversion. + * + *

The implementation in this file is inspired from the original work done in the Spanner client + * library (SpannerCloudMonitoringExporterUtils) to export metrics. The logic has been adapted for + * Datastore's use case. + */ +class DatastoreCloudMonitoringExporterUtils { + + private static final Logger logger = + Logger.getLogger(DatastoreCloudMonitoringExporterUtils.class.getName()); + + private DatastoreCloudMonitoringExporterUtils() {} + + /** + * Converts a list of {@link MetricData} to Cloud Monitoring {@link TimeSeries}. + * + * @param collection the collection of metrics to convert. + * @param clientAttributes common client labels (e.g. {@code client_name}, {@code client_uid}) to + * attach to every metric data point. + * @return a list of converted {@link TimeSeries}. + */ + static List convertToDatastoreTimeSeries( + List collection, Map clientAttributes) { + List allTimeSeries = new ArrayList<>(); + + // Metrics should already been filtered for Gax and Datastore related ones + for (MetricData metricData : collection) { + // TODO(b/405457573): The monitored resource is currently written to `global` because the + // Firestore namespace in Cloud Monitoring has not been deployed yet. Once the namespace + // is available, database_id and location labels should be added here using + // RESOURCE_LABEL_DATABASE_ID and RESOURCE_LABEL_LOCATION respectively. + + // Map OTel resource attributes to the specific monitored resource labels. + MonitoredResource.Builder monitoredResourceBuilder = + MonitoredResource.newBuilder().setType(DATASTORE_RESOURCE_TYPE); + + Attributes resourceAttributes = metricData.getResource().getAttributes(); + String resourceProjectId = resourceAttributes.get(TelemetryConstants.PROJECT_ID_KEY); + // String resourceDatabaseId = + // resourceAttributes.get(TelemetryConstants.DATABASE_ID_KEY); + // String resourceLocation = resourceAttributes.get(TelemetryConstants.LOCATION_ID_KEY); + + if (resourceProjectId != null) { + monitoredResourceBuilder.putLabels( + TelemetryConstants.RESOURCE_LABEL_PROJECT_ID, resourceProjectId); + } + + // Convert each point in the metric data to a TimeSeries. + metricData.getData().getPoints().stream() + .map( + pointData -> + convertPointToDatastoreTimeSeries( + metricData, pointData, monitoredResourceBuilder, clientAttributes)) + .forEach(allTimeSeries::add); + } + return allTimeSeries; + } + + /** + * Converts an individual {@link PointData} to a {@link TimeSeries}. + * + *

{@code clientAttributes} (e.g. {@code client_name}, {@code client_uid}) are injected here + * rather than being looked up from a singleton so that this method is testable in isolation. The + * caller ({@link DatastoreCloudMonitoringExporter}) is responsible for supplying them from {@link + * BuiltInDatastoreMetricsProvider#buildClientAttributes()}. + */ + private static TimeSeries convertPointToDatastoreTimeSeries( + MetricData metricData, + PointData pointData, + MonitoredResource.Builder monitoredResourceBuilder, + Map clientAttributes) { + MetricKind metricKind = convertMetricKind(metricData); + TimeSeries.Builder builder = + TimeSeries.newBuilder() + .setMetricKind(metricKind) + .setValueType(convertValueType(metricData.getType())); + Metric.Builder metricBuilder = Metric.newBuilder().setType(metricData.getName()); + + Attributes attributes = pointData.getAttributes(); + + // Convert attribute keys by replacing "." with "_" for Cloud Monitoring compatibility. + for (AttributeKey key : attributes.asMap().keySet()) { + metricBuilder.putLabels(key.getKey().replace(".", "_"), String.valueOf(attributes.get(key))); + } + + // Attach common client attributes (service, client_uid) to each metric. + clientAttributes.forEach(metricBuilder::putLabels); + + builder.setResource(monitoredResourceBuilder.build()); + builder.setMetric(metricBuilder.build()); + + // Define the time interval for the metric point. + TimeInterval timeInterval = + TimeInterval.newBuilder() + .setStartTime( + // For GAUGE metrics, start and end time are identical. + metricKind == MetricKind.GAUGE + ? Timestamps.fromNanos(pointData.getEpochNanos()) + : Timestamps.fromNanos(pointData.getStartEpochNanos())) + .setEndTime(Timestamps.fromNanos(pointData.getEpochNanos())) + .build(); + + builder.addPoints(createPoint(metricData.getType(), pointData, timeInterval)); + + return builder.build(); + } + + /** Maps OpenTelemetry metric type to Cloud Monitoring {@link MetricKind}. */ + private static MetricKind convertMetricKind(MetricData metricData) { + switch (metricData.getType()) { + case HISTOGRAM: + return convertHistogramType(metricData.getHistogramData()); + case LONG_GAUGE: + case DOUBLE_GAUGE: + return GAUGE; + case LONG_SUM: + return convertSumDataType(metricData.getLongSumData()); + case DOUBLE_SUM: + return convertSumDataType(metricData.getDoubleSumData()); + default: + return UNRECOGNIZED; + } + } + + /** + * Returns {@link com.google.api.MetricDescriptor.MetricKind#CUMULATIVE} for cumulative + * histograms, or {@link com.google.api.MetricDescriptor.MetricKind#UNRECOGNIZED} for delta + * histograms. Cloud Monitoring only accepts cumulative histograms; delta histograms from + * short-lived OTel SDK instances would produce incomplete data and are intentionally dropped. + */ + private static MetricKind convertHistogramType(HistogramData histogramData) { + if (histogramData.getAggregationTemporality() == AggregationTemporality.CUMULATIVE) { + return CUMULATIVE; + } + return UNRECOGNIZED; + } + + /** + * Maps an OTel {@link SumData} to a Cloud Monitoring {@link MetricKind}. + * + *

Non-monotonic sums (values that can decrease) are mapped to {@code GAUGE} because Cloud + * Monitoring {@code CUMULATIVE} metrics must be strictly monotonically increasing. Monotonic + * cumulative sums map to {@code CUMULATIVE}; delta sums are not supported and return {@code + * UNRECOGNIZED}. + */ + private static MetricKind convertSumDataType(SumData sum) { + // Non-monotonic sums are treated as GAUGE. + if (!sum.isMonotonic()) { + return GAUGE; + } + if (sum.getAggregationTemporality() == AggregationTemporality.CUMULATIVE) { + return CUMULATIVE; + } + return UNRECOGNIZED; + } + + /** Maps OpenTelemetry metric data type to Cloud Monitoring {@link ValueType}. */ + private static ValueType convertValueType(MetricDataType metricDataType) { + switch (metricDataType) { + case LONG_GAUGE: + case LONG_SUM: + return INT64; + case DOUBLE_GAUGE: + case DOUBLE_SUM: + return DOUBLE; + case HISTOGRAM: + return DISTRIBUTION; + default: + return ValueType.UNRECOGNIZED; + } + } + + /** Creates a Cloud Monitoring {@link Point} from OpenTelemetry {@link PointData}. */ + private static Point createPoint( + MetricDataType type, PointData pointData, TimeInterval timeInterval) { + Point.Builder builder = Point.newBuilder().setInterval(timeInterval); + switch (type) { + case HISTOGRAM: + return builder + .setValue( + TypedValue.newBuilder() + .setDistributionValue(convertHistogramData((HistogramPointData) pointData)) + .build()) + .build(); + case DOUBLE_GAUGE: + case DOUBLE_SUM: + return builder + .setValue( + TypedValue.newBuilder() + .setDoubleValue(((DoublePointData) pointData).getValue()) + .build()) + .build(); + case LONG_GAUGE: + case LONG_SUM: + return builder + .setValue(TypedValue.newBuilder().setInt64Value(((LongPointData) pointData).getValue())) + .build(); + default: + logger.log(Level.WARNING, "unsupported metric type"); + return builder.build(); + } + } + + /** Converts OpenTelemetry histogram data to Cloud Monitoring {@link Distribution}. */ + private static Distribution convertHistogramData(HistogramPointData pointData) { + return Distribution.newBuilder() + .setCount(pointData.getCount()) + .setMean(pointData.getCount() == 0L ? 0.0D : pointData.getSum() / pointData.getCount()) + .setBucketOptions( + BucketOptions.newBuilder() + .setExplicitBuckets(Explicit.newBuilder().addAllBounds(pointData.getBoundaries()))) + .addAllBucketCounts(pointData.getCounts()) + .build(); + } +} diff --git a/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DatastoreMetricsRecorder.java b/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DatastoreMetricsRecorder.java index e1e18b3104f6..b3eda8632be9 100644 --- a/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DatastoreMetricsRecorder.java +++ b/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/DatastoreMetricsRecorder.java @@ -22,7 +22,11 @@ import com.google.cloud.datastore.DatastoreOptions; import io.opentelemetry.api.GlobalOpenTelemetry; import io.opentelemetry.api.OpenTelemetry; +import java.util.ArrayList; +import java.util.List; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import javax.annotation.Nonnull; /** @@ -39,6 +43,21 @@ @InternalExtensionOnly public interface DatastoreMetricsRecorder extends MetricsRecorder { + Logger logger = Logger.getLogger(DatastoreMetricsRecorder.class.getName()); + + /** + * Releases any resources held by this recorder. + * + *

For built-in recorders that own a private {@link io.opentelemetry.sdk.OpenTelemetrySdk} + * instance, this will flush and shut down the underlying {@link + * io.opentelemetry.sdk.metrics.SdkMeterProvider}. For recorders backed by a user-provided {@link + * io.opentelemetry.api.OpenTelemetry} instance, this is a no-op since the caller owns that + * instance's lifecycle. + * + *

This method should be called from {@link com.google.cloud.datastore.DatastoreImpl#close()}. + */ + default void close() {} + /** Records the total latency of a transaction in milliseconds. */ void recordTransactionLatency(double latencyMs, Map attributes); @@ -49,25 +68,67 @@ public interface DatastoreMetricsRecorder extends MetricsRecorder { * Returns a {@link DatastoreMetricsRecorder} instance based on the provided {@link * DatastoreOptions}. * - *

If the user has enabled metrics and provided an {@link OpenTelemetry} instance (or {@link - * GlobalOpenTelemetry} is used as fallback), an {@link OpenTelemetryDatastoreMetricsRecorder} is - * returned. Otherwise a {@link NoOpDatastoreMetricsRecorder} is returned. + *

This factory method creates a {@link CompositeDatastoreMetricsRecorder} that delegates to + * multiple backends: + * + *

    + *
  • Default provider: Always exports metrics to Google Cloud Monitoring via a + * privately-constructed {@link io.opentelemetry.sdk.OpenTelemetrySdk} with a {@link + * DatastoreCloudMonitoringExporter}, unless explicitly disabled via {@link + * DatastoreOpenTelemetryOptions#isExportBuiltinMetricsToGoogleCloudMonitoring()}. + *
  • Custom provider: If the user has enabled metrics and provided an {@link + * OpenTelemetry} instance (or {@link GlobalOpenTelemetry} is used as fallback), metrics are + * also recorded to that backend. + *
* - * @param datastoreOptions the {@link DatastoreOptions} configuring the Datastore client - * @return a {@link DatastoreMetricsRecorder} for the configured backend + * @param options the {@link DatastoreOptions} configuring the Datastore client + * @param builtInOtel the {@link OpenTelemetry} built in Otel object + * @return a {@link DatastoreMetricsRecorder} that fans out to all configured backends */ - static DatastoreMetricsRecorder getInstance(@Nonnull DatastoreOptions datastoreOptions) { - DatastoreOpenTelemetryOptions otelOptions = datastoreOptions.getOpenTelemetryOptions(); + static DatastoreMetricsRecorder getInstance( + @Nonnull DatastoreOptions options, OpenTelemetry builtInOtel) { + List recorders = new ArrayList<>(); + + // No need to send metrics when using an emulator + String emulatorHost = System.getenv(DatastoreOptions.LOCAL_HOST_ENV_VAR); + boolean emulatorEnabled = emulatorHost != null && !emulatorHost.isEmpty(); + + if (emulatorEnabled) { + logger.log(Level.FINE, "Emulator detected in Datastore. Metrics are not being recorded."); + return new CompositeDatastoreMetricsRecorder(recorders); + } + + DatastoreOpenTelemetryOptions otelOptions = options.getOpenTelemetryOptions(); + + // When using a local emulator, there is no need to configure a built-in Otel instance + if (otelOptions.isExportBuiltinMetricsToGoogleCloudMonitoring()) { + try { + if (builtInOtel != null) { + recorders.add( + new OpenTelemetryDatastoreMetricsRecorder( + builtInOtel, TelemetryConstants.METRIC_PREFIX)); + } + } catch (Exception e) { + logger.log( + Level.WARNING, + "Failed to create built-in metrics provider for Cloud Monitoring exporting.", + e); + } + } + // If the user has enabled metrics, we will attempt to export metrics to their + // configured backend. We will first check their supplied Otel object, then check + // the global Otel config. + // Note: Metrics will not be sent if an emulator is enabled. if (otelOptions.isMetricsEnabled()) { OpenTelemetry customOtel = otelOptions.getOpenTelemetry(); if (customOtel == null) { customOtel = GlobalOpenTelemetry.get(); } - return new OpenTelemetryDatastoreMetricsRecorder( - customOtel, TelemetryConstants.METRIC_PREFIX); + recorders.add( + new OpenTelemetryDatastoreMetricsRecorder(customOtel, TelemetryConstants.METRIC_PREFIX)); } - return new NoOpDatastoreMetricsRecorder(); + return new CompositeDatastoreMetricsRecorder(recorders); } } diff --git a/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/NoOpDatastoreMetricsRecorder.java b/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/NoOpDatastoreMetricsRecorder.java deleted file mode 100644 index a3cf325acc93..000000000000 --- a/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/NoOpDatastoreMetricsRecorder.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright 2026 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.cloud.datastore.telemetry; - -import com.google.api.core.InternalApi; -import java.util.Map; - -/** - * A no-op implementation of {@link DatastoreMetricsRecorder}. - * - *

Used to stub out metrics instrumentation when metrics are disabled or when no valid recorder - * could be initialized. - * - *

WARNING: This class is intended for internal use only. It was made public to be used across - * packages as a default. It should not be used by external customers and its API may change without - * notice. - */ -@InternalApi -public class NoOpDatastoreMetricsRecorder implements DatastoreMetricsRecorder { - - @Override - public void recordTransactionLatency(double latencyMs, Map attributes) { - /* No-Op OTel Operation */ - } - - @Override - public void recordTransactionAttemptCount(long count, Map attributes) { - /* No-Op OTel Operation */ - } - - @Override - public void recordAttemptLatency(double latencyMs, Map attributes) { - /* No-Op OTel Operation */ - } - - @Override - public void recordAttemptCount(long count, Map attributes) { - /* No-Op OTel Operation */ - } - - @Override - public void recordOperationLatency(double latencyMs, Map attributes) { - /* No-Op OTel Operation */ - } - - @Override - public void recordOperationCount(long count, Map attributes) { - /* No-Op OTel Operation */ - } -} diff --git a/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/OpenTelemetryDatastoreMetricsRecorder.java b/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/OpenTelemetryDatastoreMetricsRecorder.java index 550dc1df9a7d..c1380736ded5 100644 --- a/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/OpenTelemetryDatastoreMetricsRecorder.java +++ b/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/OpenTelemetryDatastoreMetricsRecorder.java @@ -23,7 +23,10 @@ import io.opentelemetry.api.metrics.DoubleHistogram; import io.opentelemetry.api.metrics.LongCounter; import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.sdk.OpenTelemetrySdk; import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; import javax.annotation.Nonnull; /** @@ -43,17 +46,24 @@ class OpenTelemetryDatastoreMetricsRecorder extends OpenTelemetryMetricsRecorder implements DatastoreMetricsRecorder { + private static final Logger logger = + Logger.getLogger(OpenTelemetryDatastoreMetricsRecorder.class.getName()); + private final OpenTelemetry openTelemetry; // Datastore-specific transaction metrics (registered under the Datastore meter). private final DoubleHistogram transactionLatency; private final LongCounter transactionAttemptCount; - // Note: Standard GAX RPC metrics (operation_latency, attempt_latency, etc.) are handled by the - // base OpenTelemetryMetricsRecorder class. Those metrics are inherited from the parent classes. - // However, the internal metrics expect plural suffixes (e.g. `latencies` instead of `latency`). - // The discrepancy between the singular GAX names and the plural internal Cloud Monitoring names - // is handled by configuring OpenTelemetry Views. + /** + * Creates a recorder based on the Otel configuration. + * + *

Note: Standard GAX RPC metrics (operation_latency, attempt_latency, etc.) are handled by the + * base OpenTelemetryMetricsRecorder class. Those metrics are inherited from the parent classes. + * However, the internal metrics expect plural suffixes (e.g. `latencies` instead of `latency`). + * The discrepancy between the singular GAX names and the plural internal Cloud Monitoring names + * is handled by configuring OpenTelemetry Views. + */ OpenTelemetryDatastoreMetricsRecorder(@Nonnull OpenTelemetry openTelemetry, String metricPrefix) { super(openTelemetry, metricPrefix); this.openTelemetry = openTelemetry; @@ -78,6 +88,22 @@ OpenTelemetry getOpenTelemetry() { return openTelemetry; } + /** + * Closes this recorder. If this recorder owns the underlying {@link OpenTelemetry} instance + * (i.e., it was created by the built-in metrics provider), it will be shut down, flushing any + * pending metrics. If the instance was provided by the user, this is a no-op. + */ + @Override + public void close() { + if (openTelemetry instanceof OpenTelemetrySdk) { + try { + ((OpenTelemetrySdk) openTelemetry).close(); + } catch (Exception e) { + logger.log(Level.WARNING, "Failed to close built-in OpenTelemetry SDK instance.", e); + } + } + } + @Override public void recordTransactionLatency(double latencyMs, Map attributes) { transactionLatency.record(latencyMs, toOtelAttributes(attributes)); diff --git a/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TelemetryConstants.java b/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TelemetryConstants.java index 802b35ff456f..f61bee5cb309 100644 --- a/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TelemetryConstants.java +++ b/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TelemetryConstants.java @@ -17,8 +17,10 @@ package com.google.cloud.datastore.telemetry; import com.google.api.core.InternalApi; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import io.opentelemetry.api.common.AttributeKey; +import java.util.Map; import java.util.Set; /** @@ -31,17 +33,46 @@ @InternalApi public class TelemetryConstants { - // The Firestore namespace has not been deployed yet. Must target the custom namespace - // until this is implemented. + // TODO(b/405457573): The Firestore namespace has not been deployed yet. Must target the + // custom namespace until this is implemented. public static final String METRIC_PREFIX = "custom.googleapis.com/internal/client"; public static final String DATASTORE_METER_NAME = "java-datastore"; + // Short names used to build GAX_METRICS and metric full-path constants below. + public static final String METRIC_NAME_SHORT_OPERATION_LATENCY = "operation_latency"; + public static final String METRIC_NAME_SHORT_ATTEMPT_LATENCY = "attempt_latency"; + public static final String METRIC_NAME_SHORT_OPERATION_COUNT = "operation_count"; + public static final String METRIC_NAME_SHORT_ATTEMPT_COUNT = "attempt_count"; + + public static final String METRIC_NAME_SHORT_OPERATION_LATENCIES = "operation_latencies"; + public static final String METRIC_NAME_SHORT_ATTEMPT_LATENCIES = "attempt_latencies"; + + /** + * Mapping from the singular metric names recorded by the GAX library to the pluralized metric + * names required by the internal Cloud Monitoring metric descriptors. + */ + public static final Map GAX_METRIC_NAME_MAP = + ImmutableMap.of( + METRIC_NAME_SHORT_OPERATION_LATENCY, METRIC_NAME_SHORT_OPERATION_LATENCIES, + METRIC_NAME_SHORT_ATTEMPT_LATENCY, METRIC_NAME_SHORT_ATTEMPT_LATENCIES, + METRIC_NAME_SHORT_OPERATION_COUNT, METRIC_NAME_SHORT_OPERATION_COUNT, + METRIC_NAME_SHORT_ATTEMPT_COUNT, METRIC_NAME_SHORT_ATTEMPT_COUNT); + + // Short metric names (without prefix) for the four metrics recorded by the GAX layer. + // Used by DatastoreBuiltInMetricsView to register OTel views that capture and rename + // GAX-emitted metrics for the built-in Cloud Monitoring export pipeline. + public static final Set GAX_METRICS = GAX_METRIC_NAME_MAP.keySet(); + + // The subset of GAX_METRICS that are histograms (e.g. latency metrics) + public static final Set GAX_HISTOGRAM_METRICS = + ImmutableSet.of(METRIC_NAME_SHORT_OPERATION_LATENCY, METRIC_NAME_SHORT_ATTEMPT_LATENCY); + // Monitored resource type for Cloud Monitoring public static final String DATASTORE_RESOURCE_TYPE = "global"; // Resource label keys for the monitored resource - // The Firestore namespace has not been deployed yet. Must target the global - // Monitored Resource until this is implemented. + // TODO(b/405457573): The Firestore namespace has not been deployed yet. Must + // target the global Monitored Resource until this is implemented. public static final String RESOURCE_LABEL_PROJECT_ID = "project_id"; public static final String RESOURCE_LABEL_DATABASE_ID = "database_id"; public static final String RESOURCE_LABEL_LOCATION = "location"; @@ -109,26 +140,32 @@ public class TelemetryConstants { /** * Metric name for the total latency of an operation (one full RPC call including retries). * - *

The plural form ({@code operation_latencies}) is intentional: it matches the internal Cloud - * Monitoring metric descriptor name. {@link OpenTelemetryDatastoreMetricsRecorder} overrides the - * inherited GAX method to record to this name rather than the singular GAX default. + *

Singular/plural naming: This constant uses the singular form ({@code + * operation_latency}) to match the name recorded by the GAX layer and used by custom OTel + * backends (e.g., an in-memory reader in tests). The built-in Cloud Monitoring export path + * renames the metric to the plural form ({@code operation_latencies}) via an OTel View in {@link + * DatastoreBuiltInMetricsView}. This split is intentional and consistent with other Google Cloud + * client libraries. */ - public static final String METRIC_NAME_OPERATION_LATENCY = METRIC_PREFIX + "/operation_latencies"; + public static final String METRIC_NAME_OPERATION_LATENCY = + METRIC_PREFIX + "/" + METRIC_NAME_SHORT_OPERATION_LATENCY; /** * Metric name for the latency of a single RPC attempt. * - *

The plural form ({@code attempt_latencies}) is intentional: it matches the internal Cloud - * Monitoring metric descriptor name. {@link OpenTelemetryDatastoreMetricsRecorder} overrides the - * inherited GAX method to record to this name rather than the singular GAX default. + *

See {@link #METRIC_NAME_OPERATION_LATENCY} for the singular/plural naming rationale. The + * built-in Cloud Monitoring export renames this to {@code attempt_latencies} via an OTel View. */ - public static final String METRIC_NAME_ATTEMPT_LATENCY = METRIC_PREFIX + "/attempt_latencies"; + public static final String METRIC_NAME_ATTEMPT_LATENCY = + METRIC_PREFIX + "/" + METRIC_NAME_SHORT_ATTEMPT_LATENCY; /** Metric name for the count of operations. */ - public static final String METRIC_NAME_OPERATION_COUNT = METRIC_PREFIX + "/operation_count"; + public static final String METRIC_NAME_OPERATION_COUNT = + METRIC_PREFIX + "/" + METRIC_NAME_SHORT_OPERATION_COUNT; /** Metric name for the count of RPC attempts. */ - public static final String METRIC_NAME_ATTEMPT_COUNT = METRIC_PREFIX + "/attempt_count"; + public static final String METRIC_NAME_ATTEMPT_COUNT = + METRIC_PREFIX + "/" + METRIC_NAME_SHORT_ATTEMPT_COUNT; static final String METHOD_SERVICE_NAME = "Datastore"; diff --git a/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TelemetryUtils.java b/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TelemetryUtils.java index 79fc8f0ec2af..4733c3fbcf85 100644 --- a/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TelemetryUtils.java +++ b/java-datastore/google-cloud-datastore/src/main/java/com/google/cloud/datastore/telemetry/TelemetryUtils.java @@ -42,11 +42,15 @@ private TelemetryUtils() {} * @param status The status of the operation or attempt. * @return The map of attributes. */ - public static Map buildMetricAttributes(String methodName, String status) { + public static Map buildMetricAttributes( + String methodName, String status, String databaseId) { Map attributes = new HashMap<>(); attributes.put(TelemetryConstants.ATTRIBUTES_KEY_METHOD, methodName); attributes.put(TelemetryConstants.ATTRIBUTES_KEY_STATUS, status); attributes.put(TelemetryConstants.ATTRIBUTES_KEY_SERVICE, TelemetryConstants.SERVICE_VALUE); + if (databaseId != null) { + attributes.put(TelemetryConstants.ATTRIBUTES_KEY_DATABASE_ID, databaseId); + } return attributes; } @@ -65,9 +69,10 @@ public static void recordOperationMetrics( DatastoreMetricsRecorder metricsRecorder, Stopwatch operationStopwatch, String methodName, - String status) { + String status, + String databaseId) { if (methodName != null) { - Map attributes = buildMetricAttributes(methodName, status); + Map attributes = buildMetricAttributes(methodName, status, databaseId); metricsRecorder.recordOperationLatency( operationStopwatch.elapsed(TimeUnit.MILLISECONDS), attributes); metricsRecorder.recordOperationCount(1, attributes); @@ -87,7 +92,10 @@ public static void recordOperationMetrics( * @return A wrapped callable that includes attempt-level metrics recording. */ public static Callable attemptMetricsCallable( - Callable callable, DatastoreMetricsRecorder metricsRecorder, String methodName) { + Callable callable, + DatastoreMetricsRecorder metricsRecorder, + String methodName, + String databaseId) { return () -> { Stopwatch stopwatch = Stopwatch.createStarted(); String status = StatusCode.Code.UNKNOWN.toString(); @@ -99,7 +107,7 @@ public static Callable attemptMetricsCallable( status = DatastoreException.extractStatusCode(e); throw e; } finally { - Map attributes = buildMetricAttributes(methodName, status); + Map attributes = buildMetricAttributes(methodName, status, databaseId); metricsRecorder.recordAttemptLatency(stopwatch.elapsed(TimeUnit.MILLISECONDS), attributes); metricsRecorder.recordAttemptCount(1, attributes); } diff --git a/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/DatastoreImplMetricsTest.java b/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/DatastoreImplMetricsTest.java index 8a611526f0c3..16c27dac97fa 100644 --- a/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/DatastoreImplMetricsTest.java +++ b/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/DatastoreImplMetricsTest.java @@ -25,6 +25,8 @@ import com.google.cloud.datastore.spi.v1.DatastoreRpc; import com.google.cloud.datastore.telemetry.DatastoreMetricsRecorder; import com.google.cloud.datastore.telemetry.TelemetryConstants; +import com.google.cloud.grpc.GrpcTransportOptions; +import com.google.cloud.http.HttpTransportOptions; import com.google.datastore.v1.BeginTransactionRequest; import com.google.datastore.v1.BeginTransactionResponse; import com.google.datastore.v1.CommitRequest; @@ -113,9 +115,9 @@ public void setUp() { .build()); if (TelemetryConstants.Transport.GRPC.equals(transport)) { - builder.setTransportOptions(com.google.cloud.grpc.GrpcTransportOptions.newBuilder().build()); + builder.setTransportOptions(GrpcTransportOptions.newBuilder().build()); } else { - builder.setTransportOptions(com.google.cloud.http.HttpTransportOptions.newBuilder().build()); + builder.setTransportOptions(HttpTransportOptions.newBuilder().build()); } DatastoreOptions datastoreOptions = builder.build(); diff --git a/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/DatastoreOptionsTest.java b/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/DatastoreOptionsTest.java index c5e5186200ac..930526a88788 100644 --- a/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/DatastoreOptionsTest.java +++ b/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/DatastoreOptionsTest.java @@ -310,4 +310,15 @@ public void testToBuilder() { assertNotEquals(original, newOptions); assertNotEquals(original.hashCode(), newOptions.hashCode()); } + + @Test + public void builtInMetricsExport_isDisabledByDefault() { + DatastoreOptions defaultOptions = + DatastoreOptions.newBuilder().setProjectId(PROJECT_ID).build(); + assertThat( + defaultOptions + .getOpenTelemetryOptions() + .isExportBuiltinMetricsToGoogleCloudMonitoring()) + .isFalse(); + } } diff --git a/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/ITDatastoreBuiltInAndCustomMetrics.java b/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/ITDatastoreBuiltInAndCustomMetrics.java new file mode 100644 index 000000000000..655095202176 --- /dev/null +++ b/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/ITDatastoreBuiltInAndCustomMetrics.java @@ -0,0 +1,353 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.datastore; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; +import static org.junit.Assume.assumeNotNull; + +import com.google.cloud.TransportOptions; +import com.google.cloud.datastore.telemetry.TelemetryConstants; +import com.google.cloud.grpc.GrpcTransportOptions; +import com.google.cloud.http.HttpTransportOptions; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.data.HistogramPointData; +import io.opentelemetry.sdk.metrics.data.LongPointData; +import io.opentelemetry.sdk.metrics.data.MetricData; +import io.opentelemetry.sdk.metrics.data.PointData; +import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader; +import java.util.Arrays; +import java.util.Collection; +import java.util.Optional; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +/** + * Integration test that verifies a user-configured OpenTelemetry backend (backed by {@link + * InMemoryMetricReader}) correctly captures metrics emitted by the Datastore SDK. + * + *

What this test covers

+ * + *
    + *
  1. Custom OTel backend (in-memory) — {@link InMemoryMetricReader} captures every metric + * emitted by the Datastore SDK. After each operation the test collects all metrics + * synchronously and asserts that expected names, types, and attributes are present. Because + * the in-memory reader is under full test control, these assertions are deterministic. + *
+ * + *

Following the strategy used in tracing tests, this test relies on {@link InMemoryMetricReader} + * to verify that metrics are correctly generated with expected attributes. + */ +@RunWith(Parameterized.class) +@SuppressWarnings("checkstyle:abbreviationaswordinname") +public class ITDatastoreBuiltInAndCustomMetrics { + + private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); + private static final String DATABASE_ID = + System.getenv().getOrDefault("DATASTORE_DATABASE_ID", ""); + private boolean isDatastoreClosed = false; + + private final TransportOptions transportOptions; + + public ITDatastoreBuiltInAndCustomMetrics(TransportOptions transportOptions) { + this.transportOptions = transportOptions; + } + + @Parameterized.Parameters(name = "transport: {0}") + public static Iterable data() { + return Arrays.asList( + new Object[][] { + {DatastoreOptions.getDefaultGrpcTransportOptions()}, + {DatastoreOptions.getDefaultHttpTransportOptions()} + }); + } + + /** + * Delta temporality is used so that each {@link InMemoryMetricReader#collectAllMetrics()} call + * returns only the new data points recorded since the last collection, and automatically resets + * the in-memory state. This makes per-test assertions independent and avoids cross-test + * contamination. + */ + private InMemoryMetricReader metricReader; + + private SdkMeterProvider customMeterProvider; + private Datastore datastore; + private String kind; + + @Before + public void setUp() { + // Skip the test gracefully if the required environment variable is not configured. + assumeNotNull("GOOGLE_CLOUD_PROJECT must be set to run this IT test", PROJECT_ID); + + kind = "Kind-" + java.util.UUID.randomUUID().toString().substring(0, 8); + + // Build a user-configured OTel backend that records to an in-memory reader for assertions. + metricReader = InMemoryMetricReader.createDelta(); + customMeterProvider = SdkMeterProvider.builder().registerMetricReader(metricReader).build(); + OpenTelemetrySdk customOtel = + OpenTelemetrySdk.builder().setMeterProvider(customMeterProvider).build(); + + // Do not enable setExportBuiltinMetricsToGoogleCloudMonitoring(false) for this IT + // as we will only rely on in-memory to collect the metrics for this test + DatastoreOptions.Builder builder = + DatastoreOptions.newBuilder() + .setProjectId(PROJECT_ID) + .setDatabaseId(DATABASE_ID) + .setOpenTelemetryOptions( + DatastoreOpenTelemetryOptions.newBuilder() + .setMetricsEnabled(true) + .setOpenTelemetry(customOtel) + .setExportBuiltinMetricsToGoogleCloudMonitoring(false) + .build()); + + if (transportOptions instanceof GrpcTransportOptions) { + builder.setTransportOptions((GrpcTransportOptions) transportOptions); + } else { + builder.setTransportOptions((HttpTransportOptions) transportOptions); + } + + datastore = builder.build().getService(); + + // Drain any metrics emitted during client initialisation so test assertions only capture + // data from the operations performed within the test method itself. + metricReader.collectAllMetrics(); + } + + @After + public void tearDown() throws Exception { + if (datastore != null && !isDatastoreClosed) { + Key key = datastore.newKeyFactory().setKind(kind).newKey("metrics-it-entity"); + try { + datastore.delete(key); + } catch (Exception e) { + // ignore if fails, we are cleaning up + } + datastore.close(); + } + if (customMeterProvider != null) { + customMeterProvider.close(); + } + } + + /** + * Verifies that a transaction operation records {@code transaction_latency} and {@code + * transaction_attempt_count} metrics in the custom (in-memory) OTel backend. + * + *

These are Datastore-specific metrics emitted by the SDK layer (not the GAX layer), so they + * validate that the Datastore-level recording path is working end-to-end. + */ + @Test + public void transactionOperation_recordsTransactionMetricsInCustomBackend() { + Key key = datastore.newKeyFactory().setKind(kind).newKey("metrics-it-entity"); + Entity initial = Entity.newBuilder(key).set("value", 0L).build(); + datastore.put(initial); + + datastore.runInTransaction( + tx -> { + Entity current = tx.get(key); + tx.put(Entity.newBuilder(current).set("value", current.getLong("value") + 1).build()); + return null; + }); + + Collection metrics = metricReader.collectAllMetrics(); + + // --- transaction_latency --- + Optional transactionLatencyMetric = + findMetric(metrics, TelemetryConstants.METRIC_NAME_TRANSACTION_LATENCY); + assertWithMessage("transaction_latency metric was recorded") + .that(transactionLatencyMetric.isPresent()) + .isTrue(); + + HistogramPointData latencyPoint = + transactionLatencyMetric.get().getHistogramData().getPoints().stream() + .findFirst() + .orElse(null); + assertThat(latencyPoint).isNotNull(); + assertThat(latencyPoint.getCount()).isEqualTo(1); + assertWithMessage("status=OK on transaction_latency") + .that( + dataContainsStringAttribute( + latencyPoint, TelemetryConstants.ATTRIBUTES_KEY_STATUS, "OK")) + .isTrue(); + assertWithMessage("method=Transaction.Run on transaction_latency") + .that( + dataContainsStringAttribute( + latencyPoint, + TelemetryConstants.ATTRIBUTES_KEY_METHOD, + TelemetryConstants.METHOD_TRANSACTION_RUN)) + .isTrue(); + assertWithMessage("database_id attribute on transaction_latency") + .that( + dataContainsStringAttribute( + latencyPoint, TelemetryConstants.ATTRIBUTES_KEY_DATABASE_ID, DATABASE_ID)) + .isTrue(); + + // --- transaction_attempt_count --- + Optional attemptCountMetric = + findMetric(metrics, TelemetryConstants.METRIC_NAME_TRANSACTION_ATTEMPT_COUNT); + assertWithMessage("transaction_attempt_count metric was recorded") + .that(attemptCountMetric.isPresent()) + .isTrue(); + + LongPointData attemptPoint = + attemptCountMetric.get().getLongSumData().getPoints().stream().findFirst().orElse(null); + assertThat(attemptPoint).isNotNull(); + assertThat(attemptPoint.getValue()).isEqualTo(1); + assertWithMessage("status=OK on transaction_attempt_count") + .that( + dataContainsStringAttribute( + attemptPoint, TelemetryConstants.ATTRIBUTES_KEY_STATUS, "OK")) + .isTrue(); + assertWithMessage("method=Transaction.Commit on transaction_attempt_count") + .that( + dataContainsStringAttribute( + attemptPoint, + TelemetryConstants.ATTRIBUTES_KEY_METHOD, + TelemetryConstants.METHOD_TRANSACTION_COMMIT)) + .isTrue(); + } + + /** + * Verifies that a lookup RPC records {@code operation_latency}, {@code attempt_latency}, {@code + * operation_count}, and {@code attempt_count} in the custom (in-memory) OTel backend. + * + *

These GAX-layer metrics are recorded by {@link com.google.cloud.datastore.telemetry + * .TelemetryUtils} and exercise the code path that was previously gated behind a {@code !GRPC} + * transport guard (now removed). This assertion confirms that all four RPC-level metrics are + * recorded regardless of transport. + */ + @Test + public void lookupRpc_recordsGaxMetricsInCustomBackend() { + // Issue a lookup for a key that does not exist; it still produces all four RPC-level metrics. + Key key = datastore.newKeyFactory().setKind(kind).newKey("does-not-exist"); + datastore.get(key); + + Collection metrics = metricReader.collectAllMetrics(); + + // --- operation_latency --- + Optional operationLatencyMetric = + findMetric(metrics, TelemetryConstants.METRIC_NAME_OPERATION_LATENCY); + assertWithMessage("operation_latency metric was recorded") + .that(operationLatencyMetric.isPresent()) + .isTrue(); + HistogramPointData opPoint = + operationLatencyMetric.get().getHistogramData().getPoints().stream() + .filter( + p -> + dataContainsStringAttribute( + p, + TelemetryConstants.ATTRIBUTES_KEY_METHOD, + TelemetryConstants.METHOD_LOOKUP)) + .findFirst() + .orElse(null); + assertWithMessage("operation_latency point for Lookup method").that(opPoint).isNotNull(); + assertThat(opPoint.getCount()).isAtLeast(1); + assertWithMessage("status=OK on operation_latency") + .that(dataContainsStringAttribute(opPoint, TelemetryConstants.ATTRIBUTES_KEY_STATUS, "OK")) + .isTrue(); + + // --- attempt_latency --- + Optional attemptLatencyMetric = + findMetric(metrics, TelemetryConstants.METRIC_NAME_ATTEMPT_LATENCY); + assertWithMessage("attempt_latency metric was recorded") + .that(attemptLatencyMetric.isPresent()) + .isTrue(); + + // --- operation_count --- + Optional operationCountMetric = + findMetric(metrics, TelemetryConstants.METRIC_NAME_OPERATION_COUNT); + assertWithMessage("operation_count metric was recorded") + .that(operationCountMetric.isPresent()) + .isTrue(); + + // --- attempt_count --- + Optional attemptCountMetric = + findMetric(metrics, TelemetryConstants.METRIC_NAME_ATTEMPT_COUNT); + assertWithMessage("attempt_count metric was recorded") + .that(attemptCountMetric.isPresent()) + .isTrue(); + } + + /** + * Verifies that all six expected metrics appear in the custom (in-memory) OTel backend after a + * combined transaction-plus-lookup workload. This is the primary "composite" scenario: both the + * SDK-layer metrics (transaction) and the GAX-layer metrics (operation/attempt) are captured in a + * single test run, confirming the full fan-out routing is correct. + */ + @Test + public void combinedWorkload_recordsAllSixMetricsInCustomBackend() { + Key key = datastore.newKeyFactory().setKind(kind).newKey("metrics-it-entity"); + Entity initial = Entity.newBuilder(key).set("value", 0L).build(); + + // Step 1: plain put (records operation_latency, attempt_latency, etc.) + datastore.put(initial); + + // Step 2: transaction (records transaction_latency, transaction_attempt_count on top) + datastore.runInTransaction( + tx -> { + Entity current = tx.get(key); + tx.put(Entity.newBuilder(current).set("value", current.getLong("value") + 1).build()); + return null; + }); + + // Step 3: standalone lookup + datastore.get(key); + + Collection metrics = metricReader.collectAllMetrics(); + + assertWithMessage("operation_latency present") + .that(findMetric(metrics, TelemetryConstants.METRIC_NAME_OPERATION_LATENCY).isPresent()) + .isTrue(); + assertWithMessage("attempt_latency present") + .that(findMetric(metrics, TelemetryConstants.METRIC_NAME_ATTEMPT_LATENCY).isPresent()) + .isTrue(); + assertWithMessage("operation_count present") + .that(findMetric(metrics, TelemetryConstants.METRIC_NAME_OPERATION_COUNT).isPresent()) + .isTrue(); + assertWithMessage("attempt_count present") + .that(findMetric(metrics, TelemetryConstants.METRIC_NAME_ATTEMPT_COUNT).isPresent()) + .isTrue(); + assertWithMessage("transaction_latency present") + .that(findMetric(metrics, TelemetryConstants.METRIC_NAME_TRANSACTION_LATENCY).isPresent()) + .isTrue(); + assertWithMessage("transaction_attempt_count present") + .that( + findMetric(metrics, TelemetryConstants.METRIC_NAME_TRANSACTION_ATTEMPT_COUNT) + .isPresent()) + .isTrue(); + } + + // --------------------------------------------------------------------------- + // Helpers + // --------------------------------------------------------------------------- + + private static Optional findMetric( + Collection metrics, String metricName) { + return metrics.stream().filter(m -> m.getName().equals(metricName)).findFirst(); + } + + private static boolean dataContainsStringAttribute( + PointData point, String attributeKey, String expectedValue) { + String actual = point.getAttributes().get(AttributeKey.stringKey(attributeKey)); + return expectedValue.equals(actual); + } +} diff --git a/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/BuiltInDatastoreMetricsProviderTest.java b/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/BuiltInDatastoreMetricsProviderTest.java new file mode 100644 index 000000000000..655be0b6f84d --- /dev/null +++ b/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/BuiltInDatastoreMetricsProviderTest.java @@ -0,0 +1,140 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.datastore.telemetry; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.auth.CredentialTypeForMetrics; +import com.google.auth.Credentials; +import com.google.cloud.NoCredentials; +import com.google.cloud.datastore.DatastoreOptions; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import java.util.Map; +import org.easymock.EasyMock; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class BuiltInDatastoreMetricsProviderTest { + + private static final String PROJECT_ID = "project-id"; + + @Test + public void testBuildClientAttributes() { + Map attributes = BuiltInDatastoreMetricsProvider.buildClientAttributes(); + assertThat(attributes).containsKey(TelemetryConstants.CLIENT_UID_KEY.getKey()); + assertThat(attributes.get(TelemetryConstants.CLIENT_UID_KEY.getKey())).isNotEmpty(); + assertThat(attributes).containsKey(TelemetryConstants.SERVICE_KEY.getKey()); + assertThat(attributes.get(TelemetryConstants.SERVICE_KEY.getKey())) + .isEqualTo(TelemetryConstants.SERVICE_VALUE); + } + + @Test + public void testCreateOpenTelemetry_returnsNoOp() { + DatastoreOptions options = + DatastoreOptions.newBuilder() + .setProjectId(PROJECT_ID) + .setDatabaseId("test-db") + .setCredentials(NoCredentials.getInstance()) + .build(); + OpenTelemetry otel = BuiltInDatastoreMetricsProvider.INSTANCE.createOpenTelemetry(options); + assertThat(otel).isInstanceOf(OpenTelemetry.noop().getClass()); + } + + @Test + public void testCreateOpenTelemetry_returnsNonNull() { + Credentials credentials = EasyMock.mock(Credentials.class); + EasyMock.expect(credentials.getMetricsCredentialType()) + .andReturn(CredentialTypeForMetrics.DO_NOT_SEND); + EasyMock.replay(credentials); + + DatastoreOptions options = + DatastoreOptions.newBuilder() + .setProjectId(PROJECT_ID) + .setDatabaseId("test-db") + .setCredentials(credentials) + .build(); + + OpenTelemetry otel = BuiltInDatastoreMetricsProvider.INSTANCE.createOpenTelemetry(options); + try { + assertThat(otel).isNotNull(); + } finally { + if (otel instanceof OpenTelemetrySdk) { + ((OpenTelemetrySdk) otel).close(); + } + } + } + + @Test + public void testCreateOpenTelemetry_eachCallReturnsDistinctInstance() { + Credentials credentials1 = EasyMock.mock(Credentials.class); + EasyMock.expect(credentials1.getMetricsCredentialType()) + .andReturn(CredentialTypeForMetrics.DO_NOT_SEND); + Credentials credentials2 = EasyMock.mock(Credentials.class); + EasyMock.expect(credentials2.getMetricsCredentialType()) + .andReturn(CredentialTypeForMetrics.DO_NOT_SEND); + EasyMock.replay(credentials1, credentials2); + + DatastoreOptions options1 = + DatastoreOptions.newBuilder() + .setProjectId(PROJECT_ID) + .setDatabaseId("test-db") + .setCredentials(credentials1) + .build(); + + DatastoreOptions options2 = + DatastoreOptions.newBuilder() + .setProjectId(PROJECT_ID) + .setDatabaseId("test-db") + .setCredentials(credentials2) + .build(); + + OpenTelemetry otel1 = BuiltInDatastoreMetricsProvider.INSTANCE.createOpenTelemetry(options1); + OpenTelemetry otel2 = BuiltInDatastoreMetricsProvider.INSTANCE.createOpenTelemetry(options2); + try { + assertThat(otel1).isNotNull(); + assertThat(otel2).isNotNull(); + assertThat(otel1).isNotSameInstanceAs(otel2); + } finally { + if (otel1 instanceof OpenTelemetrySdk) { + ((OpenTelemetrySdk) otel1).close(); + } + if (otel2 instanceof OpenTelemetrySdk) { + ((OpenTelemetrySdk) otel2).close(); + } + } + } + + @Test + public void testCreateOpenTelemetry_withEmulatorHostProperty_returnsNoOp() { + System.setProperty(DatastoreOptions.LOCAL_HOST_ENV_VAR, "localhost:8081"); + try { + DatastoreOptions options = + DatastoreOptions.newBuilder() + .setProjectId(PROJECT_ID) + .setDatabaseId("test-db") + .setCredentials(NoCredentials.getInstance()) + .build(); + OpenTelemetry otel = BuiltInDatastoreMetricsProvider.INSTANCE.createOpenTelemetry(options); + assertThat(otel).isInstanceOf(OpenTelemetry.noop().getClass()); + } finally { + System.clearProperty(DatastoreOptions.LOCAL_HOST_ENV_VAR); + } + } +} diff --git a/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/CompositeDatastoreMetricsRecorderTest.java b/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/CompositeDatastoreMetricsRecorderTest.java new file mode 100644 index 000000000000..d26f0f05b2f6 --- /dev/null +++ b/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/CompositeDatastoreMetricsRecorderTest.java @@ -0,0 +1,88 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.datastore.telemetry; + +import static com.google.common.truth.Truth.assertThat; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +@RunWith(JUnit4.class) +public class CompositeDatastoreMetricsRecorderTest { + + @Test + public void testCloseLIFOAndExceptionSafe() { + List closeOrder = new ArrayList<>(); + DatastoreMetricsRecorder recorder1 = new MockRecorder(1, closeOrder, false); + DatastoreMetricsRecorder recorder2 = new MockRecorder(2, closeOrder, true); // will throw + DatastoreMetricsRecorder recorder3 = new MockRecorder(3, closeOrder, false); + + CompositeDatastoreMetricsRecorder composite = + new CompositeDatastoreMetricsRecorder(Arrays.asList(recorder1, recorder2, recorder3)); + + composite.close(); + + // LIFO order means 3, then 2, then 1. + // Even though 2 throws, 1 should still be closed. + assertThat(closeOrder).containsExactly(3, 2, 1).inOrder(); + } + + private static class MockRecorder implements DatastoreMetricsRecorder { + private final int id; + private final List closeOrder; + private final boolean throwOnClose; + + MockRecorder(int id, List closeOrder, boolean throwOnClose) { + this.id = id; + this.closeOrder = closeOrder; + this.throwOnClose = throwOnClose; + } + + @Override + public void close() { + closeOrder.add(id); + if (throwOnClose) { + throw new RuntimeException("Mock exception on close"); + } + } + + @Override + public void recordTransactionLatency( + double latencyMs, java.util.Map attributes) {} + + @Override + public void recordTransactionAttemptCount( + long count, java.util.Map attributes) {} + + @Override + public void recordAttemptLatency(double latencyMs, java.util.Map attributes) {} + + @Override + public void recordAttemptCount(long count, java.util.Map attributes) {} + + @Override + public void recordOperationLatency( + double latencyMs, java.util.Map attributes) {} + + @Override + public void recordOperationCount(long count, java.util.Map attributes) {} + } +} diff --git a/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/DatastoreCloudMonitoringExporterTest.java b/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/DatastoreCloudMonitoringExporterTest.java new file mode 100644 index 000000000000..9391585a12a9 --- /dev/null +++ b/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/DatastoreCloudMonitoringExporterTest.java @@ -0,0 +1,198 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.datastore.telemetry; + +import static com.google.cloud.datastore.telemetry.TelemetryConstants.CLIENT_UID_KEY; +import static com.google.cloud.datastore.telemetry.TelemetryConstants.DATABASE_ID_KEY; +import static com.google.cloud.datastore.telemetry.TelemetryConstants.LOCATION_ID_KEY; +import static com.google.cloud.datastore.telemetry.TelemetryConstants.METRIC_NAME_SHORT_OPERATION_COUNT; +import static com.google.cloud.datastore.telemetry.TelemetryConstants.METRIC_PREFIX; +import static com.google.cloud.datastore.telemetry.TelemetryConstants.PROJECT_ID_KEY; +import static com.google.cloud.datastore.telemetry.TelemetryConstants.RESOURCE_LABEL_PROJECT_ID; +import static com.google.cloud.datastore.telemetry.TelemetryConstants.SERVICE_KEY; +import static com.google.cloud.datastore.telemetry.TelemetryConstants.SERVICE_VALUE; +import static com.google.common.truth.Truth.assertThat; +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; + +import com.google.api.core.ApiFuture; +import com.google.api.core.ApiFutures; +import com.google.api.gax.rpc.UnaryCallable; +import com.google.api.gax.tracing.OpenTelemetryMetricsRecorder; +import com.google.cloud.monitoring.v3.MetricServiceClient; +import com.google.cloud.monitoring.v3.stub.MetricServiceStub; +import com.google.common.collect.ImmutableList; +import com.google.monitoring.v3.CreateTimeSeriesRequest; +import com.google.monitoring.v3.TimeSeries; +import com.google.protobuf.Empty; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.sdk.common.InstrumentationScopeInfo; +import io.opentelemetry.sdk.metrics.data.AggregationTemporality; +import io.opentelemetry.sdk.metrics.data.LongPointData; +import io.opentelemetry.sdk.metrics.data.MetricData; +import io.opentelemetry.sdk.metrics.internal.data.ImmutableLongPointData; +import io.opentelemetry.sdk.metrics.internal.data.ImmutableMetricData; +import io.opentelemetry.sdk.metrics.internal.data.ImmutableSumData; +import io.opentelemetry.sdk.resources.Resource; +import java.util.Collections; +import java.util.Map; +import org.easymock.Capture; +import org.easymock.EasyMock; +import org.junit.Before; +import org.junit.Test; + +public class DatastoreCloudMonitoringExporterTest { + + private static final String PROJECT_ID = "test-project"; + private static final String LOCATION_ID = "global"; + private static final String DATABASE_ID = "test-db"; + + private MetricServiceStub mockMetricServiceStub; + private MetricServiceClient fakeMetricServiceClient; + private DatastoreCloudMonitoringExporter exporter; + + private Attributes attributes; + private Attributes resourceAttributes; + private Resource resource; + private InstrumentationScopeInfo scope; + private String clientUid; + + @Before + public void setUp() { + mockMetricServiceStub = createMock(MetricServiceStub.class); + fakeMetricServiceClient = new FakeMetricServiceClient(mockMetricServiceStub); + + Map clientAttributes = BuiltInDatastoreMetricsProvider.buildClientAttributes(); + this.clientUid = clientAttributes.get(CLIENT_UID_KEY.getKey()); + + exporter = + new DatastoreCloudMonitoringExporter( + PROJECT_ID + ":" + DATABASE_ID + ":0", + PROJECT_ID, + fakeMetricServiceClient, + clientAttributes); + + attributes = + Attributes.builder() + .put(DATABASE_ID_KEY, DATABASE_ID) + .put(CLIENT_UID_KEY, this.clientUid) + .build(); + + resourceAttributes = + Attributes.builder() + .put(PROJECT_ID_KEY, PROJECT_ID) + .put(DATABASE_ID_KEY, DATABASE_ID) + .put(LOCATION_ID_KEY, LOCATION_ID) + .build(); + resource = Resource.create(resourceAttributes); + + scope = InstrumentationScopeInfo.create(OpenTelemetryMetricsRecorder.GAX_METER_NAME); + } + + @Test + public void testExportingSumData() { + Capture capture = EasyMock.newCapture(); + + UnaryCallable mockCallable = createMock(UnaryCallable.class); + expect(mockMetricServiceStub.isShutdown()).andReturn(false).anyTimes(); + expect(mockMetricServiceStub.createTimeSeriesCallable()).andReturn(mockCallable); + ApiFuture future = ApiFutures.immediateFuture(Empty.getDefaultInstance()); + expect(mockCallable.futureCall(EasyMock.capture(capture))).andReturn(future); + + replay(mockMetricServiceStub, mockCallable); + + long fakeValue = 11L; + long startEpoch = 10; + long endEpoch = 15; + LongPointData longPointData = + ImmutableLongPointData.create(startEpoch, endEpoch, attributes, fakeValue); + + MetricData longData = + ImmutableMetricData.createLongSum( + resource, + scope, + METRIC_PREFIX + "/" + METRIC_NAME_SHORT_OPERATION_COUNT, + "description", + "1", + ImmutableSumData.create( + true, AggregationTemporality.CUMULATIVE, ImmutableList.of(longPointData))); + + exporter.export(Collections.singletonList(longData)); + + CreateTimeSeriesRequest request = capture.getValue(); + assertThat(request.getTimeSeriesList()).hasSize(1); + + TimeSeries timeSeries = request.getTimeSeriesList().get(0); + + assertThat(timeSeries.getResource().getLabelsMap()) + .containsExactly(RESOURCE_LABEL_PROJECT_ID, PROJECT_ID); + + assertThat(timeSeries.getMetric().getLabelsMap()) + .containsExactly( + DATABASE_ID_KEY.getKey(), + DATABASE_ID, + CLIENT_UID_KEY.getKey(), + this.clientUid, + SERVICE_KEY.getKey(), + SERVICE_VALUE); + + assertThat(timeSeries.getPoints(0).getValue().getInt64Value()).isEqualTo(fakeValue); + + verify(mockMetricServiceStub, mockCallable); + } + + @Test + public void testClientCacheReferenceCounting() { + MetricServiceClient mockClient = createMock(MetricServiceClient.class); + expect(mockClient.isShutdown()).andReturn(false).anyTimes(); + mockClient.shutdown(); + EasyMock.expectLastCall(); // Expect shutdown when refCount reaches 0 + replay(mockClient); + + String key = PROJECT_ID + ":" + DATABASE_ID + ":0"; + DatastoreCloudMonitoringExporter.CachedMetricsClient cachedMetricsClient = + new DatastoreCloudMonitoringExporter.CachedMetricsClient(mockClient); + cachedMetricsClient.refCount.set(2); // Simulate 2 references + DatastoreCloudMonitoringExporter.METRICS_CLIENT_CACHE.put(key, cachedMetricsClient); + + DatastoreCloudMonitoringExporter exporter1 = + new DatastoreCloudMonitoringExporter(key, PROJECT_ID, mockClient, Collections.emptyMap()); + + // First shutdown should decrement refCount to 1, but not close client + exporter1.shutdown(); + assertThat(cachedMetricsClient.refCount.get()).isEqualTo(1); + assertThat(DatastoreCloudMonitoringExporter.METRICS_CLIENT_CACHE.containsKey(key)).isTrue(); + + DatastoreCloudMonitoringExporter exporter2 = + new DatastoreCloudMonitoringExporter(key, PROJECT_ID, mockClient, Collections.emptyMap()); + + // Second shutdown should decrement refCount to 0, close client, and remove from cache + exporter2.shutdown(); + assertThat(cachedMetricsClient.refCount.get()).isEqualTo(0); + assertThat(DatastoreCloudMonitoringExporter.METRICS_CLIENT_CACHE.containsKey(key)).isFalse(); + + verify(mockClient); + } + + private static class FakeMetricServiceClient extends MetricServiceClient { + protected FakeMetricServiceClient(MetricServiceStub stub) { + super(stub); + } + } +} diff --git a/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/DatastoreMetricsRecorderTest.java b/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/DatastoreMetricsRecorderTest.java index 1c1f76ddc156..f495e0dfa7c9 100644 --- a/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/DatastoreMetricsRecorderTest.java +++ b/java-datastore/google-cloud-datastore/src/test/java/com/google/cloud/datastore/telemetry/DatastoreMetricsRecorderTest.java @@ -19,17 +19,14 @@ import com.google.cloud.NoCredentials; import com.google.cloud.datastore.DatastoreOpenTelemetryOptions; -import com.google.cloud.datastore.DatastoreOpenTelemetryOptionsTestHelper; import com.google.cloud.datastore.DatastoreOptions; import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.sdk.OpenTelemetrySdk; -import io.opentelemetry.sdk.metrics.SdkMeterProvider; -import io.opentelemetry.sdk.testing.exporter.InMemoryMetricReader; +import org.easymock.EasyMock; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -/** Tests for {@link DatastoreMetricsRecorder#getInstance(DatastoreOptions)}. */ +/** Tests for {@link DatastoreMetricsRecorder#getInstance(DatastoreOptions, OpenTelemetry)}. */ @RunWith(JUnit4.class) public class DatastoreMetricsRecorderTest { @@ -42,51 +39,70 @@ private DatastoreOptions.Builder baseOptions() { } @Test - public void defaultOptions_returnsNoOp() { - // metricsEnabled defaults to false, so getInstance() should return NoOp - DatastoreOptions options = baseOptions().build(); - DatastoreMetricsRecorder recorder = DatastoreMetricsRecorder.getInstance(options); - assertThat(recorder).isInstanceOf(NoOpDatastoreMetricsRecorder.class); + public void defaultOptionsWithBuiltInMetricsDisabled_returnsNoRecorders() { + // When both custom metrics and built-in metrics export are disabled, + // there should be no recorders + DatastoreOptions options = + baseOptions() + .setOpenTelemetryOptions( + DatastoreOpenTelemetryOptions.newBuilder() + .setExportBuiltinMetricsToGoogleCloudMonitoring(false) + .build()) + .build(); + OpenTelemetry builtInOtel = EasyMock.createMock(OpenTelemetry.class); + EasyMock.replay(builtInOtel); + DatastoreMetricsRecorder recorder = DatastoreMetricsRecorder.getInstance(options, builtInOtel); + assertThat(recorder).isInstanceOf(CompositeDatastoreMetricsRecorder.class); + CompositeDatastoreMetricsRecorder compositeRecorder = + (CompositeDatastoreMetricsRecorder) recorder; + assertThat(compositeRecorder.getMetricRecorders().size()).isEqualTo(0); } @Test - public void tracingEnabledButMetricsDisabled_returnsNoOp() { + public void tracingEnabledButMetricsDisabledAndBuiltInDisabled_returnsNoRecorders() { // Enabling tracing alone should not enable metrics DatastoreOptions options = baseOptions() .setOpenTelemetryOptions( - DatastoreOpenTelemetryOptions.newBuilder().setTracingEnabled(true).build()) + DatastoreOpenTelemetryOptions.newBuilder() + .setTracingEnabled(true) + .setExportBuiltinMetricsToGoogleCloudMonitoring(false) + .build()) .build(); - DatastoreMetricsRecorder recorder = DatastoreMetricsRecorder.getInstance(options); - assertThat(recorder).isInstanceOf(NoOpDatastoreMetricsRecorder.class); + OpenTelemetry builtInOtel = EasyMock.createMock(OpenTelemetry.class); + EasyMock.replay(builtInOtel); + DatastoreMetricsRecorder recorder = DatastoreMetricsRecorder.getInstance(options, builtInOtel); + assertThat(recorder).isInstanceOf(CompositeDatastoreMetricsRecorder.class); + CompositeDatastoreMetricsRecorder compositeRecorder = + (CompositeDatastoreMetricsRecorder) recorder; + assertThat(compositeRecorder.getMetricRecorders().size()).isEqualTo(0); } @Test - public void metricsEnabled_withCustomOtel_returnsOpenTelemetryRecorder() { - InMemoryMetricReader metricReader = InMemoryMetricReader.create(); - SdkMeterProvider meterProvider = - SdkMeterProvider.builder().registerMetricReader(metricReader).build(); - OpenTelemetry openTelemetry = - OpenTelemetrySdk.builder().setMeterProvider(meterProvider).build(); - - // Use DatastoreOpenTelemetryOptionsTestHelper since setMetricsEnabled() is package-private - // and this test lives in the telemetry sub-package. + public void defaultOptionsWithBuiltInMetricsEnabled_butNoCredentials_returnsOneRecorder() { + // Explicitly enable built-in metrics export DatastoreOptions options = - baseOptions() + baseOptions() // Uses NoCredentials by default .setOpenTelemetryOptions( - DatastoreOpenTelemetryOptionsTestHelper.withMetricsEnabled(openTelemetry)) + DatastoreOpenTelemetryOptions.newBuilder() + .setExportBuiltinMetricsToGoogleCloudMonitoring(true) + .build()) .build(); - DatastoreMetricsRecorder recorder = DatastoreMetricsRecorder.getInstance(options); - assertThat(recorder).isInstanceOf(OpenTelemetryDatastoreMetricsRecorder.class); + DatastoreMetricsRecorder recorder = + DatastoreMetricsRecorder.getInstance(options, OpenTelemetry.noop()); + + // Since baseOptions() uses NoCredentials, the provider returns OpenTelemetry.noop(). + // This NoOp instance is passed to getInstance, which adds it to the recorders list. + // So we have 1 recorder (the NoOp one). + assertThat(recorder).isInstanceOf(CompositeDatastoreMetricsRecorder.class); + CompositeDatastoreMetricsRecorder compositeRecorder = + (CompositeDatastoreMetricsRecorder) recorder; + assertThat(compositeRecorder.getMetricRecorders().size()).isEqualTo(1); } @Test public void openTelemetryRecorderCreatedWithExplicitOpenTelemetry() { - InMemoryMetricReader metricReader = InMemoryMetricReader.create(); - SdkMeterProvider meterProvider = - SdkMeterProvider.builder().registerMetricReader(metricReader).build(); - OpenTelemetry openTelemetry = - OpenTelemetrySdk.builder().setMeterProvider(meterProvider).build(); + OpenTelemetry openTelemetry = OpenTelemetry.noop(); OpenTelemetryDatastoreMetricsRecorder recorder = new OpenTelemetryDatastoreMetricsRecorder(openTelemetry, TelemetryConstants.METRIC_PREFIX); diff --git a/java-datastore/samples/snippets/pom.xml b/java-datastore/samples/snippets/pom.xml index 8ed08dea5f3b..5ef3e9ca533e 100644 --- a/java-datastore/samples/snippets/pom.xml +++ b/java-datastore/samples/snippets/pom.xml @@ -41,6 +41,7 @@ com.google.cloud google-cloud-datastore + 2.40.0 diff --git a/java-datastore/samples/snippets/src/main/java/com/example/datastore/DatastoreMetricsSample.java b/java-datastore/samples/snippets/src/main/java/com/example/datastore/DatastoreMetricsSample.java new file mode 100644 index 000000000000..27b00466d9a3 --- /dev/null +++ b/java-datastore/samples/snippets/src/main/java/com/example/datastore/DatastoreMetricsSample.java @@ -0,0 +1,152 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.datastore; + +// [START datastore_client_side_metrics] +import com.google.cloud.datastore.Datastore; +import com.google.cloud.datastore.DatastoreOpenTelemetryOptions; +import com.google.cloud.datastore.DatastoreOptions; +import com.google.cloud.datastore.Entity; +import com.google.cloud.datastore.Key; +import com.google.cloud.datastore.Transaction; + +import java.util.UUID; + +/** + * Demonstrates default client-side metrics for the Datastore Java client library using a + * transaction flow. + * + *

Usage: DatastoreMetricsSample + * + *

Client-side metrics are automatically exported to Google Cloud Monitoring under the {@code + * custom.googleapis.com/internal/client} metric prefix, using the {@code + * firestore.googleapis.com/Database} monitored resource. The {@code service} metric label is set to + * {@code datastore.googleapis.com} to distinguish Datastore traffic from Firestore. + * + *

Built-in metrics are currently disabled by default until the Cloud Monitoring namespace is + * fully deployed. To enable them, set {@link + * DatastoreOpenTelemetryOptions.Builder#setExportBuiltinMetricsToGoogleCloudMonitoring(boolean)} to + * {@code true}, or set the environment variable {@code DATASTORE_ENABLE_METRICS=true}. + * + *

Metrics recorded by this sample: + * + *

    + *
  • {@code transaction_latency} — end-to-end latency of the transaction including retries. + *
  • {@code transaction_attempt_count} — number of commit attempts for the transaction. + *
+ * + *

To verify metrics in Cloud Monitoring after running this sample, navigate to: + * Cloud Console → Monitoring → Metrics Explorer and filter by: + * + * 

+ *   Metric  : custom.googleapis.com/internal/client/transaction_latency
+ *   Resource: firestore.googleapis.com/Database
+ *   Label   : service = datastore.googleapis.com
+ *
+ */ +public class DatastoreMetricsSample { + + public static void main(String[] args) throws Exception { + if (args.length != 2) { + System.err.println("Usage: DatastoreMetricsSample "); + System.exit(1); + } + String projectId = args[0]; + String databaseId = args[1]; + String kind = "Kind-" + UUID.randomUUID().toString().substring(0, 8); + + runSample(projectId, databaseId, kind); + } + + static void runSample(String projectId, String databaseId, String kind) throws Exception { + // [START datastore_client_side_metrics_default] + // Built-in metrics are disabled by default. Enable them explicitly. + DatastoreOptions options = + DatastoreOptions.newBuilder() + .setProjectId(projectId) + .setDatabaseId(databaseId) + .setOpenTelemetryOptions( + DatastoreOpenTelemetryOptions.newBuilder() + .setExportBuiltinMetricsToGoogleCloudMonitoring(true) + .build()) + .build(); + // [END datastore_client_side_metrics_default] + + try (Datastore datastore = options.getService()) { + System.out.printf( + "Connected to project=%s database=%s%n", projectId, databaseId); + System.out.println( + "Built-in metrics are explicitly enabled and will be exported to" + + " Google Cloud Monitoring under custom.googleapis.com/internal/client/*"); + + runTransactionFlow(datastore, kind); + + // The periodic metric reader flushes accumulated metrics to Cloud Monitoring on a fixed + // interval (default: 60 seconds). A final flush also runs when the JVM shuts down via the + // registered shutdown hook. + System.out.println( + "\nTransaction flow complete. Metrics will be flushed to Cloud Monitoring."); + System.out.println( + "Check Cloud Monitoring > Metrics Explorer for:" + + " custom.googleapis.com/internal/client/transaction_latency"); + + // Give some time for the periodic metric reader to flush metrics to Cloud Monitoring. + System.out.println("Waiting 5 seconds for metrics to flush..."); + Thread.sleep(5000); + } + } + + /** + * Runs a full transaction flow: writes an entity, reads it back within a transaction, updates it, + * then deletes it. This exercises the read-modify-write pattern that records both {@code + * transaction_latency} and {@code transaction_attempt_count} metrics. + */ + static void runTransactionFlow(Datastore datastore, String kind) { + Key key = datastore.newKeyFactory().setKind(kind).newKey("metrics-sample-entity"); + + // Step 1: Insert the entity outside any transaction to establish a baseline. + Entity initial = Entity.newBuilder(key).set("status", "created").set("value", 0L).build(); + datastore.put(initial); + System.out.printf("Inserted entity: kind=%s key=%s%n", kind, key.getName()); + + // Step 2: Read-modify-write inside a transaction. + // This is the core pattern that generates transaction_latency and + // transaction_attempt_count metrics. + Entity updated = + datastore.runInTransaction( + transaction -> { + Entity current = transaction.get(key); + Entity modified = + Entity.newBuilder(current) + .set("status", "updated") + .set("value", current.getLong("value") + 1) + .build(); + transaction.put(modified); + return modified; + }); + System.out.printf( + "Transaction committed: status=%s value=%d%n", + updated.getString("status"), updated.getLong("value")); + System.out.println( + " → transaction_latency and transaction_attempt_count metrics recorded."); + + // Step 3: Clean up. + datastore.delete(key); + System.out.printf("Deleted entity: kind=%s key=%s%n", kind, key.getName()); + } +} +// [END datastore_client_side_metrics] diff --git a/java-datastore/samples/snippets/src/test/java/com/example/datastore/DatastoreMetricsSampleIT.java b/java-datastore/samples/snippets/src/test/java/com/example/datastore/DatastoreMetricsSampleIT.java new file mode 100644 index 000000000000..52d46cdf712e --- /dev/null +++ b/java-datastore/samples/snippets/src/test/java/com/example/datastore/DatastoreMetricsSampleIT.java @@ -0,0 +1,109 @@ +/* + * Copyright 2026 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.datastore; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.cloud.datastore.Datastore; +import com.google.cloud.datastore.DatastoreOptions; +import com.google.cloud.datastore.Key; +import com.rule.SystemsOutRule; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** + * Integration test for {@link DatastoreMetricsSample}. + * + *

Requires a real GCP project with valid Application Default Credentials. Set the following + * environment variables before running: + * + *

    + *
  • {@code GOOGLE_CLOUD_PROJECT} — GCP project ID + *
  • {@code DATASTORE_DATABASE_ID} — Datastore database ID (defaults to {@code ""} for the + * default database) + *
+ * + *

To verify that metrics appeared in Cloud Monitoring after this test runs, navigate to: + * + * 

+ *   Cloud Console → Monitoring → Metrics Explorer
+ *   Metric  : custom.googleapis.com/internal/client/transaction_latency
+ *   Resource: firestore.googleapis.com/Database
+ *   Label   : service = datastore.googleapis.com
+ *
+ */ +@RunWith(JUnit4.class) +@SuppressWarnings("checkstyle:abbreviationaswordinname") +public class DatastoreMetricsSampleIT { + + private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); + private static final String DATABASE_ID = + System.getenv().getOrDefault("DATASTORE_DATABASE_ID", ""); + + @Rule public final SystemsOutRule systemsOutRule = new SystemsOutRule(); + + private Datastore datastore; + private String kind; + + @Before + public void setUp() { + kind = "Kind-" + java.util.UUID.randomUUID().toString().substring(0, 8); + datastore = + DatastoreOptions.newBuilder() + .setProjectId(PROJECT_ID) + .setDatabaseId(DATABASE_ID) + .build() + .getService(); + cleanUp(); + } + + @After + public void tearDown() { + cleanUp(); + System.setOut(null); + } + + @Test + public void testTransactionFlowRecordsMetrics() throws Exception { + DatastoreMetricsSample.runSample(PROJECT_ID, DATABASE_ID, kind); + + systemsOutRule.assertContains("Built-in metrics are explicitly enabled"); + systemsOutRule.assertContains("Inserted entity"); + systemsOutRule.assertContains("Transaction committed"); + systemsOutRule.assertContains("transaction_latency and transaction_attempt_count metrics recorded"); + systemsOutRule.assertContains("Deleted entity"); + systemsOutRule.assertContains("Metrics will be flushed to Cloud Monitoring"); + } + + @Test + public void testRunTransactionFlow_updatesEntityCorrectly() { + DatastoreMetricsSample.runTransactionFlow(datastore, kind); + + // Entity is deleted at the end of the flow; confirm it no longer exists. + Key key = datastore.newKeyFactory().setKind(kind).newKey("metrics-sample-entity"); + assertThat(datastore.get(key)).isNull(); + } + + private void cleanUp() { + Key key = datastore.newKeyFactory().setKind(kind).newKey("metrics-sample-entity"); + datastore.delete(key); + } +}