feat: Adding gfe_latencies metric to built-in metrics

google-cloud-spanner/pom.xml

@@ -371,7 +371,7 @@
       <artifactId>junit</artifactId>
       <scope>test</scope>
     </dependency>
-    
+
     <!-- Executor tests - The 'provided' scope is overwritten to compile time scope for the profile 'executor-tests' -->
     <dependency>
       <groupId>com.google.api.grpc</groupId>

google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java

@@ -34,9 +34,9 @@
 public class BuiltInMetricsConstant {
 
   public static final String METER_NAME = "spanner.googleapis.com/internal/client";
-
   public static final String GAX_METER_NAME = OpenTelemetryMetricsRecorder.GAX_METER_NAME;
-
+  static final String SPANNER_METER_NAME = "spanner-java";
+  static final String GFE_LATENCIES_NAME = "gfe_latencies";
   static final String OPERATION_LATENCIES_NAME = "operation_latencies";
   static final String ATTEMPT_LATENCIES_NAME = "attempt_latencies";
   static final String OPERATION_LATENCY_NAME = "operation_latency";
@@ -49,7 +49,8 @@ public class BuiltInMetricsConstant {
               OPERATION_LATENCIES_NAME,
               ATTEMPT_LATENCIES_NAME,
               OPERATION_COUNT_NAME,
-              ATTEMPT_COUNT_NAME)
+              ATTEMPT_COUNT_NAME,
+              GFE_LATENCIES_NAME)
           .stream()
           .map(m -> METER_NAME + '/' + m)
           .collect(Collectors.toSet());
@@ -114,27 +115,39 @@ static Map<InstrumentSelector, View> getAllViews() {
     ImmutableMap.Builder<InstrumentSelector, View> views = ImmutableMap.builder();
     defineView(
         views,
+        BuiltInMetricsConstant.GAX_METER_NAME,
         BuiltInMetricsConstant.OPERATION_LATENCY_NAME,
         BuiltInMetricsConstant.OPERATION_LATENCIES_NAME,
         BuiltInMetricsConstant.AGGREGATION_WITH_MILLIS_HISTOGRAM,
         InstrumentType.HISTOGRAM,
         "ms");
     defineView(
         views,
+        BuiltInMetricsConstant.GAX_METER_NAME,
         BuiltInMetricsConstant.ATTEMPT_LATENCY_NAME,
         BuiltInMetricsConstant.ATTEMPT_LATENCIES_NAME,
         BuiltInMetricsConstant.AGGREGATION_WITH_MILLIS_HISTOGRAM,
         InstrumentType.HISTOGRAM,
         "ms");
     defineView(
         views,
+        BuiltInMetricsConstant.SPANNER_METER_NAME,
+        BuiltInMetricsConstant.GFE_LATENCIES_NAME,
+        BuiltInMetricsConstant.GFE_LATENCIES_NAME,
+        BuiltInMetricsConstant.AGGREGATION_WITH_MILLIS_HISTOGRAM,
+        InstrumentType.HISTOGRAM,
+        "ms");
+    defineView(
+        views,
+        BuiltInMetricsConstant.GAX_METER_NAME,
         BuiltInMetricsConstant.OPERATION_COUNT_NAME,
         BuiltInMetricsConstant.OPERATION_COUNT_NAME,
         Aggregation.sum(),
         InstrumentType.COUNTER,
         "1");
     defineView(
         views,
+        BuiltInMetricsConstant.GAX_METER_NAME,
         BuiltInMetricsConstant.ATTEMPT_COUNT_NAME,
         BuiltInMetricsConstant.ATTEMPT_COUNT_NAME,
         Aggregation.sum(),
@@ -145,6 +158,7 @@ static Map<InstrumentSelector, View> getAllViews() {
 
   private static void defineView(
       ImmutableMap.Builder<InstrumentSelector, View> viewMap,
+      String meterName,
       String metricName,
       String metricViewName,
       Aggregation aggregation,
@@ -153,7 +167,7 @@ private static void defineView(
     InstrumentSelector selector =
         InstrumentSelector.builder()
             .setName(BuiltInMetricsConstant.METER_NAME + '/' + metricName)
-            .setMeterName(BuiltInMetricsConstant.GAX_METER_NAME)
+            .setMeterName(meterName)
             .setType(type)
             .setUnit(unit)
             .build();

google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsProvider.java → google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsProvider.java

@@ -46,25 +46,24 @@
 import java.util.logging.Logger;
 import javax.annotation.Nullable;
 
-final class BuiltInOpenTelemetryMetricsProvider {
+final class BuiltInMetricsProvider {
 
-  static BuiltInOpenTelemetryMetricsProvider INSTANCE = new BuiltInOpenTelemetryMetricsProvider();
+  static BuiltInMetricsProvider INSTANCE = new BuiltInMetricsProvider();
 
-  private static final Logger logger =
-      Logger.getLogger(BuiltInOpenTelemetryMetricsProvider.class.getName());
+  private static final Logger logger = Logger.getLogger(BuiltInMetricsProvider.class.getName());
 
   private static String taskId;
 
   private OpenTelemetry openTelemetry;
 
-  private BuiltInOpenTelemetryMetricsProvider() {}
+  private BuiltInMetricsProvider() {}
 
   OpenTelemetry getOrCreateOpenTelemetry(
       String projectId, @Nullable Credentials credentials, @Nullable String monitoringHost) {
     try {
       if (this.openTelemetry == null) {
         SdkMeterProviderBuilder sdkMeterProviderBuilder = SdkMeterProvider.builder();
-        BuiltInOpenTelemetryMetricsView.registerBuiltinMetrics(
+        BuiltInMetricsView.registerBuiltinMetrics(
             SpannerCloudMonitoringExporter.create(projectId, credentials, monitoringHost),
             sdkMeterProviderBuilder);
         SdkMeterProvider sdkMeterProvider = sdkMeterProviderBuilder.build();

google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsRecorder.java

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2025 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.spanner;
+
+import com.google.api.gax.core.GaxProperties;
+import com.google.api.gax.tracing.OpenTelemetryMetricsRecorder;
+import com.google.common.base.Preconditions;
+import io.opentelemetry.api.OpenTelemetry;
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.common.AttributesBuilder;
+import io.opentelemetry.api.metrics.DoubleHistogram;
+import io.opentelemetry.api.metrics.Meter;
+import java.util.Map;
+
+/**
+ * Implementation for recording built in metrics.
+ *
+ * <p>This class extends the {@link OpenTelemetryMetricsRecorder} which implements the *
+ * measurements related to the lifecyle of an RPC.
+ */
+class BuiltInMetricsRecorder extends OpenTelemetryMetricsRecorder {
+
+  private final DoubleHistogram gfeLatencyRecorder;
+
+  /**
+   * Creates the following instruments for the following metrics:
+   *
+   * <ul>
+   *   <li>GFE Latency: Histogram
+   * </ul>
+   *
+   * @param openTelemetry OpenTelemetry instance
+   * @param serviceName Service Name
+   */
+  BuiltInMetricsRecorder(OpenTelemetry openTelemetry, String serviceName) {
+    super(openTelemetry, serviceName);
+    Meter meter =
+        openTelemetry
+            .meterBuilder(BuiltInMetricsConstant.SPANNER_METER_NAME)
+            .setInstrumentationVersion(GaxProperties.getLibraryVersion(getClass()))
+            .build();
+    this.gfeLatencyRecorder =
+        meter
+            .histogramBuilder(serviceName + '/' + BuiltInMetricsConstant.GFE_LATENCIES_NAME)
+            .setDescription(
+                "Latency between Google's network receiving an RPC and reading back the first byte of the response")
+            .setUnit("ms")
+            .build();
+  }
+
+  /**
+   * Record the latency between Google's network receiving an RPC and reading back the first byte of
+   * the response. Data is stored in a Histogram.
+   *
+   * @param gfeLatency Attempt Latency in ms
+   * @param attributes Map of the attributes to store
+   */
+  void recordGFELatency(double gfeLatency, Map<String, String> attributes) {
+    gfeLatencyRecorder.record(gfeLatency, toOtelAttributes(attributes));
+  }
+
+  Attributes toOtelAttributes(Map<String, String> attributes) {
+    Preconditions.checkNotNull(attributes, "Attributes map cannot be null");
+    AttributesBuilder attributesBuilder = Attributes.builder();
+    attributes.forEach(attributesBuilder::put);
+    return attributesBuilder.build();
+  }
+}

google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsTracer.java

@@ -0,0 +1,156 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.spanner;
+
+import com.google.api.gax.rpc.ApiException;
+import com.google.api.gax.rpc.StatusCode;
+import com.google.api.gax.tracing.ApiTracer;
+import com.google.api.gax.tracing.MethodName;
+import com.google.api.gax.tracing.MetricsTracer;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.CancellationException;
+import javax.annotation.Nullable;
+
+/**
+ * Implements built-in metrics tracer.
+ *
+ * <p>This class extends the {@link MetricsTracer} which computes generic metrics that can be
+ * observed in the lifecycle of an RPC operation.
+ */
+class BuiltInMetricsTracer extends MetricsTracer implements ApiTracer {
+
+  private final BuiltInMetricsRecorder builtInOpenTelemetryMetricsRecorder;
+  // These are RPC specific attributes and pertain to a specific API Trace
+  private final Map<String, String> attributes = new HashMap<>();
+
+  private Long gfeLatency = null;
+
+  BuiltInMetricsTracer(
+      MethodName methodName, BuiltInMetricsRecorder builtInOpenTelemetryMetricsRecorder) {
+    super(methodName, builtInOpenTelemetryMetricsRecorder);
+    this.builtInOpenTelemetryMetricsRecorder = builtInOpenTelemetryMetricsRecorder;
+    this.attributes.put(METHOD_ATTRIBUTE, methodName.toString());
+  }
+
+  /**
+   * Adds an annotation that the attempt succeeded. Successful attempt add "OK" value to the status
+   * attribute key.
+   */
+  @Override
+  public void attemptSucceeded() {
+    super.attemptSucceeded();
+    if (gfeLatency != null) {
+      attributes.put(STATUS_ATTRIBUTE, StatusCode.Code.OK.toString());
+      builtInOpenTelemetryMetricsRecorder.recordGFELatency(gfeLatency, attributes);
+    }
+  }
+
+  /**
+   * Add an annotation that the attempt was cancelled by the user. Cancelled attempt add "CANCELLED"
+   * to the status attribute key.
+   */
+  @Override
+  public void attemptCancelled() {
+    super.attemptCancelled();
+    if (gfeLatency != null) {
+      attributes.put(STATUS_ATTRIBUTE, StatusCode.Code.CANCELLED.toString());
+      builtInOpenTelemetryMetricsRecorder.recordGFELatency(gfeLatency, attributes);
+    }
+  }
+
+  /**
+   * Adds an annotation that the attempt failed, but another attempt will be made after the delay.
+   *
+   * @param error the error that caused the attempt to fail.
+   * @param delay the amount of time to wait before the next attempt will start.
+   *     <p>Failed attempt extracts the error from the throwable and adds it to the status attribute
+   *     key.
+   */
+  @Override
+  public void attemptFailedDuration(Throwable error, java.time.Duration delay) {
+    super.attemptFailedDuration(error, delay);
+    if (gfeLatency != null) {
+      attributes.put(STATUS_ATTRIBUTE, extractStatus(error));
+      builtInOpenTelemetryMetricsRecorder.recordGFELatency(gfeLatency, attributes);
+    }
+  }
+
+  /**
+   * Adds an annotation that the attempt failed and that no further attempts will be made because
+   * retry limits have been reached. This extracts the error from the throwable and adds it to the
+   * status attribute key.
+   *
+   * @param error the last error received before retries were exhausted.
+   */
+  @Override
+  public void attemptFailedRetriesExhausted(Throwable error) {
+    super.attemptFailedRetriesExhausted(error);
+    if (gfeLatency != null) {
+      attributes.put(STATUS_ATTRIBUTE, extractStatus(error));
+      builtInOpenTelemetryMetricsRecorder.recordGFELatency(gfeLatency, attributes);
+    }
+  }
+
+  /**
+   * Adds an annotation that the attempt failed and that no further attempts will be made because
+   * the last error was not retryable. This extracts the error from the throwable and adds it to the
+   * status attribute key.
+   *
+   * @param error the error that caused the final attempt to fail.
+   */
+  @Override
+  public void attemptPermanentFailure(Throwable error) {
+    super.attemptPermanentFailure(error);
+    if (gfeLatency != null) {
+      attributes.put(STATUS_ATTRIBUTE, extractStatus(error));
+      builtInOpenTelemetryMetricsRecorder.recordGFELatency(gfeLatency, attributes);
+    }
+  }
+
+  void recordGFELatency(Long gfeLatency) {
+    this.gfeLatency = gfeLatency;
+  }
+
+  @Override
+  public void addAttributes(Map<String, String> attributes) {
+    super.addAttributes(attributes);
+    this.attributes.putAll(attributes);
+  };
+
+  @Override
+  public void addAttributes(String key, String value) {
+    super.addAttributes(key, value);
+    this.attributes.put(key, value);
+  }
+
+  private static String extractStatus(@Nullable Throwable error) {
+    final String statusString;
+
+    if (error == null) {
+      return StatusCode.Code.OK.toString();
+    } else if (error instanceof CancellationException) {
+      statusString = StatusCode.Code.CANCELLED.toString();
+    } else if (error instanceof ApiException) {
+      statusString = ((ApiException) error).getStatusCode().getCode().toString();
+    } else {
+      statusString = StatusCode.Code.UNKNOWN.toString();
+    }
+
+    return statusString;
+  }
+}