Merge "Camera: Add support for readout timestamp" into android13-gsi

8 files changed, 178 insertions, 7 deletions

diff --git a/core/java/android/hardware/camera2/CameraCaptureSession.java b/core/java/android/hardware/camera2/CameraCaptureSession.java
index 691690c09e0e..5dbe9bff328e 100644
--- a/core/java/android/hardware/camera2/CameraCaptureSession.java
+++ b/core/java/android/hardware/camera2/CameraCaptureSession.java
@@ -1234,6 +1234,41 @@ public abstract class CameraCaptureSession implements AutoCloseable {
         }
 
         /**
+         * This method is called when the camera device has started reading out the output
+         * image for the request, at the beginning of the sensor image readout.
+         *
+         * <p>For a capture request, this callback is invoked right after
+         * {@link #onCaptureStarted}. Unlike {@link #onCaptureStarted}, instead of passing
+         * a timestamp of start of exposure, this callback passes a timestamp of start of
+         * camera data readout. This is useful because for a camera running at fixed frame
+         * rate, the start of readout is at fixed interval, but not necessary for the start
+         * of exposure.</p>
+         *
+         * <p>This timestamp may not match {@link CaptureResult#SENSOR_TIMESTAMP the result
+         * timestamp field}. It will, however, match the timestamp of buffers sent to the
+         * output surfaces with {@link OutputConfiguration#TIMESTAMP_BASE_READOUT_SENSOR}
+         * timestamp base.</p>
+         *
+         * <p>This callback will be called only if {@link
+         * CameraCharacteristics#SENSOR_READOUT_TIMESTAMP} is
+         * {@link CameraMetadata#SENSOR_READOUT_TIMESTAMP_HARDWARE}, and it's called
+         * right after {@link #onCaptureStarted}.</p>
+         *
+         * @param session the session returned by {@link CameraDevice#createCaptureSession}
+         * @param request the request for the readout that just began
+         * @param timestamp the timestamp at start of readout for a regular request, or
+         *                  the timestamp at the input image's start of readout for a
+         *                  reprocess request, in nanoseconds.
+         * @param frameNumber the frame number for this capture
+         *
+         * @hide
+         */
+        public void onReadoutStarted(@NonNull CameraCaptureSession session,
+                @NonNull CaptureRequest request, long timestamp, long frameNumber) {
+            // default empty implementation
+        }
+
+        /**
          * This method is called when some results from an image capture are
          * available.
          *
diff --git a/core/java/android/hardware/camera2/CameraCharacteristics.java b/core/java/android/hardware/camera2/CameraCharacteristics.java
index 98a8cbd8a73c..dac55ae08448 100644
--- a/core/java/android/hardware/camera2/CameraCharacteristics.java
+++ b/core/java/android/hardware/camera2/CameraCharacteristics.java
@@ -4415,6 +4415,40 @@ public final class CameraCharacteristics extends CameraMetadata<CameraCharacteri
             new Key<android.graphics.Rect[]>("android.sensor.opticalBlackRegions", android.graphics.Rect[].class);
 
     /**
+     * <p>Whether or not the camera device supports readout timestamp and
+     * onReadoutStarted callback.</p>
+     * <p>If this tag is HARDWARE, the camera device calls onReadoutStarted in addition to the
+     * onCaptureStarted callback for each capture. The timestamp passed into the callback
+     * is the start of camera image readout rather than the start of the exposure. In
+     * addition, the application can configure an
+     * {@link android.hardware.camera2.params.OutputConfiguration } with
+     * TIMESTAMP_BASE_READOUT_SENSOR timestamp base, in which case, the timestamp of the
+     * output surface matches the timestamp from the corresponding onReadoutStarted callback.</p>
+     * <p>The readout timestamp is beneficial for video recording, because the encoder favors
+     * uniform timestamps, and the readout timestamps better reflect the cadence camera sensors
+     * output data.</p>
+     * <p>If this tag is HARDWARE, the camera device produces the start-of-exposure and
+     * start-of-readout together. As a result, the onReadoutStarted is called right after
+     * onCaptureStarted. The difference in start-of-readout and start-of-exposure is the sensor
+     * exposure time, plus certain constant offset. The offset is usually due to camera sensor
+     * level crop, and it remains constant for a given camera sensor mode.</p>
+     * <p><b>Possible values:</b></p>
+     * <ul>
+     *   <li>{@link #SENSOR_READOUT_TIMESTAMP_NOT_SUPPORTED NOT_SUPPORTED}</li>
+     *   <li>{@link #SENSOR_READOUT_TIMESTAMP_HARDWARE HARDWARE}</li>
+     * </ul>
+     *
+     * <p>This key is available on all devices.</p>
+     * @see #SENSOR_READOUT_TIMESTAMP_NOT_SUPPORTED
+     * @see #SENSOR_READOUT_TIMESTAMP_HARDWARE
+     * @hide
+     */
+    @PublicKey
+    @NonNull
+    public static final Key<Integer> SENSOR_READOUT_TIMESTAMP =
+            new Key<Integer>("android.sensor.readoutTimestamp", int.class);
+
+    /**
      * <p>List of lens shading modes for {@link CaptureRequest#SHADING_MODE android.shading.mode} that are supported by this camera device.</p>
      * <p>This list contains lens shading modes that can be set for the camera device.
      * Camera devices that support the MANUAL_POST_PROCESSING capability will always
diff --git a/core/java/android/hardware/camera2/CameraMetadata.java b/core/java/android/hardware/camera2/CameraMetadata.java
index eb8c73aced39..c67a560b5885 100644
--- a/core/java/android/hardware/camera2/CameraMetadata.java
+++ b/core/java/android/hardware/camera2/CameraMetadata.java
@@ -1658,6 +1658,28 @@ public abstract class CameraMetadata<TKey> {
     public static final int SENSOR_REFERENCE_ILLUMINANT1_ISO_STUDIO_TUNGSTEN = 24;
 
     //
+    // Enumeration values for CameraCharacteristics#SENSOR_READOUT_TIMESTAMP
+    //
+
+    /**
+     * <p>This camera device doesn't support readout timestamp and onReadoutStarted
+     * callback.</p>
+     * @see CameraCharacteristics#SENSOR_READOUT_TIMESTAMP
+     * @hide
+     */
+    public static final int SENSOR_READOUT_TIMESTAMP_NOT_SUPPORTED = 0;
+
+    /**
+     * <p>This camera device supports the onReadoutStarted callback as well as outputting
+     * readout timestamp for streams with TIMESTAMP_BASE_READOUT_SENSOR timestamp base. The
+     * readout timestamp is generated by the camera hardware and it has the same accuracy
+     * and timing characteristics of the start-of-exposure time.</p>
+     * @see CameraCharacteristics#SENSOR_READOUT_TIMESTAMP
+     * @hide
+     */
+    public static final int SENSOR_READOUT_TIMESTAMP_HARDWARE = 1;
+
+    //
     // Enumeration values for CameraCharacteristics#LED_AVAILABLE_LEDS
     //
 
diff --git a/core/java/android/hardware/camera2/impl/CameraCaptureSessionImpl.java b/core/java/android/hardware/camera2/impl/CameraCaptureSessionImpl.java
index 9a9163c724ff..b9eba9c1d541 100644
--- a/core/java/android/hardware/camera2/impl/CameraCaptureSessionImpl.java
+++ b/core/java/android/hardware/camera2/impl/CameraCaptureSessionImpl.java
@@ -657,6 +657,21 @@ public class CameraCaptureSessionImpl extends CameraCaptureSession
             }
 
             @Override
+            public void onReadoutStarted(CameraDevice camera,
+                    CaptureRequest request, long timestamp, long frameNumber) {
+                if ((callback != null) && (executor != null)) {
+                    final long ident = Binder.clearCallingIdentity();
+                    try {
+                        executor.execute(() -> callback.onReadoutStarted(
+                                    CameraCaptureSessionImpl.this, request, timestamp,
+                                    frameNumber));
+                    } finally {
+                        Binder.restoreCallingIdentity(ident);
+                    }
+                }
+            }
+
+            @Override
             public void onCapturePartial(CameraDevice camera,
                     CaptureRequest request, android.hardware.camera2.CaptureResult result) {
                 if ((callback != null) && (executor != null)) {
diff --git a/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java b/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java
index 3cb0c93d8409..a6c79b3a289f 100644
--- a/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java
+++ b/core/java/android/hardware/camera2/impl/CameraDeviceImpl.java
@@ -2025,12 +2025,16 @@ public class CameraDeviceImpl extends CameraDevice
                     resultExtras.getLastCompletedReprocessFrameNumber();
             final long lastCompletedZslFrameNumber =
                     resultExtras.getLastCompletedZslFrameNumber();
+            final boolean hasReadoutTimestamp = resultExtras.hasReadoutTimestamp();
+            final long readoutTimestamp = resultExtras.getReadoutTimestamp();
 
             if (DEBUG) {
                 Log.d(TAG, "Capture started for id " + requestId + " frame number " + frameNumber
                         + ": completedRegularFrameNumber " + lastCompletedRegularFrameNumber
                         + ", completedReprocessFrameNUmber " + lastCompletedReprocessFrameNumber
-                        + ", completedZslFrameNumber " + lastCompletedZslFrameNumber);
+                        + ", completedZslFrameNumber " + lastCompletedZslFrameNumber
+                        + ", hasReadoutTimestamp " + hasReadoutTimestamp
+                        + (hasReadoutTimestamp ? ", readoutTimestamp " + readoutTimestamp : "")) ;
             }
             final CaptureCallbackHolder holder;
 
@@ -2082,14 +2086,26 @@ public class CameraDeviceImpl extends CameraDevice
                                                 CameraDeviceImpl.this,
                                                 holder.getRequest(i),
                                                 timestamp - (subsequenceId - i) *
-                                                NANO_PER_SECOND/fpsRange.getUpper(),
+                                                NANO_PER_SECOND / fpsRange.getUpper(),
                                                 frameNumber - (subsequenceId - i));
+                                            if (hasReadoutTimestamp) {
+                                                holder.getCallback().onReadoutStarted(
+                                                    CameraDeviceImpl.this,
+                                                    holder.getRequest(i),
+                                                    readoutTimestamp - (subsequenceId - i) *
+                                                    NANO_PER_SECOND / fpsRange.getUpper(),
+                                                    frameNumber - (subsequenceId - i));
+                                            }
                                         }
                                     } else {
                                         holder.getCallback().onCaptureStarted(
-                                            CameraDeviceImpl.this,
-                                            holder.getRequest(resultExtras.getSubsequenceId()),
+                                            CameraDeviceImpl.this, request,
                                             timestamp, frameNumber);
+                                        if (hasReadoutTimestamp) {
+                                            holder.getCallback().onReadoutStarted(
+                                                CameraDeviceImpl.this, request,
+                                                readoutTimestamp, frameNumber);
+                                        }
                                     }
                                 }
                             }
diff --git a/core/java/android/hardware/camera2/impl/CaptureCallback.java b/core/java/android/hardware/camera2/impl/CaptureCallback.java
index 6defe63b1766..b064e6a1f975 100644
--- a/core/java/android/hardware/camera2/impl/CaptureCallback.java
+++ b/core/java/android/hardware/camera2/impl/CaptureCallback.java
@@ -66,6 +66,13 @@ public abstract class CaptureCallback {
             CaptureRequest request, long timestamp, long frameNumber);
 
     /**
+     * This method is called when the camera device has started reading out the output
+     * image for the request, at the beginning of the sensor image readout.
+     */
+    public abstract void onReadoutStarted(CameraDevice camera,
+            CaptureRequest request, long timestamp, long frameNumber);
+
+    /**
      * This method is called when some results from an image capture are
      * available.
      *
diff --git a/core/java/android/hardware/camera2/impl/CaptureResultExtras.java b/core/java/android/hardware/camera2/impl/CaptureResultExtras.java
index 5d9da73fd5c0..8bf94986a490 100644
--- a/core/java/android/hardware/camera2/impl/CaptureResultExtras.java
+++ b/core/java/android/hardware/camera2/impl/CaptureResultExtras.java
@@ -33,6 +33,8 @@ public class CaptureResultExtras implements Parcelable {
     private long lastCompletedRegularFrameNumber;
     private long lastCompletedReprocessFrameNumber;
     private long lastCompletedZslFrameNumber;
+    private boolean hasReadoutTimestamp;
+    private long readoutTimestamp;
 
     public static final @android.annotation.NonNull Parcelable.Creator<CaptureResultExtras> CREATOR =
             new Parcelable.Creator<CaptureResultExtras>() {
@@ -56,7 +58,8 @@ public class CaptureResultExtras implements Parcelable {
                                int partialResultCount, int errorStreamId,
                                String errorPhysicalCameraId, long lastCompletedRegularFrameNumber,
                                long lastCompletedReprocessFrameNumber,
-                               long lastCompletedZslFrameNumber) {
+                               long lastCompletedZslFrameNumber, boolean hasReadoutTimestamp,
+                               long readoutTimestamp) {
         this.requestId = requestId;
         this.subsequenceId = subsequenceId;
         this.afTriggerId = afTriggerId;
@@ -68,6 +71,8 @@ public class CaptureResultExtras implements Parcelable {
         this.lastCompletedRegularFrameNumber = lastCompletedRegularFrameNumber;
         this.lastCompletedReprocessFrameNumber = lastCompletedReprocessFrameNumber;
         this.lastCompletedZslFrameNumber = lastCompletedZslFrameNumber;
+        this.hasReadoutTimestamp = hasReadoutTimestamp;
+        this.readoutTimestamp = readoutTimestamp;
     }
 
     @Override
@@ -93,6 +98,10 @@ public class CaptureResultExtras implements Parcelable {
         dest.writeLong(lastCompletedRegularFrameNumber);
         dest.writeLong(lastCompletedReprocessFrameNumber);
         dest.writeLong(lastCompletedZslFrameNumber);
+        dest.writeBoolean(hasReadoutTimestamp);
+        if (hasReadoutTimestamp) {
+            dest.writeLong(readoutTimestamp);
+        }
     }
 
     public void readFromParcel(Parcel in) {
@@ -110,6 +119,10 @@ public class CaptureResultExtras implements Parcelable {
         lastCompletedRegularFrameNumber = in.readLong();
         lastCompletedReprocessFrameNumber = in.readLong();
         lastCompletedZslFrameNumber = in.readLong();
+        hasReadoutTimestamp = in.readBoolean();
+        if (hasReadoutTimestamp) {
+            readoutTimestamp = in.readLong();
+        }
     }
 
     public String getErrorPhysicalCameraId() {
@@ -155,4 +168,12 @@ public class CaptureResultExtras implements Parcelable {
     public long getLastCompletedZslFrameNumber() {
         return lastCompletedZslFrameNumber;
     }
+
+    public boolean hasReadoutTimestamp() {
+        return hasReadoutTimestamp;
+    }
+
+    public long getReadoutTimestamp() {
+        return readoutTimestamp;
+    }
 }
diff --git a/core/java/android/hardware/camera2/params/OutputConfiguration.java b/core/java/android/hardware/camera2/params/OutputConfiguration.java
index 4a2517763aae..9e8703779863 100644
--- a/core/java/android/hardware/camera2/params/OutputConfiguration.java
+++ b/core/java/android/hardware/camera2/params/OutputConfiguration.java
@@ -245,6 +245,26 @@ public final class OutputConfiguration implements Parcelable {
      */
     public static final int TIMESTAMP_BASE_CHOREOGRAPHER_SYNCED = 4;
 
+    /**
+     * Timestamp is the start of readout in the same time domain as TIMESTAMP_BASE_SENSOR.
+     *
+     * <p>The start of the camera sensor readout after exposure. For a rolling shutter camera
+     * sensor, the timestamp is typically equal to the start of exposure time +
+     * exposure time + certain fixed offset. The fixed offset could be due to camera sensor
+     * level crop. The benefit of using readout time is that when camera runs in a fixed
+     * frame rate, the timestamp intervals between frames are constant.</p>
+     *
+     * <p>This timestamp is in the same time domain as in TIMESTAMP_BASE_SENSOR, with the exception
+     * that one is start of exposure, and the other is start of readout.</p>
+     *
+     * <p>This timestamp base is supported only if {@link
+     * CameraCharacteristics#SENSOR_READOUT_TIMESTAMP} is
+     * {@link CameraMetadata#SENSOR_READOUT_TIMESTAMP_HARDWARE}.</p>
+     *
+     * @hide
+     */
+    public static final int TIMESTAMP_BASE_READOUT_SENSOR = 5;
+
     /** @hide */
     @Retention(RetentionPolicy.SOURCE)
     @IntDef(prefix = {"TIMESTAMP_BASE_"}, value =
@@ -252,7 +272,8 @@ public final class OutputConfiguration implements Parcelable {
          TIMESTAMP_BASE_SENSOR,
          TIMESTAMP_BASE_MONOTONIC,
          TIMESTAMP_BASE_REALTIME,
-         TIMESTAMP_BASE_CHOREOGRAPHER_SYNCED})
+         TIMESTAMP_BASE_CHOREOGRAPHER_SYNCED,
+         TIMESTAMP_BASE_READOUT_SENSOR})
     public @interface TimestampBase {};
 
     /** @hide */
@@ -974,7 +995,7 @@ public final class OutputConfiguration implements Parcelable {
     public void setTimestampBase(@TimestampBase int timestampBase) {
         // Verify that the value is in range
         if (timestampBase < TIMESTAMP_BASE_DEFAULT ||
-                timestampBase > TIMESTAMP_BASE_CHOREOGRAPHER_SYNCED) {
+                timestampBase > TIMESTAMP_BASE_READOUT_SENSOR) {
             throw new IllegalArgumentException("Not a valid timestamp base value " +
                     timestampBase);
         }