/*
 * Copyright (C) 2014 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package android.hardware.camera2.legacy;

import android.hardware.camera2.impl.CameraDeviceImpl;
import android.util.Log;
import android.util.MutableLong;
import android.util.Pair;
import android.view.Surface;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;

Collect timestamps and state for each CaptureRequest as it passes through the Legacy camera pipeline.
/** * Collect timestamps and state for each {@link CaptureRequest} as it passes through * the Legacy camera pipeline. */
public class CaptureCollector { private static final String TAG = "CaptureCollector"; private static final boolean DEBUG = false; private static final int FLAG_RECEIVED_JPEG = 1; private static final int FLAG_RECEIVED_JPEG_TS = 2; private static final int FLAG_RECEIVED_PREVIEW = 4; private static final int FLAG_RECEIVED_PREVIEW_TS = 8; private static final int FLAG_RECEIVED_ALL_JPEG = FLAG_RECEIVED_JPEG | FLAG_RECEIVED_JPEG_TS; private static final int FLAG_RECEIVED_ALL_PREVIEW = FLAG_RECEIVED_PREVIEW | FLAG_RECEIVED_PREVIEW_TS; private static final int MAX_JPEGS_IN_FLIGHT = 1; private class CaptureHolder implements Comparable<CaptureHolder>{ private final RequestHolder mRequest; private final LegacyRequest mLegacy; public final boolean needsJpeg; public final boolean needsPreview; private long mTimestamp = 0; private int mReceivedFlags = 0; private boolean mHasStarted = false; private boolean mFailedJpeg = false; private boolean mFailedPreview = false; private boolean mCompleted = false; private boolean mPreviewCompleted = false; public CaptureHolder(RequestHolder request, LegacyRequest legacyHolder) { mRequest = request; mLegacy = legacyHolder; needsJpeg = request.hasJpegTargets(); needsPreview = request.hasPreviewTargets(); } public boolean isPreviewCompleted() { return (mReceivedFlags & FLAG_RECEIVED_ALL_PREVIEW) == FLAG_RECEIVED_ALL_PREVIEW; } public boolean isJpegCompleted() { return (mReceivedFlags & FLAG_RECEIVED_ALL_JPEG) == FLAG_RECEIVED_ALL_JPEG; } public boolean isCompleted() { return (needsJpeg == isJpegCompleted()) && (needsPreview == isPreviewCompleted()); } public void tryComplete() { if (!mPreviewCompleted && needsPreview && isPreviewCompleted()) { CaptureCollector.this.onPreviewCompleted(); mPreviewCompleted = true; } if (isCompleted() && !mCompleted) { if (mFailedPreview || mFailedJpeg) { if (!mHasStarted) { // Send a request error if the capture has not yet started. mRequest.failRequest(); CaptureCollector.this.mDeviceState.setCaptureStart(mRequest, mTimestamp, CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_REQUEST); } else { // Send buffer dropped errors for each pending buffer if the request has // started. for (Surface targetSurface : mRequest.getRequest().getTargets() ) { try { if (mRequest.jpegType(targetSurface)) { if (mFailedJpeg) { CaptureCollector.this.mDeviceState.setCaptureResult(mRequest, /*result*/null, CameraDeviceImpl.CameraDeviceCallbacks. ERROR_CAMERA_BUFFER, targetSurface); } } else { // preview buffer if (mFailedPreview) { CaptureCollector.this.mDeviceState.setCaptureResult(mRequest, /*result*/null, CameraDeviceImpl.CameraDeviceCallbacks. ERROR_CAMERA_BUFFER, targetSurface); } } } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) { Log.e(TAG, "Unexpected exception when querying Surface: " + e); } } } } CaptureCollector.this.onRequestCompleted(CaptureHolder.this); mCompleted = true; } } public void setJpegTimestamp(long timestamp) { if (DEBUG) { Log.d(TAG, "setJpegTimestamp - called for request " + mRequest.getRequestId()); } if (!needsJpeg) { throw new IllegalStateException( "setJpegTimestamp called for capture with no jpeg targets."); } if (isCompleted()) { throw new IllegalStateException( "setJpegTimestamp called on already completed request."); } mReceivedFlags |= FLAG_RECEIVED_JPEG_TS; if (mTimestamp == 0) { mTimestamp = timestamp; } if (!mHasStarted) { mHasStarted = true; CaptureCollector.this.mDeviceState.setCaptureStart(mRequest, mTimestamp, CameraDeviceState.NO_CAPTURE_ERROR); } tryComplete(); } public void setJpegProduced() { if (DEBUG) { Log.d(TAG, "setJpegProduced - called for request " + mRequest.getRequestId()); } if (!needsJpeg) { throw new IllegalStateException( "setJpegProduced called for capture with no jpeg targets."); } if (isCompleted()) { throw new IllegalStateException( "setJpegProduced called on already completed request."); } mReceivedFlags |= FLAG_RECEIVED_JPEG; tryComplete(); } public void setJpegFailed() { if (DEBUG) { Log.d(TAG, "setJpegFailed - called for request " + mRequest.getRequestId()); } if (!needsJpeg || isJpegCompleted()) { return; } mFailedJpeg = true; mReceivedFlags |= FLAG_RECEIVED_JPEG; mReceivedFlags |= FLAG_RECEIVED_JPEG_TS; tryComplete(); } public void setPreviewTimestamp(long timestamp) { if (DEBUG) { Log.d(TAG, "setPreviewTimestamp - called for request " + mRequest.getRequestId()); } if (!needsPreview) { throw new IllegalStateException( "setPreviewTimestamp called for capture with no preview targets."); } if (isCompleted()) { throw new IllegalStateException( "setPreviewTimestamp called on already completed request."); } mReceivedFlags |= FLAG_RECEIVED_PREVIEW_TS; if (mTimestamp == 0) { mTimestamp = timestamp; } if (!needsJpeg) { if (!mHasStarted) { mHasStarted = true; CaptureCollector.this.mDeviceState.setCaptureStart(mRequest, mTimestamp, CameraDeviceState.NO_CAPTURE_ERROR); } } tryComplete(); } public void setPreviewProduced() { if (DEBUG) { Log.d(TAG, "setPreviewProduced - called for request " + mRequest.getRequestId()); } if (!needsPreview) { throw new IllegalStateException( "setPreviewProduced called for capture with no preview targets."); } if (isCompleted()) { throw new IllegalStateException( "setPreviewProduced called on already completed request."); } mReceivedFlags |= FLAG_RECEIVED_PREVIEW; tryComplete(); } public void setPreviewFailed() { if (DEBUG) { Log.d(TAG, "setPreviewFailed - called for request " + mRequest.getRequestId()); } if (!needsPreview || isPreviewCompleted()) { return; } mFailedPreview = true; mReceivedFlags |= FLAG_RECEIVED_PREVIEW; mReceivedFlags |= FLAG_RECEIVED_PREVIEW_TS; tryComplete(); } // Comparison and equals based on frame number. @Override public int compareTo(CaptureHolder captureHolder) { return (mRequest.getFrameNumber() > captureHolder.mRequest.getFrameNumber()) ? 1 : ((mRequest.getFrameNumber() == captureHolder.mRequest.getFrameNumber()) ? 0 : -1); } // Comparison and equals based on frame number. @Override public boolean equals(Object o) { return o instanceof CaptureHolder && compareTo((CaptureHolder) o) == 0; } } private final TreeSet<CaptureHolder> mActiveRequests; private final ArrayDeque<CaptureHolder> mJpegCaptureQueue; private final ArrayDeque<CaptureHolder> mJpegProduceQueue; private final ArrayDeque<CaptureHolder> mPreviewCaptureQueue; private final ArrayDeque<CaptureHolder> mPreviewProduceQueue; private final ArrayList<CaptureHolder> mCompletedRequests = new ArrayList<>(); private final ReentrantLock mLock = new ReentrantLock(); private final Condition mIsEmpty; private final Condition mPreviewsEmpty; private final Condition mNotFull; private final CameraDeviceState mDeviceState; private int mInFlight = 0; private int mInFlightPreviews = 0; private final int mMaxInFlight;
Create a new CaptureCollector that can modify the given CameraDeviceState.
Params:
  • maxInFlight – max allowed in-flight requests.
  • deviceState – the CameraDeviceState to update as requests are processed.
/** * Create a new {@link CaptureCollector} that can modify the given {@link CameraDeviceState}. * * @param maxInFlight max allowed in-flight requests. * @param deviceState the {@link CameraDeviceState} to update as requests are processed. */
public CaptureCollector(int maxInFlight, CameraDeviceState deviceState) { mMaxInFlight = maxInFlight; mJpegCaptureQueue = new ArrayDeque<>(MAX_JPEGS_IN_FLIGHT); mJpegProduceQueue = new ArrayDeque<>(MAX_JPEGS_IN_FLIGHT); mPreviewCaptureQueue = new ArrayDeque<>(mMaxInFlight); mPreviewProduceQueue = new ArrayDeque<>(mMaxInFlight); mActiveRequests = new TreeSet<>(); mIsEmpty = mLock.newCondition(); mNotFull = mLock.newCondition(); mPreviewsEmpty = mLock.newCondition(); mDeviceState = deviceState; }
Queue a new request.

For requests that use the Camera1 API preview output stream, this will block if there are already maxInFlight requests in progress (until at least one prior request has completed). For requests that use the Camera1 API jpeg callbacks, this will block until all prior requests have been completed to avoid stopping preview for Camera.takePicture before prior preview requests have been completed.

Params:
  • holder – the RequestHolder for this request.
  • legacy – the LegacyRequest for this request; this will not be mutated.
  • timeout – a timeout to use for this call.
  • unit – the units to use for the timeout.
Throws:
Returns:false if this method timed out.
/** * Queue a new request. * * <p> * For requests that use the Camera1 API preview output stream, this will block if there are * already {@code maxInFlight} requests in progress (until at least one prior request has * completed). For requests that use the Camera1 API jpeg callbacks, this will block until * all prior requests have been completed to avoid stopping preview for * {@link android.hardware.Camera#takePicture} before prior preview requests have been * completed. * </p> * @param holder the {@link RequestHolder} for this request. * @param legacy the {@link LegacyRequest} for this request; this will not be mutated. * @param timeout a timeout to use for this call. * @param unit the units to use for the timeout. * @return {@code false} if this method timed out. * @throws InterruptedException if this thread is interrupted. */
public boolean queueRequest(RequestHolder holder, LegacyRequest legacy, long timeout, TimeUnit unit) throws InterruptedException { CaptureHolder h = new CaptureHolder(holder, legacy); long nanos = unit.toNanos(timeout); final ReentrantLock lock = this.mLock; lock.lock(); try { if (DEBUG) { Log.d(TAG, "queueRequest for request " + holder.getRequestId() + " - " + mInFlight + " requests remain in flight."); } if (!(h.needsJpeg || h.needsPreview)) { throw new IllegalStateException("Request must target at least one output surface!"); } if (h.needsJpeg) { // Wait for all current requests to finish before queueing jpeg. while (mInFlight > 0) { if (nanos <= 0) { return false; } nanos = mIsEmpty.awaitNanos(nanos); } mJpegCaptureQueue.add(h); mJpegProduceQueue.add(h); } if (h.needsPreview) { while (mInFlight >= mMaxInFlight) { if (nanos <= 0) { return false; } nanos = mNotFull.awaitNanos(nanos); } mPreviewCaptureQueue.add(h); mPreviewProduceQueue.add(h); mInFlightPreviews++; } mActiveRequests.add(h); mInFlight++; return true; } finally { lock.unlock(); } }
Wait all queued requests to complete.
Params:
  • timeout – a timeout to use for this call.
  • unit – the units to use for the timeout.
Throws:
Returns:false if this method timed out.
/** * Wait all queued requests to complete. * * @param timeout a timeout to use for this call. * @param unit the units to use for the timeout. * @return {@code false} if this method timed out. * @throws InterruptedException if this thread is interrupted. */
public boolean waitForEmpty(long timeout, TimeUnit unit) throws InterruptedException { long nanos = unit.toNanos(timeout); final ReentrantLock lock = this.mLock; lock.lock(); try { while (mInFlight > 0) { if (nanos <= 0) { return false; } nanos = mIsEmpty.awaitNanos(nanos); } return true; } finally { lock.unlock(); } }
Wait all queued requests that use the Camera1 API preview output to complete.
Params:
  • timeout – a timeout to use for this call.
  • unit – the units to use for the timeout.
Throws:
Returns:false if this method timed out.
/** * Wait all queued requests that use the Camera1 API preview output to complete. * * @param timeout a timeout to use for this call. * @param unit the units to use for the timeout. * @return {@code false} if this method timed out. * @throws InterruptedException if this thread is interrupted. */
public boolean waitForPreviewsEmpty(long timeout, TimeUnit unit) throws InterruptedException { long nanos = unit.toNanos(timeout); final ReentrantLock lock = this.mLock; lock.lock(); try { while (mInFlightPreviews > 0) { if (nanos <= 0) { return false; } nanos = mPreviewsEmpty.awaitNanos(nanos); } return true; } finally { lock.unlock(); } }
Wait for the specified request to be completed (all buffers available).

May not wait for the same request more than once, since a successful wait will erase the history of that request.

Params:
  • holder – the RequestHolder for this request.
  • timeout – a timeout to use for this call.
  • unit – the units to use for the timeout.
  • timestamp – the timestamp of the request will be written out to here, in ns
Throws:
Returns:false if this method timed out.
/** * Wait for the specified request to be completed (all buffers available). * * <p>May not wait for the same request more than once, since a successful wait * will erase the history of that request.</p> * * @param holder the {@link RequestHolder} for this request. * @param timeout a timeout to use for this call. * @param unit the units to use for the timeout. * @param timestamp the timestamp of the request will be written out to here, in ns * * @return {@code false} if this method timed out. * * @throws InterruptedException if this thread is interrupted. */
public boolean waitForRequestCompleted(RequestHolder holder, long timeout, TimeUnit unit, MutableLong timestamp) throws InterruptedException { long nanos = unit.toNanos(timeout); final ReentrantLock lock = this.mLock; lock.lock(); try { while (!removeRequestIfCompleted(holder, /*out*/timestamp)) { if (nanos <= 0) { return false; } nanos = mNotFull.awaitNanos(nanos); } return true; } finally { lock.unlock(); } } private boolean removeRequestIfCompleted(RequestHolder holder, MutableLong timestamp) { int i = 0; for (CaptureHolder h : mCompletedRequests) { if (h.mRequest.equals(holder)) { timestamp.value = h.mTimestamp; mCompletedRequests.remove(i); return true; } i++; } return false; }
Called to alert the CaptureCollector that the jpeg capture has begun.
Params:
  • timestamp – the time of the jpeg capture.
Returns:the RequestHolder for the request associated with this capture.
/** * Called to alert the {@link CaptureCollector} that the jpeg capture has begun. * * @param timestamp the time of the jpeg capture. * @return the {@link RequestHolder} for the request associated with this capture. */
public RequestHolder jpegCaptured(long timestamp) { final ReentrantLock lock = this.mLock; lock.lock(); try { CaptureHolder h = mJpegCaptureQueue.poll(); if (h == null) { Log.w(TAG, "jpegCaptured called with no jpeg request on queue!"); return null; } h.setJpegTimestamp(timestamp); return h.mRequest; } finally { lock.unlock(); } }
Called to alert the CaptureCollector that the jpeg capture has completed.
Returns:a pair containing the RequestHolder and the timestamp of the capture.
/** * Called to alert the {@link CaptureCollector} that the jpeg capture has completed. * * @return a pair containing the {@link RequestHolder} and the timestamp of the capture. */
public Pair<RequestHolder, Long> jpegProduced() { final ReentrantLock lock = this.mLock; lock.lock(); try { CaptureHolder h = mJpegProduceQueue.poll(); if (h == null) { Log.w(TAG, "jpegProduced called with no jpeg request on queue!"); return null; } h.setJpegProduced(); return new Pair<>(h.mRequest, h.mTimestamp); } finally { lock.unlock(); } }
Check if there are any pending capture requests that use the Camera1 API preview output.
Returns:true if there are pending preview requests.
/** * Check if there are any pending capture requests that use the Camera1 API preview output. * * @return {@code true} if there are pending preview requests. */
public boolean hasPendingPreviewCaptures() { final ReentrantLock lock = this.mLock; lock.lock(); try { return !mPreviewCaptureQueue.isEmpty(); } finally { lock.unlock(); } }
Called to alert the CaptureCollector that the preview capture has begun.
Params:
  • timestamp – the time of the preview capture.
Returns:a pair containing the RequestHolder and the timestamp of the capture.
/** * Called to alert the {@link CaptureCollector} that the preview capture has begun. * * @param timestamp the time of the preview capture. * @return a pair containing the {@link RequestHolder} and the timestamp of the capture. */
public Pair<RequestHolder, Long> previewCaptured(long timestamp) { final ReentrantLock lock = this.mLock; lock.lock(); try { CaptureHolder h = mPreviewCaptureQueue.poll(); if (h == null) { if (DEBUG) { Log.d(TAG, "previewCaptured called with no preview request on queue!"); } return null; } h.setPreviewTimestamp(timestamp); return new Pair<>(h.mRequest, h.mTimestamp); } finally { lock.unlock(); } }
Called to alert the CaptureCollector that the preview capture has completed.
Returns:the RequestHolder for the request associated with this capture.
/** * Called to alert the {@link CaptureCollector} that the preview capture has completed. * * @return the {@link RequestHolder} for the request associated with this capture. */
public RequestHolder previewProduced() { final ReentrantLock lock = this.mLock; lock.lock(); try { CaptureHolder h = mPreviewProduceQueue.poll(); if (h == null) { Log.w(TAG, "previewProduced called with no preview request on queue!"); return null; } h.setPreviewProduced(); return h.mRequest; } finally { lock.unlock(); } }
Called to alert the CaptureCollector that the next pending preview capture has failed.
/** * Called to alert the {@link CaptureCollector} that the next pending preview capture has failed. */
public void failNextPreview() { final ReentrantLock lock = this.mLock; lock.lock(); try { CaptureHolder h1 = mPreviewCaptureQueue.peek(); CaptureHolder h2 = mPreviewProduceQueue.peek(); // Find the request with the lowest frame number. CaptureHolder h = (h1 == null) ? h2 : ((h2 == null) ? h1 : ((h1.compareTo(h2) <= 0) ? h1 : h2)); if (h != null) { mPreviewCaptureQueue.remove(h); mPreviewProduceQueue.remove(h); mActiveRequests.remove(h); h.setPreviewFailed(); } } finally { lock.unlock(); } }
Called to alert the CaptureCollector that the next pending jpeg capture has failed.
/** * Called to alert the {@link CaptureCollector} that the next pending jpeg capture has failed. */
public void failNextJpeg() { final ReentrantLock lock = this.mLock; lock.lock(); try { CaptureHolder h1 = mJpegCaptureQueue.peek(); CaptureHolder h2 = mJpegProduceQueue.peek(); // Find the request with the lowest frame number. CaptureHolder h = (h1 == null) ? h2 : ((h2 == null) ? h1 : ((h1.compareTo(h2) <= 0) ? h1 : h2)); if (h != null) { mJpegCaptureQueue.remove(h); mJpegProduceQueue.remove(h); mActiveRequests.remove(h); h.setJpegFailed(); } } finally { lock.unlock(); } }
Called to alert the CaptureCollector all pending captures have failed.
/** * Called to alert the {@link CaptureCollector} all pending captures have failed. */
public void failAll() { final ReentrantLock lock = this.mLock; lock.lock(); try { CaptureHolder h; while ((h = mActiveRequests.pollFirst()) != null) { h.setPreviewFailed(); h.setJpegFailed(); } mPreviewCaptureQueue.clear(); mPreviewProduceQueue.clear(); mJpegCaptureQueue.clear(); mJpegProduceQueue.clear(); } finally { lock.unlock(); } } private void onPreviewCompleted() { mInFlightPreviews--; if (mInFlightPreviews < 0) { throw new IllegalStateException( "More preview captures completed than requests queued."); } if (mInFlightPreviews == 0) { mPreviewsEmpty.signalAll(); } } private void onRequestCompleted(CaptureHolder capture) { RequestHolder request = capture.mRequest; mInFlight--; if (DEBUG) { Log.d(TAG, "Completed request " + request.getRequestId() + ", " + mInFlight + " requests remain in flight."); } if (mInFlight < 0) { throw new IllegalStateException( "More captures completed than requests queued."); } mCompletedRequests.add(capture); mActiveRequests.remove(capture); mNotFull.signalAll(); if (mInFlight == 0) { mIsEmpty.signalAll(); } } }