Android: Use a spawning queue for child processes while we are at capacity.

content/public/android/java/src/org/chromium/content/browser/ChildProcessLauncher.java

 // Copyright 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 package org.chromium.content.browser;
 
 import android.content.Context;
 import android.content.pm.ApplicationInfo;
 import android.content.pm.PackageManager;
 import android.graphics.SurfaceTexture;
 import android.os.RemoteException;
 import android.util.Log;
 import android.util.Pair;
 import android.view.Surface;
 
 import org.chromium.base.CalledByNative;
 import org.chromium.base.JNINamespace;
 import org.chromium.base.ThreadUtils;
 import org.chromium.base.TraceEvent;
 import org.chromium.base.VisibleForTesting;
 import org.chromium.base.library_loader.Linker;
 import org.chromium.content.app.ChildProcessService;
 import org.chromium.content.app.ChromiumLinkerParams;
 import org.chromium.content.app.PrivilegedProcessService;
 import org.chromium.content.app.SandboxedProcessService;
 import org.chromium.content.common.IChildProcessCallback;
 import org.chromium.content.common.SurfaceWrapper;
 
 import java.util.ArrayList;
+import java.util.LinkedList;
 import java.util.Map;
+import java.util.Queue;
 import java.util.concurrent.ConcurrentHashMap;
 
 /**
  * This class provides the method to start/stop ChildProcess called by native.
  */
 @JNINamespace("content")
 public class ChildProcessLauncher {
     private static final String TAG = "ChildProcessLauncher";
 
     static final int CALLBACK_FOR_UNKNOWN_PROCESS = 0;
     static final int CALLBACK_FOR_GPU_PROCESS = 1;
     static final int CALLBACK_FOR_RENDERER_PROCESS = 2;
     static final int CALLBACK_FOR_UTILITY_PROCESS = 3;
 
     private static final String SWITCH_PROCESS_TYPE = "type";
     private static final String SWITCH_RENDERER_PROCESS = "renderer";
     private static final String SWITCH_UTILITY_PROCESS = "utility";
     private static final String SWITCH_GPU_PROCESS = "gpu-process";
 
     private static class ChildConnectionAllocator {
         // Connections to services. Indices of the array correspond to the service numbers.
         private final ChildProcessConnection[] mChildProcessConnections;
 
-        // The list of free (not bound) service indices. When looking for a free service, the first
-        // index in that list should be used. When a service is unbound, its index is added to the
-        // end of the list. This is so that we avoid immediately reusing the freed service (see
-        // http://crbug.com/164069): the framework might keep a service process alive when it's been
-        // unbound for a short time. If a new connection to the same service is bound at that point,
-        // the process is reused and bad things happen (mostly static variables are set when we
-        // don't expect them to).
+        // The list of free (not bound) service indices.
         // SHOULD BE ACCESSED WITH mConnectionLock.
         private final ArrayList<Integer> mFreeConnectionIndices;
         private final Object mConnectionLock = new Object();
 
         private Class<? extends ChildProcessService> mChildClass;
         private final boolean mInSandbox;
 
         public ChildConnectionAllocator(boolean inSandbox, int numChildServices) {
             mChildProcessConnections = new ChildProcessConnectionImpl[numChildServices];
             mFreeConnectionIndices = new ArrayList<Integer>(numChildServices);
             for (int i = 0; i < numChildServices; i++) {
                 mFreeConnectionIndices.add(i);
             }
             mChildClass =
                 inSandbox ? SandboxedProcessService.class : PrivilegedProcessService.class;
             mInSandbox = inSandbox;
         }
 
         public ChildProcessConnection allocate(
                 Context context, ChildProcessConnection.DeathCallback deathCallback,
                 ChromiumLinkerParams chromiumLinkerParams) {
             synchronized (mConnectionLock) {
                 if (mFreeConnectionIndices.isEmpty()) {
-                    Log.e(TAG, "Ran out of services to allocate.");
-                    assert false;
+                    Log.d(TAG, "Ran out of services to allocate.");
                     return null;
                 }
                 int slot = mFreeConnectionIndices.remove(0);
                 assert mChildProcessConnections[slot] == null;
                 mChildProcessConnections[slot] = new ChildProcessConnectionImpl(context, slot,
                         mInSandbox, deathCallback, mChildClass, chromiumLinkerParams);
                 Log.d(TAG, "Allocator allocated a connection, sandbox: " + mInSandbox
                         + ", slot: " + slot);
                 return mChildProcessConnections[slot];
             }
@@ skipping 18 matching lines @@
             }
         }
 
         /** @return the count of connections managed by the allocator */
         @VisibleForTesting
         int allocatedConnectionsCountForTesting() {
             return mChildProcessConnections.length - mFreeConnectionIndices.size();
         }
     }
 
+    private static class PendingSpawnData {
+        private final Context mContext;
+        private final String[] mCommandLine;
+        private final int mChildProcessId;
+        private final FileDescriptorInfo[] mFilesToBeMapped;
+        private final long mClientContext;
+        private final int mCallbackType;
+        private final boolean mInSandbox;
+
+        private PendingSpawnData(
+                Context context,
+                String[] commandLine,
+                int childProcessId,
+                FileDescriptorInfo[] filesToBeMapped,
+                long clientContext,
+                int callbackType,
+                boolean inSandbox) {
+            mContext = context;
+            mCommandLine = commandLine;
+            mChildProcessId = childProcessId;
+            mFilesToBeMapped = filesToBeMapped;
+            mClientContext = clientContext;
+            mCallbackType = callbackType;
+            mInSandbox = inSandbox;
+        }
+
+        private Context context() {
+            return mContext;
+        }
+        private String[] commandLine() {
+            return mCommandLine;
+        }
+        private int childProcessId() {
+            return mChildProcessId;
+        }
+        private FileDescriptorInfo[] filesToBeMapped() {
+            return mFilesToBeMapped;
+        }
+        private long clientContext() {
+            return mClientContext;
+        }
+        private int callbackType() {
+            return mCallbackType;
+        }
+        private boolean inSandbox() {
+            return mInSandbox;
+        }
+    }
+
+    private static class PendingSpawnQueue {
+        // The list of pending process spawn requests and its lock.
+        private static Queue<PendingSpawnData> sPendingSpawns =
+                new LinkedList<PendingSpawnData>();
+        static final Object sPendingSpawnsLock = new Object();
+
+        /**
+         * Queue up a spawn requests to be processed once a free service is available.
+         * Called when a spawn is requested while we are at the capacity.
+         */
+        public void enqueue(final PendingSpawnData pendingSpawn) {
+            synchronized (sPendingSpawnsLock) {
+                sPendingSpawns.add(pendingSpawn);
+            }
+        }
+
+        /**
+         * Pop the next request from the queue. Called when a free service is available.
+         * @return the next spawn request waiting in the queue.
+         */
+        public PendingSpawnData dequeue() {
+            synchronized (sPendingSpawnsLock) {
+                return sPendingSpawns.poll();
+            }
+        }
+
+        /** @return the count of pending spawns in the queue */
+        public int size() {
+            synchronized (sPendingSpawnsLock) {
+                return sPendingSpawns.size();
+            }
+        }
+    }
+
+    private static final PendingSpawnQueue sPendingSpawnQueue = new PendingSpawnQueue();
+
     // Service class for child process. As the default value it uses SandboxedProcessService0 and
     // PrivilegedProcessService0.
     private static ChildConnectionAllocator sSandboxedChildConnectionAllocator;
     private static ChildConnectionAllocator sPrivilegedChildConnectionAllocator;
 
     private static final String NUM_SANDBOXED_SERVICES_KEY =
             "org.chromium.content.browser.NUM_SANDBOXED_SERVICES";
     private static final String NUM_PRIVILEGED_SERVICES_KEY =
             "org.chromium.content.browser.NUM_PRIVILEGED_SERVICES";
 
@@ skipping 77 matching lines @@
             String[] commandLine, boolean inSandbox) {
         ChromiumLinkerParams chromiumLinkerParams = getLinkerParamsForNewConnection();
         ChildProcessConnection connection =
                 allocateConnection(context, inSandbox, chromiumLinkerParams);
         if (connection != null) {
             connection.start(commandLine);
         }
         return connection;
     }
 
+    private static final long FREE_CONNECTION_DELAY_MILLIS = 1;
+
     private static void freeConnection(ChildProcessConnection connection) {
-        getConnectionAllocator(connection.isInSandbox()).free(connection);
+        // Freeing a service should be delayed. This is so that we avoid immediately reusing the
+        // freed service (see http://crbug.com/164069): the framework might keep a service process
+        // alive when it's been unbound for a short time. If a new connection to the same service
+        // is bound at that point, the process is reused and bad things happen (mostly static
+        // variables are set when we don't expect them to).
+        final ChildProcessConnection conn = connection;
+        ThreadUtils.postOnUiThreadDelayed(new Runnable() {
+            @Override
+            public void run() {
+                getConnectionAllocator(conn.isInSandbox()).free(conn);
+
+                final PendingSpawnData pendingSpawn = sPendingSpawnQueue.dequeue();
+                if (pendingSpawn != null) {
+                    new Thread(new Runnable() {
+                        @Override
+                        public void run() {
+                            startInternal(pendingSpawn.context(), pendingSpawn.commandLine(),
+                                    pendingSpawn.childProcessId(), pendingSpawn.filesToBeMapped(),
+                                    pendingSpawn.clientContext(), pendingSpawn.callbackType(),
+                                    pendingSpawn.inSandbox());
+                        }
+                    }).start();
+                }
+            }
+        }, FREE_CONNECTION_DELAY_MILLIS);
     }
 
     // Represents an invalid process handle; same as base/process/process.h kNullProcessHandle.
     private static final int NULL_PROCESS_HANDLE = 0;
 
     // Map from pid to ChildService connection.
     private static Map<Integer, ChildProcessConnection> sServiceMap =
             new ConcurrentHashMap<Integer, ChildProcessConnection>();
 
     // A pre-allocated and pre-bound connection ready for connection setup, or null.
@@ skipping 147 matching lines @@
      */
     @CalledByNative
     static void start(
             Context context,
             final String[] commandLine,
             int childProcessId,
             int[] fileIds,
             int[] fileFds,
             boolean[] fileAutoClose,
             long clientContext) {
+        assert fileIds.length == fileFds.length && fileFds.length == fileAutoClose.length;
+        FileDescriptorInfo[] filesToBeMapped = new FileDescriptorInfo[fileFds.length];
+        for (int i = 0; i < fileFds.length; i++) {
+            filesToBeMapped[i] =
+                    new FileDescriptorInfo(fileIds[i], fileFds[i], fileAutoClose[i]);
+        }
+        assert clientContext != 0;
+
+        int callbackType = CALLBACK_FOR_UNKNOWN_PROCESS;
+        boolean inSandbox = true;
+        String processType = getSwitchValue(commandLine, SWITCH_PROCESS_TYPE);
+        if (SWITCH_RENDERER_PROCESS.equals(processType)) {
+            callbackType = CALLBACK_FOR_RENDERER_PROCESS;
+        } else if (SWITCH_GPU_PROCESS.equals(processType)) {
+            callbackType = CALLBACK_FOR_GPU_PROCESS;
+            inSandbox = false;
+        } else if (SWITCH_UTILITY_PROCESS.equals(processType)) {
+            // We only support sandboxed right now.
+            callbackType = CALLBACK_FOR_UTILITY_PROCESS;
+        } else {
+            assert false;
+        }
+
+        startInternal(context, commandLine, childProcessId, filesToBeMapped, clientContext,
+                callbackType, inSandbox);
+    }
+
+    private static void startInternal(
+            Context context,
+            final String[] commandLine,
+            int childProcessId,
+            FileDescriptorInfo[] filesToBeMapped,
+            long clientContext,
+            int callbackType,
+            boolean inSandbox) {
         try {
-            TraceEvent.begin("ChildProcessLauncher.start");
-            assert fileIds.length == fileFds.length && fileFds.length == fileAutoClose.length;
-            FileDescriptorInfo[] filesToBeMapped = new FileDescriptorInfo[fileFds.length];
-            for (int i = 0; i < fileFds.length; i++) {
-                filesToBeMapped[i] =
-                        new FileDescriptorInfo(fileIds[i], fileFds[i], fileAutoClose[i]);
-            }
-            assert clientContext != 0;
-
-            int callbackType = CALLBACK_FOR_UNKNOWN_PROCESS;
-            boolean inSandbox = true;
-            String processType = getSwitchValue(commandLine, SWITCH_PROCESS_TYPE);
-            if (SWITCH_RENDERER_PROCESS.equals(processType)) {
-                callbackType = CALLBACK_FOR_RENDERER_PROCESS;
-            } else if (SWITCH_GPU_PROCESS.equals(processType)) {
-                callbackType = CALLBACK_FOR_GPU_PROCESS;
-                inSandbox = false;
-            } else if (SWITCH_UTILITY_PROCESS.equals(processType)) {
-                // We only support sandboxed right now.
-                callbackType = CALLBACK_FOR_UTILITY_PROCESS;
-            } else {
-                assert false;
-            }
+            TraceEvent.begin("ChildProcessLauncher.startInternal");
 
             ChildProcessConnection allocatedConnection = null;
             synchronized (ChildProcessLauncher.class) {
                 if (inSandbox) {
                     allocatedConnection = sSpareSandboxedConnection;
                     sSpareSandboxedConnection = null;
                 }
             }
             if (allocatedConnection == null) {
                 allocatedConnection = allocateBoundConnection(context, commandLine, inSandbox);
                 if (allocatedConnection == null) {
-                    // Notify the native code so it can free the heap allocated callback.
-                    nativeOnChildProcessStarted(clientContext, 0);
-                    Log.e(TAG, "Allocation of new service failed.");
+                    Log.d(TAG, "Allocation of new service failed. Queuing up pending spawn.");
+                    sPendingSpawnQueue.enqueue(new PendingSpawnData(context, commandLine,
+                            childProcessId, filesToBeMapped, clientContext, callbackType,
+                            inSandbox));
                     return;
                 }
             }
 
             Log.d(TAG, "Setting up connection to process: slot="
                     + allocatedConnection.getServiceNumber());
             triggerConnectionSetup(allocatedConnection, commandLine, childProcessId,
                     filesToBeMapped, callbackType, clientContext);
         } finally {
-            TraceEvent.end("ChildProcessLauncher.start");
+            TraceEvent.end("ChildProcessLauncher.startInternal");
         }
     }
 
     @VisibleForTesting
     static void triggerConnectionSetup(
             final ChildProcessConnection connection,
             String[] commandLine,
             int childProcessId,
             FileDescriptorInfo[] filesToBeMapped,
             int callbackType,
@@ skipping 125 matching lines @@
         if (pid > 0 && !nativeIsSingleProcess()) {
             Log.w(TAG, message + ", pid=" + pid);
         }
     }
 
     @VisibleForTesting
     static ChildProcessConnection allocateBoundConnectionForTesting(Context context) {
         return allocateBoundConnection(context, null, true);
     }
 
+    /**
+     * Queue up a spawn requests for testing.
+     */
+    @VisibleForTesting
+    static void enqueuePendingSpawnForTesting(Context context) {
+        sPendingSpawnQueue.enqueue(new PendingSpawnData(context, new String[0], 1,
+                new FileDescriptorInfo[0], 0, CALLBACK_FOR_RENDERER_PROCESS, true));
+    }
+
     /** @return the count of sandboxed connections managed by the allocator */
     @VisibleForTesting
     static int allocatedConnectionsCountForTesting(Context context) {
         initConnectionAllocatorsIfNecessary(context);
         return sSandboxedChildConnectionAllocator.allocatedConnectionsCountForTesting();
     }
 
     /** @return the count of services set up and working */
     @VisibleForTesting
     static int connectedServicesCountForTesting() {
         return sServiceMap.size();
     }
 
+    /** @return the count of pending spawns in the queue */
+    @VisibleForTesting
+    static int pendingSpawnsCountForTesting() {
+        return sPendingSpawnQueue.size();
+    }
+
     /**
      * Kills the child process for testing.
      * @return true iff the process was killed as expected
      */
     @VisibleForTesting
     public static boolean crashProcessForTesting(int pid) {
         if (sServiceMap.get(pid) == null) return false;
 
         try {
             ((ChildProcessConnectionImpl) sServiceMap.get(pid)).crashServiceForTesting();
         } catch (RemoteException ex) {
             return false;
         }
 
         return true;
     }
 
     private static native void nativeOnChildProcessStarted(long clientContext, int pid);
     private static native void nativeEstablishSurfacePeer(
             int pid, Surface surface, int primaryID, int secondaryID);
     private static native boolean nativeIsSingleProcess();
 }