Source code

001/*
002 * Copyright (C) 2012 The Guava Authors
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
005 * in compliance with the License. You may obtain a copy of the License at
006 *
007 * http://www.apache.org/licenses/LICENSE-2.0
008 *
009 * Unless required by applicable law or agreed to in writing, software distributed under the License
010 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
011 * or implied. See the License for the specific language governing permissions and limitations under
012 * the License.
013 */
014
015package com.google.common.util.concurrent;
016
017import static com.google.common.base.Preconditions.checkArgument;
018import static com.google.common.base.Preconditions.checkNotNull;
019import static com.google.common.base.Preconditions.checkState;
020import static com.google.common.base.Predicates.equalTo;
021import static com.google.common.base.Predicates.in;
022import static com.google.common.base.Predicates.instanceOf;
023import static com.google.common.base.Predicates.not;
024import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
025import static com.google.common.util.concurrent.Service.State.FAILED;
026import static com.google.common.util.concurrent.Service.State.NEW;
027import static com.google.common.util.concurrent.Service.State.RUNNING;
028import static com.google.common.util.concurrent.Service.State.STARTING;
029import static com.google.common.util.concurrent.Service.State.STOPPING;
030import static com.google.common.util.concurrent.Service.State.TERMINATED;
031import static java.util.concurrent.TimeUnit.MILLISECONDS;
032
033import com.google.common.annotations.GwtIncompatible;
034import com.google.common.annotations.J2ktIncompatible;
035import com.google.common.base.Function;
036import com.google.common.base.MoreObjects;
037import com.google.common.base.Stopwatch;
038import com.google.common.collect.Collections2;
039import com.google.common.collect.ImmutableCollection;
040import com.google.common.collect.ImmutableList;
041import com.google.common.collect.ImmutableMap;
042import com.google.common.collect.ImmutableSet;
043import com.google.common.collect.ImmutableSetMultimap;
044import com.google.common.collect.Lists;
045import com.google.common.collect.Maps;
046import com.google.common.collect.MultimapBuilder;
047import com.google.common.collect.Multimaps;
048import com.google.common.collect.Multiset;
049import com.google.common.collect.Ordering;
050import com.google.common.collect.SetMultimap;
051import com.google.common.util.concurrent.Service.State;
052import com.google.errorprone.annotations.CanIgnoreReturnValue;
053import com.google.errorprone.annotations.concurrent.GuardedBy;
054import com.google.j2objc.annotations.WeakOuter;
055import java.lang.ref.WeakReference;
056import java.util.Collections;
057import java.util.EnumSet;
058import java.util.List;
059import java.util.Map;
060import java.util.Map.Entry;
061import java.util.concurrent.Executor;
062import java.util.concurrent.TimeUnit;
063import java.util.concurrent.TimeoutException;
064import java.util.logging.Level;
065import java.util.logging.Logger;
066
067/**
068 * A manager for monitoring and controlling a set of {@linkplain Service services}. This class
069 * provides methods for {@linkplain #startAsync() starting}, {@linkplain #stopAsync() stopping} and
070 * {@linkplain #servicesByState inspecting} a collection of {@linkplain Service services}.
071 * Additionally, users can monitor state transitions with the {@linkplain Listener listener}
072 * mechanism.
073 *
074 * <p>While it is recommended that service lifecycles be managed via this class, state transitions
075 * initiated via other mechanisms do not impact the correctness of its methods. For example, if the
076 * services are started by some mechanism besides {@link #startAsync}, the listeners will be invoked
077 * when appropriate and {@link #awaitHealthy} will still work as expected.
078 *
079 * <p>Here is a simple example of how to use a {@code ServiceManager} to start a server.
080 *
081 * <pre>{@code
082 * class Server {
083 *   public static void main(String[] args) {
084 *     Set<Service> services = ...;
085 *     ServiceManager manager = new ServiceManager(services);
086 *     manager.addListener(new Listener() {
087 *         public void stopped() {}
088 *         public void healthy() {
089 *           // Services have been initialized and are healthy, start accepting requests...
090 *         }
091 *         public void failure(Service service) {
092 *           // Something failed, at this point we could log it, notify a load balancer, or take
093 *           // some other action.  For now we will just exit.
094 *           System.exit(1);
095 *         }
096 *       },
097 *       MoreExecutors.directExecutor());
098 *
099 *     Runtime.getRuntime().addShutdownHook(new Thread() {
100 *       public void run() {
101 *         // Give the services 5 seconds to stop to ensure that we are responsive to shutdown
102 *         // requests.
103 *         try {
104 *           manager.stopAsync().awaitStopped(5, TimeUnit.SECONDS);
105 *         } catch (TimeoutException timeout) {
106 *           // stopping timed out
107 *         }
108 *       }
109 *     });
110 *     manager.startAsync();  // start all the services asynchronously
111 *   }
112 * }
113 * }</pre>
114 *
115 * <p>This class uses the ServiceManager's methods to start all of its services, to respond to
116 * service failure and to ensure that when the JVM is shutting down all the services are stopped.
117 *
118 * @author Luke Sandberg
119 * @since 14.0
120 */
121@J2ktIncompatible
122@GwtIncompatible
123@ElementTypesAreNonnullByDefault
124public final class ServiceManager implements ServiceManagerBridge {
125  private static final Logger logger = Logger.getLogger(ServiceManager.class.getName());
126  private static final ListenerCallQueue.Event<Listener> HEALTHY_EVENT =
127      new ListenerCallQueue.Event<Listener>() {
128        @Override
129        public void call(Listener listener) {
130          listener.healthy();
131        }
132
133        @Override
134        public String toString() {
135          return "healthy()";
136        }
137      };
138  private static final ListenerCallQueue.Event<Listener> STOPPED_EVENT =
139      new ListenerCallQueue.Event<Listener>() {
140        @Override
141        public void call(Listener listener) {
142          listener.stopped();
143        }
144
145        @Override
146        public String toString() {
147          return "stopped()";
148        }
149      };
150
151  /**
152   * A listener for the aggregate state changes of the services that are under management. Users
153   * that need to listen to more fine-grained events (such as when each particular {@linkplain
154   * Service service} starts, or terminates), should attach {@linkplain Service.Listener service
155   * listeners} to each individual service.
156   *
157   * @author Luke Sandberg
158   * @since 15.0 (present as an interface in 14.0)
159   */
160  public abstract static class Listener {
161    /**
162     * Called when the service initially becomes healthy.
163     *
164     * <p>This will be called at most once after all the services have entered the {@linkplain
165     * State#RUNNING running} state. If any services fail during start up or {@linkplain
166     * State#FAILED fail}/{@linkplain State#TERMINATED terminate} before all other services have
167     * started {@linkplain State#RUNNING running} then this method will not be called.
168     */
169    public void healthy() {}
170
171    /**
172     * Called when the all of the component services have reached a terminal state, either
173     * {@linkplain State#TERMINATED terminated} or {@linkplain State#FAILED failed}.
174     */
175    public void stopped() {}
176
177    /**
178     * Called when a component service has {@linkplain State#FAILED failed}.
179     *
180     * @param service The service that failed.
181     */
182    public void failure(Service service) {}
183  }
184
185  /**
186   * An encapsulation of all of the state that is accessed by the {@linkplain ServiceListener
187   * service listeners}. This is extracted into its own object so that {@link ServiceListener} could
188   * be made {@code static} and its instances can be safely constructed and added in the {@link
189   * ServiceManager} constructor without having to close over the partially constructed {@link
190   * ServiceManager} instance (i.e. avoid leaking a pointer to {@code this}).
191   */
192  private final ServiceManagerState state;
193
194  private final ImmutableList<Service> services;
195
196  /**
197   * Constructs a new instance for managing the given services.
198   *
199   * @param services The services to manage
200   * @throws IllegalArgumentException if not all services are {@linkplain State#NEW new} or if there
201   *     are any duplicate services.
202   */
203  public ServiceManager(Iterable<? extends Service> services) {
204    ImmutableList<Service> copy = ImmutableList.copyOf(services);
205    if (copy.isEmpty()) {
206      // Having no services causes the manager to behave strangely. Notably, listeners are never
207      // fired. To avoid this we substitute a placeholder service.
208      logger.log(
209          Level.WARNING,
210          "ServiceManager configured with no services.  Is your application configured properly?",
211          new EmptyServiceManagerWarning());
212      copy = ImmutableList.<Service>of(new NoOpService());
213    }
214    this.state = new ServiceManagerState(copy);
215    this.services = copy;
216    WeakReference<ServiceManagerState> stateReference = new WeakReference<>(state);
217    for (Service service : copy) {
218      service.addListener(new ServiceListener(service, stateReference), directExecutor());
219      // We check the state after adding the listener as a way to ensure that our listener was added
220      // to a NEW service.
221      checkArgument(service.state() == NEW, "Can only manage NEW services, %s", service);
222    }
223    // We have installed all of our listeners and after this point any state transition should be
224    // correct.
225    this.state.markReady();
226  }
227
228  /**
229   * Registers a {@link Listener} to be {@linkplain Executor#execute executed} on the given
230   * executor. The listener will not have previous state changes replayed, so it is suggested that
231   * listeners are added before any of the managed services are {@linkplain Service#startAsync
232   * started}.
233   *
234   * <p>{@code addListener} guarantees execution ordering across calls to a given listener but not
235   * across calls to multiple listeners. Specifically, a given listener will have its callbacks
236   * invoked in the same order as the underlying service enters those states. Additionally, at most
237   * one of the listener's callbacks will execute at once. However, multiple listeners' callbacks
238   * may execute concurrently, and listeners may execute in an order different from the one in which
239   * they were registered.
240   *
241   * <p>RuntimeExceptions thrown by a listener will be caught and logged. Any exception thrown
242   * during {@code Executor.execute} (e.g., a {@code RejectedExecutionException}) will be caught and
243   * logged.
244   *
245   * <p>When selecting an executor, note that {@code directExecutor} is dangerous in some cases. See
246   * the discussion in the {@link ListenableFuture#addListener ListenableFuture.addListener}
247   * documentation.
248   *
249   * @param listener the listener to run when the manager changes state
250   * @param executor the executor in which the listeners callback methods will be run.
251   */
252  public void addListener(Listener listener, Executor executor) {
253    state.addListener(listener, executor);
254  }
255
256  /**
257   * Initiates service {@linkplain Service#startAsync startup} on all the services being managed. It
258   * is only valid to call this method if all of the services are {@linkplain State#NEW new}.
259   *
260   * @return this
261   * @throws IllegalStateException if any of the Services are not {@link State#NEW new} when the
262   *     method is called.
263   */
264  @CanIgnoreReturnValue
265  public ServiceManager startAsync() {
266    for (Service service : services) {
267      checkState(service.state() == NEW, "Not all services are NEW, cannot start %s", this);
268    }
269    for (Service service : services) {
270      try {
271        state.tryStartTiming(service);
272        service.startAsync();
273      } catch (IllegalStateException e) {
274        // This can happen if the service has already been started or stopped (e.g. by another
275        // service or listener). Our contract says it is safe to call this method if
276        // all services were NEW when it was called, and this has already been verified above, so we
277        // don't propagate the exception.
278        logger.log(Level.WARNING, "Unable to start Service " + service, e);
279      }
280    }
281    return this;
282  }
283
284  /**
285   * Waits for the {@link ServiceManager} to become {@linkplain #isHealthy() healthy}. The manager
286   * will become healthy after all the component services have reached the {@linkplain State#RUNNING
287   * running} state.
288   *
289   * @throws IllegalStateException if the service manager reaches a state from which it cannot
290   *     become {@linkplain #isHealthy() healthy}.
291   */
292  public void awaitHealthy() {
293    state.awaitHealthy();
294  }
295
296  /**
297   * Waits for the {@link ServiceManager} to become {@linkplain #isHealthy() healthy} for no more
298   * than the given time. The manager will become healthy after all the component services have
299   * reached the {@linkplain State#RUNNING running} state.
300   *
301   * @param timeout the maximum time to wait
302   * @param unit the time unit of the timeout argument
303   * @throws TimeoutException if not all of the services have finished starting within the deadline
304   * @throws IllegalStateException if the service manager reaches a state from which it cannot
305   *     become {@linkplain #isHealthy() healthy}.
306   */
307  @SuppressWarnings("GoodTime") // should accept a java.time.Duration
308  public void awaitHealthy(long timeout, TimeUnit unit) throws TimeoutException {
309    state.awaitHealthy(timeout, unit);
310  }
311
312  /**
313   * Initiates service {@linkplain Service#stopAsync shutdown} if necessary on all the services
314   * being managed.
315   *
316   * @return this
317   */
318  @CanIgnoreReturnValue
319  public ServiceManager stopAsync() {
320    for (Service service : services) {
321      service.stopAsync();
322    }
323    return this;
324  }
325
326  /**
327   * Waits for the all the services to reach a terminal state. After this method returns all
328   * services will either be {@linkplain Service.State#TERMINATED terminated} or {@linkplain
329   * Service.State#FAILED failed}.
330   */
331  public void awaitStopped() {
332    state.awaitStopped();
333  }
334
335  /**
336   * Waits for the all the services to reach a terminal state for no more than the given time. After
337   * this method returns all services will either be {@linkplain Service.State#TERMINATED
338   * terminated} or {@linkplain Service.State#FAILED failed}.
339   *
340   * @param timeout the maximum time to wait
341   * @param unit the time unit of the timeout argument
342   * @throws TimeoutException if not all of the services have stopped within the deadline
343   */
344  @SuppressWarnings("GoodTime") // should accept a java.time.Duration
345  public void awaitStopped(long timeout, TimeUnit unit) throws TimeoutException {
346    state.awaitStopped(timeout, unit);
347  }
348
349  /**
350   * Returns true if all services are currently in the {@linkplain State#RUNNING running} state.
351   *
352   * <p>Users who want more detailed information should use the {@link #servicesByState} method to
353   * get detailed information about which services are not running.
354   */
355  public boolean isHealthy() {
356    for (Service service : services) {
357      if (!service.isRunning()) {
358        return false;
359      }
360    }
361    return true;
362  }
363
364  /**
365   * Provides a snapshot of the current state of all the services under management.
366   *
367   * <p>N.B. This snapshot is guaranteed to be consistent, i.e. the set of states returned will
368   * correspond to a point in time view of the services.
369   *
370   * @since 29.0 (present with return type {@code ImmutableMultimap} since 14.0)
371   */
372  @Override
373  public ImmutableSetMultimap<State, Service> servicesByState() {
374    return state.servicesByState();
375  }
376
377  /**
378   * Returns the service load times. This value will only return startup times for services that
379   * have finished starting.
380   *
381   * @return Map of services and their corresponding startup time in millis, the map entries will be
382   *     ordered by startup time.
383   */
384  public ImmutableMap<Service, Long> startupTimes() {
385    return state.startupTimes();
386  }
387
388  @Override
389  public String toString() {
390    return MoreObjects.toStringHelper(ServiceManager.class)
391        .add("services", Collections2.filter(services, not(instanceOf(NoOpService.class))))
392        .toString();
393  }
394
395  /**
396   * An encapsulation of all the mutable state of the {@link ServiceManager} that needs to be
397   * accessed by instances of {@link ServiceListener}.
398   */
399  private static final class ServiceManagerState {
400    final Monitor monitor = new Monitor();
401
402    @GuardedBy("monitor")
403    final SetMultimap<State, Service> servicesByState =
404        MultimapBuilder.enumKeys(State.class).linkedHashSetValues().build();
405
406    @GuardedBy("monitor")
407    final Multiset<State> states = servicesByState.keys();
408
409    @GuardedBy("monitor")
410    final Map<Service, Stopwatch> startupTimers = Maps.newIdentityHashMap();
411
412    /**
413     * These two booleans are used to mark the state as ready to start.
414     *
415     * <p>{@link #ready}: is set by {@link #markReady} to indicate that all listeners have been
416     * correctly installed
417     *
418     * <p>{@link #transitioned}: is set by {@link #transitionService} to indicate that some
419     * transition has been performed.
420     *
421     * <p>Together, they allow us to enforce that all services have their listeners installed prior
422     * to any service performing a transition, then we can fail in the ServiceManager constructor
423     * rather than in a Service.Listener callback.
424     */
425    @GuardedBy("monitor")
426    boolean ready;
427
428    @GuardedBy("monitor")
429    boolean transitioned;
430
431    final int numberOfServices;
432
433    /**
434     * Controls how long to wait for all the services to either become healthy or reach a state from
435     * which it is guaranteed that it can never become healthy.
436     */
437    final Monitor.Guard awaitHealthGuard = new AwaitHealthGuard();
438
439    @WeakOuter
440    final class AwaitHealthGuard extends Monitor.Guard {
441      AwaitHealthGuard() {
442        super(ServiceManagerState.this.monitor);
443      }
444
445      @Override
446      @GuardedBy("ServiceManagerState.this.monitor")
447      public boolean isSatisfied() {
448        // All services have started or some service has terminated/failed.
449        return states.count(RUNNING) == numberOfServices
450            || states.contains(STOPPING)
451            || states.contains(TERMINATED)
452            || states.contains(FAILED);
453      }
454    }
455
456    /** Controls how long to wait for all services to reach a terminal state. */
457    final Monitor.Guard stoppedGuard = new StoppedGuard();
458
459    @WeakOuter
460    final class StoppedGuard extends Monitor.Guard {
461      StoppedGuard() {
462        super(ServiceManagerState.this.monitor);
463      }
464
465      @Override
466      @GuardedBy("ServiceManagerState.this.monitor")
467      public boolean isSatisfied() {
468        return states.count(TERMINATED) + states.count(FAILED) == numberOfServices;
469      }
470    }
471
472    /** The listeners to notify during a state transition. */
473    final ListenerCallQueue<Listener> listeners = new ListenerCallQueue<>();
474
475    /**
476     * It is implicitly assumed that all the services are NEW and that they will all remain NEW
477     * until all the Listeners are installed and {@link #markReady()} is called. It is our caller's
478     * responsibility to only call {@link #markReady()} if all services were new at the time this
479     * method was called and when all the listeners were installed.
480     */
481    ServiceManagerState(ImmutableCollection<Service> services) {
482      this.numberOfServices = services.size();
483      servicesByState.putAll(NEW, services);
484    }
485
486    /**
487     * Attempts to start the timer immediately prior to the service being started via {@link
488     * Service#startAsync()}.
489     */
490    void tryStartTiming(Service service) {
491      monitor.enter();
492      try {
493        Stopwatch stopwatch = startupTimers.get(service);
494        if (stopwatch == null) {
495          startupTimers.put(service, Stopwatch.createStarted());
496        }
497      } finally {
498        monitor.leave();
499      }
500    }
501
502    /**
503     * Marks the {@link State} as ready to receive transitions. Returns true if no transitions have
504     * been observed yet.
505     */
506    void markReady() {
507      monitor.enter();
508      try {
509        if (!transitioned) {
510          // nothing has transitioned since construction, good.
511          ready = true;
512        } else {
513          // This should be an extremely rare race condition.
514          List<Service> servicesInBadStates = Lists.newArrayList();
515          for (Service service : servicesByState().values()) {
516            if (service.state() != NEW) {
517              servicesInBadStates.add(service);
518            }
519          }
520          throw new IllegalArgumentException(
521              "Services started transitioning asynchronously before "
522                  + "the ServiceManager was constructed: "
523                  + servicesInBadStates);
524        }
525      } finally {
526        monitor.leave();
527      }
528    }
529
530    void addListener(Listener listener, Executor executor) {
531      listeners.addListener(listener, executor);
532    }
533
534    void awaitHealthy() {
535      monitor.enterWhenUninterruptibly(awaitHealthGuard);
536      try {
537        checkHealthy();
538      } finally {
539        monitor.leave();
540      }
541    }
542
543    void awaitHealthy(long timeout, TimeUnit unit) throws TimeoutException {
544      monitor.enter();
545      try {
546        if (!monitor.waitForUninterruptibly(awaitHealthGuard, timeout, unit)) {
547          throw new TimeoutException(
548              "Timeout waiting for the services to become healthy. The "
549                  + "following services have not started: "
550                  + Multimaps.filterKeys(servicesByState, in(ImmutableSet.of(NEW, STARTING))));
551        }
552        checkHealthy();
553      } finally {
554        monitor.leave();
555      }
556    }
557
558    void awaitStopped() {
559      monitor.enterWhenUninterruptibly(stoppedGuard);
560      monitor.leave();
561    }
562
563    void awaitStopped(long timeout, TimeUnit unit) throws TimeoutException {
564      monitor.enter();
565      try {
566        if (!monitor.waitForUninterruptibly(stoppedGuard, timeout, unit)) {
567          throw new TimeoutException(
568              "Timeout waiting for the services to stop. The following "
569                  + "services have not stopped: "
570                  + Multimaps.filterKeys(servicesByState, not(in(EnumSet.of(TERMINATED, FAILED)))));
571        }
572      } finally {
573        monitor.leave();
574      }
575    }
576
577    ImmutableSetMultimap<State, Service> servicesByState() {
578      ImmutableSetMultimap.Builder<State, Service> builder = ImmutableSetMultimap.builder();
579      monitor.enter();
580      try {
581        for (Entry<State, Service> entry : servicesByState.entries()) {
582          if (!(entry.getValue() instanceof NoOpService)) {
583            builder.put(entry);
584          }
585        }
586      } finally {
587        monitor.leave();
588      }
589      return builder.build();
590    }
591
592    ImmutableMap<Service, Long> startupTimes() {
593      List<Entry<Service, Long>> loadTimes;
594      monitor.enter();
595      try {
596        loadTimes = Lists.newArrayListWithCapacity(startupTimers.size());
597        // N.B. There will only be an entry in the map if the service has started
598        for (Entry<Service, Stopwatch> entry : startupTimers.entrySet()) {
599          Service service = entry.getKey();
600          Stopwatch stopwatch = entry.getValue();
601          if (!stopwatch.isRunning() && !(service instanceof NoOpService)) {
602            loadTimes.add(Maps.immutableEntry(service, stopwatch.elapsed(MILLISECONDS)));
603          }
604        }
605      } finally {
606        monitor.leave();
607      }
608      Collections.sort(
609          loadTimes,
610          Ordering.natural()
611              .onResultOf(
612                  new Function<Entry<Service, Long>, Long>() {
613                    @Override
614                    public Long apply(Entry<Service, Long> input) {
615                      return input.getValue();
616                    }
617                  }));
618      return ImmutableMap.copyOf(loadTimes);
619    }
620
621    /**
622     * Updates the state with the given service transition.
623     *
624     * <p>This method performs the main logic of ServiceManager in the following steps.
625     *
626     * <ol>
627     *   <li>Update the {@link #servicesByState()}
628     *   <li>Update the {@link #startupTimers}
629     *   <li>Based on the new state queue listeners to run
630     *   <li>Run the listeners (outside of the lock)
631     * </ol>
632     */
633    void transitionService(final Service service, State from, State to) {
634      checkNotNull(service);
635      checkArgument(from != to);
636      monitor.enter();
637      try {
638        transitioned = true;
639        if (!ready) {
640          return;
641        }
642        // Update state.
643        checkState(
644            servicesByState.remove(from, service),
645            "Service %s not at the expected location in the state map %s",
646            service,
647            from);
648        checkState(
649            servicesByState.put(to, service),
650            "Service %s in the state map unexpectedly at %s",
651            service,
652            to);
653        // Update the timer
654        Stopwatch stopwatch = startupTimers.get(service);
655        if (stopwatch == null) {
656          // This means the service was started by some means other than ServiceManager.startAsync
657          stopwatch = Stopwatch.createStarted();
658          startupTimers.put(service, stopwatch);
659        }
660        if (to.compareTo(RUNNING) >= 0 && stopwatch.isRunning()) {
661          // N.B. if we miss the STARTING event then we may never record a startup time.
662          stopwatch.stop();
663          if (!(service instanceof NoOpService)) {
664            logger.log(Level.FINE, "Started {0} in {1}.", new Object[] {service, stopwatch});
665          }
666        }
667        // Queue our listeners
668
669        // Did a service fail?
670        if (to == FAILED) {
671          enqueueFailedEvent(service);
672        }
673
674        if (states.count(RUNNING) == numberOfServices) {
675          // This means that the manager is currently healthy. N.B. If other threads call isHealthy
676          // they are not guaranteed to get 'true', because any service could fail right now.
677          enqueueHealthyEvent();
678        } else if (states.count(TERMINATED) + states.count(FAILED) == numberOfServices) {
679          enqueueStoppedEvent();
680        }
681      } finally {
682        monitor.leave();
683        // Run our executors outside of the lock
684        dispatchListenerEvents();
685      }
686    }
687
688    void enqueueStoppedEvent() {
689      listeners.enqueue(STOPPED_EVENT);
690    }
691
692    void enqueueHealthyEvent() {
693      listeners.enqueue(HEALTHY_EVENT);
694    }
695
696    void enqueueFailedEvent(final Service service) {
697      listeners.enqueue(
698          new ListenerCallQueue.Event<Listener>() {
699            @Override
700            public void call(Listener listener) {
701              listener.failure(service);
702            }
703
704            @Override
705            public String toString() {
706              return "failed({service=" + service + "})";
707            }
708          });
709    }
710
711    /** Attempts to execute all the listeners in {@link #listeners}. */
712    void dispatchListenerEvents() {
713      checkState(
714          !monitor.isOccupiedByCurrentThread(),
715          "It is incorrect to execute listeners with the monitor held.");
716      listeners.dispatch();
717    }
718
719    @GuardedBy("monitor")
720    void checkHealthy() {
721      if (states.count(RUNNING) != numberOfServices) {
722        IllegalStateException exception =
723            new IllegalStateException(
724                "Expected to be healthy after starting. The following services are not running: "
725                    + Multimaps.filterKeys(servicesByState, not(equalTo(RUNNING))));
726        throw exception;
727      }
728    }
729  }
730
731  /**
732   * A {@link Service} that wraps another service and times how long it takes for it to start and
733   * also calls the {@link ServiceManagerState#transitionService(Service, State, State)}, to record
734   * the state transitions.
735   */
736  private static final class ServiceListener extends Service.Listener {
737    final Service service;
738    // We store the state in a weak reference to ensure that if something went wrong while
739    // constructing the ServiceManager we don't pointlessly keep updating the state.
740    final WeakReference<ServiceManagerState> state;
741
742    ServiceListener(Service service, WeakReference<ServiceManagerState> state) {
743      this.service = service;
744      this.state = state;
745    }
746
747    @Override
748    public void starting() {
749      ServiceManagerState state = this.state.get();
750      if (state != null) {
751        state.transitionService(service, NEW, STARTING);
752        if (!(service instanceof NoOpService)) {
753          logger.log(Level.FINE, "Starting {0}.", service);
754        }
755      }
756    }
757
758    @Override
759    public void running() {
760      ServiceManagerState state = this.state.get();
761      if (state != null) {
762        state.transitionService(service, STARTING, RUNNING);
763      }
764    }
765
766    @Override
767    public void stopping(State from) {
768      ServiceManagerState state = this.state.get();
769      if (state != null) {
770        state.transitionService(service, from, STOPPING);
771      }
772    }
773
774    @Override
775    public void terminated(State from) {
776      ServiceManagerState state = this.state.get();
777      if (state != null) {
778        if (!(service instanceof NoOpService)) {
779          logger.log(
780              Level.FINE,
781              "Service {0} has terminated. Previous state was: {1}",
782              new Object[] {service, from});
783        }
784        state.transitionService(service, from, TERMINATED);
785      }
786    }
787
788    @Override
789    public void failed(State from, Throwable failure) {
790      ServiceManagerState state = this.state.get();
791      if (state != null) {
792        // Log before the transition, so that if the process exits in response to server failure,
793        // there is a higher likelihood that the cause will be in the logs.
794        boolean log = !(service instanceof NoOpService);
795        if (log) {
796          logger.log(
797              Level.SEVERE,
798              "Service " + service + " has failed in the " + from + " state.",
799              failure);
800        }
801        state.transitionService(service, from, FAILED);
802      }
803    }
804  }
805
806  /**
807   * A {@link Service} instance that does nothing. This is only useful as a placeholder to ensure
808   * that the {@link ServiceManager} functions properly even when it is managing no services.
809   *
810   * <p>The use of this class is considered an implementation detail of ServiceManager and as such
811   * it is excluded from {@link #servicesByState}, {@link #startupTimes}, {@link #toString} and all
812   * logging statements.
813   */
814  private static final class NoOpService extends AbstractService {
815    @Override
816    protected void doStart() {
817      notifyStarted();
818    }
819
820    @Override
821    protected void doStop() {
822      notifyStopped();
823    }
824  }
825
826  /** This is never thrown but only used for logging. */
827  private static final class EmptyServiceManagerWarning extends Throwable {}
828}