001/** 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017package org.apache.camel.impl; 018 019import java.util.HashMap; 020import java.util.Map; 021import java.util.concurrent.ScheduledExecutorService; 022import java.util.concurrent.TimeUnit; 023 024import org.apache.camel.Endpoint; 025import org.apache.camel.Exchange; 026import org.apache.camel.FailedToCreateConsumerException; 027import org.apache.camel.LoggingLevel; 028import org.apache.camel.PollingConsumerPollingStrategy; 029import org.apache.camel.Processor; 030import org.apache.camel.Suspendable; 031import org.apache.camel.spi.PollingConsumerPollStrategy; 032import org.apache.camel.spi.ScheduledPollConsumerScheduler; 033import org.apache.camel.util.IntrospectionSupport; 034import org.apache.camel.util.ObjectHelper; 035import org.apache.camel.util.ServiceHelper; 036import org.slf4j.Logger; 037import org.slf4j.LoggerFactory; 038 039/** 040 * A useful base class for any consumer which is polling based 041 */ 042public abstract class ScheduledPollConsumer extends DefaultConsumer implements Runnable, Suspendable, PollingConsumerPollingStrategy { 043 private static final Logger LOG = LoggerFactory.getLogger(ScheduledPollConsumer.class); 044 045 private ScheduledPollConsumerScheduler scheduler; 046 private ScheduledExecutorService scheduledExecutorService; 047 048 // if adding more options then align with org.apache.camel.impl.ScheduledPollEndpoint 049 private boolean startScheduler = true; 050 private long initialDelay = 1000; 051 private long delay = 500; 052 private TimeUnit timeUnit = TimeUnit.MILLISECONDS; 053 private boolean useFixedDelay = true; 054 private PollingConsumerPollStrategy pollStrategy = new DefaultPollingConsumerPollStrategy(); 055 private LoggingLevel runLoggingLevel = LoggingLevel.TRACE; 056 private boolean sendEmptyMessageWhenIdle; 057 private boolean greedy; 058 private int backoffMultiplier; 059 private int backoffIdleThreshold; 060 private int backoffErrorThreshold; 061 private Map<String, Object> schedulerProperties; 062 063 // state during running 064 private volatile boolean polling; 065 private volatile int backoffCounter; 066 private volatile long idleCounter; 067 private volatile long errorCounter; 068 069 public ScheduledPollConsumer(Endpoint endpoint, Processor processor) { 070 super(endpoint, processor); 071 } 072 073 public ScheduledPollConsumer(Endpoint endpoint, Processor processor, ScheduledExecutorService scheduledExecutorService) { 074 super(endpoint, processor); 075 // we have been given an existing thread pool, so we should not manage its lifecycle 076 // so we should keep shutdownExecutor as false 077 this.scheduledExecutorService = scheduledExecutorService; 078 ObjectHelper.notNull(scheduledExecutorService, "scheduledExecutorService"); 079 } 080 081 /** 082 * Invoked whenever we should be polled 083 */ 084 public void run() { 085 // avoid this thread to throw exceptions because the thread pool wont re-schedule a new thread 086 try { 087 // log starting 088 if (LoggingLevel.ERROR == runLoggingLevel) { 089 LOG.error("Scheduled task started on: {}", this.getEndpoint()); 090 } else if (LoggingLevel.WARN == runLoggingLevel) { 091 LOG.warn("Scheduled task started on: {}", this.getEndpoint()); 092 } else if (LoggingLevel.INFO == runLoggingLevel) { 093 LOG.info("Scheduled task started on: {}", this.getEndpoint()); 094 } else if (LoggingLevel.DEBUG == runLoggingLevel) { 095 LOG.debug("Scheduled task started on: {}", this.getEndpoint()); 096 } else { 097 LOG.trace("Scheduled task started on: {}", this.getEndpoint()); 098 } 099 100 // execute scheduled task 101 doRun(); 102 103 // log completed 104 if (LoggingLevel.ERROR == runLoggingLevel) { 105 LOG.error("Scheduled task completed on: {}", this.getEndpoint()); 106 } else if (LoggingLevel.WARN == runLoggingLevel) { 107 LOG.warn("Scheduled task completed on: {}", this.getEndpoint()); 108 } else if (LoggingLevel.INFO == runLoggingLevel) { 109 LOG.info("Scheduled task completed on: {}", this.getEndpoint()); 110 } else if (LoggingLevel.DEBUG == runLoggingLevel) { 111 LOG.debug("Scheduled task completed on: {}", this.getEndpoint()); 112 } else { 113 LOG.trace("Scheduled task completed on: {}", this.getEndpoint()); 114 } 115 116 } catch (Error e) { 117 // must catch Error, to ensure the task is re-scheduled 118 LOG.error("Error occurred during running scheduled task on: " + this.getEndpoint() + ", due: " + e.getMessage(), e); 119 } 120 } 121 122 private void doRun() { 123 if (isSuspended()) { 124 LOG.trace("Cannot start to poll: {} as its suspended", this.getEndpoint()); 125 return; 126 } 127 128 // should we backoff if its enabled, and either the idle or error counter is > the threshold 129 if (backoffMultiplier > 0 130 // either idle or error threshold could be not in use, so check for that and use MAX_VALUE if not in use 131 && (idleCounter >= (backoffIdleThreshold > 0 ? backoffIdleThreshold : Integer.MAX_VALUE)) 132 || errorCounter >= (backoffErrorThreshold > 0 ? backoffErrorThreshold : Integer.MAX_VALUE)) { 133 if (backoffCounter++ < backoffMultiplier) { 134 // yes we should backoff 135 if (idleCounter > 0) { 136 LOG.debug("doRun() backoff due subsequent {} idles (backoff at {}/{})", new Object[]{idleCounter, backoffCounter, backoffMultiplier}); 137 } else { 138 LOG.debug("doRun() backoff due subsequent {} errors (backoff at {}/{})", new Object[]{errorCounter, backoffCounter, backoffMultiplier}); 139 } 140 return; 141 } else { 142 // we are finished with backoff so reset counters 143 idleCounter = 0; 144 errorCounter = 0; 145 backoffCounter = 0; 146 LOG.trace("doRun() backoff finished, resetting counters."); 147 } 148 } 149 150 int retryCounter = -1; 151 boolean done = false; 152 Throwable cause = null; 153 int polledMessages = 0; 154 155 while (!done) { 156 try { 157 cause = null; 158 // eager assume we are done 159 done = true; 160 if (isPollAllowed()) { 161 162 if (retryCounter == -1) { 163 LOG.trace("Starting to poll: {}", this.getEndpoint()); 164 } else { 165 LOG.debug("Retrying attempt {} to poll: {}", retryCounter, this.getEndpoint()); 166 } 167 168 // mark we are polling which should also include the begin/poll/commit 169 polling = true; 170 try { 171 boolean begin = pollStrategy.begin(this, getEndpoint()); 172 if (begin) { 173 retryCounter++; 174 polledMessages = poll(); 175 LOG.trace("Polled {} messages", polledMessages); 176 177 if (polledMessages == 0 && isSendEmptyMessageWhenIdle()) { 178 // send an "empty" exchange 179 processEmptyMessage(); 180 } 181 182 pollStrategy.commit(this, getEndpoint(), polledMessages); 183 184 if (polledMessages > 0 && isGreedy()) { 185 done = false; 186 retryCounter = -1; 187 LOG.trace("Greedy polling after processing {} messages", polledMessages); 188 } 189 } else { 190 LOG.debug("Cannot begin polling as pollStrategy returned false: {}", pollStrategy); 191 } 192 } finally { 193 polling = false; 194 } 195 } 196 197 LOG.trace("Finished polling: {}", this.getEndpoint()); 198 } catch (Exception e) { 199 try { 200 boolean retry = pollStrategy.rollback(this, getEndpoint(), retryCounter, e); 201 if (retry) { 202 // do not set cause as we retry 203 done = false; 204 } else { 205 cause = e; 206 done = true; 207 } 208 } catch (Throwable t) { 209 cause = t; 210 done = true; 211 } 212 } catch (Throwable t) { 213 cause = t; 214 done = true; 215 } 216 217 if (cause != null && isRunAllowed()) { 218 // let exception handler deal with the caused exception 219 // but suppress this during shutdown as the logs may get flooded with exceptions during shutdown/forced shutdown 220 try { 221 getExceptionHandler().handleException("Consumer " + this + " failed polling endpoint: " + getEndpoint() 222 + ". Will try again at next poll", cause); 223 } catch (Throwable e) { 224 LOG.warn("Error handling exception. This exception will be ignored.", e); 225 } 226 } 227 } 228 229 if (cause != null) { 230 idleCounter = 0; 231 errorCounter++; 232 } else { 233 idleCounter = polledMessages == 0 ? ++idleCounter : 0; 234 errorCounter = 0; 235 } 236 LOG.trace("doRun() done with idleCounter={}, errorCounter={}", idleCounter, errorCounter); 237 238 // avoid this thread to throw exceptions because the thread pool wont re-schedule a new thread 239 } 240 241 /** 242 * No messages to poll so send an empty message instead. 243 * 244 * @throws Exception is thrown if error processing the empty message. 245 */ 246 protected void processEmptyMessage() throws Exception { 247 Exchange exchange = getEndpoint().createExchange(); 248 log.debug("Sending empty message as there were no messages from polling: {}", this.getEndpoint()); 249 getProcessor().process(exchange); 250 } 251 252 // Properties 253 // ------------------------------------------------------------------------- 254 255 protected boolean isPollAllowed() { 256 return isRunAllowed() && !isSuspended(); 257 } 258 259 /** 260 * Whether polling is currently in progress 261 */ 262 protected boolean isPolling() { 263 return polling; 264 } 265 266 public ScheduledPollConsumerScheduler getScheduler() { 267 return scheduler; 268 } 269 270 public void setScheduler(ScheduledPollConsumerScheduler scheduler) { 271 this.scheduler = scheduler; 272 } 273 274 public Map<String, Object> getSchedulerProperties() { 275 return schedulerProperties; 276 } 277 278 public void setSchedulerProperties(Map<String, Object> schedulerProperties) { 279 this.schedulerProperties = schedulerProperties; 280 } 281 282 public long getInitialDelay() { 283 return initialDelay; 284 } 285 286 public void setInitialDelay(long initialDelay) { 287 this.initialDelay = initialDelay; 288 } 289 290 public long getDelay() { 291 return delay; 292 } 293 294 public void setDelay(long delay) { 295 this.delay = delay; 296 } 297 298 public TimeUnit getTimeUnit() { 299 return timeUnit; 300 } 301 302 public void setTimeUnit(TimeUnit timeUnit) { 303 this.timeUnit = timeUnit; 304 } 305 306 public boolean isUseFixedDelay() { 307 return useFixedDelay; 308 } 309 310 public void setUseFixedDelay(boolean useFixedDelay) { 311 this.useFixedDelay = useFixedDelay; 312 } 313 314 public LoggingLevel getRunLoggingLevel() { 315 return runLoggingLevel; 316 } 317 318 public void setRunLoggingLevel(LoggingLevel runLoggingLevel) { 319 this.runLoggingLevel = runLoggingLevel; 320 } 321 322 public PollingConsumerPollStrategy getPollStrategy() { 323 return pollStrategy; 324 } 325 326 public void setPollStrategy(PollingConsumerPollStrategy pollStrategy) { 327 this.pollStrategy = pollStrategy; 328 } 329 330 public boolean isStartScheduler() { 331 return startScheduler; 332 } 333 334 public void setStartScheduler(boolean startScheduler) { 335 this.startScheduler = startScheduler; 336 } 337 338 public void setSendEmptyMessageWhenIdle(boolean sendEmptyMessageWhenIdle) { 339 this.sendEmptyMessageWhenIdle = sendEmptyMessageWhenIdle; 340 } 341 342 public boolean isSendEmptyMessageWhenIdle() { 343 return sendEmptyMessageWhenIdle; 344 } 345 346 public boolean isGreedy() { 347 return greedy; 348 } 349 350 public void setGreedy(boolean greedy) { 351 this.greedy = greedy; 352 } 353 354 public int getBackoffCounter() { 355 return backoffCounter; 356 } 357 358 public int getBackoffMultiplier() { 359 return backoffMultiplier; 360 } 361 362 public void setBackoffMultiplier(int backoffMultiplier) { 363 this.backoffMultiplier = backoffMultiplier; 364 } 365 366 public int getBackoffIdleThreshold() { 367 return backoffIdleThreshold; 368 } 369 370 public void setBackoffIdleThreshold(int backoffIdleThreshold) { 371 this.backoffIdleThreshold = backoffIdleThreshold; 372 } 373 374 public int getBackoffErrorThreshold() { 375 return backoffErrorThreshold; 376 } 377 378 public void setBackoffErrorThreshold(int backoffErrorThreshold) { 379 this.backoffErrorThreshold = backoffErrorThreshold; 380 } 381 382 public ScheduledExecutorService getScheduledExecutorService() { 383 return scheduledExecutorService; 384 } 385 386 public boolean isSchedulerStarted() { 387 return scheduler.isSchedulerStarted(); 388 } 389 390 public void setScheduledExecutorService(ScheduledExecutorService scheduledExecutorService) { 391 this.scheduledExecutorService = scheduledExecutorService; 392 } 393 394 // Implementation methods 395 // ------------------------------------------------------------------------- 396 397 /** 398 * The polling method which is invoked periodically to poll this consumer 399 * 400 * @return number of messages polled, will be <tt>0</tt> if no message was polled at all. 401 * @throws Exception can be thrown if an exception occurred during polling 402 */ 403 protected abstract int poll() throws Exception; 404 405 @Override 406 protected void doStart() throws Exception { 407 super.doStart(); 408 409 // validate that if backoff multiplier is in use, the threshold values is set correctly 410 if (backoffMultiplier > 0) { 411 if (backoffIdleThreshold <= 0 && backoffErrorThreshold <= 0) { 412 throw new IllegalArgumentException("backoffIdleThreshold and/or backoffErrorThreshold must be configured to a positive value when using backoffMultiplier"); 413 } 414 LOG.debug("Using backoff[multiplier={}, idleThreshold={}, errorThreshold={}] on {}", new Object[]{backoffMultiplier, backoffIdleThreshold, backoffErrorThreshold, getEndpoint()}); 415 } 416 417 if (scheduler == null) { 418 scheduler = new DefaultScheduledPollConsumerScheduler(scheduledExecutorService); 419 } 420 scheduler.setCamelContext(getEndpoint().getCamelContext()); 421 scheduler.onInit(this); 422 scheduler.scheduleTask(this); 423 424 // configure scheduler with options from this consumer 425 Map<String, Object> properties = new HashMap<String, Object>(); 426 IntrospectionSupport.getProperties(this, properties, null); 427 IntrospectionSupport.setProperties(getEndpoint().getCamelContext().getTypeConverter(), scheduler, properties); 428 if (schedulerProperties != null && !schedulerProperties.isEmpty()) { 429 // need to use a copy in case the consumer is restarted so we keep the properties 430 Map<String, Object> copy = new HashMap<String, Object>(schedulerProperties); 431 IntrospectionSupport.setProperties(getEndpoint().getCamelContext().getTypeConverter(), scheduler, copy); 432 if (copy.size() > 0) { 433 throw new FailedToCreateConsumerException(getEndpoint(), "There are " + copy.size() 434 + " scheduler parameters that couldn't be set on the endpoint." 435 + " Check the uri if the parameters are spelt correctly and that they are properties of the endpoint." 436 + " Unknown parameters=[" + copy + "]"); 437 } 438 } 439 440 ObjectHelper.notNull(scheduler, "scheduler", this); 441 ObjectHelper.notNull(pollStrategy, "pollStrategy", this); 442 443 ServiceHelper.startService(scheduler); 444 445 if (isStartScheduler()) { 446 startScheduler(); 447 } 448 } 449 450 /** 451 * Starts the scheduler. 452 * <p/> 453 * If the scheduler is already started, then this is a noop method call. 454 */ 455 public void startScheduler() { 456 scheduler.startScheduler(); 457 } 458 459 @Override 460 protected void doStop() throws Exception { 461 if (scheduler != null) { 462 scheduler.unscheduleTask(); 463 ServiceHelper.stopAndShutdownServices(scheduler); 464 } 465 466 // clear counters 467 backoffCounter = 0; 468 idleCounter = 0; 469 errorCounter = 0; 470 471 super.doStop(); 472 } 473 474 @Override 475 protected void doShutdown() throws Exception { 476 ServiceHelper.stopAndShutdownServices(scheduler); 477 super.doShutdown(); 478 } 479 480 @Override 481 protected void doSuspend() throws Exception { 482 // dont stop/cancel the future task since we just check in the run method 483 } 484 485 @Override 486 public void onInit() throws Exception { 487 // make sure the scheduler is starter 488 startScheduler = true; 489 } 490 491 @Override 492 public long beforePoll(long timeout) throws Exception { 493 LOG.trace("Before poll {}", getEndpoint()); 494 // resume or start our self 495 if (!ServiceHelper.resumeService(this)) { 496 ServiceHelper.startService(this); 497 } 498 499 // ensure at least timeout is as long as one poll delay 500 return Math.max(timeout, getDelay()); 501 } 502 503 @Override 504 public void afterPoll() throws Exception { 505 LOG.trace("After poll {}", getEndpoint()); 506 // suspend or stop our self 507 if (!ServiceHelper.suspendService(this)) { 508 ServiceHelper.stopService(this); 509 } 510 } 511 512}