Source code

001/**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.camel.impl;
018
019import java.io.File;
020import java.lang.management.ManagementFactory;
021import java.lang.management.MemoryMXBean;
022import java.util.LinkedHashSet;
023import java.util.Set;
024import java.util.UUID;
025
026import org.apache.camel.CamelContext;
027import org.apache.camel.CamelContextAware;
028import org.apache.camel.Exchange;
029import org.apache.camel.Message;
030import org.apache.camel.StreamCache;
031import org.apache.camel.spi.StreamCachingStrategy;
032import org.apache.camel.util.FilePathResolver;
033import org.apache.camel.util.FileUtil;
034import org.apache.camel.util.IOHelper;
035import org.slf4j.Logger;
036import org.slf4j.LoggerFactory;
037
038/**
039 * Default implementation of {@link StreamCachingStrategy}
040 */
041public class DefaultStreamCachingStrategy extends org.apache.camel.support.ServiceSupport implements CamelContextAware, StreamCachingStrategy {
042
043    @Deprecated
044    public static final String THRESHOLD = "CamelCachedOutputStreamThreshold";
045    @Deprecated
046    public static final String BUFFER_SIZE = "CamelCachedOutputStreamBufferSize";
047    @Deprecated
048    public static final String TEMP_DIR = "CamelCachedOutputStreamOutputDirectory";
049    @Deprecated
050    public static final String CIPHER_TRANSFORMATION = "CamelCachedOutputStreamCipherTransformation";
051
052    private static final Logger LOG = LoggerFactory.getLogger(DefaultStreamCachingStrategy.class);
053
054    private CamelContext camelContext;
055    private boolean enabled;
056    private File spoolDirectory;
057    private transient String spoolDirectoryName = "${java.io.tmpdir}/camel/camel-tmp-#uuid#";
058    private long spoolThreshold = StreamCache.DEFAULT_SPOOL_THRESHOLD;
059    private int spoolUsedHeapMemoryThreshold;
060    private SpoolUsedHeapMemoryLimit spoolUsedHeapMemoryLimit;
061    private String spoolChiper;
062    private int bufferSize = IOHelper.DEFAULT_BUFFER_SIZE;
063    private boolean removeSpoolDirectoryWhenStopping = true;
064    private final UtilizationStatistics statistics = new UtilizationStatistics();
065    private final Set<SpoolRule> spoolRules = new LinkedHashSet<SpoolRule>();
066    private boolean anySpoolRules;
067
068    public CamelContext getCamelContext() {
069        return camelContext;
070    }
071
072    public void setCamelContext(CamelContext camelContext) {
073        this.camelContext = camelContext;
074    }
075
076    public boolean isEnabled() {
077        return enabled;
078    }
079
080    public void setEnabled(boolean enabled) {
081        this.enabled = enabled;
082    }
083
084    public void setSpoolDirectory(String path) {
085        this.spoolDirectoryName = path;
086    }
087
088    public void setSpoolDirectory(File path) {
089        this.spoolDirectory = path;
090    }
091
092    public File getSpoolDirectory() {
093        return spoolDirectory;
094    }
095
096    public long getSpoolThreshold() {
097        return spoolThreshold;
098    }
099
100    public int getSpoolUsedHeapMemoryThreshold() {
101        return spoolUsedHeapMemoryThreshold;
102    }
103
104    public void setSpoolUsedHeapMemoryThreshold(int spoolHeapMemoryWatermarkThreshold) {
105        this.spoolUsedHeapMemoryThreshold = spoolHeapMemoryWatermarkThreshold;
106    }
107
108    public SpoolUsedHeapMemoryLimit getSpoolUsedHeapMemoryLimit() {
109        return spoolUsedHeapMemoryLimit;
110    }
111
112    public void setSpoolUsedHeapMemoryLimit(SpoolUsedHeapMemoryLimit spoolUsedHeapMemoryLimit) {
113        this.spoolUsedHeapMemoryLimit = spoolUsedHeapMemoryLimit;
114    }
115
116    public void setSpoolThreshold(long spoolThreshold) {
117        this.spoolThreshold = spoolThreshold;
118    }
119
120    public String getSpoolChiper() {
121        return spoolChiper;
122    }
123
124    public void setSpoolChiper(String spoolChiper) {
125        this.spoolChiper = spoolChiper;
126    }
127
128    public int getBufferSize() {
129        return bufferSize;
130    }
131
132    public void setBufferSize(int bufferSize) {
133        this.bufferSize = bufferSize;
134    }
135
136    public boolean isRemoveSpoolDirectoryWhenStopping() {
137        return removeSpoolDirectoryWhenStopping;
138    }
139
140    public void setRemoveSpoolDirectoryWhenStopping(boolean removeSpoolDirectoryWhenStopping) {
141        this.removeSpoolDirectoryWhenStopping = removeSpoolDirectoryWhenStopping;
142    }
143
144    public boolean isAnySpoolRules() {
145        return anySpoolRules;
146    }
147
148    public void setAnySpoolRules(boolean anySpoolTasks) {
149        this.anySpoolRules = anySpoolTasks;
150    }
151
152    public Statistics getStatistics() {
153        return statistics;
154    }
155
156    public boolean shouldSpoolCache(long length) {
157        if (!enabled || spoolRules.isEmpty()) {
158            return false;
159        }
160
161        boolean all = true;
162        boolean any = false;
163        for (SpoolRule rule : spoolRules) {
164            boolean result = rule.shouldSpoolCache(length);
165            if (!result) {
166                all = false;
167                if (!anySpoolRules) {
168                    // no need to check anymore
169                    break;
170                }
171            } else {
172                any = true;
173                if (anySpoolRules) {
174                    // no need to check anymore
175                    break;
176                }
177            }
178        }
179
180        boolean answer = anySpoolRules ? any : all;
181        LOG.debug("Should spool cache {} -> {}", length, answer);
182        return answer;
183    }
184
185    public void addSpoolRule(SpoolRule rule) {
186        spoolRules.add(rule);
187    }
188
189    public StreamCache cache(Exchange exchange) {
190        Message message = exchange.hasOut() ? exchange.getOut() : exchange.getIn();
191        StreamCache cache = message.getBody(StreamCache.class);
192        if (cache != null) {
193            if (LOG.isTraceEnabled()) {
194                LOG.trace("Cached stream to {} -> {}", cache.inMemory() ? "memory" : "spool", cache);
195            }
196            if (statistics.isStatisticsEnabled()) {
197                try {
198                    if (cache.inMemory()) {
199                        statistics.updateMemory(cache.length());
200                    } else {
201                        statistics.updateSpool(cache.length());
202                    }
203                } catch (Exception e) {
204                    LOG.debug("Error updating cache statistics. This exception is ignored.", e);
205                }
206            }
207        }
208        return cache;
209    }
210
211    protected String resolveSpoolDirectory(String path) {
212        String name = camelContext.getManagementNameStrategy().resolveManagementName(path, camelContext.getName(), false);
213        if (name != null) {
214            name = customResolveManagementName(name);
215        }
216        // and then check again with invalid check to ensure all ## is resolved
217        if (name != null) {
218            name = camelContext.getManagementNameStrategy().resolveManagementName(name, camelContext.getName(), true);
219        }
220        return name;
221    }
222
223    protected String customResolveManagementName(String pattern) {
224        if (pattern.contains("#uuid#")) {
225            String uuid = UUID.randomUUID().toString();
226            pattern = pattern.replaceFirst("#uuid#", uuid);
227        }
228        return FilePathResolver.resolvePath(pattern);
229    }
230
231    @Override
232    protected void doStart() throws Exception {
233        if (!enabled) {
234            LOG.debug("StreamCaching is not enabled");
235            return;
236        }
237
238        String bufferSize = camelContext.getGlobalOption(BUFFER_SIZE);
239        String hold = camelContext.getGlobalOption(THRESHOLD);
240        String chiper = camelContext.getGlobalOption(CIPHER_TRANSFORMATION);
241        String dir = camelContext.getGlobalOption(TEMP_DIR);
242
243        boolean warn = false;
244        if (bufferSize != null) {
245            warn = true;
246            this.bufferSize = camelContext.getTypeConverter().convertTo(Integer.class, bufferSize);
247        }
248        if (hold != null) {
249            warn = true;
250            this.spoolThreshold = camelContext.getTypeConverter().convertTo(Long.class, hold);
251        }
252        if (chiper != null) {
253            warn = true;
254            this.spoolChiper = chiper;
255        }
256        if (dir != null) {
257            warn = true;
258            this.spoolDirectory = camelContext.getTypeConverter().convertTo(File.class, dir);
259        }
260        if (warn) {
261            LOG.warn("Configuring of StreamCaching using CamelContext properties is deprecated - use StreamCachingStrategy instead.");
262        }
263
264        if (spoolUsedHeapMemoryThreshold > 99) {
265            throw new IllegalArgumentException("SpoolHeapMemoryWatermarkThreshold must not be higher than 99, was: " + spoolUsedHeapMemoryThreshold);
266        }
267
268        // if we can overflow to disk then make sure directory exists / is created
269        if (spoolThreshold > 0 || spoolUsedHeapMemoryThreshold > 0) {
270
271            if (spoolDirectory == null && spoolDirectoryName == null) {
272                throw new IllegalArgumentException("SpoolDirectory must be configured when using SpoolThreshold > 0");
273            }
274
275            if (spoolDirectory == null) {
276                String name = resolveSpoolDirectory(spoolDirectoryName);
277                if (name != null) {
278                    spoolDirectory = new File(name);
279                    spoolDirectoryName = null;
280                } else {
281                    throw new IllegalStateException("Cannot resolve spool directory from pattern: " + spoolDirectoryName);
282                }
283            }
284
285            if (spoolDirectory.exists()) {
286                if (spoolDirectory.isDirectory()) {
287                    LOG.debug("Using spool directory: {}", spoolDirectory);
288                } else {
289                    LOG.warn("Spool directory: {} is not a directory. This may cause problems spooling to disk for the stream caching!", spoolDirectory);
290                }
291            } else {
292                boolean created = spoolDirectory.mkdirs();
293                if (!created) {
294                    LOG.warn("Cannot create spool directory: {}. This may cause problems spooling to disk for the stream caching!", spoolDirectory);
295                } else {
296                    LOG.debug("Created spool directory: {}", spoolDirectory);
297                }
298
299            }
300
301            if (spoolThreshold > 0) {
302                spoolRules.add(new FixedThresholdSpoolRule());
303            }
304            if (spoolUsedHeapMemoryThreshold > 0) {
305                if (spoolUsedHeapMemoryLimit == null) {
306                    // use max by default
307                    spoolUsedHeapMemoryLimit = SpoolUsedHeapMemoryLimit.Max;
308                }
309                spoolRules.add(new UsedHeapMemorySpoolRule(spoolUsedHeapMemoryLimit));
310            }
311        }
312
313        LOG.debug("StreamCaching configuration {}", this.toString());
314
315        if (spoolDirectory != null) {
316            LOG.info("StreamCaching in use with spool directory: {} and rules: {}", spoolDirectory.getPath(), spoolRules.toString());
317        } else {
318            LOG.info("StreamCaching in use with rules: {}", spoolRules.toString());
319        }
320    }
321
322    @Override
323    protected void doStop() throws Exception {
324        if (spoolThreshold > 0 & spoolDirectory != null  && isRemoveSpoolDirectoryWhenStopping()) {
325            LOG.debug("Removing spool directory: {}", spoolDirectory);
326            FileUtil.removeDir(spoolDirectory);
327        }
328
329        if (LOG.isDebugEnabled() && statistics.isStatisticsEnabled()) {
330            LOG.debug("Stopping StreamCachingStrategy with statistics: {}", statistics.toString());
331        }
332
333        statistics.reset();
334    }
335
336    @Override
337    public String toString() {
338        return "DefaultStreamCachingStrategy["
339            + "spoolDirectory=" + spoolDirectory
340            + ", spoolChiper=" + spoolChiper
341            + ", spoolThreshold=" + spoolThreshold
342            + ", spoolUsedHeapMemoryThreshold=" + spoolUsedHeapMemoryThreshold
343            + ", bufferSize=" + bufferSize
344            + ", anySpoolRules=" + anySpoolRules + "]";
345    }
346
347    private final class FixedThresholdSpoolRule implements SpoolRule {
348
349        public boolean shouldSpoolCache(long length) {
350            if (spoolThreshold > 0 && length > spoolThreshold) {
351                LOG.trace("Should spool cache fixed threshold {} > {} -> true", length, spoolThreshold);
352                return true;
353            }
354            return false;
355        }
356
357        public String toString() {
358            if (spoolThreshold < 1024) {
359                return "Spool > " + spoolThreshold + " bytes body size";
360            } else {
361                return "Spool > " + (spoolThreshold >> 10) + "K body size";
362            }
363        }
364    }
365
366    private final class UsedHeapMemorySpoolRule implements SpoolRule {
367
368        private final MemoryMXBean heapUsage;
369        private final SpoolUsedHeapMemoryLimit limit;
370
371        private UsedHeapMemorySpoolRule(SpoolUsedHeapMemoryLimit limit) {
372            this.limit = limit;
373            this.heapUsage = ManagementFactory.getMemoryMXBean();
374        }
375
376        public boolean shouldSpoolCache(long length) {
377            if (spoolUsedHeapMemoryThreshold > 0) {
378                // must use double to calculate with decimals for the percentage
379                double used = heapUsage.getHeapMemoryUsage().getUsed();
380                double upper = limit == SpoolUsedHeapMemoryLimit.Committed
381                    ? heapUsage.getHeapMemoryUsage().getCommitted() : heapUsage.getHeapMemoryUsage().getMax();
382                double calc = (used / upper) * 100;
383                int percentage = (int) calc;
384
385                if (LOG.isTraceEnabled()) {
386                    long u = heapUsage.getHeapMemoryUsage().getUsed();
387                    long c = heapUsage.getHeapMemoryUsage().getCommitted();
388                    long m = heapUsage.getHeapMemoryUsage().getMax();
389                    LOG.trace("Heap memory: [used={}M ({}%), committed={}M, max={}M]", new Object[]{u >> 20, percentage, c >> 20, m >> 20});
390                }
391
392                if (percentage > spoolUsedHeapMemoryThreshold) {
393                    LOG.trace("Should spool cache heap memory threshold {} > {} -> true", percentage, spoolUsedHeapMemoryThreshold);
394                    return true;
395                }
396            }
397            return false;
398        }
399
400        public String toString() {
401            return "Spool > " + spoolUsedHeapMemoryThreshold + "% used of " + limit + " heap memory";
402        }
403    }
404
405    /**
406     * Represents utilization statistics.
407     */
408    private static final class UtilizationStatistics implements Statistics {
409
410        private boolean statisticsEnabled;
411        private volatile long memoryCounter;
412        private volatile long memorySize;
413        private volatile long memoryAverageSize;
414        private volatile long spoolCounter;
415        private volatile long spoolSize;
416        private volatile long spoolAverageSize;
417
418        synchronized void updateMemory(long size) {
419            memoryCounter++;
420            memorySize += size;
421            memoryAverageSize = memorySize / memoryCounter;
422        }
423
424        synchronized void updateSpool(long size) {
425            spoolCounter++;
426            spoolSize += size;
427            spoolAverageSize = spoolSize / spoolCounter;
428        }
429
430        public long getCacheMemoryCounter() {
431            return memoryCounter;
432        }
433
434        public long getCacheMemorySize() {
435            return memorySize;
436        }
437
438        public long getCacheMemoryAverageSize() {
439            return memoryAverageSize;
440        }
441
442        public long getCacheSpoolCounter() {
443            return spoolCounter;
444        }
445
446        public long getCacheSpoolSize() {
447            return spoolSize;
448        }
449
450        public long getCacheSpoolAverageSize() {
451            return spoolAverageSize;
452        }
453
454        public synchronized void reset() {
455            memoryCounter = 0;
456            memorySize = 0;
457            memoryAverageSize = 0;
458            spoolCounter = 0;
459            spoolSize = 0;
460            spoolAverageSize = 0;
461        }
462
463        public boolean isStatisticsEnabled() {
464            return statisticsEnabled;
465        }
466
467        public void setStatisticsEnabled(boolean statisticsEnabled) {
468            this.statisticsEnabled = statisticsEnabled;
469        }
470
471        public String toString() {
472            return String.format("[memoryCounter=%s, memorySize=%s, memoryAverageSize=%s, spoolCounter=%s, spoolSize=%s, spoolAverageSize=%s]",
473                    memoryCounter, memorySize, memoryAverageSize, spoolCounter, spoolSize, spoolAverageSize);
474        }
475    }
476
477}