001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements. See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License. You may obtain a copy of the License at
008 *
009 * http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017 package org.apache.camel.component.file.remote;
018
019 import java.io.ByteArrayOutputStream;
020 import java.io.IOException;
021 import java.util.concurrent.ScheduledExecutorService;
022
023 import org.apache.camel.Processor;
024 import org.apache.camel.component.file.FileComponent;
025 import org.apache.camel.util.ObjectHelper;
026 import org.apache.commons.net.ftp.FTPClient;
027 import org.apache.commons.net.ftp.FTPFile;
028
029 public class FtpConsumer extends RemoteFileConsumer<RemoteFileExchange> {
030
031 private FtpEndpoint endpoint;
032 private FTPClient client;
033 private boolean loggedIn;
034
035 public FtpConsumer(FtpEndpoint endpoint, Processor processor, FTPClient client) {
036 super(endpoint, processor);
037 this.endpoint = endpoint;
038 this.client = client;
039 }
040
041 public FtpConsumer(FtpEndpoint endpoint, Processor processor, FTPClient client,
042 ScheduledExecutorService executor) {
043 super(endpoint, processor, executor);
044 this.endpoint = endpoint;
045 this.client = client;
046 }
047
048 protected void doStart() throws Exception {
049 log.info("Starting");
050 super.doStart();
051 }
052
053 protected void doStop() throws Exception {
054 log.info("Stopping");
055 // disconnect when stopping
056 try {
057 disconnect();
058 } catch (Exception e) {
059 // ignore just log a warning
060 String message = "Could not disconnect from " + remoteServer()
061 + ". Reason: " + client.getReplyString() + ". Code: " + client.getReplyCode();
062 log.warn(message);
063 }
064 super.doStop();
065 }
066
067 protected void connectIfNecessary() throws IOException {
068 if (!client.isConnected() || !loggedIn) {
069 if (log.isDebugEnabled()) {
070 log.debug("Not connected/logged in, connecting to " + remoteServer());
071 }
072
073 try {
074 loggedIn = FtpUtils.connect(client, endpoint.getConfiguration());
075 } catch (java.net.ConnectException ce) {
076 // Wait a bit to make sure we have something to connect to and
077 // try once more.
078 try {
079 Thread.sleep(3000);
080 } catch (InterruptedException ie) {
081 // ignore.
082 }
083 loggedIn = FtpUtils.connect(client, endpoint.getConfiguration());
084 }
085 if (!loggedIn) {
086 return;
087 }
088 }
089
090 log.info("Connected and logged in to " + remoteServer());
091 }
092
093 protected void disconnect() throws IOException {
094 loggedIn = false;
095 log.debug("Disconnecting from " + remoteServer());
096 FtpUtils.disconnect(client);
097 }
098
099 protected void poll() throws Exception {
100 if (log.isTraceEnabled()) {
101 log.trace("Polling " + endpoint.getConfiguration());
102 }
103
104 try {
105 connectIfNecessary();
106
107 if (!loggedIn) {
108 String message = "Could not connect/login to " + endpoint.getConfiguration();
109 log.warn(message);
110 throw new FtpOperationFailedException(client.getReplyCode(), client.getReplyString(), message);
111 }
112
113 final String fileName = endpoint.getConfiguration().getFile();
114 if (endpoint.getConfiguration().isDirectory()) {
115 pollDirectory(fileName);
116 } else {
117 int index = fileName.lastIndexOf('/');
118 if (index > -1) {
119 // cd to the folder of the filename
120 client.changeWorkingDirectory(fileName.substring(0, index));
121 }
122 // list the files in the fold and poll the first file
123 final FTPFile[] files = client.listFiles(fileName.substring(index + 1));
124 if (files != null && files.length > 0) {
125 pollFile(files[0]);
126 }
127 }
128
129 lastPollTime = System.currentTimeMillis();
130
131 } catch (Exception e) {
132 loggedIn = false;
133 if (isStopping() || isStopped()) {
134 // if we are stopping then ignore any exception during a poll
135 log.warn("Consumer is stopping. Ignoring caught exception: "
136 + e.getClass().getCanonicalName() + " message: " + e.getMessage());
137 } else {
138 log.warn("Exception occurred during polling: "
139 + e.getClass().getCanonicalName() + " message: " + e.getMessage());
140 disconnect();
141 // Rethrow to signify that we didn't poll
142 throw e;
143 }
144 }
145 }
146
147 protected void pollDirectory(String dir) throws Exception {
148 if (log.isTraceEnabled()) {
149 log.trace("Polling directory: " + dir);
150 }
151 String currentDir = client.printWorkingDirectory();
152
153 client.changeWorkingDirectory(dir);
154 for (FTPFile ftpFile : client.listFiles()) {
155 if (ftpFile.isFile()) {
156 pollFile(ftpFile);
157 } else if (ftpFile.isDirectory()) {
158 if (isRecursive()) {
159 pollDirectory(getFullFileName(ftpFile));
160 }
161 } else {
162 log.debug("Unsupported type of FTPFile: " + ftpFile + " (not a file or directory). It is skipped.");
163 }
164 }
165
166 // change back to original current dir
167 client.changeWorkingDirectory(currentDir);
168 }
169
170 protected String getFullFileName(FTPFile ftpFile) throws IOException {
171 return client.printWorkingDirectory() + "/" + ftpFile.getName();
172 }
173
174 private void pollFile(FTPFile ftpFile) throws Exception {
175 if (ftpFile == null) {
176 return;
177 }
178
179 if (log.isTraceEnabled()) {
180 log.trace("Polling file: " + ftpFile);
181 }
182
183 // if using last polltime for timestamp matcing (to be removed in Camel 2.0)
184 boolean timestampMatched = true;
185 if (isTimestamp()) {
186 // TODO do we need to adjust the TZ? can we?
187 long ts = ftpFile.getTimestamp().getTimeInMillis();
188 timestampMatched = ts > lastPollTime;
189 if (log.isTraceEnabled()) {
190 log.trace("The file is to old + " + ftpFile + ". lastPollTime=" + lastPollTime + " > fileTimestamp=" + ts);
191 }
192 }
193
194 if (timestampMatched && isMatched(ftpFile)) {
195 String fullFileName = getFullFileName(ftpFile);
196
197 // is we use exclusive read then acquire the exclusive read (waiting until we got it)
198 if (exclusiveReadLock) {
199 acquireExclusiveReadLock(client, ftpFile);
200 }
201
202 // retrieve the file
203 final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
204 try {
205 client.retrieveFile(ftpFile.getName(), byteArrayOutputStream);
206 if (log.isDebugEnabled()) {
207 log.debug("Retrieved file: " + ftpFile.getName() + " from: " + remoteServer());
208 }
209 } finally {
210 // close stream to avoid leaking FTP connections
211 ObjectHelper.close(byteArrayOutputStream, "retrieve: " + ftpFile.getName(), log);
212 }
213
214 RemoteFileExchange exchange = endpoint.createExchange(fullFileName, ftpFile.getName(),
215 ftpFile.getSize(), byteArrayOutputStream);
216
217 if (isSetNames()) {
218 // set the filename in the special header filename marker to the ftp filename
219 String ftpBasePath = endpoint.getConfiguration().getFile();
220 String relativePath = fullFileName.substring(ftpBasePath.length() + 1);
221 relativePath = relativePath.replaceFirst("/", "");
222
223 if (log.isDebugEnabled()) {
224 log.debug("Setting exchange filename to " + relativePath);
225 }
226 exchange.getIn().setHeader(FileComponent.HEADER_FILE_NAME, relativePath);
227 }
228
229
230 // all success so lets process it
231 getProcessor().process(exchange);
232
233 if (exchange.isFailed()) {
234 if (log.isDebugEnabled()) {
235 log.debug("Processing of exchange failed, so cannot do FTP post command such as move or delete: " + exchange);
236 }
237 } else {
238 // after processing then do post command such as delete or move
239 if (deleteFile) {
240 // delete file after consuming
241 if (log.isDebugEnabled()) {
242 log.debug("Deleting file: " + ftpFile.getName() + " from: " + remoteServer());
243 }
244 boolean deleted = client.deleteFile(ftpFile.getName());
245 if (!deleted) {
246 String message = "Can not delete file: " + ftpFile.getName() + " from: " + remoteServer();
247 throw new FtpOperationFailedException(client.getReplyCode(), client.getReplyString(), message);
248 }
249 } else if (isMoveFile()) {
250 String fromName = ftpFile.getName();
251 String toName = getMoveFileName(fromName, exchange);
252 if (log.isDebugEnabled()) {
253 log.debug("Moving file: " + fromName + " to: " + toName);
254 }
255
256 // delete any existing file
257 boolean deleted = client.deleteFile(toName);
258 if (!deleted) {
259 // if we could not delete any existing file then maybe the folder is missing
260 // build folder if needed
261 int lastPathIndex = toName.lastIndexOf('/');
262 if (lastPathIndex != -1) {
263 String directory = toName.substring(0, lastPathIndex);
264 if (!FtpUtils.buildDirectory(client, directory)) {
265 log.warn("Can not build directory: " + directory + " (maybe because of denied permissions)");
266 }
267 }
268 }
269
270 // try to rename
271 boolean success = client.rename(fromName, toName);
272 if (!success) {
273 String message = "Can not move file: " + fromName + " to: " + toName;
274 throw new FtpOperationFailedException(client.getReplyCode(), client.getReplyString(), message);
275 }
276 }
277 }
278
279 }
280 }
281
282 protected void acquireExclusiveReadLock(FTPClient client, FTPFile ftpFile) throws IOException {
283 if (log.isTraceEnabled()) {
284 log.trace("Waiting for exclusive read lock to file: " + ftpFile);
285 }
286
287 // the trick is to try to rename the file, if we can rename then we have exclusive read
288 // since its a remote file we can not use java.nio to get a RW lock
289 String originalName = ftpFile.getName();
290 String newName = originalName + ".camelExclusiveReadLock";
291 boolean exclusive = false;
292 while (!exclusive) {
293 exclusive = client.rename(originalName, newName);
294 if (exclusive) {
295 if (log.isDebugEnabled()) {
296 log.debug("Acquired exclusive read lock to file: " + originalName);
297 }
298 // rename it back so we can read it
299 client.rename(newName, originalName);
300 } else {
301 log.trace("Exclusive read lock not granted. Sleeping for 1000 millis.");
302 try {
303 Thread.sleep(1000);
304 } catch (InterruptedException e) {
305 // ignore
306 }
307 }
308 }
309 }
310
311 protected String getFileName(Object file) {
312 FTPFile ftpFile = (FTPFile) file;
313 return ftpFile.getName();
314 }
315
316 }