001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements. See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License. You may obtain a copy of the License at
008 *
009 * http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017 package org.apache.camel.component.file.remote;
018
019 import java.io.ByteArrayOutputStream;
020 import java.io.IOException;
021 import java.util.Vector;
022 import java.util.concurrent.ScheduledExecutorService;
023
024 import com.jcraft.jsch.ChannelSftp;
025 import com.jcraft.jsch.JSchException;
026 import com.jcraft.jsch.Session;
027 import com.jcraft.jsch.SftpException;
028
029 import org.apache.camel.Processor;
030 import org.apache.camel.component.file.FileComponent;
031 import org.apache.camel.util.ObjectHelper;
032
033 public class SftpConsumer extends RemoteFileConsumer<RemoteFileExchange> {
034 private final SftpEndpoint endpoint;
035
036 private ChannelSftp channel;
037 private Session session;
038
039 public SftpConsumer(SftpEndpoint endpoint, Processor processor, Session session) {
040 super(endpoint, processor);
041 this.endpoint = endpoint;
042 this.session = session;
043 }
044
045 public SftpConsumer(SftpEndpoint endpoint, Processor processor, Session session, ScheduledExecutorService executor) {
046 super(endpoint, processor, executor);
047 this.endpoint = endpoint;
048 this.session = session;
049 }
050
051 protected void doStart() throws Exception {
052 log.info("Starting");
053 super.doStart();
054 }
055
056 protected void doStop() throws Exception {
057 log.info("Stopping");
058 // disconnect when stopping
059 try {
060 disconnect();
061 } catch (Exception e) {
062 // ignore just log a warning
063 log.warn("Exception occurred during disconecting from " + remoteServer() + ". "
064 + e.getClass().getCanonicalName() + " message: " + e.getMessage());
065 }
066 super.doStop();
067 }
068
069 protected void connectIfNecessary() throws JSchException {
070 if (channel == null || !channel.isConnected()) {
071 if (session == null || !session.isConnected()) {
072 log.trace("Session isn't connected, trying to recreate and connect.");
073 session = endpoint.createSession();
074 session.connect();
075 }
076 log.trace("Channel isn't connected, trying to recreate and connect.");
077 channel = endpoint.createChannelSftp(session);
078 channel.connect();
079 log.info("Connected to " + remoteServer());
080 }
081 }
082
083 protected void disconnect() throws JSchException {
084 log.debug("Disconnecting from " + remoteServer());
085 if (session != null) {
086 session.disconnect();
087 }
088 if (channel != null) {
089 channel.disconnect();
090 }
091 }
092
093 protected void poll() throws Exception {
094 if (log.isTraceEnabled()) {
095 log.trace("Polling " + endpoint.getConfiguration());
096 }
097 connectIfNecessary();
098 // If the attempt to connect isn't successful, then the thrown
099 // exception will signify that we couldn't poll
100 try {
101 final String fileName = endpoint.getConfiguration().getFile();
102 if (endpoint.getConfiguration().isDirectory()) {
103 pollDirectory(fileName);
104 } else {
105 int index = fileName.lastIndexOf('/');
106 if (index > -1) {
107 // cd to the folder of the filename
108 channel.cd(fileName.substring(0, index));
109 }
110
111 // list the files in the fold and poll the first file
112 final Vector files = channel.ls(fileName.substring(index + 1));
113 final ChannelSftp.LsEntry file = (ChannelSftp.LsEntry) files.get(0);
114 pollFile(file);
115 }
116 lastPollTime = System.currentTimeMillis();
117 } catch (Exception e) {
118 if (isStopping() || isStopped()) {
119 // if we are stopping then ignore any exception during a poll
120 log.warn("Consumer is stopping. Ignoring caught exception: "
121 + e.getClass().getCanonicalName() + " message: " + e.getMessage());
122 } else {
123 log.warn("Exception occurred during polling: "
124 + e.getClass().getCanonicalName() + " message: " + e.getMessage());
125 disconnect();
126 // Rethrow to signify that we didn't poll
127 throw e;
128 }
129 }
130 }
131
132 protected void pollDirectory(String dir) throws Exception {
133 if (log.isTraceEnabled()) {
134 log.trace("Polling directory: " + dir);
135 }
136 String currentDir = channel.pwd();
137
138 channel.cd(dir);
139 Vector files = channel.ls(".");
140 for (int i = 0; i < files.size(); i++) {
141 ChannelSftp.LsEntry sftpFile = (ChannelSftp.LsEntry)files.get(i);
142 if (sftpFile.getFilename().startsWith(".")) {
143 // skip
144 } else if (sftpFile.getAttrs().isDir()) {
145 if (isRecursive()) {
146 pollDirectory(getFullFileName(sftpFile));
147 }
148 } else {
149 pollFile(sftpFile);
150 }
151 }
152
153 // change back to original current dir
154 channel.cd(currentDir);
155 }
156
157 protected String getFullFileName(ChannelSftp.LsEntry sftpFile) throws IOException, SftpException {
158 return channel.pwd() + "/" + sftpFile.getFilename();
159 }
160
161 private void pollFile(ChannelSftp.LsEntry sftpFile) throws Exception {
162 if (log.isTraceEnabled()) {
163 log.trace("Polling file: " + sftpFile);
164 }
165
166 // if using last polltime for timestamp matcing (to be removed in Camel 2.0)
167 boolean timestampMatched = true;
168 if (isTimestamp()) {
169 // TODO do we need to adjust the TZ? can we?
170 long ts = sftpFile.getAttrs().getMTime() * 1000L;
171 timestampMatched = ts > lastPollTime;
172 if (log.isTraceEnabled()) {
173 log.trace("The file is to old + " + sftpFile + ". lastPollTime=" + lastPollTime + " > fileTimestamp=" + ts);
174 }
175 }
176
177 if (timestampMatched && isMatched(sftpFile)) {
178 String fullFileName = getFullFileName(sftpFile);
179
180 // is we use exclusive read then acquire the exclusive read (waiting until we got it)
181 if (exclusiveReadLock) {
182 acquireExclusiveReadLock(sftpFile);
183 }
184
185 // retrieve the file
186 final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
187 try {
188 channel.get(sftpFile.getFilename(), byteArrayOutputStream);
189 if (log.isDebugEnabled()) {
190 log.debug("Retrieved file: " + sftpFile.getFilename() + " from: " + remoteServer());
191 }
192 } finally {
193 // close stream to avoid leaking SFTP connections
194 ObjectHelper.close(byteArrayOutputStream, "retrieve: " + sftpFile.getFilename(), log);
195 }
196
197 RemoteFileExchange exchange = endpoint.createExchange(getFullFileName(sftpFile),
198 sftpFile.getFilename(), sftpFile.getAttrs().getSize(), byteArrayOutputStream);
199
200 if (isSetNames()) {
201 String ftpBasePath = endpoint.getConfiguration().getFile();
202 String relativePath = fullFileName.substring(ftpBasePath.length() + 1);
203 relativePath = relativePath.replaceFirst("/", "");
204
205 if (log.isDebugEnabled()) {
206 log.debug("Setting exchange filename to " + relativePath);
207 }
208 exchange.getIn().setHeader(FileComponent.HEADER_FILE_NAME, relativePath);
209 }
210
211 // all success so lets process it
212 getProcessor().process(exchange);
213
214 if (exchange.isFailed()) {
215 if (log.isDebugEnabled()) {
216 log.debug("Processing of exchange failed, so cannot do FTP post command such as move or delete: " + exchange);
217 }
218 } else {
219 // after processing then do post command such as delete or move
220 if (deleteFile) {
221 // delete file after consuming
222 if (log.isDebugEnabled()) {
223 log.debug("Deleting file: " + sftpFile.getFilename() + " from: " + remoteServer());
224 }
225 deleteFile(sftpFile.getFilename());
226 } else if (isMoveFile()) {
227 String fromName = sftpFile.getFilename();
228 String toName = getMoveFileName(fromName, exchange);
229 if (log.isDebugEnabled()) {
230 log.debug("Moving file: " + fromName + " to: " + toName);
231 }
232
233 // delete any existing file
234 boolean deleted = deleteFile(toName);
235 if (!deleted) {
236 // if we could not delete any existing file then maybe the folder is missing
237 // build folder if needed
238 int lastPathIndex = toName.lastIndexOf('/');
239 if (lastPathIndex != -1) {
240 String directory = toName.substring(0, lastPathIndex);
241 if (!SftpUtils.buildDirectory(channel, directory)) {
242 log.warn("Can not build directory: " + directory + " (maybe because of denied permissions)");
243 }
244 }
245 }
246
247 // try to rename
248 try {
249 channel.rename(fromName, toName);
250 } catch (SftpException e) {
251 // ignore just log a warning
252 log.warn("Can not move file: " + fromName + " to: " + toName);
253 }
254 }
255 }
256 }
257 }
258
259 private boolean deleteFile(String filename) {
260 try {
261 channel.rm(filename);
262 return true;
263 } catch (SftpException e) {
264 // ignore just log a warning
265 log.warn("Could not delete file: " + filename + " from: " + remoteServer());
266 return false;
267 }
268 }
269
270 protected void acquireExclusiveReadLock(ChannelSftp.LsEntry sftpFile) throws SftpException {
271 if (log.isTraceEnabled()) {
272 log.trace("Waiting for exclusive read lock to file: " + sftpFile);
273 }
274
275 // the trick is to try to rename the file, if we can rename then we have exclusive read
276 // since its a remote file we can not use java.nio to get a RW access
277 String originalName = sftpFile.getFilename();
278 String newName = originalName + ".camelExclusiveReadLock";
279 boolean exclusive = false;
280 while (!exclusive) {
281 try {
282 channel.rename(originalName, newName);
283 exclusive = true;
284 } catch (SftpException e) {
285 // ignore we can not rename it
286 }
287
288 if (exclusive) {
289 if (log.isDebugEnabled()) {
290 log.debug("Acquired exclusive read lock to file: " + originalName);
291 }
292 // rename it back so we can read it
293 channel.rename(newName, originalName);
294 } else {
295 log.trace("Exclusive read lock not granted. Sleeping for 1000 millis");
296 try {
297 Thread.sleep(1000);
298 } catch (InterruptedException e) {
299 // ignore
300 }
301 }
302 }
303 }
304
305 protected String getFileName(Object file) {
306 ChannelSftp.LsEntry sftpFile = (ChannelSftp.LsEntry) file;
307 return sftpFile.getFilename();
308 }
309
310 }