Source code

001/*
002 * SonarQube, open source software quality management tool.
003 * Copyright (C) 2008-2013 SonarSource
004 * mailto:contact AT sonarsource DOT com
005 *
006 * SonarQube is free software; you can redistribute it and/or
007 * modify it under the terms of the GNU Lesser General Public
008 * License as published by the Free Software Foundation; either
009 * version 3 of the License, or (at your option) any later version.
010 *
011 * SonarQube is distributed in the hope that it will be useful,
012 * but WITHOUT ANY WARRANTY; without even the implied warranty of
013 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
014 * Lesser General Public License for more details.
015 *
016 * You should have received a copy of the GNU Lesser General Public License
017 * along with this program; if not, write to the Free Software Foundation,
018 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
019 */
020package org.sonar.api.utils;
021
022import com.google.common.annotations.VisibleForTesting;
023import com.google.common.base.Joiner;
024import com.google.common.base.Strings;
025import com.google.common.collect.ImmutableList;
026import com.google.common.collect.Lists;
027import com.google.common.io.ByteStreams;
028import com.google.common.io.CharStreams;
029import com.google.common.io.Files;
030import com.google.common.io.InputSupplier;
031import org.apache.commons.codec.binary.Base64;
032import org.apache.commons.io.FileUtils;
033import org.apache.commons.io.IOUtils;
034import org.slf4j.LoggerFactory;
035import org.sonar.api.BatchComponent;
036import org.sonar.api.ServerComponent;
037import org.sonar.api.config.Settings;
038import org.sonar.api.platform.Server;
039
040import javax.annotation.Nullable;
041
042import java.io.File;
043import java.io.IOException;
044import java.io.InputStream;
045import java.net.Authenticator;
046import java.net.HttpURLConnection;
047import java.net.PasswordAuthentication;
048import java.net.Proxy;
049import java.net.ProxySelector;
050import java.net.URI;
051import java.nio.charset.Charset;
052import java.util.List;
053import java.util.Map;
054
055/**
056 * This component downloads HTTP files
057 *
058 * @since 2.2
059 */
060public class HttpDownloader extends UriReader.SchemeProcessor implements BatchComponent, ServerComponent {
061  public static final int TIMEOUT_MILLISECONDS = 20 * 1000;
062
063  private final BaseHttpDownloader downloader;
064  private final Integer readTimeout;
065
066  public HttpDownloader(Server server, Settings settings) {
067    this(server, settings, null);
068  }
069
070  public HttpDownloader(Server server, Settings settings, @Nullable Integer readTimeout) {
071    this.readTimeout = readTimeout;
072    downloader = new BaseHttpDownloader(settings.getProperties(), server.getVersion());
073  }
074
075  public HttpDownloader(Settings settings) {
076    this(settings, null);
077  }
078
079  public HttpDownloader(Settings settings, @Nullable Integer readTimeout) {
080    this.readTimeout = readTimeout;
081    downloader = new BaseHttpDownloader(settings.getProperties(), null);
082  }
083
084  @Override
085  String description(URI uri) {
086    return String.format("%s (%s)", uri.toString(), getProxySynthesis(uri));
087  }
088
089  @Override
090  String[] getSupportedSchemes() {
091    return new String[] {"http", "https"};
092  }
093
094  @Override
095  byte[] readBytes(URI uri) {
096    return download(uri);
097  }
098
099  @Override
100  String readString(URI uri, Charset charset) {
101    try {
102      return CharStreams.toString(CharStreams.newReaderSupplier(downloader.newInputSupplier(uri, this.readTimeout), charset));
103    } catch (IOException e) {
104      throw failToDownload(uri, e);
105    }
106  }
107
108  public String downloadPlainText(URI uri, String encoding) {
109    return readString(uri, Charset.forName(encoding));
110  }
111
112  public byte[] download(URI uri) {
113    try {
114      return ByteStreams.toByteArray(downloader.newInputSupplier(uri, this.readTimeout));
115    } catch (IOException e) {
116      throw failToDownload(uri, e);
117    }
118  }
119
120  public String getProxySynthesis(URI uri) {
121    return downloader.getProxySynthesis(uri);
122  }
123
124  public InputStream openStream(URI uri) {
125    try {
126      return downloader.newInputSupplier(uri, this.readTimeout).getInput();
127    } catch (IOException e) {
128      throw failToDownload(uri, e);
129    }
130  }
131
132  public void download(URI uri, File toFile) {
133    try {
134      Files.copy(downloader.newInputSupplier(uri, this.readTimeout), toFile);
135    } catch (IOException e) {
136      FileUtils.deleteQuietly(toFile);
137      throw failToDownload(uri, e);
138    }
139  }
140
141  private SonarException failToDownload(URI uri, IOException e) {
142    throw new SonarException(String.format("Fail to download: %s (%s)", uri, getProxySynthesis(uri)), e);
143  }
144
145  public static class BaseHttpDownloader {
146    private static final List<String> PROXY_SETTINGS = ImmutableList.of(
147        "http.proxyHost", "http.proxyPort", "http.nonProxyHosts",
148        "http.auth.ntlm.domain", "socksProxyHost", "socksProxyPort");
149
150    private String userAgent;
151
152    public BaseHttpDownloader(Map<String, String> settings, String userAgent) {
153      initProxy(settings);
154      initUserAgent(userAgent);
155    }
156
157    private void initProxy(Map<String, String> settings) {
158      propagateProxySystemProperties(settings);
159      if (requiresProxyAuthentication(settings)) {
160        registerProxyCredentials(settings);
161      }
162    }
163
164    private void initUserAgent(String sonarVersion) {
165      userAgent = (sonarVersion == null ? "Sonar" : String.format("Sonar %s", sonarVersion));
166      System.setProperty("http.agent", userAgent);
167    }
168
169    private String getProxySynthesis(URI uri) {
170      return getProxySynthesis(uri, ProxySelector.getDefault());
171    }
172
173    @VisibleForTesting
174    static String getProxySynthesis(URI uri, ProxySelector proxySelector) {
175      List<Proxy> proxies = proxySelector.select(uri);
176      if (proxies.size() == 1 && proxies.get(0).type().equals(Proxy.Type.DIRECT)) {
177        return "no proxy";
178      }
179
180      List<String> descriptions = Lists.newArrayList();
181      for (Proxy proxy : proxies) {
182        if (proxy.type() != Proxy.Type.DIRECT) {
183          descriptions.add("proxy: " + proxy.address());
184        }
185      }
186
187      return Joiner.on(", ").join(descriptions);
188    }
189
190    private void registerProxyCredentials(Map<String, String> settings) {
191      Authenticator.setDefault(new ProxyAuthenticator(
192          settings.get("http.proxyUser"),
193          settings.get("http.proxyPassword")));
194    }
195
196    private boolean requiresProxyAuthentication(Map<String, String> settings) {
197      return settings.containsKey("http.proxyUser");
198    }
199
200    private void propagateProxySystemProperties(Map<String, String> settings) {
201      for (String key : PROXY_SETTINGS) {
202        if (settings.containsKey(key)) {
203          System.setProperty(key, settings.get(key));
204        }
205      }
206    }
207
208    public InputSupplier<InputStream> newInputSupplier(URI uri) {
209      return new HttpInputSupplier(uri, userAgent, null, null, TIMEOUT_MILLISECONDS);
210    }
211
212    public InputSupplier<InputStream> newInputSupplier(URI uri, @Nullable Integer readTimeoutMillis) {
213      if (readTimeoutMillis != null) {
214        return new HttpInputSupplier(uri, userAgent, null, null, readTimeoutMillis);
215      }
216      return new HttpInputSupplier(uri, userAgent, null, null, TIMEOUT_MILLISECONDS);
217    }
218
219    public InputSupplier<InputStream> newInputSupplier(URI uri, String login, String password) {
220      return new HttpInputSupplier(uri, userAgent, login, password, TIMEOUT_MILLISECONDS);
221    }
222
223    public InputSupplier<InputStream> newInputSupplier(URI uri, String login, String password, @Nullable Integer readTimeoutMillis) {
224      if (readTimeoutMillis != null) {
225        return new HttpInputSupplier(uri, userAgent, login, password, readTimeoutMillis);
226      }
227      return new HttpInputSupplier(uri, userAgent, login, password, TIMEOUT_MILLISECONDS);
228    }
229
230    private static class HttpInputSupplier implements InputSupplier<InputStream> {
231      private final String login;
232      private final String password;
233      private final URI uri;
234      private final String userAgent;
235      private final int readTimeoutMillis;
236
237      HttpInputSupplier(URI uri, String userAgent, String login, String password, int readTimeoutMillis) {
238        this.uri = uri;
239        this.userAgent = userAgent;
240        this.login = login;
241        this.password = password;
242        this.readTimeoutMillis = readTimeoutMillis;
243      }
244
245      public InputStream getInput() throws IOException {
246        LoggerFactory.getLogger(getClass()).debug("Download: " + uri + " (" + getProxySynthesis(uri, ProxySelector.getDefault()) + ")");
247
248        HttpURLConnection connection = (HttpURLConnection) uri.toURL().openConnection();
249        if (!Strings.isNullOrEmpty(login)) {
250          String encoded = new String(Base64.encodeBase64((login + ":" + password).getBytes()));
251          connection.setRequestProperty("Authorization", "Basic " + encoded);
252        }
253        connection.setConnectTimeout(TIMEOUT_MILLISECONDS);
254        connection.setReadTimeout(readTimeoutMillis);
255        connection.setUseCaches(true);
256        connection.setInstanceFollowRedirects(true);
257        connection.setRequestProperty("User-Agent", userAgent);
258
259        int responseCode = connection.getResponseCode();
260        if (responseCode >= 400) {
261          InputStream errorResponse = null;
262          try {
263            errorResponse = connection.getErrorStream();
264            if (errorResponse != null) {
265              String errorResponseContent = IOUtils.toString(errorResponse);
266              throw new HttpException(uri, responseCode, errorResponseContent);
267            }
268            else {
269              throw new HttpException(uri, responseCode);
270            }
271          } finally {
272            IOUtils.closeQuietly(errorResponse);
273          }
274        }
275
276        return connection.getInputStream();
277      }
278    }
279
280    private static class ProxyAuthenticator extends Authenticator {
281      private final PasswordAuthentication auth;
282
283      ProxyAuthenticator(String user, String password) {
284        auth = new PasswordAuthentication(user, password == null ? new char[0] : password.toCharArray());
285      }
286
287      @Override
288      protected PasswordAuthentication getPasswordAuthentication() {
289        return auth;
290      }
291    }
292  }
293
294  public static class HttpException extends RuntimeException {
295    private final URI uri;
296    private final int responseCode;
297    private final String responseContent;
298
299    public HttpException(URI uri, int responseContent) {
300      this(uri, responseContent, null);
301    }
302
303    public HttpException(URI uri, int responseCode, String responseContent) {
304      super("Fail to download [" + uri + "]. Response code: " + responseCode);
305      this.uri = uri;
306      this.responseCode = responseCode;
307      this.responseContent = responseContent;
308    }
309
310    public int getResponseCode() {
311      return responseCode;
312    }
313
314    public URI getUri() {
315      return uri;
316    }
317
318    public String getResponseContent() {
319      return responseContent;
320    }
321  }
322}