001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements. See the NOTICE file distributed with this
004 * work for additional information regarding copyright ownership. The ASF
005 * licenses this file to you under the Apache License, Version 2.0 (the
006 * "License"); you may not use this file except in compliance with the License.
007 * You may obtain a copy of the License at
008 *
009 * http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
013 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
014 * License for the specific language governing permissions and limitations under
015 * the License.
016 */
017 package org.apache.hadoop.security;
018
019 import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
020
021 import java.io.IOException;
022 import java.net.InetAddress;
023 import java.net.InetSocketAddress;
024 import java.net.URI;
025 import java.net.URL;
026 import java.net.URLConnection;
027 import java.net.UnknownHostException;
028 import java.security.AccessController;
029 import java.security.PrivilegedAction;
030 import java.security.PrivilegedExceptionAction;
031 import java.util.Arrays;
032 import java.util.List;
033 import java.util.Locale;
034 import java.util.ServiceLoader;
035 import java.util.Set;
036
037 import javax.security.auth.Subject;
038 import javax.security.auth.kerberos.KerberosPrincipal;
039 import javax.security.auth.kerberos.KerberosTicket;
040
041 import org.apache.commons.logging.Log;
042 import org.apache.commons.logging.LogFactory;
043 import org.apache.hadoop.classification.InterfaceAudience;
044 import org.apache.hadoop.classification.InterfaceStability;
045 import org.apache.hadoop.conf.Configuration;
046 import org.apache.hadoop.fs.CommonConfigurationKeys;
047 import org.apache.hadoop.http.HttpConfig;
048 import org.apache.hadoop.io.Text;
049 import org.apache.hadoop.net.NetUtils;
050 import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
051 import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
052 import org.apache.hadoop.security.authentication.client.AuthenticationException;
053 import org.apache.hadoop.security.ssl.SSLFactory;
054 import org.apache.hadoop.security.token.Token;
055 import org.apache.hadoop.security.token.TokenInfo;
056
057 import com.google.common.annotations.VisibleForTesting;
058
059 //this will need to be replaced someday when there is a suitable replacement
060 import sun.net.dns.ResolverConfiguration;
061 import sun.net.util.IPAddressUtil;
062
063 @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
064 @InterfaceStability.Evolving
065 public class SecurityUtil {
066 public static final Log LOG = LogFactory.getLog(SecurityUtil.class);
067 public static final String HOSTNAME_PATTERN = "_HOST";
068
069 // controls whether buildTokenService will use an ip or host/ip as given
070 // by the user
071 @VisibleForTesting
072 static boolean useIpForTokenService;
073 @VisibleForTesting
074 static HostResolver hostResolver;
075
076 private static SSLFactory sslFactory;
077
078 static {
079 Configuration conf = new Configuration();
080 boolean useIp = conf.getBoolean(
081 CommonConfigurationKeys.HADOOP_SECURITY_TOKEN_SERVICE_USE_IP,
082 CommonConfigurationKeys.HADOOP_SECURITY_TOKEN_SERVICE_USE_IP_DEFAULT);
083 setTokenServiceUseIp(useIp);
084 if (HttpConfig.isSecure()) {
085 sslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, conf);
086 try {
087 sslFactory.init();
088 } catch (Exception ex) {
089 throw new RuntimeException(ex);
090 }
091 }
092 }
093
094 /**
095 * For use only by tests and initialization
096 */
097 @InterfaceAudience.Private
098 static void setTokenServiceUseIp(boolean flag) {
099 useIpForTokenService = flag;
100 hostResolver = !useIpForTokenService
101 ? new QualifiedHostResolver()
102 : new StandardHostResolver();
103 }
104
105 /**
106 * Find the original TGT within the current subject's credentials. Cross-realm
107 * TGT's of the form "krbtgt/[email protected]" may be present.
108 *
109 * @return The TGT from the current subject
110 * @throws IOException
111 * if TGT can't be found
112 */
113 private static KerberosTicket getTgtFromSubject() throws IOException {
114 Subject current = Subject.getSubject(AccessController.getContext());
115 if (current == null) {
116 throw new IOException(
117 "Can't get TGT from current Subject, because it is null");
118 }
119 Set<KerberosTicket> tickets = current
120 .getPrivateCredentials(KerberosTicket.class);
121 for (KerberosTicket t : tickets) {
122 if (isOriginalTGT(t))
123 return t;
124 }
125 throw new IOException("Failed to find TGT from current Subject:"+current);
126 }
127
128 /**
129 * TGS must have the server principal of the form "krbtgt/FOO@FOO".
130 * @param principal
131 * @return true or false
132 */
133 static boolean
134 isTGSPrincipal(KerberosPrincipal principal) {
135 if (principal == null)
136 return false;
137 if (principal.getName().equals("krbtgt/" + principal.getRealm() +
138 "@" + principal.getRealm())) {
139 return true;
140 }
141 return false;
142 }
143
144 /**
145 * Check whether the server principal is the TGS's principal
146 * @param ticket the original TGT (the ticket that is obtained when a
147 * kinit is done)
148 * @return true or false
149 */
150 protected static boolean isOriginalTGT(KerberosTicket ticket) {
151 return isTGSPrincipal(ticket.getServer());
152 }
153
154 /**
155 * Convert Kerberos principal name pattern to valid Kerberos principal
156 * names. It replaces hostname pattern with hostname, which should be
157 * fully-qualified domain name. If hostname is null or "0.0.0.0", it uses
158 * dynamically looked-up fqdn of the current host instead.
159 *
160 * @param principalConfig
161 * the Kerberos principal name conf value to convert
162 * @param hostname
163 * the fully-qualified domain name used for substitution
164 * @return converted Kerberos principal name
165 * @throws IOException if the client address cannot be determined
166 */
167 @InterfaceAudience.Public
168 @InterfaceStability.Evolving
169 public static String getServerPrincipal(String principalConfig,
170 String hostname) throws IOException {
171 String[] components = getComponents(principalConfig);
172 if (components == null || components.length != 3
173 || !components[1].equals(HOSTNAME_PATTERN)) {
174 return principalConfig;
175 } else {
176 return replacePattern(components, hostname);
177 }
178 }
179
180 /**
181 * Convert Kerberos principal name pattern to valid Kerberos principal names.
182 * This method is similar to {@link #getServerPrincipal(String, String)},
183 * except 1) the reverse DNS lookup from addr to hostname is done only when
184 * necessary, 2) param addr can't be null (no default behavior of using local
185 * hostname when addr is null).
186 *
187 * @param principalConfig
188 * Kerberos principal name pattern to convert
189 * @param addr
190 * InetAddress of the host used for substitution
191 * @return converted Kerberos principal name
192 * @throws IOException if the client address cannot be determined
193 */
194 @InterfaceAudience.Public
195 @InterfaceStability.Evolving
196 public static String getServerPrincipal(String principalConfig,
197 InetAddress addr) throws IOException {
198 String[] components = getComponents(principalConfig);
199 if (components == null || components.length != 3
200 || !components[1].equals(HOSTNAME_PATTERN)) {
201 return principalConfig;
202 } else {
203 if (addr == null) {
204 throw new IOException("Can't replace " + HOSTNAME_PATTERN
205 + " pattern since client address is null");
206 }
207 return replacePattern(components, addr.getCanonicalHostName());
208 }
209 }
210
211 private static String[] getComponents(String principalConfig) {
212 if (principalConfig == null)
213 return null;
214 return principalConfig.split("[/@]");
215 }
216
217 private static String replacePattern(String[] components, String hostname)
218 throws IOException {
219 String fqdn = hostname;
220 if (fqdn == null || fqdn.equals("") || fqdn.equals("0.0.0.0")) {
221 fqdn = getLocalHostName();
222 }
223 return components[0] + "/" + fqdn.toLowerCase(Locale.US) + "@" + components[2];
224 }
225
226 static String getLocalHostName() throws UnknownHostException {
227 return InetAddress.getLocalHost().getCanonicalHostName();
228 }
229
230 /**
231 * Login as a principal specified in config. Substitute $host in
232 * user's Kerberos principal name with a dynamically looked-up fully-qualified
233 * domain name of the current host.
234 *
235 * @param conf
236 * conf to use
237 * @param keytabFileKey
238 * the key to look for keytab file in conf
239 * @param userNameKey
240 * the key to look for user's Kerberos principal name in conf
241 * @throws IOException if login fails
242 */
243 @InterfaceAudience.Public
244 @InterfaceStability.Evolving
245 public static void login(final Configuration conf,
246 final String keytabFileKey, final String userNameKey) throws IOException {
247 login(conf, keytabFileKey, userNameKey, getLocalHostName());
248 }
249
250 /**
251 * Login as a principal specified in config. Substitute $host in user's Kerberos principal
252 * name with hostname. If non-secure mode - return. If no keytab available -
253 * bail out with an exception
254 *
255 * @param conf
256 * conf to use
257 * @param keytabFileKey
258 * the key to look for keytab file in conf
259 * @param userNameKey
260 * the key to look for user's Kerberos principal name in conf
261 * @param hostname
262 * hostname to use for substitution
263 * @throws IOException if the config doesn't specify a keytab
264 */
265 @InterfaceAudience.Public
266 @InterfaceStability.Evolving
267 public static void login(final Configuration conf,
268 final String keytabFileKey, final String userNameKey, String hostname)
269 throws IOException {
270
271 if(! UserGroupInformation.isSecurityEnabled())
272 return;
273
274 String keytabFilename = conf.get(keytabFileKey);
275 if (keytabFilename == null || keytabFilename.length() == 0) {
276 throw new IOException("Running in secure mode, but config doesn't have a keytab");
277 }
278
279 String principalConfig = conf.get(userNameKey, System
280 .getProperty("user.name"));
281 String principalName = SecurityUtil.getServerPrincipal(principalConfig,
282 hostname);
283 UserGroupInformation.loginUserFromKeytab(principalName, keytabFilename);
284 }
285
286 /**
287 * create the service name for a Delegation token
288 * @param uri of the service
289 * @param defPort is used if the uri lacks a port
290 * @return the token service, or null if no authority
291 * @see #buildTokenService(InetSocketAddress)
292 */
293 public static String buildDTServiceName(URI uri, int defPort) {
294 String authority = uri.getAuthority();
295 if (authority == null) {
296 return null;
297 }
298 InetSocketAddress addr = NetUtils.createSocketAddr(authority, defPort);
299 return buildTokenService(addr).toString();
300 }
301
302 /**
303 * Get the host name from the principal name of format <service>/host@realm.
304 * @param principalName principal name of format as described above
305 * @return host name if the the string conforms to the above format, else null
306 */
307 public static String getHostFromPrincipal(String principalName) {
308 return new HadoopKerberosName(principalName).getHostName();
309 }
310
311 private static ServiceLoader<SecurityInfo> securityInfoProviders =
312 ServiceLoader.load(SecurityInfo.class);
313 private static SecurityInfo[] testProviders = new SecurityInfo[0];
314
315 /**
316 * Test setup method to register additional providers.
317 * @param providers a list of high priority providers to use
318 */
319 @InterfaceAudience.Private
320 public static void setSecurityInfoProviders(SecurityInfo... providers) {
321 testProviders = providers;
322 }
323
324 /**
325 * Look up the KerberosInfo for a given protocol. It searches all known
326 * SecurityInfo providers.
327 * @param protocol the protocol class to get the information for
328 * @param conf configuration object
329 * @return the KerberosInfo or null if it has no KerberosInfo defined
330 */
331 public static KerberosInfo
332 getKerberosInfo(Class<?> protocol, Configuration conf) {
333 synchronized (testProviders) {
334 for(SecurityInfo provider: testProviders) {
335 KerberosInfo result = provider.getKerberosInfo(protocol, conf);
336 if (result != null) {
337 return result;
338 }
339 }
340 }
341
342 synchronized (securityInfoProviders) {
343 for(SecurityInfo provider: securityInfoProviders) {
344 KerberosInfo result = provider.getKerberosInfo(protocol, conf);
345 if (result != null) {
346 return result;
347 }
348 }
349 }
350 return null;
351 }
352
353 /**
354 * Look up the TokenInfo for a given protocol. It searches all known
355 * SecurityInfo providers.
356 * @param protocol The protocol class to get the information for.
357 * @param conf Configuration object
358 * @return the TokenInfo or null if it has no KerberosInfo defined
359 */
360 public static TokenInfo getTokenInfo(Class<?> protocol, Configuration conf) {
361 synchronized (testProviders) {
362 for(SecurityInfo provider: testProviders) {
363 TokenInfo result = provider.getTokenInfo(protocol, conf);
364 if (result != null) {
365 return result;
366 }
367 }
368 }
369
370 synchronized (securityInfoProviders) {
371 for(SecurityInfo provider: securityInfoProviders) {
372 TokenInfo result = provider.getTokenInfo(protocol, conf);
373 if (result != null) {
374 return result;
375 }
376 }
377 }
378
379 return null;
380 }
381
382 /**
383 * Decode the given token's service field into an InetAddress
384 * @param token from which to obtain the service
385 * @return InetAddress for the service
386 */
387 public static InetSocketAddress getTokenServiceAddr(Token<?> token) {
388 return NetUtils.createSocketAddr(token.getService().toString());
389 }
390
391 /**
392 * Set the given token's service to the format expected by the RPC client
393 * @param token a delegation token
394 * @param addr the socket for the rpc connection
395 */
396 public static void setTokenService(Token<?> token, InetSocketAddress addr) {
397 Text service = buildTokenService(addr);
398 if (token != null) {
399 token.setService(service);
400 if (LOG.isDebugEnabled()) {
401 LOG.debug("Acquired token "+token); // Token#toString() prints service
402 }
403 } else {
404 LOG.warn("Failed to get token for service "+service);
405 }
406 }
407
408 /**
409 * Construct the service key for a token
410 * @param addr InetSocketAddress of remote connection with a token
411 * @return "ip:port" or "host:port" depending on the value of
412 * hadoop.security.token.service.use_ip
413 */
414 public static Text buildTokenService(InetSocketAddress addr) {
415 String host = null;
416 if (useIpForTokenService) {
417 if (addr.isUnresolved()) { // host has no ip address
418 throw new IllegalArgumentException(
419 new UnknownHostException(addr.getHostName())
420 );
421 }
422 host = addr.getAddress().getHostAddress();
423 } else {
424 host = addr.getHostName().toLowerCase();
425 }
426 return new Text(host + ":" + addr.getPort());
427 }
428
429 /**
430 * Construct the service key for a token
431 * @param uri of remote connection with a token
432 * @return "ip:port" or "host:port" depending on the value of
433 * hadoop.security.token.service.use_ip
434 */
435 public static Text buildTokenService(URI uri) {
436 return buildTokenService(NetUtils.createSocketAddr(uri.getAuthority()));
437 }
438
439 /**
440 * Perform the given action as the daemon's login user. If the login
441 * user cannot be determined, this will log a FATAL error and exit
442 * the whole JVM.
443 */
444 public static <T> T doAsLoginUserOrFatal(PrivilegedAction<T> action) {
445 if (UserGroupInformation.isSecurityEnabled()) {
446 UserGroupInformation ugi = null;
447 try {
448 ugi = UserGroupInformation.getLoginUser();
449 } catch (IOException e) {
450 LOG.fatal("Exception while getting login user", e);
451 e.printStackTrace();
452 Runtime.getRuntime().exit(-1);
453 }
454 return ugi.doAs(action);
455 } else {
456 return action.run();
457 }
458 }
459
460 /**
461 * Perform the given action as the daemon's login user. If an
462 * InterruptedException is thrown, it is converted to an IOException.
463 *
464 * @param action the action to perform
465 * @return the result of the action
466 * @throws IOException in the event of error
467 */
468 public static <T> T doAsLoginUser(PrivilegedExceptionAction<T> action)
469 throws IOException {
470 return doAsUser(UserGroupInformation.getLoginUser(), action);
471 }
472
473 /**
474 * Perform the given action as the daemon's current user. If an
475 * InterruptedException is thrown, it is converted to an IOException.
476 *
477 * @param action the action to perform
478 * @return the result of the action
479 * @throws IOException in the event of error
480 */
481 public static <T> T doAsCurrentUser(PrivilegedExceptionAction<T> action)
482 throws IOException {
483 return doAsUser(UserGroupInformation.getCurrentUser(), action);
484 }
485
486 private static <T> T doAsUser(UserGroupInformation ugi,
487 PrivilegedExceptionAction<T> action) throws IOException {
488 try {
489 return ugi.doAs(action);
490 } catch (InterruptedException ie) {
491 throw new IOException(ie);
492 }
493 }
494
495 /**
496 * Open a (if need be) secure connection to a URL in a secure environment
497 * that is using SPNEGO to authenticate its URLs. All Namenode and Secondary
498 * Namenode URLs that are protected via SPNEGO should be accessed via this
499 * method.
500 *
501 * @param url to authenticate via SPNEGO.
502 * @return A connection that has been authenticated via SPNEGO
503 * @throws IOException If unable to authenticate via SPNEGO
504 */
505 public static URLConnection openSecureHttpConnection(URL url) throws IOException {
506 if (!HttpConfig.isSecure() && !UserGroupInformation.isSecurityEnabled()) {
507 return url.openConnection();
508 }
509
510 AuthenticatedURL.Token token = new AuthenticatedURL.Token();
511 try {
512 return new AuthenticatedURL(null, sslFactory).openConnection(url, token);
513 } catch (AuthenticationException e) {
514 throw new IOException("Exception trying to open authenticated connection to "
515 + url, e);
516 }
517 }
518
519 /**
520 * Resolves a host subject to the security requirements determined by
521 * hadoop.security.token.service.use_ip.
522 *
523 * @param hostname host or ip to resolve
524 * @return a resolved host
525 * @throws UnknownHostException if the host doesn't exist
526 */
527 @InterfaceAudience.Private
528 public static
529 InetAddress getByName(String hostname) throws UnknownHostException {
530 return hostResolver.getByName(hostname);
531 }
532
533 interface HostResolver {
534 InetAddress getByName(String host) throws UnknownHostException;
535 }
536
537 /**
538 * Uses standard java host resolution
539 */
540 static class StandardHostResolver implements HostResolver {
541 @Override
542 public InetAddress getByName(String host) throws UnknownHostException {
543 return InetAddress.getByName(host);
544 }
545 }
546
547 /**
548 * This an alternate resolver with important properties that the standard
549 * java resolver lacks:
550 * 1) The hostname is fully qualified. This avoids security issues if not
551 * all hosts in the cluster do not share the same search domains. It
552 * also prevents other hosts from performing unnecessary dns searches.
553 * In contrast, InetAddress simply returns the host as given.
554 * 2) The InetAddress is instantiated with an exact host and IP to prevent
555 * further unnecessary lookups. InetAddress may perform an unnecessary
556 * reverse lookup for an IP.
557 * 3) A call to getHostName() will always return the qualified hostname, or
558 * more importantly, the IP if instantiated with an IP. This avoids
559 * unnecessary dns timeouts if the host is not resolvable.
560 * 4) Point 3 also ensures that if the host is re-resolved, ex. during a
561 * connection re-attempt, that a reverse lookup to host and forward
562 * lookup to IP is not performed since the reverse/forward mappings may
563 * not always return the same IP. If the client initiated a connection
564 * with an IP, then that IP is all that should ever be contacted.
565 *
566 * NOTE: this resolver is only used if:
567 * hadoop.security.token.service.use_ip=false
568 */
569 protected static class QualifiedHostResolver implements HostResolver {
570 @SuppressWarnings("unchecked")
571 private List<String> searchDomains =
572 ResolverConfiguration.open().searchlist();
573
574 /**
575 * Create an InetAddress with a fully qualified hostname of the given
576 * hostname. InetAddress does not qualify an incomplete hostname that
577 * is resolved via the domain search list.
578 * {@link InetAddress#getCanonicalHostName()} will fully qualify the
579 * hostname, but it always return the A record whereas the given hostname
580 * may be a CNAME.
581 *
582 * @param host a hostname or ip address
583 * @return InetAddress with the fully qualified hostname or ip
584 * @throws UnknownHostException if host does not exist
585 */
586 @Override
587 public InetAddress getByName(String host) throws UnknownHostException {
588 InetAddress addr = null;
589
590 if (IPAddressUtil.isIPv4LiteralAddress(host)) {
591 // use ipv4 address as-is
592 byte[] ip = IPAddressUtil.textToNumericFormatV4(host);
593 addr = InetAddress.getByAddress(host, ip);
594 } else if (IPAddressUtil.isIPv6LiteralAddress(host)) {
595 // use ipv6 address as-is
596 byte[] ip = IPAddressUtil.textToNumericFormatV6(host);
597 addr = InetAddress.getByAddress(host, ip);
598 } else if (host.endsWith(".")) {
599 // a rooted host ends with a dot, ex. "host."
600 // rooted hosts never use the search path, so only try an exact lookup
601 addr = getByExactName(host);
602 } else if (host.contains(".")) {
603 // the host contains a dot (domain), ex. "host.domain"
604 // try an exact host lookup, then fallback to search list
605 addr = getByExactName(host);
606 if (addr == null) {
607 addr = getByNameWithSearch(host);
608 }
609 } else {
610 // it's a simple host with no dots, ex. "host"
611 // try the search list, then fallback to exact host
612 InetAddress loopback = InetAddress.getByName(null);
613 if (host.equalsIgnoreCase(loopback.getHostName())) {
614 addr = InetAddress.getByAddress(host, loopback.getAddress());
615 } else {
616 addr = getByNameWithSearch(host);
617 if (addr == null) {
618 addr = getByExactName(host);
619 }
620 }
621 }
622 // unresolvable!
623 if (addr == null) {
624 throw new UnknownHostException(host);
625 }
626 return addr;
627 }
628
629 InetAddress getByExactName(String host) {
630 InetAddress addr = null;
631 // InetAddress will use the search list unless the host is rooted
632 // with a trailing dot. The trailing dot will disable any use of the
633 // search path in a lower level resolver. See RFC 1535.
634 String fqHost = host;
635 if (!fqHost.endsWith(".")) fqHost += ".";
636 try {
637 addr = getInetAddressByName(fqHost);
638 // can't leave the hostname as rooted or other parts of the system
639 // malfunction, ex. kerberos principals are lacking proper host
640 // equivalence for rooted/non-rooted hostnames
641 addr = InetAddress.getByAddress(host, addr.getAddress());
642 } catch (UnknownHostException e) {
643 // ignore, caller will throw if necessary
644 }
645 return addr;
646 }
647
648 InetAddress getByNameWithSearch(String host) {
649 InetAddress addr = null;
650 if (host.endsWith(".")) { // already qualified?
651 addr = getByExactName(host);
652 } else {
653 for (String domain : searchDomains) {
654 String dot = !domain.startsWith(".") ? "." : "";
655 addr = getByExactName(host + dot + domain);
656 if (addr != null) break;
657 }
658 }
659 return addr;
660 }
661
662 // implemented as a separate method to facilitate unit testing
663 InetAddress getInetAddressByName(String host) throws UnknownHostException {
664 return InetAddress.getByName(host);
665 }
666
667 void setSearchDomains(String ... domains) {
668 searchDomains = Arrays.asList(domains);
669 }
670 }
671
672 public static AuthenticationMethod getAuthenticationMethod(Configuration conf) {
673 String value = conf.get(HADOOP_SECURITY_AUTHENTICATION, "simple");
674 try {
675 return Enum.valueOf(AuthenticationMethod.class, value.toUpperCase());
676 } catch (IllegalArgumentException iae) {
677 throw new IllegalArgumentException("Invalid attribute value for " +
678 HADOOP_SECURITY_AUTHENTICATION + " of " + value);
679 }
680 }
681
682 public static void setAuthenticationMethod(
683 AuthenticationMethod authenticationMethod, Configuration conf) {
684 if (authenticationMethod == null) {
685 authenticationMethod = AuthenticationMethod.SIMPLE;
686 }
687 conf.set(HADOOP_SECURITY_AUTHENTICATION,
688 authenticationMethod.toString().toLowerCase());
689 }
690 }