001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.conf;
020
021 import java.io.BufferedInputStream;
022 import java.io.DataInput;
023 import java.io.DataOutput;
024 import java.io.File;
025 import java.io.FileInputStream;
026 import java.io.IOException;
027 import java.io.InputStream;
028 import java.io.InputStreamReader;
029 import java.io.OutputStream;
030 import java.io.OutputStreamWriter;
031 import java.io.Reader;
032 import java.io.Writer;
033 import java.net.InetSocketAddress;
034 import java.net.URL;
035 import java.util.ArrayList;
036 import java.util.Arrays;
037 import java.util.Collection;
038 import java.util.Collections;
039 import java.util.Enumeration;
040 import java.util.HashMap;
041 import java.util.HashSet;
042 import java.util.Iterator;
043 import java.util.List;
044 import java.util.ListIterator;
045 import java.util.Map;
046 import java.util.Properties;
047 import java.util.Set;
048 import java.util.StringTokenizer;
049 import java.util.WeakHashMap;
050 import java.util.concurrent.CopyOnWriteArrayList;
051 import java.util.regex.Matcher;
052 import java.util.regex.Pattern;
053 import java.util.regex.PatternSyntaxException;
054
055 import javax.xml.parsers.DocumentBuilder;
056 import javax.xml.parsers.DocumentBuilderFactory;
057 import javax.xml.parsers.ParserConfigurationException;
058 import javax.xml.transform.Transformer;
059 import javax.xml.transform.TransformerException;
060 import javax.xml.transform.TransformerFactory;
061 import javax.xml.transform.dom.DOMSource;
062 import javax.xml.transform.stream.StreamResult;
063
064 import org.apache.commons.logging.Log;
065 import org.apache.commons.logging.LogFactory;
066 import org.apache.hadoop.classification.InterfaceAudience;
067 import org.apache.hadoop.classification.InterfaceStability;
068 import org.apache.hadoop.fs.FileSystem;
069 import org.apache.hadoop.fs.Path;
070 import org.apache.hadoop.fs.CommonConfigurationKeys;
071 import org.apache.hadoop.io.Writable;
072 import org.apache.hadoop.io.WritableUtils;
073 import org.apache.hadoop.net.NetUtils;
074 import org.apache.hadoop.util.ReflectionUtils;
075 import org.apache.hadoop.util.StringUtils;
076 import org.codehaus.jackson.JsonFactory;
077 import org.codehaus.jackson.JsonGenerator;
078 import org.w3c.dom.Comment;
079 import org.w3c.dom.DOMException;
080 import org.w3c.dom.Document;
081 import org.w3c.dom.Element;
082 import org.w3c.dom.Node;
083 import org.w3c.dom.NodeList;
084 import org.w3c.dom.Text;
085 import org.xml.sax.SAXException;
086
087 /**
088 * Provides access to configuration parameters.
089 *
090 * <h4 id="Resources">Resources</h4>
091 *
092 * <p>Configurations are specified by resources. A resource contains a set of
093 * name/value pairs as XML data. Each resource is named by either a
094 * <code>String</code> or by a {@link Path}. If named by a <code>String</code>,
095 * then the classpath is examined for a file with that name. If named by a
096 * <code>Path</code>, then the local filesystem is examined directly, without
097 * referring to the classpath.
098 *
099 * <p>Unless explicitly turned off, Hadoop by default specifies two
100 * resources, loaded in-order from the classpath: <ol>
101 * <li><tt><a href="{@docRoot}/../core-default.html">core-default.xml</a>
102 * </tt>: Read-only defaults for hadoop.</li>
103 * <li><tt>core-site.xml</tt>: Site-specific configuration for a given hadoop
104 * installation.</li>
105 * </ol>
106 * Applications may add additional resources, which are loaded
107 * subsequent to these resources in the order they are added.
108 *
109 * <h4 id="FinalParams">Final Parameters</h4>
110 *
111 * <p>Configuration parameters may be declared <i>final</i>.
112 * Once a resource declares a value final, no subsequently-loaded
113 * resource can alter that value.
114 * For example, one might define a final parameter with:
115 * <tt><pre>
116 * <property>
117 * <name>dfs.client.buffer.dir</name>
118 * <value>/tmp/hadoop/dfs/client</value>
119 * <b><final>true</final></b>
120 * </property></pre></tt>
121 *
122 * Administrators typically define parameters as final in
123 * <tt>core-site.xml</tt> for values that user applications may not alter.
124 *
125 * <h4 id="VariableExpansion">Variable Expansion</h4>
126 *
127 * <p>Value strings are first processed for <i>variable expansion</i>. The
128 * available properties are:<ol>
129 * <li>Other properties defined in this Configuration; and, if a name is
130 * undefined here,</li>
131 * <li>Properties in {@link System#getProperties()}.</li>
132 * </ol>
133 *
134 * <p>For example, if a configuration resource contains the following property
135 * definitions:
136 * <tt><pre>
137 * <property>
138 * <name>basedir</name>
139 * <value>/user/${<i>user.name</i>}</value>
140 * </property>
141 *
142 * <property>
143 * <name>tempdir</name>
144 * <value>${<i>basedir</i>}/tmp</value>
145 * </property></pre></tt>
146 *
147 * When <tt>conf.get("tempdir")</tt> is called, then <tt>${<i>basedir</i>}</tt>
148 * will be resolved to another property in this Configuration, while
149 * <tt>${<i>user.name</i>}</tt> would then ordinarily be resolved to the value
150 * of the System property with that name.
151 */
152 @InterfaceAudience.Public
153 @InterfaceStability.Stable
154 public class Configuration implements Iterable<Map.Entry<String,String>>,
155 Writable {
156 private static final Log LOG =
157 LogFactory.getLog(Configuration.class);
158
159 private boolean quietmode = true;
160
161 /**
162 * List of configuration resources.
163 */
164 private ArrayList<Object> resources = new ArrayList<Object>();
165
166 /**
167 * The value reported as the setting resource when a key is set
168 * by code rather than a file resource.
169 */
170 static final String UNKNOWN_RESOURCE = "Unknown";
171
172 /**
173 * List of configuration parameters marked <b>final</b>.
174 */
175 private Set<String> finalParameters = new HashSet<String>();
176
177 private boolean loadDefaults = true;
178
179 /**
180 * Configuration objects
181 */
182 private static final WeakHashMap<Configuration,Object> REGISTRY =
183 new WeakHashMap<Configuration,Object>();
184
185 /**
186 * List of default Resources. Resources are loaded in the order of the list
187 * entries
188 */
189 private static final CopyOnWriteArrayList<String> defaultResources =
190 new CopyOnWriteArrayList<String>();
191
192 private static final Map<ClassLoader, Map<String, Class<?>>>
193 CACHE_CLASSES = new WeakHashMap<ClassLoader, Map<String, Class<?>>>();
194
195 /**
196 * Sentinel value to store negative cache results in {@link #CACHE_CLASSES}.
197 */
198 private static final Class<?> NEGATIVE_CACHE_SENTINEL =
199 NegativeCacheSentinel.class;
200
201 /**
202 * Stores the mapping of key to the resource which modifies or loads
203 * the key most recently
204 */
205 private HashMap<String, String> updatingResource;
206
207 /**
208 * Class to keep the information about the keys which replace the deprecated
209 * ones.
210 *
211 * This class stores the new keys which replace the deprecated keys and also
212 * gives a provision to have a custom message for each of the deprecated key
213 * that is being replaced. It also provides method to get the appropriate
214 * warning message which can be logged whenever the deprecated key is used.
215 */
216 private static class DeprecatedKeyInfo {
217 private String[] newKeys;
218 private String customMessage;
219 private boolean accessed;
220 DeprecatedKeyInfo(String[] newKeys, String customMessage) {
221 this.newKeys = newKeys;
222 this.customMessage = customMessage;
223 accessed = false;
224 }
225
226 /**
227 * Method to provide the warning message. It gives the custom message if
228 * non-null, and default message otherwise.
229 * @param key the associated deprecated key.
230 * @return message that is to be logged when a deprecated key is used.
231 */
232 private final String getWarningMessage(String key) {
233 String warningMessage;
234 if(customMessage == null) {
235 StringBuilder message = new StringBuilder(key);
236 String deprecatedKeySuffix = " is deprecated. Instead, use ";
237 message.append(deprecatedKeySuffix);
238 for (int i = 0; i < newKeys.length; i++) {
239 message.append(newKeys[i]);
240 if(i != newKeys.length-1) {
241 message.append(", ");
242 }
243 }
244 warningMessage = message.toString();
245 }
246 else {
247 warningMessage = customMessage;
248 }
249 accessed = true;
250 return warningMessage;
251 }
252 }
253
254 /**
255 * Stores the deprecated keys, the new keys which replace the deprecated keys
256 * and custom message(if any provided).
257 */
258 private static Map<String, DeprecatedKeyInfo> deprecatedKeyMap =
259 new HashMap<String, DeprecatedKeyInfo>();
260
261 /**
262 * Stores a mapping from superseding keys to the keys which they deprecate.
263 */
264 private static Map<String, String> reverseDeprecatedKeyMap =
265 new HashMap<String, String>();
266
267 /**
268 * Adds the deprecated key to the deprecation map.
269 * It does not override any existing entries in the deprecation map.
270 * This is to be used only by the developers in order to add deprecation of
271 * keys, and attempts to call this method after loading resources once,
272 * would lead to <tt>UnsupportedOperationException</tt>
273 *
274 * If a key is deprecated in favor of multiple keys, they are all treated as
275 * aliases of each other, and setting any one of them resets all the others
276 * to the new value.
277 *
278 * @param key
279 * @param newKeys
280 * @param customMessage
281 * @deprecated use {@link #addDeprecation(String key, String newKey,
282 String customMessage)} instead
283 */
284 @Deprecated
285 public synchronized static void addDeprecation(String key, String[] newKeys,
286 String customMessage) {
287 if (key == null || key.length() == 0 ||
288 newKeys == null || newKeys.length == 0) {
289 throw new IllegalArgumentException();
290 }
291 if (!isDeprecated(key)) {
292 DeprecatedKeyInfo newKeyInfo;
293 newKeyInfo = new DeprecatedKeyInfo(newKeys, customMessage);
294 deprecatedKeyMap.put(key, newKeyInfo);
295 for (String newKey : newKeys) {
296 reverseDeprecatedKeyMap.put(newKey, key);
297 }
298 }
299 }
300
301 /**
302 * Adds the deprecated key to the deprecation map.
303 * It does not override any existing entries in the deprecation map.
304 * This is to be used only by the developers in order to add deprecation of
305 * keys, and attempts to call this method after loading resources once,
306 * would lead to <tt>UnsupportedOperationException</tt>
307 *
308 * @param key
309 * @param newKey
310 * @param customMessage
311 */
312 public synchronized static void addDeprecation(String key, String newKey,
313 String customMessage) {
314 addDeprecation(key, new String[] {newKey}, customMessage);
315 }
316
317 /**
318 * Adds the deprecated key to the deprecation map when no custom message
319 * is provided.
320 * It does not override any existing entries in the deprecation map.
321 * This is to be used only by the developers in order to add deprecation of
322 * keys, and attempts to call this method after loading resources once,
323 * would lead to <tt>UnsupportedOperationException</tt>
324 *
325 * If a key is deprecated in favor of multiple keys, they are all treated as
326 * aliases of each other, and setting any one of them resets all the others
327 * to the new value.
328 *
329 * @param key Key that is to be deprecated
330 * @param newKeys list of keys that take up the values of deprecated key
331 * @deprecated use {@link #addDeprecation(String key, String newKey)} instead
332 */
333 @Deprecated
334 public synchronized static void addDeprecation(String key, String[] newKeys) {
335 addDeprecation(key, newKeys, null);
336 }
337
338 /**
339 * Adds the deprecated key to the deprecation map when no custom message
340 * is provided.
341 * It does not override any existing entries in the deprecation map.
342 * This is to be used only by the developers in order to add deprecation of
343 * keys, and attempts to call this method after loading resources once,
344 * would lead to <tt>UnsupportedOperationException</tt>
345 *
346 * @param key Key that is to be deprecated
347 * @param newKey key that takes up the value of deprecated key
348 */
349 public synchronized static void addDeprecation(String key, String newKey) {
350 addDeprecation(key, new String[] {newKey}, null);
351 }
352
353 /**
354 * checks whether the given <code>key</code> is deprecated.
355 *
356 * @param key the parameter which is to be checked for deprecation
357 * @return <code>true</code> if the key is deprecated and
358 * <code>false</code> otherwise.
359 */
360 public static boolean isDeprecated(String key) {
361 return deprecatedKeyMap.containsKey(key);
362 }
363
364 /**
365 * Returns the alternate name for a key if the property name is deprecated
366 * or if deprecates a property name.
367 *
368 * @param name property name.
369 * @return alternate name.
370 */
371 private String[] getAlternateNames(String name) {
372 String oldName, altNames[] = null;
373 DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name);
374 if (keyInfo == null) {
375 altNames = (reverseDeprecatedKeyMap.get(name) != null ) ?
376 new String [] {reverseDeprecatedKeyMap.get(name)} : null;
377 if(altNames != null && altNames.length > 0) {
378 //To help look for other new configs for this deprecated config
379 keyInfo = deprecatedKeyMap.get(altNames[0]);
380 }
381 }
382 if(keyInfo != null && keyInfo.newKeys.length > 0) {
383 List<String> list = new ArrayList<String>();
384 if(altNames != null) {
385 list.addAll(Arrays.asList(altNames));
386 }
387 list.addAll(Arrays.asList(keyInfo.newKeys));
388 altNames = list.toArray(new String[list.size()]);
389 }
390 return altNames;
391 }
392
393 /**
394 * Checks for the presence of the property <code>name</code> in the
395 * deprecation map. Returns the first of the list of new keys if present
396 * in the deprecation map or the <code>name</code> itself. If the property
397 * is not presently set but the property map contains an entry for the
398 * deprecated key, the value of the deprecated key is set as the value for
399 * the provided property name.
400 *
401 * @param name the property name
402 * @return the first property in the list of properties mapping
403 * the <code>name</code> or the <code>name</code> itself.
404 */
405 private String[] handleDeprecation(String name) {
406 ArrayList<String > names = new ArrayList<String>();
407 if (isDeprecated(name)) {
408 DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name);
409 warnOnceIfDeprecated(name);
410 for (String newKey : keyInfo.newKeys) {
411 if(newKey != null) {
412 names.add(newKey);
413 }
414 }
415 }
416 if(names.size() == 0) {
417 names.add(name);
418 }
419 for(String n : names) {
420 String deprecatedKey = reverseDeprecatedKeyMap.get(n);
421 if (deprecatedKey != null && !getOverlay().containsKey(n) &&
422 getOverlay().containsKey(deprecatedKey)) {
423 getProps().setProperty(n, getOverlay().getProperty(deprecatedKey));
424 getOverlay().setProperty(n, getOverlay().getProperty(deprecatedKey));
425 }
426 }
427 return names.toArray(new String[names.size()]);
428 }
429
430 private void handleDeprecation() {
431 LOG.debug("Handling deprecation for all properties in config...");
432 Set<Object> keys = new HashSet<Object>();
433 keys.addAll(getProps().keySet());
434 for (Object item: keys) {
435 LOG.debug("Handling deprecation for " + (String)item);
436 handleDeprecation((String)item);
437 }
438 }
439
440 static{
441 //print deprecation warning if hadoop-site.xml is found in classpath
442 ClassLoader cL = Thread.currentThread().getContextClassLoader();
443 if (cL == null) {
444 cL = Configuration.class.getClassLoader();
445 }
446 if(cL.getResource("hadoop-site.xml")!=null) {
447 LOG.warn("DEPRECATED: hadoop-site.xml found in the classpath. " +
448 "Usage of hadoop-site.xml is deprecated. Instead use core-site.xml, "
449 + "mapred-site.xml and hdfs-site.xml to override properties of " +
450 "core-default.xml, mapred-default.xml and hdfs-default.xml " +
451 "respectively");
452 }
453 addDefaultResource("core-default.xml");
454 addDefaultResource("core-site.xml");
455 //Add code for managing deprecated key mapping
456 //for example
457 //addDeprecation("oldKey1",new String[]{"newkey1","newkey2"});
458 //adds deprecation for oldKey1 to two new keys(newkey1, newkey2).
459 //so get or set of oldKey1 will correctly populate/access values of
460 //newkey1 and newkey2
461 addDeprecatedKeys();
462 }
463
464 private Properties properties;
465 private Properties overlay;
466 private ClassLoader classLoader;
467 {
468 classLoader = Thread.currentThread().getContextClassLoader();
469 if (classLoader == null) {
470 classLoader = Configuration.class.getClassLoader();
471 }
472 }
473
474 /** A new configuration. */
475 public Configuration() {
476 this(true);
477 }
478
479 /** A new configuration where the behavior of reading from the default
480 * resources can be turned off.
481 *
482 * If the parameter {@code loadDefaults} is false, the new instance
483 * will not load resources from the default files.
484 * @param loadDefaults specifies whether to load from the default files
485 */
486 public Configuration(boolean loadDefaults) {
487 this.loadDefaults = loadDefaults;
488 updatingResource = new HashMap<String, String>();
489 synchronized(Configuration.class) {
490 REGISTRY.put(this, null);
491 }
492 }
493
494 /**
495 * A new configuration with the same settings cloned from another.
496 *
497 * @param other the configuration from which to clone settings.
498 */
499 @SuppressWarnings("unchecked")
500 public Configuration(Configuration other) {
501 this.resources = (ArrayList)other.resources.clone();
502 synchronized(other) {
503 if (other.properties != null) {
504 this.properties = (Properties)other.properties.clone();
505 }
506
507 if (other.overlay!=null) {
508 this.overlay = (Properties)other.overlay.clone();
509 }
510
511 this.updatingResource = new HashMap<String, String>(other.updatingResource);
512 }
513
514 this.finalParameters = new HashSet<String>(other.finalParameters);
515 synchronized(Configuration.class) {
516 REGISTRY.put(this, null);
517 }
518 this.classLoader = other.classLoader;
519 this.loadDefaults = other.loadDefaults;
520 setQuietMode(other.getQuietMode());
521 }
522
523 /**
524 * Add a default resource. Resources are loaded in the order of the resources
525 * added.
526 * @param name file name. File should be present in the classpath.
527 */
528 public static synchronized void addDefaultResource(String name) {
529 if(!defaultResources.contains(name)) {
530 defaultResources.add(name);
531 for(Configuration conf : REGISTRY.keySet()) {
532 if(conf.loadDefaults) {
533 conf.reloadConfiguration();
534 }
535 }
536 }
537 }
538
539 /**
540 * Add a configuration resource.
541 *
542 * The properties of this resource will override properties of previously
543 * added resources, unless they were marked <a href="#Final">final</a>.
544 *
545 * @param name resource to be added, the classpath is examined for a file
546 * with that name.
547 */
548 public void addResource(String name) {
549 addResourceObject(name);
550 }
551
552 /**
553 * Add a configuration resource.
554 *
555 * The properties of this resource will override properties of previously
556 * added resources, unless they were marked <a href="#Final">final</a>.
557 *
558 * @param url url of the resource to be added, the local filesystem is
559 * examined directly to find the resource, without referring to
560 * the classpath.
561 */
562 public void addResource(URL url) {
563 addResourceObject(url);
564 }
565
566 /**
567 * Add a configuration resource.
568 *
569 * The properties of this resource will override properties of previously
570 * added resources, unless they were marked <a href="#Final">final</a>.
571 *
572 * @param file file-path of resource to be added, the local filesystem is
573 * examined directly to find the resource, without referring to
574 * the classpath.
575 */
576 public void addResource(Path file) {
577 addResourceObject(file);
578 }
579
580 /**
581 * Add a configuration resource.
582 *
583 * The properties of this resource will override properties of previously
584 * added resources, unless they were marked <a href="#Final">final</a>.
585 *
586 * @param in InputStream to deserialize the object from.
587 */
588 public void addResource(InputStream in) {
589 addResourceObject(in);
590 }
591
592
593 /**
594 * Reload configuration from previously added resources.
595 *
596 * This method will clear all the configuration read from the added
597 * resources, and final parameters. This will make the resources to
598 * be read again before accessing the values. Values that are added
599 * via set methods will overlay values read from the resources.
600 */
601 public synchronized void reloadConfiguration() {
602 properties = null; // trigger reload
603 finalParameters.clear(); // clear site-limits
604 }
605
606 private synchronized void addResourceObject(Object resource) {
607 resources.add(resource); // add to resources
608 reloadConfiguration();
609 }
610
611 private static Pattern varPat = Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}");
612 private static int MAX_SUBST = 20;
613
614 private String substituteVars(String expr) {
615 if (expr == null) {
616 return null;
617 }
618 Matcher match = varPat.matcher("");
619 String eval = expr;
620 for(int s=0; s<MAX_SUBST; s++) {
621 match.reset(eval);
622 if (!match.find()) {
623 return eval;
624 }
625 String var = match.group();
626 var = var.substring(2, var.length()-1); // remove ${ .. }
627 String val = null;
628 try {
629 val = System.getProperty(var);
630 } catch(SecurityException se) {
631 LOG.warn("Unexpected SecurityException in Configuration", se);
632 }
633 if (val == null) {
634 val = getRaw(var);
635 }
636 if (val == null) {
637 return eval; // return literal ${var}: var is unbound
638 }
639 // substitute
640 eval = eval.substring(0, match.start())+val+eval.substring(match.end());
641 }
642 throw new IllegalStateException("Variable substitution depth too large: "
643 + MAX_SUBST + " " + expr);
644 }
645
646 /**
647 * Get the value of the <code>name</code> property, <code>null</code> if
648 * no such property exists. If the key is deprecated, it returns the value of
649 * the first key which replaces the deprecated key and is not null
650 *
651 * Values are processed for <a href="#VariableExpansion">variable expansion</a>
652 * before being returned.
653 *
654 * @param name the property name.
655 * @return the value of the <code>name</code> or its replacing property,
656 * or null if no such property exists.
657 */
658 public String get(String name) {
659 String[] names = handleDeprecation(name);
660 String result = null;
661 for(String n : names) {
662 result = substituteVars(getProps().getProperty(n));
663 }
664 return result;
665 }
666
667 /**
668 * Get the value of the <code>name</code> property as a trimmed <code>String</code>,
669 * <code>null</code> if no such property exists.
670 * If the key is deprecated, it returns the value of
671 * the first key which replaces the deprecated key and is not null
672 *
673 * Values are processed for <a href="#VariableExpansion">variable expansion</a>
674 * before being returned.
675 *
676 * @param name the property name.
677 * @return the value of the <code>name</code> or its replacing property,
678 * or null if no such property exists.
679 */
680 public String getTrimmed(String name) {
681 String value = get(name);
682
683 if (null == value) {
684 return null;
685 } else {
686 return value.trim();
687 }
688 }
689
690 /**
691 * Get the value of the <code>name</code> property, without doing
692 * <a href="#VariableExpansion">variable expansion</a>.If the key is
693 * deprecated, it returns the value of the first key which replaces
694 * the deprecated key and is not null.
695 *
696 * @param name the property name.
697 * @return the value of the <code>name</code> property or
698 * its replacing property and null if no such property exists.
699 */
700 public String getRaw(String name) {
701 String[] names = handleDeprecation(name);
702 String result = null;
703 for(String n : names) {
704 result = getProps().getProperty(n);
705 }
706 return result;
707 }
708
709 /**
710 * Set the <code>value</code> of the <code>name</code> property. If
711 * <code>name</code> is deprecated or there is a deprecated name associated to it,
712 * it sets the value to both names.
713 *
714 * @param name property name.
715 * @param value property value.
716 */
717 public void set(String name, String value) {
718 if (deprecatedKeyMap.isEmpty()) {
719 getProps();
720 }
721 getOverlay().setProperty(name, value);
722 getProps().setProperty(name, value);
723 updatingResource.put(name, UNKNOWN_RESOURCE);
724 String[] altNames = getAlternateNames(name);
725 if (altNames != null && altNames.length > 0) {
726 for(String altName : altNames) {
727 getOverlay().setProperty(altName, value);
728 getProps().setProperty(altName, value);
729 }
730 }
731 warnOnceIfDeprecated(name);
732 }
733
734 private void warnOnceIfDeprecated(String name) {
735 DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name);
736 if (keyInfo != null && !keyInfo.accessed) {
737 LOG.warn(keyInfo.getWarningMessage(name));
738 }
739 }
740
741 /**
742 * Unset a previously set property.
743 */
744 public synchronized void unset(String name) {
745 String[] altNames = getAlternateNames(name);
746 getOverlay().remove(name);
747 getProps().remove(name);
748 if (altNames !=null && altNames.length > 0) {
749 for(String altName : altNames) {
750 getOverlay().remove(altName);
751 getProps().remove(altName);
752 }
753 }
754 }
755
756 /**
757 * Sets a property if it is currently unset.
758 * @param name the property name
759 * @param value the new value
760 */
761 public synchronized void setIfUnset(String name, String value) {
762 if (get(name) == null) {
763 set(name, value);
764 }
765 }
766
767 private synchronized Properties getOverlay() {
768 if (overlay==null){
769 overlay=new Properties();
770 }
771 return overlay;
772 }
773
774 /**
775 * Get the value of the <code>name</code>. If the key is deprecated,
776 * it returns the value of the first key which replaces the deprecated key
777 * and is not null.
778 * If no such property exists,
779 * then <code>defaultValue</code> is returned.
780 *
781 * @param name property name.
782 * @param defaultValue default value.
783 * @return property value, or <code>defaultValue</code> if the property
784 * doesn't exist.
785 */
786 public String get(String name, String defaultValue) {
787 String[] names = handleDeprecation(name);
788 String result = null;
789 for(String n : names) {
790 result = substituteVars(getProps().getProperty(n, defaultValue));
791 }
792 return result;
793 }
794
795 /**
796 * Get the value of the <code>name</code> property as an <code>int</code>.
797 *
798 * If no such property exists, the provided default value is returned,
799 * or if the specified value is not a valid <code>int</code>,
800 * then an error is thrown.
801 *
802 * @param name property name.
803 * @param defaultValue default value.
804 * @throws NumberFormatException when the value is invalid
805 * @return property value as an <code>int</code>,
806 * or <code>defaultValue</code>.
807 */
808 public int getInt(String name, int defaultValue) {
809 String valueString = getTrimmed(name);
810 if (valueString == null)
811 return defaultValue;
812 String hexString = getHexDigits(valueString);
813 if (hexString != null) {
814 return Integer.parseInt(hexString, 16);
815 }
816 return Integer.parseInt(valueString);
817 }
818
819 /**
820 * Set the value of the <code>name</code> property to an <code>int</code>.
821 *
822 * @param name property name.
823 * @param value <code>int</code> value of the property.
824 */
825 public void setInt(String name, int value) {
826 set(name, Integer.toString(value));
827 }
828
829
830 /**
831 * Get the value of the <code>name</code> property as a <code>long</code>.
832 * If no such property exists, the provided default value is returned,
833 * or if the specified value is not a valid <code>long</code>,
834 * then an error is thrown.
835 *
836 * @param name property name.
837 * @param defaultValue default value.
838 * @throws NumberFormatException when the value is invalid
839 * @return property value as a <code>long</code>,
840 * or <code>defaultValue</code>.
841 */
842 public long getLong(String name, long defaultValue) {
843 String valueString = getTrimmed(name);
844 if (valueString == null)
845 return defaultValue;
846 String hexString = getHexDigits(valueString);
847 if (hexString != null) {
848 return Long.parseLong(hexString, 16);
849 }
850 return Long.parseLong(valueString);
851 }
852
853 /**
854 * Get the value of the <code>name</code> property as a <code>long</code> or
855 * human readable format. If no such property exists, the provided default
856 * value is returned, or if the specified value is not a valid
857 * <code>long</code> or human readable format, then an error is thrown. You
858 * can use the following suffix (case insensitive): k(kilo), m(mega), g(giga),
859 * t(tera), p(peta), e(exa)
860 *
861 * @param name property name.
862 * @param defaultValue default value.
863 * @throws NumberFormatException when the value is invalid
864 * @return property value as a <code>long</code>,
865 * or <code>defaultValue</code>.
866 */
867 public long getLongBytes(String name, long defaultValue) {
868 String valueString = getTrimmed(name);
869 if (valueString == null)
870 return defaultValue;
871 return StringUtils.TraditionalBinaryPrefix.string2long(valueString);
872 }
873
874 private String getHexDigits(String value) {
875 boolean negative = false;
876 String str = value;
877 String hexString = null;
878 if (value.startsWith("-")) {
879 negative = true;
880 str = value.substring(1);
881 }
882 if (str.startsWith("0x") || str.startsWith("0X")) {
883 hexString = str.substring(2);
884 if (negative) {
885 hexString = "-" + hexString;
886 }
887 return hexString;
888 }
889 return null;
890 }
891
892 /**
893 * Set the value of the <code>name</code> property to a <code>long</code>.
894 *
895 * @param name property name.
896 * @param value <code>long</code> value of the property.
897 */
898 public void setLong(String name, long value) {
899 set(name, Long.toString(value));
900 }
901
902 /**
903 * Get the value of the <code>name</code> property as a <code>float</code>.
904 * If no such property exists, the provided default value is returned,
905 * or if the specified value is not a valid <code>float</code>,
906 * then an error is thrown.
907 *
908 * @param name property name.
909 * @param defaultValue default value.
910 * @throws NumberFormatException when the value is invalid
911 * @return property value as a <code>float</code>,
912 * or <code>defaultValue</code>.
913 */
914 public float getFloat(String name, float defaultValue) {
915 String valueString = getTrimmed(name);
916 if (valueString == null)
917 return defaultValue;
918 return Float.parseFloat(valueString);
919 }
920 /**
921 * Set the value of the <code>name</code> property to a <code>float</code>.
922 *
923 * @param name property name.
924 * @param value property value.
925 */
926 public void setFloat(String name, float value) {
927 set(name,Float.toString(value));
928 }
929
930 /**
931 * Get the value of the <code>name</code> property as a <code>boolean</code>.
932 * If no such property is specified, or if the specified value is not a valid
933 * <code>boolean</code>, then <code>defaultValue</code> is returned.
934 *
935 * @param name property name.
936 * @param defaultValue default value.
937 * @return property value as a <code>boolean</code>,
938 * or <code>defaultValue</code>.
939 */
940 public boolean getBoolean(String name, boolean defaultValue) {
941 String valueString = getTrimmed(name);
942 if (null == valueString || "".equals(valueString)) {
943 return defaultValue;
944 }
945
946 valueString = valueString.toLowerCase();
947
948 if ("true".equals(valueString))
949 return true;
950 else if ("false".equals(valueString))
951 return false;
952 else return defaultValue;
953 }
954
955 /**
956 * Set the value of the <code>name</code> property to a <code>boolean</code>.
957 *
958 * @param name property name.
959 * @param value <code>boolean</code> value of the property.
960 */
961 public void setBoolean(String name, boolean value) {
962 set(name, Boolean.toString(value));
963 }
964
965 /**
966 * Set the given property, if it is currently unset.
967 * @param name property name
968 * @param value new value
969 */
970 public void setBooleanIfUnset(String name, boolean value) {
971 setIfUnset(name, Boolean.toString(value));
972 }
973
974 /**
975 * Set the value of the <code>name</code> property to the given type. This
976 * is equivalent to <code>set(<name>, value.toString())</code>.
977 * @param name property name
978 * @param value new value
979 */
980 public <T extends Enum<T>> void setEnum(String name, T value) {
981 set(name, value.toString());
982 }
983
984 /**
985 * Return value matching this enumerated type.
986 * @param name Property name
987 * @param defaultValue Value returned if no mapping exists
988 * @throws IllegalArgumentException If mapping is illegal for the type
989 * provided
990 */
991 public <T extends Enum<T>> T getEnum(String name, T defaultValue) {
992 final String val = get(name);
993 return null == val
994 ? defaultValue
995 : Enum.valueOf(defaultValue.getDeclaringClass(), val);
996 }
997
998 /**
999 * Get the value of the <code>name</code> property as a <code>Pattern</code>.
1000 * If no such property is specified, or if the specified value is not a valid
1001 * <code>Pattern</code>, then <code>DefaultValue</code> is returned.
1002 *
1003 * @param name property name
1004 * @param defaultValue default value
1005 * @return property value as a compiled Pattern, or defaultValue
1006 */
1007 public Pattern getPattern(String name, Pattern defaultValue) {
1008 String valString = get(name);
1009 if (null == valString || "".equals(valString)) {
1010 return defaultValue;
1011 }
1012 try {
1013 return Pattern.compile(valString);
1014 } catch (PatternSyntaxException pse) {
1015 LOG.warn("Regular expression '" + valString + "' for property '" +
1016 name + "' not valid. Using default", pse);
1017 return defaultValue;
1018 }
1019 }
1020
1021 /**
1022 * Set the given property to <code>Pattern</code>.
1023 * If the pattern is passed as null, sets the empty pattern which results in
1024 * further calls to getPattern(...) returning the default value.
1025 *
1026 * @param name property name
1027 * @param pattern new value
1028 */
1029 public void setPattern(String name, Pattern pattern) {
1030 if (null == pattern) {
1031 set(name, null);
1032 } else {
1033 set(name, pattern.pattern());
1034 }
1035 }
1036
1037 /**
1038 * A class that represents a set of positive integer ranges. It parses
1039 * strings of the form: "2-3,5,7-" where ranges are separated by comma and
1040 * the lower/upper bounds are separated by dash. Either the lower or upper
1041 * bound may be omitted meaning all values up to or over. So the string
1042 * above means 2, 3, 5, and 7, 8, 9, ...
1043 */
1044 public static class IntegerRanges implements Iterable<Integer>{
1045 private static class Range {
1046 int start;
1047 int end;
1048 }
1049
1050 private static class RangeNumberIterator implements Iterator<Integer> {
1051 Iterator<Range> internal;
1052 int at;
1053 int end;
1054
1055 public RangeNumberIterator(List<Range> ranges) {
1056 if (ranges != null) {
1057 internal = ranges.iterator();
1058 }
1059 at = -1;
1060 end = -2;
1061 }
1062
1063 @Override
1064 public boolean hasNext() {
1065 if (at <= end) {
1066 return true;
1067 } else if (internal != null){
1068 return internal.hasNext();
1069 }
1070 return false;
1071 }
1072
1073 @Override
1074 public Integer next() {
1075 if (at <= end) {
1076 at++;
1077 return at - 1;
1078 } else if (internal != null){
1079 Range found = internal.next();
1080 if (found != null) {
1081 at = found.start;
1082 end = found.end;
1083 at++;
1084 return at - 1;
1085 }
1086 }
1087 return null;
1088 }
1089
1090 @Override
1091 public void remove() {
1092 throw new UnsupportedOperationException();
1093 }
1094 };
1095
1096 List<Range> ranges = new ArrayList<Range>();
1097
1098 public IntegerRanges() {
1099 }
1100
1101 public IntegerRanges(String newValue) {
1102 StringTokenizer itr = new StringTokenizer(newValue, ",");
1103 while (itr.hasMoreTokens()) {
1104 String rng = itr.nextToken().trim();
1105 String[] parts = rng.split("-", 3);
1106 if (parts.length < 1 || parts.length > 2) {
1107 throw new IllegalArgumentException("integer range badly formed: " +
1108 rng);
1109 }
1110 Range r = new Range();
1111 r.start = convertToInt(parts[0], 0);
1112 if (parts.length == 2) {
1113 r.end = convertToInt(parts[1], Integer.MAX_VALUE);
1114 } else {
1115 r.end = r.start;
1116 }
1117 if (r.start > r.end) {
1118 throw new IllegalArgumentException("IntegerRange from " + r.start +
1119 " to " + r.end + " is invalid");
1120 }
1121 ranges.add(r);
1122 }
1123 }
1124
1125 /**
1126 * Convert a string to an int treating empty strings as the default value.
1127 * @param value the string value
1128 * @param defaultValue the value for if the string is empty
1129 * @return the desired integer
1130 */
1131 private static int convertToInt(String value, int defaultValue) {
1132 String trim = value.trim();
1133 if (trim.length() == 0) {
1134 return defaultValue;
1135 }
1136 return Integer.parseInt(trim);
1137 }
1138
1139 /**
1140 * Is the given value in the set of ranges
1141 * @param value the value to check
1142 * @return is the value in the ranges?
1143 */
1144 public boolean isIncluded(int value) {
1145 for(Range r: ranges) {
1146 if (r.start <= value && value <= r.end) {
1147 return true;
1148 }
1149 }
1150 return false;
1151 }
1152
1153 /**
1154 * @return true if there are no values in this range, else false.
1155 */
1156 public boolean isEmpty() {
1157 return ranges == null || ranges.isEmpty();
1158 }
1159
1160 @Override
1161 public String toString() {
1162 StringBuilder result = new StringBuilder();
1163 boolean first = true;
1164 for(Range r: ranges) {
1165 if (first) {
1166 first = false;
1167 } else {
1168 result.append(',');
1169 }
1170 result.append(r.start);
1171 result.append('-');
1172 result.append(r.end);
1173 }
1174 return result.toString();
1175 }
1176
1177 @Override
1178 public Iterator<Integer> iterator() {
1179 return new RangeNumberIterator(ranges);
1180 }
1181
1182 }
1183
1184 /**
1185 * Parse the given attribute as a set of integer ranges
1186 * @param name the attribute name
1187 * @param defaultValue the default value if it is not set
1188 * @return a new set of ranges from the configured value
1189 */
1190 public IntegerRanges getRange(String name, String defaultValue) {
1191 return new IntegerRanges(get(name, defaultValue));
1192 }
1193
1194 /**
1195 * Get the comma delimited values of the <code>name</code> property as
1196 * a collection of <code>String</code>s.
1197 * If no such property is specified then empty collection is returned.
1198 * <p>
1199 * This is an optimized version of {@link #getStrings(String)}
1200 *
1201 * @param name property name.
1202 * @return property value as a collection of <code>String</code>s.
1203 */
1204 public Collection<String> getStringCollection(String name) {
1205 String valueString = get(name);
1206 return StringUtils.getStringCollection(valueString);
1207 }
1208
1209 /**
1210 * Get the comma delimited values of the <code>name</code> property as
1211 * an array of <code>String</code>s.
1212 * If no such property is specified then <code>null</code> is returned.
1213 *
1214 * @param name property name.
1215 * @return property value as an array of <code>String</code>s,
1216 * or <code>null</code>.
1217 */
1218 public String[] getStrings(String name) {
1219 String valueString = get(name);
1220 return StringUtils.getStrings(valueString);
1221 }
1222
1223 /**
1224 * Get the comma delimited values of the <code>name</code> property as
1225 * an array of <code>String</code>s.
1226 * If no such property is specified then default value is returned.
1227 *
1228 * @param name property name.
1229 * @param defaultValue The default value
1230 * @return property value as an array of <code>String</code>s,
1231 * or default value.
1232 */
1233 public String[] getStrings(String name, String... defaultValue) {
1234 String valueString = get(name);
1235 if (valueString == null) {
1236 return defaultValue;
1237 } else {
1238 return StringUtils.getStrings(valueString);
1239 }
1240 }
1241
1242 /**
1243 * Get the comma delimited values of the <code>name</code> property as
1244 * a collection of <code>String</code>s, trimmed of the leading and trailing whitespace.
1245 * If no such property is specified then empty <code>Collection</code> is returned.
1246 *
1247 * @param name property name.
1248 * @return property value as a collection of <code>String</code>s, or empty <code>Collection</code>
1249 */
1250 public Collection<String> getTrimmedStringCollection(String name) {
1251 String valueString = get(name);
1252 if (null == valueString) {
1253 Collection<String> empty = new ArrayList<String>();
1254 return empty;
1255 }
1256 return StringUtils.getTrimmedStringCollection(valueString);
1257 }
1258
1259 /**
1260 * Get the comma delimited values of the <code>name</code> property as
1261 * an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
1262 * If no such property is specified then an empty array is returned.
1263 *
1264 * @param name property name.
1265 * @return property value as an array of trimmed <code>String</code>s,
1266 * or empty array.
1267 */
1268 public String[] getTrimmedStrings(String name) {
1269 String valueString = get(name);
1270 return StringUtils.getTrimmedStrings(valueString);
1271 }
1272
1273 /**
1274 * Get the comma delimited values of the <code>name</code> property as
1275 * an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
1276 * If no such property is specified then default value is returned.
1277 *
1278 * @param name property name.
1279 * @param defaultValue The default value
1280 * @return property value as an array of trimmed <code>String</code>s,
1281 * or default value.
1282 */
1283 public String[] getTrimmedStrings(String name, String... defaultValue) {
1284 String valueString = get(name);
1285 if (null == valueString) {
1286 return defaultValue;
1287 } else {
1288 return StringUtils.getTrimmedStrings(valueString);
1289 }
1290 }
1291
1292 /**
1293 * Set the array of string values for the <code>name</code> property as
1294 * as comma delimited values.
1295 *
1296 * @param name property name.
1297 * @param values The values
1298 */
1299 public void setStrings(String name, String... values) {
1300 set(name, StringUtils.arrayToString(values));
1301 }
1302
1303 /**
1304 * Get the socket address for <code>name</code> property as a
1305 * <code>InetSocketAddress</code>.
1306 * @param name property name.
1307 * @param defaultAddress the default value
1308 * @param defaultPort the default port
1309 * @return InetSocketAddress
1310 */
1311 public InetSocketAddress getSocketAddr(
1312 String name, String defaultAddress, int defaultPort) {
1313 final String address = get(name, defaultAddress);
1314 return NetUtils.createSocketAddr(address, defaultPort, name);
1315 }
1316
1317 /**
1318 * Set the socket address for the <code>name</code> property as
1319 * a <code>host:port</code>.
1320 */
1321 public void setSocketAddr(String name, InetSocketAddress addr) {
1322 set(name, NetUtils.getHostPortString(addr));
1323 }
1324
1325 /**
1326 * Set the socket address a client can use to connect for the
1327 * <code>name</code> property as a <code>host:port</code>. The wildcard
1328 * address is replaced with the local host's address.
1329 * @param name property name.
1330 * @param addr InetSocketAddress of a listener to store in the given property
1331 * @return InetSocketAddress for clients to connect
1332 */
1333 public InetSocketAddress updateConnectAddr(String name,
1334 InetSocketAddress addr) {
1335 final InetSocketAddress connectAddr = NetUtils.getConnectAddress(addr);
1336 setSocketAddr(name, connectAddr);
1337 return connectAddr;
1338 }
1339
1340 /**
1341 * Load a class by name.
1342 *
1343 * @param name the class name.
1344 * @return the class object.
1345 * @throws ClassNotFoundException if the class is not found.
1346 */
1347 public Class<?> getClassByName(String name) throws ClassNotFoundException {
1348 Class<?> ret = getClassByNameOrNull(name);
1349 if (ret == null) {
1350 throw new ClassNotFoundException("Class " + name + " not found");
1351 }
1352 return ret;
1353 }
1354
1355 /**
1356 * Load a class by name, returning null rather than throwing an exception
1357 * if it couldn't be loaded. This is to avoid the overhead of creating
1358 * an exception.
1359 *
1360 * @param name the class name
1361 * @return the class object, or null if it could not be found.
1362 */
1363 public Class<?> getClassByNameOrNull(String name) {
1364 Map<String, Class<?>> map;
1365
1366 synchronized (CACHE_CLASSES) {
1367 map = CACHE_CLASSES.get(classLoader);
1368 if (map == null) {
1369 map = Collections.synchronizedMap(
1370 new WeakHashMap<String, Class<?>>());
1371 CACHE_CLASSES.put(classLoader, map);
1372 }
1373 }
1374
1375 Class<?> clazz = map.get(name);
1376 if (clazz == null) {
1377 try {
1378 clazz = Class.forName(name, true, classLoader);
1379 } catch (ClassNotFoundException e) {
1380 // Leave a marker that the class isn't found
1381 map.put(name, NEGATIVE_CACHE_SENTINEL);
1382 return null;
1383 }
1384 // two putters can race here, but they'll put the same class
1385 map.put(name, clazz);
1386 return clazz;
1387 } else if (clazz == NEGATIVE_CACHE_SENTINEL) {
1388 return null; // not found
1389 } else {
1390 // cache hit
1391 return clazz;
1392 }
1393 }
1394
1395 /**
1396 * Get the value of the <code>name</code> property
1397 * as an array of <code>Class</code>.
1398 * The value of the property specifies a list of comma separated class names.
1399 * If no such property is specified, then <code>defaultValue</code> is
1400 * returned.
1401 *
1402 * @param name the property name.
1403 * @param defaultValue default value.
1404 * @return property value as a <code>Class[]</code>,
1405 * or <code>defaultValue</code>.
1406 */
1407 public Class<?>[] getClasses(String name, Class<?> ... defaultValue) {
1408 String[] classnames = getTrimmedStrings(name);
1409 if (classnames == null)
1410 return defaultValue;
1411 try {
1412 Class<?>[] classes = new Class<?>[classnames.length];
1413 for(int i = 0; i < classnames.length; i++) {
1414 classes[i] = getClassByName(classnames[i]);
1415 }
1416 return classes;
1417 } catch (ClassNotFoundException e) {
1418 throw new RuntimeException(e);
1419 }
1420 }
1421
1422 /**
1423 * Get the value of the <code>name</code> property as a <code>Class</code>.
1424 * If no such property is specified, then <code>defaultValue</code> is
1425 * returned.
1426 *
1427 * @param name the class name.
1428 * @param defaultValue default value.
1429 * @return property value as a <code>Class</code>,
1430 * or <code>defaultValue</code>.
1431 */
1432 public Class<?> getClass(String name, Class<?> defaultValue) {
1433 String valueString = getTrimmed(name);
1434 if (valueString == null)
1435 return defaultValue;
1436 try {
1437 return getClassByName(valueString);
1438 } catch (ClassNotFoundException e) {
1439 throw new RuntimeException(e);
1440 }
1441 }
1442
1443 /**
1444 * Get the value of the <code>name</code> property as a <code>Class</code>
1445 * implementing the interface specified by <code>xface</code>.
1446 *
1447 * If no such property is specified, then <code>defaultValue</code> is
1448 * returned.
1449 *
1450 * An exception is thrown if the returned class does not implement the named
1451 * interface.
1452 *
1453 * @param name the class name.
1454 * @param defaultValue default value.
1455 * @param xface the interface implemented by the named class.
1456 * @return property value as a <code>Class</code>,
1457 * or <code>defaultValue</code>.
1458 */
1459 public <U> Class<? extends U> getClass(String name,
1460 Class<? extends U> defaultValue,
1461 Class<U> xface) {
1462 try {
1463 Class<?> theClass = getClass(name, defaultValue);
1464 if (theClass != null && !xface.isAssignableFrom(theClass))
1465 throw new RuntimeException(theClass+" not "+xface.getName());
1466 else if (theClass != null)
1467 return theClass.asSubclass(xface);
1468 else
1469 return null;
1470 } catch (Exception e) {
1471 throw new RuntimeException(e);
1472 }
1473 }
1474
1475 /**
1476 * Get the value of the <code>name</code> property as a <code>List</code>
1477 * of objects implementing the interface specified by <code>xface</code>.
1478 *
1479 * An exception is thrown if any of the classes does not exist, or if it does
1480 * not implement the named interface.
1481 *
1482 * @param name the property name.
1483 * @param xface the interface implemented by the classes named by
1484 * <code>name</code>.
1485 * @return a <code>List</code> of objects implementing <code>xface</code>.
1486 */
1487 @SuppressWarnings("unchecked")
1488 public <U> List<U> getInstances(String name, Class<U> xface) {
1489 List<U> ret = new ArrayList<U>();
1490 Class<?>[] classes = getClasses(name);
1491 for (Class<?> cl: classes) {
1492 if (!xface.isAssignableFrom(cl)) {
1493 throw new RuntimeException(cl + " does not implement " + xface);
1494 }
1495 ret.add((U)ReflectionUtils.newInstance(cl, this));
1496 }
1497 return ret;
1498 }
1499
1500 /**
1501 * Set the value of the <code>name</code> property to the name of a
1502 * <code>theClass</code> implementing the given interface <code>xface</code>.
1503 *
1504 * An exception is thrown if <code>theClass</code> does not implement the
1505 * interface <code>xface</code>.
1506 *
1507 * @param name property name.
1508 * @param theClass property value.
1509 * @param xface the interface implemented by the named class.
1510 */
1511 public void setClass(String name, Class<?> theClass, Class<?> xface) {
1512 if (!xface.isAssignableFrom(theClass))
1513 throw new RuntimeException(theClass+" not "+xface.getName());
1514 set(name, theClass.getName());
1515 }
1516
1517 /**
1518 * Get a local file under a directory named by <i>dirsProp</i> with
1519 * the given <i>path</i>. If <i>dirsProp</i> contains multiple directories,
1520 * then one is chosen based on <i>path</i>'s hash code. If the selected
1521 * directory does not exist, an attempt is made to create it.
1522 *
1523 * @param dirsProp directory in which to locate the file.
1524 * @param path file-path.
1525 * @return local file under the directory with the given path.
1526 */
1527 public Path getLocalPath(String dirsProp, String path)
1528 throws IOException {
1529 String[] dirs = getTrimmedStrings(dirsProp);
1530 int hashCode = path.hashCode();
1531 FileSystem fs = FileSystem.getLocal(this);
1532 for (int i = 0; i < dirs.length; i++) { // try each local dir
1533 int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length;
1534 Path file = new Path(dirs[index], path);
1535 Path dir = file.getParent();
1536 if (fs.mkdirs(dir) || fs.exists(dir)) {
1537 return file;
1538 }
1539 }
1540 LOG.warn("Could not make " + path +
1541 " in local directories from " + dirsProp);
1542 for(int i=0; i < dirs.length; i++) {
1543 int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length;
1544 LOG.warn(dirsProp + "[" + index + "]=" + dirs[index]);
1545 }
1546 throw new IOException("No valid local directories in property: "+dirsProp);
1547 }
1548
1549 /**
1550 * Get a local file name under a directory named in <i>dirsProp</i> with
1551 * the given <i>path</i>. If <i>dirsProp</i> contains multiple directories,
1552 * then one is chosen based on <i>path</i>'s hash code. If the selected
1553 * directory does not exist, an attempt is made to create it.
1554 *
1555 * @param dirsProp directory in which to locate the file.
1556 * @param path file-path.
1557 * @return local file under the directory with the given path.
1558 */
1559 public File getFile(String dirsProp, String path)
1560 throws IOException {
1561 String[] dirs = getTrimmedStrings(dirsProp);
1562 int hashCode = path.hashCode();
1563 for (int i = 0; i < dirs.length; i++) { // try each local dir
1564 int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length;
1565 File file = new File(dirs[index], path);
1566 File dir = file.getParentFile();
1567 if (dir.exists() || dir.mkdirs()) {
1568 return file;
1569 }
1570 }
1571 throw new IOException("No valid local directories in property: "+dirsProp);
1572 }
1573
1574 /**
1575 * Get the {@link URL} for the named resource.
1576 *
1577 * @param name resource name.
1578 * @return the url for the named resource.
1579 */
1580 public URL getResource(String name) {
1581 return classLoader.getResource(name);
1582 }
1583
1584 /**
1585 * Get an input stream attached to the configuration resource with the
1586 * given <code>name</code>.
1587 *
1588 * @param name configuration resource name.
1589 * @return an input stream attached to the resource.
1590 */
1591 public InputStream getConfResourceAsInputStream(String name) {
1592 try {
1593 URL url= getResource(name);
1594
1595 if (url == null) {
1596 LOG.info(name + " not found");
1597 return null;
1598 } else {
1599 LOG.info("found resource " + name + " at " + url);
1600 }
1601
1602 return url.openStream();
1603 } catch (Exception e) {
1604 return null;
1605 }
1606 }
1607
1608 /**
1609 * Get a {@link Reader} attached to the configuration resource with the
1610 * given <code>name</code>.
1611 *
1612 * @param name configuration resource name.
1613 * @return a reader attached to the resource.
1614 */
1615 public Reader getConfResourceAsReader(String name) {
1616 try {
1617 URL url= getResource(name);
1618
1619 if (url == null) {
1620 LOG.info(name + " not found");
1621 return null;
1622 } else {
1623 LOG.info("found resource " + name + " at " + url);
1624 }
1625
1626 return new InputStreamReader(url.openStream());
1627 } catch (Exception e) {
1628 return null;
1629 }
1630 }
1631
1632 protected synchronized Properties getProps() {
1633 if (properties == null) {
1634 properties = new Properties();
1635 loadResources(properties, resources, quietmode);
1636 if (overlay!= null) {
1637 properties.putAll(overlay);
1638 for (Map.Entry<Object,Object> item: overlay.entrySet()) {
1639 updatingResource.put((String) item.getKey(), UNKNOWN_RESOURCE);
1640 }
1641 }
1642 }
1643 return properties;
1644 }
1645
1646 /**
1647 * Return the number of keys in the configuration.
1648 *
1649 * @return number of keys in the configuration.
1650 */
1651 public int size() {
1652 return getProps().size();
1653 }
1654
1655 /**
1656 * Clears all keys from the configuration.
1657 */
1658 public void clear() {
1659 getProps().clear();
1660 getOverlay().clear();
1661 }
1662
1663 /**
1664 * Get an {@link Iterator} to go through the list of <code>String</code>
1665 * key-value pairs in the configuration.
1666 *
1667 * @return an iterator over the entries.
1668 */
1669 public Iterator<Map.Entry<String, String>> iterator() {
1670 // Get a copy of just the string to string pairs. After the old object
1671 // methods that allow non-strings to be put into configurations are removed,
1672 // we could replace properties with a Map<String,String> and get rid of this
1673 // code.
1674 Map<String,String> result = new HashMap<String,String>();
1675 for(Map.Entry<Object,Object> item: getProps().entrySet()) {
1676 if (item.getKey() instanceof String &&
1677 item.getValue() instanceof String) {
1678 result.put((String) item.getKey(), (String) item.getValue());
1679 }
1680 }
1681 return result.entrySet().iterator();
1682 }
1683
1684 private void loadResources(Properties properties,
1685 ArrayList resources,
1686 boolean quiet) {
1687 if(loadDefaults) {
1688 for (String resource : defaultResources) {
1689 loadResource(properties, resource, quiet);
1690 }
1691
1692 //support the hadoop-site.xml as a deprecated case
1693 if(getResource("hadoop-site.xml")!=null) {
1694 loadResource(properties, "hadoop-site.xml", quiet);
1695 }
1696 }
1697
1698 for (Object resource : resources) {
1699 loadResource(properties, resource, quiet);
1700 }
1701 }
1702
1703 private void loadResource(Properties properties, Object name, boolean quiet) {
1704 try {
1705 DocumentBuilderFactory docBuilderFactory
1706 = DocumentBuilderFactory.newInstance();
1707 //ignore all comments inside the xml file
1708 docBuilderFactory.setIgnoringComments(true);
1709
1710 //allow includes in the xml file
1711 docBuilderFactory.setNamespaceAware(true);
1712 try {
1713 docBuilderFactory.setXIncludeAware(true);
1714 } catch (UnsupportedOperationException e) {
1715 LOG.error("Failed to set setXIncludeAware(true) for parser "
1716 + docBuilderFactory
1717 + ":" + e,
1718 e);
1719 }
1720 DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
1721 Document doc = null;
1722 Element root = null;
1723
1724 if (name instanceof URL) { // an URL resource
1725 URL url = (URL)name;
1726 if (url != null) {
1727 if (!quiet) {
1728 LOG.info("parsing " + url);
1729 }
1730 doc = builder.parse(url.toString());
1731 }
1732 } else if (name instanceof String) { // a CLASSPATH resource
1733 URL url = getResource((String)name);
1734 if (url != null) {
1735 if (!quiet) {
1736 LOG.info("parsing " + url);
1737 }
1738 doc = builder.parse(url.toString());
1739 }
1740 } else if (name instanceof Path) { // a file resource
1741 // Can't use FileSystem API or we get an infinite loop
1742 // since FileSystem uses Configuration API. Use java.io.File instead.
1743 File file = new File(((Path)name).toUri().getPath())
1744 .getAbsoluteFile();
1745 if (file.exists()) {
1746 if (!quiet) {
1747 LOG.info("parsing " + file);
1748 }
1749 InputStream in = new BufferedInputStream(new FileInputStream(file));
1750 try {
1751 doc = builder.parse(in);
1752 } finally {
1753 in.close();
1754 }
1755 }
1756 } else if (name instanceof InputStream) {
1757 try {
1758 doc = builder.parse((InputStream)name);
1759 } finally {
1760 ((InputStream)name).close();
1761 }
1762 } else if (name instanceof Element) {
1763 root = (Element)name;
1764 }
1765
1766 if (doc == null && root == null) {
1767 if (quiet)
1768 return;
1769 throw new RuntimeException(name + " not found");
1770 }
1771
1772 if (root == null) {
1773 root = doc.getDocumentElement();
1774 }
1775 if (!"configuration".equals(root.getTagName()))
1776 LOG.fatal("bad conf file: top-level element not <configuration>");
1777 NodeList props = root.getChildNodes();
1778 for (int i = 0; i < props.getLength(); i++) {
1779 Node propNode = props.item(i);
1780 if (!(propNode instanceof Element))
1781 continue;
1782 Element prop = (Element)propNode;
1783 if ("configuration".equals(prop.getTagName())) {
1784 loadResource(properties, prop, quiet);
1785 continue;
1786 }
1787 if (!"property".equals(prop.getTagName()))
1788 LOG.warn("bad conf file: element not <property>");
1789 NodeList fields = prop.getChildNodes();
1790 String attr = null;
1791 String value = null;
1792 boolean finalParameter = false;
1793 for (int j = 0; j < fields.getLength(); j++) {
1794 Node fieldNode = fields.item(j);
1795 if (!(fieldNode instanceof Element))
1796 continue;
1797 Element field = (Element)fieldNode;
1798 if ("name".equals(field.getTagName()) && field.hasChildNodes())
1799 attr = ((Text)field.getFirstChild()).getData().trim();
1800 if ("value".equals(field.getTagName()) && field.hasChildNodes())
1801 value = ((Text)field.getFirstChild()).getData();
1802 if ("final".equals(field.getTagName()) && field.hasChildNodes())
1803 finalParameter = "true".equals(((Text)field.getFirstChild()).getData());
1804 }
1805
1806 // Ignore this parameter if it has already been marked as 'final'
1807 if (attr != null) {
1808 if (deprecatedKeyMap.containsKey(attr)) {
1809 DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(attr);
1810 keyInfo.accessed = false;
1811 for (String key:keyInfo.newKeys) {
1812 // update new keys with deprecated key's value
1813 loadProperty(properties, name, key, value, finalParameter);
1814 }
1815 }
1816 else {
1817 loadProperty(properties, name, attr, value, finalParameter);
1818 }
1819 }
1820 }
1821
1822 } catch (IOException e) {
1823 LOG.fatal("error parsing conf file: " + e);
1824 throw new RuntimeException(e);
1825 } catch (DOMException e) {
1826 LOG.fatal("error parsing conf file: " + e);
1827 throw new RuntimeException(e);
1828 } catch (SAXException e) {
1829 LOG.fatal("error parsing conf file: " + e);
1830 throw new RuntimeException(e);
1831 } catch (ParserConfigurationException e) {
1832 LOG.fatal("error parsing conf file: " + e);
1833 throw new RuntimeException(e);
1834 }
1835 }
1836
1837 private void loadProperty(Properties properties, Object name, String attr,
1838 String value, boolean finalParameter) {
1839 if (value != null) {
1840 if (!finalParameters.contains(attr)) {
1841 properties.setProperty(attr, value);
1842 updatingResource.put(attr, name.toString());
1843 } else {
1844 LOG.warn(name+":an attempt to override final parameter: "+attr
1845 +"; Ignoring.");
1846 }
1847 }
1848 if (finalParameter) {
1849 finalParameters.add(attr);
1850 }
1851 }
1852
1853 /**
1854 * Write out the non-default properties in this configuration to the given
1855 * {@link OutputStream}.
1856 *
1857 * @param out the output stream to write to.
1858 */
1859 public void writeXml(OutputStream out) throws IOException {
1860 writeXml(new OutputStreamWriter(out));
1861 }
1862
1863 /**
1864 * Write out the non-default properties in this configuration to the given
1865 * {@link Writer}.
1866 *
1867 * @param out the writer to write to.
1868 */
1869 public void writeXml(Writer out) throws IOException {
1870 Document doc = asXmlDocument();
1871
1872 try {
1873 DOMSource source = new DOMSource(doc);
1874 StreamResult result = new StreamResult(out);
1875 TransformerFactory transFactory = TransformerFactory.newInstance();
1876 Transformer transformer = transFactory.newTransformer();
1877
1878 // Important to not hold Configuration log while writing result, since
1879 // 'out' may be an HDFS stream which needs to lock this configuration
1880 // from another thread.
1881 transformer.transform(source, result);
1882 } catch (TransformerException te) {
1883 throw new IOException(te);
1884 }
1885 }
1886
1887 /**
1888 * Return the XML DOM corresponding to this Configuration.
1889 */
1890 private synchronized Document asXmlDocument() throws IOException {
1891 Document doc;
1892 try {
1893 doc =
1894 DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument();
1895 } catch (ParserConfigurationException pe) {
1896 throw new IOException(pe);
1897 }
1898 Element conf = doc.createElement("configuration");
1899 doc.appendChild(conf);
1900 conf.appendChild(doc.createTextNode("\n"));
1901 handleDeprecation(); //ensure properties is set and deprecation is handled
1902 for (Enumeration e = properties.keys(); e.hasMoreElements();) {
1903 String name = (String)e.nextElement();
1904 Object object = properties.get(name);
1905 String value = null;
1906 if (object instanceof String) {
1907 value = (String) object;
1908 }else {
1909 continue;
1910 }
1911 Element propNode = doc.createElement("property");
1912 conf.appendChild(propNode);
1913
1914 if (updatingResource != null) {
1915 Comment commentNode = doc.createComment(
1916 "Loaded from " + updatingResource.get(name));
1917 propNode.appendChild(commentNode);
1918 }
1919 Element nameNode = doc.createElement("name");
1920 nameNode.appendChild(doc.createTextNode(name));
1921 propNode.appendChild(nameNode);
1922
1923 Element valueNode = doc.createElement("value");
1924 valueNode.appendChild(doc.createTextNode(value));
1925 propNode.appendChild(valueNode);
1926
1927 conf.appendChild(doc.createTextNode("\n"));
1928 }
1929 return doc;
1930 }
1931
1932 /**
1933 * Writes out all the parameters and their properties (final and resource) to
1934 * the given {@link Writer}
1935 * The format of the output would be
1936 * { "properties" : [ {key1,value1,key1.isFinal,key1.resource}, {key2,value2,
1937 * key2.isFinal,key2.resource}... ] }
1938 * It does not output the parameters of the configuration object which is
1939 * loaded from an input stream.
1940 * @param out the Writer to write to
1941 * @throws IOException
1942 */
1943 public static void dumpConfiguration(Configuration config,
1944 Writer out) throws IOException {
1945 JsonFactory dumpFactory = new JsonFactory();
1946 JsonGenerator dumpGenerator = dumpFactory.createJsonGenerator(out);
1947 dumpGenerator.writeStartObject();
1948 dumpGenerator.writeFieldName("properties");
1949 dumpGenerator.writeStartArray();
1950 dumpGenerator.flush();
1951 synchronized (config) {
1952 for (Map.Entry<Object,Object> item: config.getProps().entrySet()) {
1953 dumpGenerator.writeStartObject();
1954 dumpGenerator.writeStringField("key", (String) item.getKey());
1955 dumpGenerator.writeStringField("value",
1956 config.get((String) item.getKey()));
1957 dumpGenerator.writeBooleanField("isFinal",
1958 config.finalParameters.contains(item.getKey()));
1959 dumpGenerator.writeStringField("resource",
1960 config.updatingResource.get(item.getKey()));
1961 dumpGenerator.writeEndObject();
1962 }
1963 }
1964 dumpGenerator.writeEndArray();
1965 dumpGenerator.writeEndObject();
1966 dumpGenerator.flush();
1967 }
1968
1969 /**
1970 * Get the {@link ClassLoader} for this job.
1971 *
1972 * @return the correct class loader.
1973 */
1974 public ClassLoader getClassLoader() {
1975 return classLoader;
1976 }
1977
1978 /**
1979 * Set the class loader that will be used to load the various objects.
1980 *
1981 * @param classLoader the new class loader.
1982 */
1983 public void setClassLoader(ClassLoader classLoader) {
1984 this.classLoader = classLoader;
1985 }
1986
1987 @Override
1988 public String toString() {
1989 StringBuilder sb = new StringBuilder();
1990 sb.append("Configuration: ");
1991 if(loadDefaults) {
1992 toString(defaultResources, sb);
1993 if(resources.size()>0) {
1994 sb.append(", ");
1995 }
1996 }
1997 toString(resources, sb);
1998 return sb.toString();
1999 }
2000
2001 private <T> void toString(List<T> resources, StringBuilder sb) {
2002 ListIterator<T> i = resources.listIterator();
2003 while (i.hasNext()) {
2004 if (i.nextIndex() != 0) {
2005 sb.append(", ");
2006 }
2007 sb.append(i.next());
2008 }
2009 }
2010
2011 /**
2012 * Set the quietness-mode.
2013 *
2014 * In the quiet-mode, error and informational messages might not be logged.
2015 *
2016 * @param quietmode <code>true</code> to set quiet-mode on, <code>false</code>
2017 * to turn it off.
2018 */
2019 public synchronized void setQuietMode(boolean quietmode) {
2020 this.quietmode = quietmode;
2021 }
2022
2023 synchronized boolean getQuietMode() {
2024 return this.quietmode;
2025 }
2026
2027 /** For debugging. List non-default properties to the terminal and exit. */
2028 public static void main(String[] args) throws Exception {
2029 new Configuration().writeXml(System.out);
2030 }
2031
2032 @Override
2033 public void readFields(DataInput in) throws IOException {
2034 clear();
2035 int size = WritableUtils.readVInt(in);
2036 for(int i=0; i < size; ++i) {
2037 set(org.apache.hadoop.io.Text.readString(in),
2038 org.apache.hadoop.io.Text.readString(in));
2039 }
2040 }
2041
2042 //@Override
2043 public void write(DataOutput out) throws IOException {
2044 Properties props = getProps();
2045 WritableUtils.writeVInt(out, props.size());
2046 for(Map.Entry<Object, Object> item: props.entrySet()) {
2047 org.apache.hadoop.io.Text.writeString(out, (String) item.getKey());
2048 org.apache.hadoop.io.Text.writeString(out, (String) item.getValue());
2049 }
2050 }
2051
2052 /**
2053 * get keys matching the the regex
2054 * @param regex
2055 * @return Map<String,String> with matching keys
2056 */
2057 public Map<String,String> getValByRegex(String regex) {
2058 Pattern p = Pattern.compile(regex);
2059
2060 Map<String,String> result = new HashMap<String,String>();
2061 Matcher m;
2062
2063 for(Map.Entry<Object,Object> item: getProps().entrySet()) {
2064 if (item.getKey() instanceof String &&
2065 item.getValue() instanceof String) {
2066 m = p.matcher((String)item.getKey());
2067 if(m.find()) { // match
2068 result.put((String) item.getKey(), (String) item.getValue());
2069 }
2070 }
2071 }
2072 return result;
2073 }
2074
2075 //Load deprecated keys in common
2076 private static void addDeprecatedKeys() {
2077 Configuration.addDeprecation("topology.script.file.name",
2078 new String[]{CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY});
2079 Configuration.addDeprecation("topology.script.number.args",
2080 new String[]{CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY});
2081 Configuration.addDeprecation("hadoop.configured.node.mapping",
2082 new String[]{CommonConfigurationKeys.NET_TOPOLOGY_CONFIGURED_NODE_MAPPING_KEY});
2083 Configuration.addDeprecation("topology.node.switch.mapping.impl",
2084 new String[]{CommonConfigurationKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY});
2085 Configuration.addDeprecation("dfs.df.interval",
2086 new String[]{CommonConfigurationKeys.FS_DF_INTERVAL_KEY});
2087 Configuration.addDeprecation("dfs.client.buffer.dir",
2088 new String[]{CommonConfigurationKeys.FS_CLIENT_BUFFER_DIR_KEY});
2089 Configuration.addDeprecation("hadoop.native.lib",
2090 new String[]{CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY});
2091 Configuration.addDeprecation("fs.default.name",
2092 new String[]{CommonConfigurationKeys.FS_DEFAULT_NAME_KEY});
2093 }
2094
2095 /**
2096 * A unique class which is used as a sentinel value in the caching
2097 * for getClassByName. {@see Configuration#getClassByNameOrNull(String)}
2098 */
2099 private static abstract class NegativeCacheSentinel {}
2100 }