001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.conf;
020
021 import java.io.BufferedInputStream;
022 import java.io.DataInput;
023 import java.io.DataOutput;
024 import java.io.File;
025 import java.io.FileInputStream;
026 import java.io.IOException;
027 import java.io.InputStream;
028 import java.io.InputStreamReader;
029 import java.io.OutputStream;
030 import java.io.OutputStreamWriter;
031 import java.io.Reader;
032 import java.io.Writer;
033 import java.lang.ref.WeakReference;
034 import java.net.InetSocketAddress;
035 import java.net.URL;
036 import java.util.ArrayList;
037 import java.util.Arrays;
038 import java.util.Collection;
039 import java.util.Collections;
040 import java.util.Enumeration;
041 import java.util.HashMap;
042 import java.util.HashSet;
043 import java.util.Iterator;
044 import java.util.LinkedList;
045 import java.util.List;
046 import java.util.ListIterator;
047 import java.util.Map;
048 import java.util.Map.Entry;
049 import java.util.Properties;
050 import java.util.Set;
051 import java.util.StringTokenizer;
052 import java.util.WeakHashMap;
053 import java.util.concurrent.CopyOnWriteArrayList;
054 import java.util.regex.Matcher;
055 import java.util.regex.Pattern;
056 import java.util.regex.PatternSyntaxException;
057 import java.util.concurrent.TimeUnit;
058
059 import javax.xml.parsers.DocumentBuilder;
060 import javax.xml.parsers.DocumentBuilderFactory;
061 import javax.xml.parsers.ParserConfigurationException;
062 import javax.xml.transform.Transformer;
063 import javax.xml.transform.TransformerException;
064 import javax.xml.transform.TransformerFactory;
065 import javax.xml.transform.dom.DOMSource;
066 import javax.xml.transform.stream.StreamResult;
067
068 import org.apache.commons.logging.Log;
069 import org.apache.commons.logging.LogFactory;
070 import org.apache.hadoop.classification.InterfaceAudience;
071 import org.apache.hadoop.classification.InterfaceStability;
072 import org.apache.hadoop.fs.FileSystem;
073 import org.apache.hadoop.fs.Path;
074 import org.apache.hadoop.fs.CommonConfigurationKeys;
075 import org.apache.hadoop.io.Writable;
076 import org.apache.hadoop.io.WritableUtils;
077 import org.apache.hadoop.net.NetUtils;
078 import org.apache.hadoop.util.ReflectionUtils;
079 import org.apache.hadoop.util.StringInterner;
080 import org.apache.hadoop.util.StringUtils;
081 import org.codehaus.jackson.JsonFactory;
082 import org.codehaus.jackson.JsonGenerator;
083 import org.w3c.dom.DOMException;
084 import org.w3c.dom.Document;
085 import org.w3c.dom.Element;
086 import org.w3c.dom.Node;
087 import org.w3c.dom.NodeList;
088 import org.w3c.dom.Text;
089 import org.xml.sax.SAXException;
090 import com.google.common.base.Preconditions;
091
092 /**
093 * Provides access to configuration parameters.
094 *
095 * <h4 id="Resources">Resources</h4>
096 *
097 * <p>Configurations are specified by resources. A resource contains a set of
098 * name/value pairs as XML data. Each resource is named by either a
099 * <code>String</code> or by a {@link Path}. If named by a <code>String</code>,
100 * then the classpath is examined for a file with that name. If named by a
101 * <code>Path</code>, then the local filesystem is examined directly, without
102 * referring to the classpath.
103 *
104 * <p>Unless explicitly turned off, Hadoop by default specifies two
105 * resources, loaded in-order from the classpath: <ol>
106 * <li><tt><a href="{@docRoot}/../core-default.html">core-default.xml</a>
107 * </tt>: Read-only defaults for hadoop.</li>
108 * <li><tt>core-site.xml</tt>: Site-specific configuration for a given hadoop
109 * installation.</li>
110 * </ol>
111 * Applications may add additional resources, which are loaded
112 * subsequent to these resources in the order they are added.
113 *
114 * <h4 id="FinalParams">Final Parameters</h4>
115 *
116 * <p>Configuration parameters may be declared <i>final</i>.
117 * Once a resource declares a value final, no subsequently-loaded
118 * resource can alter that value.
119 * For example, one might define a final parameter with:
120 * <tt><pre>
121 * <property>
122 * <name>dfs.hosts.include</name>
123 * <value>/etc/hadoop/conf/hosts.include</value>
124 * <b><final>true</final></b>
125 * </property></pre></tt>
126 *
127 * Administrators typically define parameters as final in
128 * <tt>core-site.xml</tt> for values that user applications may not alter.
129 *
130 * <h4 id="VariableExpansion">Variable Expansion</h4>
131 *
132 * <p>Value strings are first processed for <i>variable expansion</i>. The
133 * available properties are:<ol>
134 * <li>Other properties defined in this Configuration; and, if a name is
135 * undefined here,</li>
136 * <li>Properties in {@link System#getProperties()}.</li>
137 * </ol>
138 *
139 * <p>For example, if a configuration resource contains the following property
140 * definitions:
141 * <tt><pre>
142 * <property>
143 * <name>basedir</name>
144 * <value>/user/${<i>user.name</i>}</value>
145 * </property>
146 *
147 * <property>
148 * <name>tempdir</name>
149 * <value>${<i>basedir</i>}/tmp</value>
150 * </property></pre></tt>
151 *
152 * When <tt>conf.get("tempdir")</tt> is called, then <tt>${<i>basedir</i>}</tt>
153 * will be resolved to another property in this Configuration, while
154 * <tt>${<i>user.name</i>}</tt> would then ordinarily be resolved to the value
155 * of the System property with that name.
156 */
157 @InterfaceAudience.Public
158 @InterfaceStability.Stable
159 public class Configuration implements Iterable<Map.Entry<String,String>>,
160 Writable {
161 private static final Log LOG =
162 LogFactory.getLog(Configuration.class);
163
164 private boolean quietmode = true;
165
166 private static class Resource {
167 private final Object resource;
168 private final String name;
169
170 public Resource(Object resource) {
171 this(resource, resource.toString());
172 }
173
174 public Resource(Object resource, String name) {
175 this.resource = resource;
176 this.name = name;
177 }
178
179 public String getName(){
180 return name;
181 }
182
183 public Object getResource() {
184 return resource;
185 }
186
187 @Override
188 public String toString() {
189 return name;
190 }
191 }
192
193 /**
194 * List of configuration resources.
195 */
196 private ArrayList<Resource> resources = new ArrayList<Resource>();
197
198 /**
199 * The value reported as the setting resource when a key is set
200 * by code rather than a file resource by dumpConfiguration.
201 */
202 static final String UNKNOWN_RESOURCE = "Unknown";
203
204
205 /**
206 * List of configuration parameters marked <b>final</b>.
207 */
208 private Set<String> finalParameters = new HashSet<String>();
209
210 private boolean loadDefaults = true;
211
212 /**
213 * Configuration objects
214 */
215 private static final WeakHashMap<Configuration,Object> REGISTRY =
216 new WeakHashMap<Configuration,Object>();
217
218 /**
219 * List of default Resources. Resources are loaded in the order of the list
220 * entries
221 */
222 private static final CopyOnWriteArrayList<String> defaultResources =
223 new CopyOnWriteArrayList<String>();
224
225 private static final Map<ClassLoader, Map<String, WeakReference<Class<?>>>>
226 CACHE_CLASSES = new WeakHashMap<ClassLoader, Map<String, WeakReference<Class<?>>>>();
227
228 /**
229 * Sentinel value to store negative cache results in {@link #CACHE_CLASSES}.
230 */
231 private static final Class<?> NEGATIVE_CACHE_SENTINEL =
232 NegativeCacheSentinel.class;
233
234 /**
235 * Stores the mapping of key to the resource which modifies or loads
236 * the key most recently
237 */
238 private HashMap<String, String[]> updatingResource;
239
240 /**
241 * Class to keep the information about the keys which replace the deprecated
242 * ones.
243 *
244 * This class stores the new keys which replace the deprecated keys and also
245 * gives a provision to have a custom message for each of the deprecated key
246 * that is being replaced. It also provides method to get the appropriate
247 * warning message which can be logged whenever the deprecated key is used.
248 */
249 private static class DeprecatedKeyInfo {
250 private String[] newKeys;
251 private String customMessage;
252 private boolean accessed;
253 DeprecatedKeyInfo(String[] newKeys, String customMessage) {
254 this.newKeys = newKeys;
255 this.customMessage = customMessage;
256 accessed = false;
257 }
258
259 /**
260 * Method to provide the warning message. It gives the custom message if
261 * non-null, and default message otherwise.
262 * @param key the associated deprecated key.
263 * @return message that is to be logged when a deprecated key is used.
264 */
265 private final String getWarningMessage(String key) {
266 String warningMessage;
267 if(customMessage == null) {
268 StringBuilder message = new StringBuilder(key);
269 String deprecatedKeySuffix = " is deprecated. Instead, use ";
270 message.append(deprecatedKeySuffix);
271 for (int i = 0; i < newKeys.length; i++) {
272 message.append(newKeys[i]);
273 if(i != newKeys.length-1) {
274 message.append(", ");
275 }
276 }
277 warningMessage = message.toString();
278 }
279 else {
280 warningMessage = customMessage;
281 }
282 accessed = true;
283 return warningMessage;
284 }
285 }
286
287 /**
288 * Stores the deprecated keys, the new keys which replace the deprecated keys
289 * and custom message(if any provided).
290 */
291 private static Map<String, DeprecatedKeyInfo> deprecatedKeyMap =
292 new HashMap<String, DeprecatedKeyInfo>();
293
294 /**
295 * Stores a mapping from superseding keys to the keys which they deprecate.
296 */
297 private static Map<String, String> reverseDeprecatedKeyMap =
298 new HashMap<String, String>();
299
300 /**
301 * Adds the deprecated key to the deprecation map.
302 * It does not override any existing entries in the deprecation map.
303 * This is to be used only by the developers in order to add deprecation of
304 * keys, and attempts to call this method after loading resources once,
305 * would lead to <tt>UnsupportedOperationException</tt>
306 *
307 * If a key is deprecated in favor of multiple keys, they are all treated as
308 * aliases of each other, and setting any one of them resets all the others
309 * to the new value.
310 *
311 * @param key
312 * @param newKeys
313 * @param customMessage
314 * @deprecated use {@link #addDeprecation(String key, String newKey,
315 String customMessage)} instead
316 */
317 @Deprecated
318 public synchronized static void addDeprecation(String key, String[] newKeys,
319 String customMessage) {
320 if (key == null || key.length() == 0 ||
321 newKeys == null || newKeys.length == 0) {
322 throw new IllegalArgumentException();
323 }
324 if (!isDeprecated(key)) {
325 DeprecatedKeyInfo newKeyInfo;
326 newKeyInfo = new DeprecatedKeyInfo(newKeys, customMessage);
327 deprecatedKeyMap.put(key, newKeyInfo);
328 for (String newKey : newKeys) {
329 reverseDeprecatedKeyMap.put(newKey, key);
330 }
331 }
332 }
333
334 /**
335 * Adds the deprecated key to the deprecation map.
336 * It does not override any existing entries in the deprecation map.
337 * This is to be used only by the developers in order to add deprecation of
338 * keys, and attempts to call this method after loading resources once,
339 * would lead to <tt>UnsupportedOperationException</tt>
340 *
341 * @param key
342 * @param newKey
343 * @param customMessage
344 */
345 public synchronized static void addDeprecation(String key, String newKey,
346 String customMessage) {
347 addDeprecation(key, new String[] {newKey}, customMessage);
348 }
349
350 /**
351 * Adds the deprecated key to the deprecation map when no custom message
352 * is provided.
353 * It does not override any existing entries in the deprecation map.
354 * This is to be used only by the developers in order to add deprecation of
355 * keys, and attempts to call this method after loading resources once,
356 * would lead to <tt>UnsupportedOperationException</tt>
357 *
358 * If a key is deprecated in favor of multiple keys, they are all treated as
359 * aliases of each other, and setting any one of them resets all the others
360 * to the new value.
361 *
362 * @param key Key that is to be deprecated
363 * @param newKeys list of keys that take up the values of deprecated key
364 * @deprecated use {@link #addDeprecation(String key, String newKey)} instead
365 */
366 @Deprecated
367 public synchronized static void addDeprecation(String key, String[] newKeys) {
368 addDeprecation(key, newKeys, null);
369 }
370
371 /**
372 * Adds the deprecated key to the deprecation map when no custom message
373 * is provided.
374 * It does not override any existing entries in the deprecation map.
375 * This is to be used only by the developers in order to add deprecation of
376 * keys, and attempts to call this method after loading resources once,
377 * would lead to <tt>UnsupportedOperationException</tt>
378 *
379 * @param key Key that is to be deprecated
380 * @param newKey key that takes up the value of deprecated key
381 */
382 public synchronized static void addDeprecation(String key, String newKey) {
383 addDeprecation(key, new String[] {newKey}, null);
384 }
385
386 /**
387 * checks whether the given <code>key</code> is deprecated.
388 *
389 * @param key the parameter which is to be checked for deprecation
390 * @return <code>true</code> if the key is deprecated and
391 * <code>false</code> otherwise.
392 */
393 public static boolean isDeprecated(String key) {
394 return deprecatedKeyMap.containsKey(key);
395 }
396
397 /**
398 * Returns the alternate name for a key if the property name is deprecated
399 * or if deprecates a property name.
400 *
401 * @param name property name.
402 * @return alternate name.
403 */
404 private String[] getAlternateNames(String name) {
405 String altNames[] = null;
406 DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name);
407 if (keyInfo == null) {
408 altNames = (reverseDeprecatedKeyMap.get(name) != null ) ?
409 new String [] {reverseDeprecatedKeyMap.get(name)} : null;
410 if(altNames != null && altNames.length > 0) {
411 //To help look for other new configs for this deprecated config
412 keyInfo = deprecatedKeyMap.get(altNames[0]);
413 }
414 }
415 if(keyInfo != null && keyInfo.newKeys.length > 0) {
416 List<String> list = new ArrayList<String>();
417 if(altNames != null) {
418 list.addAll(Arrays.asList(altNames));
419 }
420 list.addAll(Arrays.asList(keyInfo.newKeys));
421 altNames = list.toArray(new String[list.size()]);
422 }
423 return altNames;
424 }
425
426 /**
427 * Checks for the presence of the property <code>name</code> in the
428 * deprecation map. Returns the first of the list of new keys if present
429 * in the deprecation map or the <code>name</code> itself. If the property
430 * is not presently set but the property map contains an entry for the
431 * deprecated key, the value of the deprecated key is set as the value for
432 * the provided property name.
433 *
434 * @param name the property name
435 * @return the first property in the list of properties mapping
436 * the <code>name</code> or the <code>name</code> itself.
437 */
438 private String[] handleDeprecation(String name) {
439 ArrayList<String > names = new ArrayList<String>();
440 if (isDeprecated(name)) {
441 DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name);
442 warnOnceIfDeprecated(name);
443 for (String newKey : keyInfo.newKeys) {
444 if(newKey != null) {
445 names.add(newKey);
446 }
447 }
448 }
449 if(names.size() == 0) {
450 names.add(name);
451 }
452 for(String n : names) {
453 String deprecatedKey = reverseDeprecatedKeyMap.get(n);
454 if (deprecatedKey != null && !getOverlay().containsKey(n) &&
455 getOverlay().containsKey(deprecatedKey)) {
456 getProps().setProperty(n, getOverlay().getProperty(deprecatedKey));
457 getOverlay().setProperty(n, getOverlay().getProperty(deprecatedKey));
458 }
459 }
460 return names.toArray(new String[names.size()]);
461 }
462
463 private void handleDeprecation() {
464 LOG.debug("Handling deprecation for all properties in config...");
465 Set<Object> keys = new HashSet<Object>();
466 keys.addAll(getProps().keySet());
467 for (Object item: keys) {
468 LOG.debug("Handling deprecation for " + (String)item);
469 handleDeprecation((String)item);
470 }
471 }
472
473 static{
474 //print deprecation warning if hadoop-site.xml is found in classpath
475 ClassLoader cL = Thread.currentThread().getContextClassLoader();
476 if (cL == null) {
477 cL = Configuration.class.getClassLoader();
478 }
479 if(cL.getResource("hadoop-site.xml")!=null) {
480 LOG.warn("DEPRECATED: hadoop-site.xml found in the classpath. " +
481 "Usage of hadoop-site.xml is deprecated. Instead use core-site.xml, "
482 + "mapred-site.xml and hdfs-site.xml to override properties of " +
483 "core-default.xml, mapred-default.xml and hdfs-default.xml " +
484 "respectively");
485 }
486 addDefaultResource("core-default.xml");
487 addDefaultResource("core-site.xml");
488 //Add code for managing deprecated key mapping
489 //for example
490 //addDeprecation("oldKey1",new String[]{"newkey1","newkey2"});
491 //adds deprecation for oldKey1 to two new keys(newkey1, newkey2).
492 //so get or set of oldKey1 will correctly populate/access values of
493 //newkey1 and newkey2
494 addDeprecatedKeys();
495 }
496
497 private Properties properties;
498 private Properties overlay;
499 private ClassLoader classLoader;
500 {
501 classLoader = Thread.currentThread().getContextClassLoader();
502 if (classLoader == null) {
503 classLoader = Configuration.class.getClassLoader();
504 }
505 }
506
507 /** A new configuration. */
508 public Configuration() {
509 this(true);
510 }
511
512 /** A new configuration where the behavior of reading from the default
513 * resources can be turned off.
514 *
515 * If the parameter {@code loadDefaults} is false, the new instance
516 * will not load resources from the default files.
517 * @param loadDefaults specifies whether to load from the default files
518 */
519 public Configuration(boolean loadDefaults) {
520 this.loadDefaults = loadDefaults;
521 updatingResource = new HashMap<String, String[]>();
522 synchronized(Configuration.class) {
523 REGISTRY.put(this, null);
524 }
525 }
526
527 /**
528 * A new configuration with the same settings cloned from another.
529 *
530 * @param other the configuration from which to clone settings.
531 */
532 @SuppressWarnings("unchecked")
533 public Configuration(Configuration other) {
534 this.resources = (ArrayList<Resource>) other.resources.clone();
535 synchronized(other) {
536 if (other.properties != null) {
537 this.properties = (Properties)other.properties.clone();
538 }
539
540 if (other.overlay!=null) {
541 this.overlay = (Properties)other.overlay.clone();
542 }
543
544 this.updatingResource = new HashMap<String, String[]>(other.updatingResource);
545 }
546
547 this.finalParameters = new HashSet<String>(other.finalParameters);
548 synchronized(Configuration.class) {
549 REGISTRY.put(this, null);
550 }
551 this.classLoader = other.classLoader;
552 this.loadDefaults = other.loadDefaults;
553 setQuietMode(other.getQuietMode());
554 }
555
556 /**
557 * Add a default resource. Resources are loaded in the order of the resources
558 * added.
559 * @param name file name. File should be present in the classpath.
560 */
561 public static synchronized void addDefaultResource(String name) {
562 if(!defaultResources.contains(name)) {
563 defaultResources.add(name);
564 for(Configuration conf : REGISTRY.keySet()) {
565 if(conf.loadDefaults) {
566 conf.reloadConfiguration();
567 }
568 }
569 }
570 }
571
572 /**
573 * Add a configuration resource.
574 *
575 * The properties of this resource will override properties of previously
576 * added resources, unless they were marked <a href="#Final">final</a>.
577 *
578 * @param name resource to be added, the classpath is examined for a file
579 * with that name.
580 */
581 public void addResource(String name) {
582 addResourceObject(new Resource(name));
583 }
584
585 /**
586 * Add a configuration resource.
587 *
588 * The properties of this resource will override properties of previously
589 * added resources, unless they were marked <a href="#Final">final</a>.
590 *
591 * @param url url of the resource to be added, the local filesystem is
592 * examined directly to find the resource, without referring to
593 * the classpath.
594 */
595 public void addResource(URL url) {
596 addResourceObject(new Resource(url));
597 }
598
599 /**
600 * Add a configuration resource.
601 *
602 * The properties of this resource will override properties of previously
603 * added resources, unless they were marked <a href="#Final">final</a>.
604 *
605 * @param file file-path of resource to be added, the local filesystem is
606 * examined directly to find the resource, without referring to
607 * the classpath.
608 */
609 public void addResource(Path file) {
610 addResourceObject(new Resource(file));
611 }
612
613 /**
614 * Add a configuration resource.
615 *
616 * The properties of this resource will override properties of previously
617 * added resources, unless they were marked <a href="#Final">final</a>.
618 *
619 * WARNING: The contents of the InputStream will be cached, by this method.
620 * So use this sparingly because it does increase the memory consumption.
621 *
622 * @param in InputStream to deserialize the object from. In will be read from
623 * when a get or set is called next. After it is read the stream will be
624 * closed.
625 */
626 public void addResource(InputStream in) {
627 addResourceObject(new Resource(in));
628 }
629
630 /**
631 * Add a configuration resource.
632 *
633 * The properties of this resource will override properties of previously
634 * added resources, unless they were marked <a href="#Final">final</a>.
635 *
636 * @param in InputStream to deserialize the object from.
637 * @param name the name of the resource because InputStream.toString is not
638 * very descriptive some times.
639 */
640 public void addResource(InputStream in, String name) {
641 addResourceObject(new Resource(in, name));
642 }
643
644
645 /**
646 * Reload configuration from previously added resources.
647 *
648 * This method will clear all the configuration read from the added
649 * resources, and final parameters. This will make the resources to
650 * be read again before accessing the values. Values that are added
651 * via set methods will overlay values read from the resources.
652 */
653 public synchronized void reloadConfiguration() {
654 properties = null; // trigger reload
655 finalParameters.clear(); // clear site-limits
656 }
657
658 private synchronized void addResourceObject(Resource resource) {
659 resources.add(resource); // add to resources
660 reloadConfiguration();
661 }
662
663 private static Pattern varPat = Pattern.compile("\\$\\{[^\\}\\$\u0020]+\\}");
664 private static int MAX_SUBST = 20;
665
666 private String substituteVars(String expr) {
667 if (expr == null) {
668 return null;
669 }
670 Matcher match = varPat.matcher("");
671 String eval = expr;
672 for(int s=0; s<MAX_SUBST; s++) {
673 match.reset(eval);
674 if (!match.find()) {
675 return eval;
676 }
677 String var = match.group();
678 var = var.substring(2, var.length()-1); // remove ${ .. }
679 String val = null;
680 try {
681 val = System.getProperty(var);
682 } catch(SecurityException se) {
683 LOG.warn("Unexpected SecurityException in Configuration", se);
684 }
685 if (val == null) {
686 val = getRaw(var);
687 }
688 if (val == null) {
689 return eval; // return literal ${var}: var is unbound
690 }
691 // substitute
692 eval = eval.substring(0, match.start())+val+eval.substring(match.end());
693 }
694 throw new IllegalStateException("Variable substitution depth too large: "
695 + MAX_SUBST + " " + expr);
696 }
697
698 /**
699 * Get the value of the <code>name</code> property, <code>null</code> if
700 * no such property exists. If the key is deprecated, it returns the value of
701 * the first key which replaces the deprecated key and is not null
702 *
703 * Values are processed for <a href="#VariableExpansion">variable expansion</a>
704 * before being returned.
705 *
706 * @param name the property name.
707 * @return the value of the <code>name</code> or its replacing property,
708 * or null if no such property exists.
709 */
710 public String get(String name) {
711 String[] names = handleDeprecation(name);
712 String result = null;
713 for(String n : names) {
714 result = substituteVars(getProps().getProperty(n));
715 }
716 return result;
717 }
718
719 /**
720 * Get the value of the <code>name</code> property as a trimmed <code>String</code>,
721 * <code>null</code> if no such property exists.
722 * If the key is deprecated, it returns the value of
723 * the first key which replaces the deprecated key and is not null
724 *
725 * Values are processed for <a href="#VariableExpansion">variable expansion</a>
726 * before being returned.
727 *
728 * @param name the property name.
729 * @return the value of the <code>name</code> or its replacing property,
730 * or null if no such property exists.
731 */
732 public String getTrimmed(String name) {
733 String value = get(name);
734
735 if (null == value) {
736 return null;
737 } else {
738 return value.trim();
739 }
740 }
741
742 /**
743 * Get the value of the <code>name</code> property as a trimmed <code>String</code>,
744 * <code>defaultValue</code> if no such property exists.
745 * See @{Configuration#getTrimmed} for more details.
746 *
747 * @param name the property name.
748 * @param defaultValue the property default value.
749 * @return the value of the <code>name</code> or defaultValue
750 * if it is not set.
751 */
752 public String getTrimmed(String name, String defaultValue) {
753 String ret = getTrimmed(name);
754 return ret == null ? defaultValue : ret;
755 }
756
757 /**
758 * Get the value of the <code>name</code> property, without doing
759 * <a href="#VariableExpansion">variable expansion</a>.If the key is
760 * deprecated, it returns the value of the first key which replaces
761 * the deprecated key and is not null.
762 *
763 * @param name the property name.
764 * @return the value of the <code>name</code> property or
765 * its replacing property and null if no such property exists.
766 */
767 public String getRaw(String name) {
768 String[] names = handleDeprecation(name);
769 String result = null;
770 for(String n : names) {
771 result = getProps().getProperty(n);
772 }
773 return result;
774 }
775
776 /**
777 * Set the <code>value</code> of the <code>name</code> property. If
778 * <code>name</code> is deprecated or there is a deprecated name associated to it,
779 * it sets the value to both names.
780 *
781 * @param name property name.
782 * @param value property value.
783 */
784 public void set(String name, String value) {
785 set(name, value, null);
786 }
787
788 /**
789 * Set the <code>value</code> of the <code>name</code> property. If
790 * <code>name</code> is deprecated or there is a deprecated name associated to it,
791 * it sets the value to both names.
792 *
793 * @param name property name.
794 * @param value property value.
795 * @param source the place that this configuration value came from
796 * (For debugging).
797 * @throws IllegalArgumentException when the value or name is null.
798 */
799 public void set(String name, String value, String source) {
800 Preconditions.checkArgument(
801 name != null,
802 "Property name must not be null");
803 Preconditions.checkArgument(
804 value != null,
805 "Property value must not be null");
806 if (deprecatedKeyMap.isEmpty()) {
807 getProps();
808 }
809 getOverlay().setProperty(name, value);
810 getProps().setProperty(name, value);
811 if(source == null) {
812 updatingResource.put(name, new String[] {"programatically"});
813 } else {
814 updatingResource.put(name, new String[] {source});
815 }
816 String[] altNames = getAlternateNames(name);
817 if (altNames != null && altNames.length > 0) {
818 String altSource = "because " + name + " is deprecated";
819 for(String altName : altNames) {
820 if(!altName.equals(name)) {
821 getOverlay().setProperty(altName, value);
822 getProps().setProperty(altName, value);
823 updatingResource.put(altName, new String[] {altSource});
824 }
825 }
826 }
827 warnOnceIfDeprecated(name);
828 }
829
830 private void warnOnceIfDeprecated(String name) {
831 DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(name);
832 if (keyInfo != null && !keyInfo.accessed) {
833 LOG.warn(keyInfo.getWarningMessage(name));
834 }
835 }
836
837 /**
838 * Unset a previously set property.
839 */
840 public synchronized void unset(String name) {
841 String[] altNames = getAlternateNames(name);
842 getOverlay().remove(name);
843 getProps().remove(name);
844 if (altNames !=null && altNames.length > 0) {
845 for(String altName : altNames) {
846 getOverlay().remove(altName);
847 getProps().remove(altName);
848 }
849 }
850 }
851
852 /**
853 * Sets a property if it is currently unset.
854 * @param name the property name
855 * @param value the new value
856 */
857 public synchronized void setIfUnset(String name, String value) {
858 if (get(name) == null) {
859 set(name, value);
860 }
861 }
862
863 private synchronized Properties getOverlay() {
864 if (overlay==null){
865 overlay=new Properties();
866 }
867 return overlay;
868 }
869
870 /**
871 * Get the value of the <code>name</code>. If the key is deprecated,
872 * it returns the value of the first key which replaces the deprecated key
873 * and is not null.
874 * If no such property exists,
875 * then <code>defaultValue</code> is returned.
876 *
877 * @param name property name.
878 * @param defaultValue default value.
879 * @return property value, or <code>defaultValue</code> if the property
880 * doesn't exist.
881 */
882 public String get(String name, String defaultValue) {
883 String[] names = handleDeprecation(name);
884 String result = null;
885 for(String n : names) {
886 result = substituteVars(getProps().getProperty(n, defaultValue));
887 }
888 return result;
889 }
890
891 /**
892 * Get the value of the <code>name</code> property as an <code>int</code>.
893 *
894 * If no such property exists, the provided default value is returned,
895 * or if the specified value is not a valid <code>int</code>,
896 * then an error is thrown.
897 *
898 * @param name property name.
899 * @param defaultValue default value.
900 * @throws NumberFormatException when the value is invalid
901 * @return property value as an <code>int</code>,
902 * or <code>defaultValue</code>.
903 */
904 public int getInt(String name, int defaultValue) {
905 String valueString = getTrimmed(name);
906 if (valueString == null)
907 return defaultValue;
908 String hexString = getHexDigits(valueString);
909 if (hexString != null) {
910 return Integer.parseInt(hexString, 16);
911 }
912 return Integer.parseInt(valueString);
913 }
914
915 /**
916 * Get the value of the <code>name</code> property as a set of comma-delimited
917 * <code>int</code> values.
918 *
919 * If no such property exists, an empty array is returned.
920 *
921 * @param name property name
922 * @return property value interpreted as an array of comma-delimited
923 * <code>int</code> values
924 */
925 public int[] getInts(String name) {
926 String[] strings = getTrimmedStrings(name);
927 int[] ints = new int[strings.length];
928 for (int i = 0; i < strings.length; i++) {
929 ints[i] = Integer.parseInt(strings[i]);
930 }
931 return ints;
932 }
933
934 /**
935 * Set the value of the <code>name</code> property to an <code>int</code>.
936 *
937 * @param name property name.
938 * @param value <code>int</code> value of the property.
939 */
940 public void setInt(String name, int value) {
941 set(name, Integer.toString(value));
942 }
943
944
945 /**
946 * Get the value of the <code>name</code> property as a <code>long</code>.
947 * If no such property exists, the provided default value is returned,
948 * or if the specified value is not a valid <code>long</code>,
949 * then an error is thrown.
950 *
951 * @param name property name.
952 * @param defaultValue default value.
953 * @throws NumberFormatException when the value is invalid
954 * @return property value as a <code>long</code>,
955 * or <code>defaultValue</code>.
956 */
957 public long getLong(String name, long defaultValue) {
958 String valueString = getTrimmed(name);
959 if (valueString == null)
960 return defaultValue;
961 String hexString = getHexDigits(valueString);
962 if (hexString != null) {
963 return Long.parseLong(hexString, 16);
964 }
965 return Long.parseLong(valueString);
966 }
967
968 /**
969 * Get the value of the <code>name</code> property as a <code>long</code> or
970 * human readable format. If no such property exists, the provided default
971 * value is returned, or if the specified value is not a valid
972 * <code>long</code> or human readable format, then an error is thrown. You
973 * can use the following suffix (case insensitive): k(kilo), m(mega), g(giga),
974 * t(tera), p(peta), e(exa)
975 *
976 * @param name property name.
977 * @param defaultValue default value.
978 * @throws NumberFormatException when the value is invalid
979 * @return property value as a <code>long</code>,
980 * or <code>defaultValue</code>.
981 */
982 public long getLongBytes(String name, long defaultValue) {
983 String valueString = getTrimmed(name);
984 if (valueString == null)
985 return defaultValue;
986 return StringUtils.TraditionalBinaryPrefix.string2long(valueString);
987 }
988
989 private String getHexDigits(String value) {
990 boolean negative = false;
991 String str = value;
992 String hexString = null;
993 if (value.startsWith("-")) {
994 negative = true;
995 str = value.substring(1);
996 }
997 if (str.startsWith("0x") || str.startsWith("0X")) {
998 hexString = str.substring(2);
999 if (negative) {
1000 hexString = "-" + hexString;
1001 }
1002 return hexString;
1003 }
1004 return null;
1005 }
1006
1007 /**
1008 * Set the value of the <code>name</code> property to a <code>long</code>.
1009 *
1010 * @param name property name.
1011 * @param value <code>long</code> value of the property.
1012 */
1013 public void setLong(String name, long value) {
1014 set(name, Long.toString(value));
1015 }
1016
1017 /**
1018 * Get the value of the <code>name</code> property as a <code>float</code>.
1019 * If no such property exists, the provided default value is returned,
1020 * or if the specified value is not a valid <code>float</code>,
1021 * then an error is thrown.
1022 *
1023 * @param name property name.
1024 * @param defaultValue default value.
1025 * @throws NumberFormatException when the value is invalid
1026 * @return property value as a <code>float</code>,
1027 * or <code>defaultValue</code>.
1028 */
1029 public float getFloat(String name, float defaultValue) {
1030 String valueString = getTrimmed(name);
1031 if (valueString == null)
1032 return defaultValue;
1033 return Float.parseFloat(valueString);
1034 }
1035
1036 /**
1037 * Set the value of the <code>name</code> property to a <code>float</code>.
1038 *
1039 * @param name property name.
1040 * @param value property value.
1041 */
1042 public void setFloat(String name, float value) {
1043 set(name,Float.toString(value));
1044 }
1045
1046 /**
1047 * Get the value of the <code>name</code> property as a <code>double</code>.
1048 * If no such property exists, the provided default value is returned,
1049 * or if the specified value is not a valid <code>double</code>,
1050 * then an error is thrown.
1051 *
1052 * @param name property name.
1053 * @param defaultValue default value.
1054 * @throws NumberFormatException when the value is invalid
1055 * @return property value as a <code>double</code>,
1056 * or <code>defaultValue</code>.
1057 */
1058 public double getDouble(String name, double defaultValue) {
1059 String valueString = getTrimmed(name);
1060 if (valueString == null)
1061 return defaultValue;
1062 return Double.parseDouble(valueString);
1063 }
1064
1065 /**
1066 * Set the value of the <code>name</code> property to a <code>double</code>.
1067 *
1068 * @param name property name.
1069 * @param value property value.
1070 */
1071 public void setDouble(String name, double value) {
1072 set(name,Double.toString(value));
1073 }
1074
1075 /**
1076 * Get the value of the <code>name</code> property as a <code>boolean</code>.
1077 * If no such property is specified, or if the specified value is not a valid
1078 * <code>boolean</code>, then <code>defaultValue</code> is returned.
1079 *
1080 * @param name property name.
1081 * @param defaultValue default value.
1082 * @return property value as a <code>boolean</code>,
1083 * or <code>defaultValue</code>.
1084 */
1085 public boolean getBoolean(String name, boolean defaultValue) {
1086 String valueString = getTrimmed(name);
1087 if (null == valueString || "".equals(valueString)) {
1088 return defaultValue;
1089 }
1090
1091 valueString = valueString.toLowerCase();
1092
1093 if ("true".equals(valueString))
1094 return true;
1095 else if ("false".equals(valueString))
1096 return false;
1097 else return defaultValue;
1098 }
1099
1100 /**
1101 * Set the value of the <code>name</code> property to a <code>boolean</code>.
1102 *
1103 * @param name property name.
1104 * @param value <code>boolean</code> value of the property.
1105 */
1106 public void setBoolean(String name, boolean value) {
1107 set(name, Boolean.toString(value));
1108 }
1109
1110 /**
1111 * Set the given property, if it is currently unset.
1112 * @param name property name
1113 * @param value new value
1114 */
1115 public void setBooleanIfUnset(String name, boolean value) {
1116 setIfUnset(name, Boolean.toString(value));
1117 }
1118
1119 /**
1120 * Set the value of the <code>name</code> property to the given type. This
1121 * is equivalent to <code>set(<name>, value.toString())</code>.
1122 * @param name property name
1123 * @param value new value
1124 */
1125 public <T extends Enum<T>> void setEnum(String name, T value) {
1126 set(name, value.toString());
1127 }
1128
1129 /**
1130 * Return value matching this enumerated type.
1131 * @param name Property name
1132 * @param defaultValue Value returned if no mapping exists
1133 * @throws IllegalArgumentException If mapping is illegal for the type
1134 * provided
1135 */
1136 public <T extends Enum<T>> T getEnum(String name, T defaultValue) {
1137 final String val = get(name);
1138 return null == val
1139 ? defaultValue
1140 : Enum.valueOf(defaultValue.getDeclaringClass(), val);
1141 }
1142
1143 enum ParsedTimeDuration {
1144 NS {
1145 TimeUnit unit() { return TimeUnit.NANOSECONDS; }
1146 String suffix() { return "ns"; }
1147 },
1148 US {
1149 TimeUnit unit() { return TimeUnit.MICROSECONDS; }
1150 String suffix() { return "us"; }
1151 },
1152 MS {
1153 TimeUnit unit() { return TimeUnit.MILLISECONDS; }
1154 String suffix() { return "ms"; }
1155 },
1156 S {
1157 TimeUnit unit() { return TimeUnit.SECONDS; }
1158 String suffix() { return "s"; }
1159 },
1160 M {
1161 TimeUnit unit() { return TimeUnit.MINUTES; }
1162 String suffix() { return "m"; }
1163 },
1164 H {
1165 TimeUnit unit() { return TimeUnit.HOURS; }
1166 String suffix() { return "h"; }
1167 },
1168 D {
1169 TimeUnit unit() { return TimeUnit.DAYS; }
1170 String suffix() { return "d"; }
1171 };
1172 abstract TimeUnit unit();
1173 abstract String suffix();
1174 static ParsedTimeDuration unitFor(String s) {
1175 for (ParsedTimeDuration ptd : values()) {
1176 // iteration order is in decl order, so SECONDS matched last
1177 if (s.endsWith(ptd.suffix())) {
1178 return ptd;
1179 }
1180 }
1181 return null;
1182 }
1183 static ParsedTimeDuration unitFor(TimeUnit unit) {
1184 for (ParsedTimeDuration ptd : values()) {
1185 if (ptd.unit() == unit) {
1186 return ptd;
1187 }
1188 }
1189 return null;
1190 }
1191 }
1192
1193 /**
1194 * Set the value of <code>name</code> to the given time duration. This
1195 * is equivalent to <code>set(<name>, value + <time suffix>)</code>.
1196 * @param name Property name
1197 * @param value Time duration
1198 * @param unit Unit of time
1199 */
1200 public void setTimeDuration(String name, long value, TimeUnit unit) {
1201 set(name, value + ParsedTimeDuration.unitFor(unit).suffix());
1202 }
1203
1204 /**
1205 * Return time duration in the given time unit. Valid units are encoded in
1206 * properties as suffixes: nanoseconds (ns), microseconds (us), milliseconds
1207 * (ms), seconds (s), minutes (m), hours (h), and days (d).
1208 * @param name Property name
1209 * @param defaultValue Value returned if no mapping exists.
1210 * @param unit Unit to convert the stored property, if it exists.
1211 * @throws NumberFormatException If the property stripped of its unit is not
1212 * a number
1213 */
1214 public long getTimeDuration(String name, long defaultValue, TimeUnit unit) {
1215 String vStr = get(name);
1216 if (null == vStr) {
1217 return defaultValue;
1218 }
1219 vStr = vStr.trim();
1220 ParsedTimeDuration vUnit = ParsedTimeDuration.unitFor(vStr);
1221 if (null == vUnit) {
1222 LOG.warn("No unit for " + name + "(" + vStr + ") assuming " + unit);
1223 vUnit = ParsedTimeDuration.unitFor(unit);
1224 } else {
1225 vStr = vStr.substring(0, vStr.lastIndexOf(vUnit.suffix()));
1226 }
1227 return unit.convert(Long.parseLong(vStr), vUnit.unit());
1228 }
1229
1230 /**
1231 * Get the value of the <code>name</code> property as a <code>Pattern</code>.
1232 * If no such property is specified, or if the specified value is not a valid
1233 * <code>Pattern</code>, then <code>DefaultValue</code> is returned.
1234 *
1235 * @param name property name
1236 * @param defaultValue default value
1237 * @return property value as a compiled Pattern, or defaultValue
1238 */
1239 public Pattern getPattern(String name, Pattern defaultValue) {
1240 String valString = get(name);
1241 if (null == valString || "".equals(valString)) {
1242 return defaultValue;
1243 }
1244 try {
1245 return Pattern.compile(valString);
1246 } catch (PatternSyntaxException pse) {
1247 LOG.warn("Regular expression '" + valString + "' for property '" +
1248 name + "' not valid. Using default", pse);
1249 return defaultValue;
1250 }
1251 }
1252
1253 /**
1254 * Set the given property to <code>Pattern</code>.
1255 * If the pattern is passed as null, sets the empty pattern which results in
1256 * further calls to getPattern(...) returning the default value.
1257 *
1258 * @param name property name
1259 * @param pattern new value
1260 */
1261 public void setPattern(String name, Pattern pattern) {
1262 if (null == pattern) {
1263 set(name, null);
1264 } else {
1265 set(name, pattern.pattern());
1266 }
1267 }
1268
1269 /**
1270 * Gets information about why a property was set. Typically this is the
1271 * path to the resource objects (file, URL, etc.) the property came from, but
1272 * it can also indicate that it was set programatically, or because of the
1273 * command line.
1274 *
1275 * @param name - The property name to get the source of.
1276 * @return null - If the property or its source wasn't found. Otherwise,
1277 * returns a list of the sources of the resource. The older sources are
1278 * the first ones in the list. So for example if a configuration is set from
1279 * the command line, and then written out to a file that is read back in the
1280 * first entry would indicate that it was set from the command line, while
1281 * the second one would indicate the file that the new configuration was read
1282 * in from.
1283 */
1284 @InterfaceStability.Unstable
1285 public synchronized String[] getPropertySources(String name) {
1286 if (properties == null) {
1287 // If properties is null, it means a resource was newly added
1288 // but the props were cleared so as to load it upon future
1289 // requests. So lets force a load by asking a properties list.
1290 getProps();
1291 }
1292 // Return a null right away if our properties still
1293 // haven't loaded or the resource mapping isn't defined
1294 if (properties == null || updatingResource == null) {
1295 return null;
1296 } else {
1297 String[] source = updatingResource.get(name);
1298 if(source == null) {
1299 return null;
1300 } else {
1301 return Arrays.copyOf(source, source.length);
1302 }
1303 }
1304 }
1305
1306 /**
1307 * A class that represents a set of positive integer ranges. It parses
1308 * strings of the form: "2-3,5,7-" where ranges are separated by comma and
1309 * the lower/upper bounds are separated by dash. Either the lower or upper
1310 * bound may be omitted meaning all values up to or over. So the string
1311 * above means 2, 3, 5, and 7, 8, 9, ...
1312 */
1313 public static class IntegerRanges implements Iterable<Integer>{
1314 private static class Range {
1315 int start;
1316 int end;
1317 }
1318
1319 private static class RangeNumberIterator implements Iterator<Integer> {
1320 Iterator<Range> internal;
1321 int at;
1322 int end;
1323
1324 public RangeNumberIterator(List<Range> ranges) {
1325 if (ranges != null) {
1326 internal = ranges.iterator();
1327 }
1328 at = -1;
1329 end = -2;
1330 }
1331
1332 @Override
1333 public boolean hasNext() {
1334 if (at <= end) {
1335 return true;
1336 } else if (internal != null){
1337 return internal.hasNext();
1338 }
1339 return false;
1340 }
1341
1342 @Override
1343 public Integer next() {
1344 if (at <= end) {
1345 at++;
1346 return at - 1;
1347 } else if (internal != null){
1348 Range found = internal.next();
1349 if (found != null) {
1350 at = found.start;
1351 end = found.end;
1352 at++;
1353 return at - 1;
1354 }
1355 }
1356 return null;
1357 }
1358
1359 @Override
1360 public void remove() {
1361 throw new UnsupportedOperationException();
1362 }
1363 };
1364
1365 List<Range> ranges = new ArrayList<Range>();
1366
1367 public IntegerRanges() {
1368 }
1369
1370 public IntegerRanges(String newValue) {
1371 StringTokenizer itr = new StringTokenizer(newValue, ",");
1372 while (itr.hasMoreTokens()) {
1373 String rng = itr.nextToken().trim();
1374 String[] parts = rng.split("-", 3);
1375 if (parts.length < 1 || parts.length > 2) {
1376 throw new IllegalArgumentException("integer range badly formed: " +
1377 rng);
1378 }
1379 Range r = new Range();
1380 r.start = convertToInt(parts[0], 0);
1381 if (parts.length == 2) {
1382 r.end = convertToInt(parts[1], Integer.MAX_VALUE);
1383 } else {
1384 r.end = r.start;
1385 }
1386 if (r.start > r.end) {
1387 throw new IllegalArgumentException("IntegerRange from " + r.start +
1388 " to " + r.end + " is invalid");
1389 }
1390 ranges.add(r);
1391 }
1392 }
1393
1394 /**
1395 * Convert a string to an int treating empty strings as the default value.
1396 * @param value the string value
1397 * @param defaultValue the value for if the string is empty
1398 * @return the desired integer
1399 */
1400 private static int convertToInt(String value, int defaultValue) {
1401 String trim = value.trim();
1402 if (trim.length() == 0) {
1403 return defaultValue;
1404 }
1405 return Integer.parseInt(trim);
1406 }
1407
1408 /**
1409 * Is the given value in the set of ranges
1410 * @param value the value to check
1411 * @return is the value in the ranges?
1412 */
1413 public boolean isIncluded(int value) {
1414 for(Range r: ranges) {
1415 if (r.start <= value && value <= r.end) {
1416 return true;
1417 }
1418 }
1419 return false;
1420 }
1421
1422 /**
1423 * @return true if there are no values in this range, else false.
1424 */
1425 public boolean isEmpty() {
1426 return ranges == null || ranges.isEmpty();
1427 }
1428
1429 @Override
1430 public String toString() {
1431 StringBuilder result = new StringBuilder();
1432 boolean first = true;
1433 for(Range r: ranges) {
1434 if (first) {
1435 first = false;
1436 } else {
1437 result.append(',');
1438 }
1439 result.append(r.start);
1440 result.append('-');
1441 result.append(r.end);
1442 }
1443 return result.toString();
1444 }
1445
1446 @Override
1447 public Iterator<Integer> iterator() {
1448 return new RangeNumberIterator(ranges);
1449 }
1450
1451 }
1452
1453 /**
1454 * Parse the given attribute as a set of integer ranges
1455 * @param name the attribute name
1456 * @param defaultValue the default value if it is not set
1457 * @return a new set of ranges from the configured value
1458 */
1459 public IntegerRanges getRange(String name, String defaultValue) {
1460 return new IntegerRanges(get(name, defaultValue));
1461 }
1462
1463 /**
1464 * Get the comma delimited values of the <code>name</code> property as
1465 * a collection of <code>String</code>s.
1466 * If no such property is specified then empty collection is returned.
1467 * <p>
1468 * This is an optimized version of {@link #getStrings(String)}
1469 *
1470 * @param name property name.
1471 * @return property value as a collection of <code>String</code>s.
1472 */
1473 public Collection<String> getStringCollection(String name) {
1474 String valueString = get(name);
1475 return StringUtils.getStringCollection(valueString);
1476 }
1477
1478 /**
1479 * Get the comma delimited values of the <code>name</code> property as
1480 * an array of <code>String</code>s.
1481 * If no such property is specified then <code>null</code> is returned.
1482 *
1483 * @param name property name.
1484 * @return property value as an array of <code>String</code>s,
1485 * or <code>null</code>.
1486 */
1487 public String[] getStrings(String name) {
1488 String valueString = get(name);
1489 return StringUtils.getStrings(valueString);
1490 }
1491
1492 /**
1493 * Get the comma delimited values of the <code>name</code> property as
1494 * an array of <code>String</code>s.
1495 * If no such property is specified then default value is returned.
1496 *
1497 * @param name property name.
1498 * @param defaultValue The default value
1499 * @return property value as an array of <code>String</code>s,
1500 * or default value.
1501 */
1502 public String[] getStrings(String name, String... defaultValue) {
1503 String valueString = get(name);
1504 if (valueString == null) {
1505 return defaultValue;
1506 } else {
1507 return StringUtils.getStrings(valueString);
1508 }
1509 }
1510
1511 /**
1512 * Get the comma delimited values of the <code>name</code> property as
1513 * a collection of <code>String</code>s, trimmed of the leading and trailing whitespace.
1514 * If no such property is specified then empty <code>Collection</code> is returned.
1515 *
1516 * @param name property name.
1517 * @return property value as a collection of <code>String</code>s, or empty <code>Collection</code>
1518 */
1519 public Collection<String> getTrimmedStringCollection(String name) {
1520 String valueString = get(name);
1521 if (null == valueString) {
1522 Collection<String> empty = new ArrayList<String>();
1523 return empty;
1524 }
1525 return StringUtils.getTrimmedStringCollection(valueString);
1526 }
1527
1528 /**
1529 * Get the comma delimited values of the <code>name</code> property as
1530 * an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
1531 * If no such property is specified then an empty array is returned.
1532 *
1533 * @param name property name.
1534 * @return property value as an array of trimmed <code>String</code>s,
1535 * or empty array.
1536 */
1537 public String[] getTrimmedStrings(String name) {
1538 String valueString = get(name);
1539 return StringUtils.getTrimmedStrings(valueString);
1540 }
1541
1542 /**
1543 * Get the comma delimited values of the <code>name</code> property as
1544 * an array of <code>String</code>s, trimmed of the leading and trailing whitespace.
1545 * If no such property is specified then default value is returned.
1546 *
1547 * @param name property name.
1548 * @param defaultValue The default value
1549 * @return property value as an array of trimmed <code>String</code>s,
1550 * or default value.
1551 */
1552 public String[] getTrimmedStrings(String name, String... defaultValue) {
1553 String valueString = get(name);
1554 if (null == valueString) {
1555 return defaultValue;
1556 } else {
1557 return StringUtils.getTrimmedStrings(valueString);
1558 }
1559 }
1560
1561 /**
1562 * Set the array of string values for the <code>name</code> property as
1563 * as comma delimited values.
1564 *
1565 * @param name property name.
1566 * @param values The values
1567 */
1568 public void setStrings(String name, String... values) {
1569 set(name, StringUtils.arrayToString(values));
1570 }
1571
1572 /**
1573 * Get the socket address for <code>name</code> property as a
1574 * <code>InetSocketAddress</code>.
1575 * @param name property name.
1576 * @param defaultAddress the default value
1577 * @param defaultPort the default port
1578 * @return InetSocketAddress
1579 */
1580 public InetSocketAddress getSocketAddr(
1581 String name, String defaultAddress, int defaultPort) {
1582 final String address = get(name, defaultAddress);
1583 return NetUtils.createSocketAddr(address, defaultPort, name);
1584 }
1585
1586 /**
1587 * Set the socket address for the <code>name</code> property as
1588 * a <code>host:port</code>.
1589 */
1590 public void setSocketAddr(String name, InetSocketAddress addr) {
1591 set(name, NetUtils.getHostPortString(addr));
1592 }
1593
1594 /**
1595 * Set the socket address a client can use to connect for the
1596 * <code>name</code> property as a <code>host:port</code>. The wildcard
1597 * address is replaced with the local host's address.
1598 * @param name property name.
1599 * @param addr InetSocketAddress of a listener to store in the given property
1600 * @return InetSocketAddress for clients to connect
1601 */
1602 public InetSocketAddress updateConnectAddr(String name,
1603 InetSocketAddress addr) {
1604 final InetSocketAddress connectAddr = NetUtils.getConnectAddress(addr);
1605 setSocketAddr(name, connectAddr);
1606 return connectAddr;
1607 }
1608
1609 /**
1610 * Load a class by name.
1611 *
1612 * @param name the class name.
1613 * @return the class object.
1614 * @throws ClassNotFoundException if the class is not found.
1615 */
1616 public Class<?> getClassByName(String name) throws ClassNotFoundException {
1617 Class<?> ret = getClassByNameOrNull(name);
1618 if (ret == null) {
1619 throw new ClassNotFoundException("Class " + name + " not found");
1620 }
1621 return ret;
1622 }
1623
1624 /**
1625 * Load a class by name, returning null rather than throwing an exception
1626 * if it couldn't be loaded. This is to avoid the overhead of creating
1627 * an exception.
1628 *
1629 * @param name the class name
1630 * @return the class object, or null if it could not be found.
1631 */
1632 public Class<?> getClassByNameOrNull(String name) {
1633 Map<String, WeakReference<Class<?>>> map;
1634
1635 synchronized (CACHE_CLASSES) {
1636 map = CACHE_CLASSES.get(classLoader);
1637 if (map == null) {
1638 map = Collections.synchronizedMap(
1639 new WeakHashMap<String, WeakReference<Class<?>>>());
1640 CACHE_CLASSES.put(classLoader, map);
1641 }
1642 }
1643
1644 Class<?> clazz = null;
1645 WeakReference<Class<?>> ref = map.get(name);
1646 if (ref != null) {
1647 clazz = ref.get();
1648 }
1649
1650 if (clazz == null) {
1651 try {
1652 clazz = Class.forName(name, true, classLoader);
1653 } catch (ClassNotFoundException e) {
1654 // Leave a marker that the class isn't found
1655 map.put(name, new WeakReference<Class<?>>(NEGATIVE_CACHE_SENTINEL));
1656 return null;
1657 }
1658 // two putters can race here, but they'll put the same class
1659 map.put(name, new WeakReference<Class<?>>(clazz));
1660 return clazz;
1661 } else if (clazz == NEGATIVE_CACHE_SENTINEL) {
1662 return null; // not found
1663 } else {
1664 // cache hit
1665 return clazz;
1666 }
1667 }
1668
1669 /**
1670 * Get the value of the <code>name</code> property
1671 * as an array of <code>Class</code>.
1672 * The value of the property specifies a list of comma separated class names.
1673 * If no such property is specified, then <code>defaultValue</code> is
1674 * returned.
1675 *
1676 * @param name the property name.
1677 * @param defaultValue default value.
1678 * @return property value as a <code>Class[]</code>,
1679 * or <code>defaultValue</code>.
1680 */
1681 public Class<?>[] getClasses(String name, Class<?> ... defaultValue) {
1682 String[] classnames = getTrimmedStrings(name);
1683 if (classnames == null)
1684 return defaultValue;
1685 try {
1686 Class<?>[] classes = new Class<?>[classnames.length];
1687 for(int i = 0; i < classnames.length; i++) {
1688 classes[i] = getClassByName(classnames[i]);
1689 }
1690 return classes;
1691 } catch (ClassNotFoundException e) {
1692 throw new RuntimeException(e);
1693 }
1694 }
1695
1696 /**
1697 * Get the value of the <code>name</code> property as a <code>Class</code>.
1698 * If no such property is specified, then <code>defaultValue</code> is
1699 * returned.
1700 *
1701 * @param name the class name.
1702 * @param defaultValue default value.
1703 * @return property value as a <code>Class</code>,
1704 * or <code>defaultValue</code>.
1705 */
1706 public Class<?> getClass(String name, Class<?> defaultValue) {
1707 String valueString = getTrimmed(name);
1708 if (valueString == null)
1709 return defaultValue;
1710 try {
1711 return getClassByName(valueString);
1712 } catch (ClassNotFoundException e) {
1713 throw new RuntimeException(e);
1714 }
1715 }
1716
1717 /**
1718 * Get the value of the <code>name</code> property as a <code>Class</code>
1719 * implementing the interface specified by <code>xface</code>.
1720 *
1721 * If no such property is specified, then <code>defaultValue</code> is
1722 * returned.
1723 *
1724 * An exception is thrown if the returned class does not implement the named
1725 * interface.
1726 *
1727 * @param name the class name.
1728 * @param defaultValue default value.
1729 * @param xface the interface implemented by the named class.
1730 * @return property value as a <code>Class</code>,
1731 * or <code>defaultValue</code>.
1732 */
1733 public <U> Class<? extends U> getClass(String name,
1734 Class<? extends U> defaultValue,
1735 Class<U> xface) {
1736 try {
1737 Class<?> theClass = getClass(name, defaultValue);
1738 if (theClass != null && !xface.isAssignableFrom(theClass))
1739 throw new RuntimeException(theClass+" not "+xface.getName());
1740 else if (theClass != null)
1741 return theClass.asSubclass(xface);
1742 else
1743 return null;
1744 } catch (Exception e) {
1745 throw new RuntimeException(e);
1746 }
1747 }
1748
1749 /**
1750 * Get the value of the <code>name</code> property as a <code>List</code>
1751 * of objects implementing the interface specified by <code>xface</code>.
1752 *
1753 * An exception is thrown if any of the classes does not exist, or if it does
1754 * not implement the named interface.
1755 *
1756 * @param name the property name.
1757 * @param xface the interface implemented by the classes named by
1758 * <code>name</code>.
1759 * @return a <code>List</code> of objects implementing <code>xface</code>.
1760 */
1761 @SuppressWarnings("unchecked")
1762 public <U> List<U> getInstances(String name, Class<U> xface) {
1763 List<U> ret = new ArrayList<U>();
1764 Class<?>[] classes = getClasses(name);
1765 for (Class<?> cl: classes) {
1766 if (!xface.isAssignableFrom(cl)) {
1767 throw new RuntimeException(cl + " does not implement " + xface);
1768 }
1769 ret.add((U)ReflectionUtils.newInstance(cl, this));
1770 }
1771 return ret;
1772 }
1773
1774 /**
1775 * Set the value of the <code>name</code> property to the name of a
1776 * <code>theClass</code> implementing the given interface <code>xface</code>.
1777 *
1778 * An exception is thrown if <code>theClass</code> does not implement the
1779 * interface <code>xface</code>.
1780 *
1781 * @param name property name.
1782 * @param theClass property value.
1783 * @param xface the interface implemented by the named class.
1784 */
1785 public void setClass(String name, Class<?> theClass, Class<?> xface) {
1786 if (!xface.isAssignableFrom(theClass))
1787 throw new RuntimeException(theClass+" not "+xface.getName());
1788 set(name, theClass.getName());
1789 }
1790
1791 /**
1792 * Get a local file under a directory named by <i>dirsProp</i> with
1793 * the given <i>path</i>. If <i>dirsProp</i> contains multiple directories,
1794 * then one is chosen based on <i>path</i>'s hash code. If the selected
1795 * directory does not exist, an attempt is made to create it.
1796 *
1797 * @param dirsProp directory in which to locate the file.
1798 * @param path file-path.
1799 * @return local file under the directory with the given path.
1800 */
1801 public Path getLocalPath(String dirsProp, String path)
1802 throws IOException {
1803 String[] dirs = getTrimmedStrings(dirsProp);
1804 int hashCode = path.hashCode();
1805 FileSystem fs = FileSystem.getLocal(this);
1806 for (int i = 0; i < dirs.length; i++) { // try each local dir
1807 int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length;
1808 Path file = new Path(dirs[index], path);
1809 Path dir = file.getParent();
1810 if (fs.mkdirs(dir) || fs.exists(dir)) {
1811 return file;
1812 }
1813 }
1814 LOG.warn("Could not make " + path +
1815 " in local directories from " + dirsProp);
1816 for(int i=0; i < dirs.length; i++) {
1817 int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length;
1818 LOG.warn(dirsProp + "[" + index + "]=" + dirs[index]);
1819 }
1820 throw new IOException("No valid local directories in property: "+dirsProp);
1821 }
1822
1823 /**
1824 * Get a local file name under a directory named in <i>dirsProp</i> with
1825 * the given <i>path</i>. If <i>dirsProp</i> contains multiple directories,
1826 * then one is chosen based on <i>path</i>'s hash code. If the selected
1827 * directory does not exist, an attempt is made to create it.
1828 *
1829 * @param dirsProp directory in which to locate the file.
1830 * @param path file-path.
1831 * @return local file under the directory with the given path.
1832 */
1833 public File getFile(String dirsProp, String path)
1834 throws IOException {
1835 String[] dirs = getTrimmedStrings(dirsProp);
1836 int hashCode = path.hashCode();
1837 for (int i = 0; i < dirs.length; i++) { // try each local dir
1838 int index = (hashCode+i & Integer.MAX_VALUE) % dirs.length;
1839 File file = new File(dirs[index], path);
1840 File dir = file.getParentFile();
1841 if (dir.exists() || dir.mkdirs()) {
1842 return file;
1843 }
1844 }
1845 throw new IOException("No valid local directories in property: "+dirsProp);
1846 }
1847
1848 /**
1849 * Get the {@link URL} for the named resource.
1850 *
1851 * @param name resource name.
1852 * @return the url for the named resource.
1853 */
1854 public URL getResource(String name) {
1855 return classLoader.getResource(name);
1856 }
1857
1858 /**
1859 * Get an input stream attached to the configuration resource with the
1860 * given <code>name</code>.
1861 *
1862 * @param name configuration resource name.
1863 * @return an input stream attached to the resource.
1864 */
1865 public InputStream getConfResourceAsInputStream(String name) {
1866 try {
1867 URL url= getResource(name);
1868
1869 if (url == null) {
1870 LOG.info(name + " not found");
1871 return null;
1872 } else {
1873 LOG.info("found resource " + name + " at " + url);
1874 }
1875
1876 return url.openStream();
1877 } catch (Exception e) {
1878 return null;
1879 }
1880 }
1881
1882 /**
1883 * Get a {@link Reader} attached to the configuration resource with the
1884 * given <code>name</code>.
1885 *
1886 * @param name configuration resource name.
1887 * @return a reader attached to the resource.
1888 */
1889 public Reader getConfResourceAsReader(String name) {
1890 try {
1891 URL url= getResource(name);
1892
1893 if (url == null) {
1894 LOG.info(name + " not found");
1895 return null;
1896 } else {
1897 LOG.info("found resource " + name + " at " + url);
1898 }
1899
1900 return new InputStreamReader(url.openStream());
1901 } catch (Exception e) {
1902 return null;
1903 }
1904 }
1905
1906 protected synchronized Properties getProps() {
1907 if (properties == null) {
1908 properties = new Properties();
1909 HashMap<String, String[]> backup =
1910 new HashMap<String, String[]>(updatingResource);
1911 loadResources(properties, resources, quietmode);
1912 if (overlay!= null) {
1913 properties.putAll(overlay);
1914 for (Map.Entry<Object,Object> item: overlay.entrySet()) {
1915 String key = (String)item.getKey();
1916 updatingResource.put(key, backup.get(key));
1917 }
1918 }
1919 }
1920 return properties;
1921 }
1922
1923 /**
1924 * Return the number of keys in the configuration.
1925 *
1926 * @return number of keys in the configuration.
1927 */
1928 public int size() {
1929 return getProps().size();
1930 }
1931
1932 /**
1933 * Clears all keys from the configuration.
1934 */
1935 public void clear() {
1936 getProps().clear();
1937 getOverlay().clear();
1938 }
1939
1940 /**
1941 * Get an {@link Iterator} to go through the list of <code>String</code>
1942 * key-value pairs in the configuration.
1943 *
1944 * @return an iterator over the entries.
1945 */
1946 @Override
1947 public Iterator<Map.Entry<String, String>> iterator() {
1948 // Get a copy of just the string to string pairs. After the old object
1949 // methods that allow non-strings to be put into configurations are removed,
1950 // we could replace properties with a Map<String,String> and get rid of this
1951 // code.
1952 Map<String,String> result = new HashMap<String,String>();
1953 for(Map.Entry<Object,Object> item: getProps().entrySet()) {
1954 if (item.getKey() instanceof String &&
1955 item.getValue() instanceof String) {
1956 result.put((String) item.getKey(), (String) item.getValue());
1957 }
1958 }
1959 return result.entrySet().iterator();
1960 }
1961
1962 private Document parse(DocumentBuilder builder, URL url)
1963 throws IOException, SAXException {
1964 if (!quietmode) {
1965 LOG.info("parsing URL " + url);
1966 }
1967 if (url == null) {
1968 return null;
1969 }
1970 return parse(builder, url.openStream(), url.toString());
1971 }
1972
1973 private Document parse(DocumentBuilder builder, InputStream is,
1974 String systemId) throws IOException, SAXException {
1975 if (!quietmode) {
1976 LOG.info("parsing input stream " + is);
1977 }
1978 if (is == null) {
1979 return null;
1980 }
1981 try {
1982 return (systemId == null) ? builder.parse(is) : builder.parse(is,
1983 systemId);
1984 } finally {
1985 is.close();
1986 }
1987 }
1988
1989 private void loadResources(Properties properties,
1990 ArrayList<Resource> resources,
1991 boolean quiet) {
1992 if(loadDefaults) {
1993 for (String resource : defaultResources) {
1994 loadResource(properties, new Resource(resource), quiet);
1995 }
1996
1997 //support the hadoop-site.xml as a deprecated case
1998 if(getResource("hadoop-site.xml")!=null) {
1999 loadResource(properties, new Resource("hadoop-site.xml"), quiet);
2000 }
2001 }
2002
2003 for (int i = 0; i < resources.size(); i++) {
2004 Resource ret = loadResource(properties, resources.get(i), quiet);
2005 if (ret != null) {
2006 resources.set(i, ret);
2007 }
2008 }
2009 }
2010
2011 private Resource loadResource(Properties properties, Resource wrapper, boolean quiet) {
2012 String name = UNKNOWN_RESOURCE;
2013 try {
2014 Object resource = wrapper.getResource();
2015 name = wrapper.getName();
2016
2017 DocumentBuilderFactory docBuilderFactory
2018 = DocumentBuilderFactory.newInstance();
2019 //ignore all comments inside the xml file
2020 docBuilderFactory.setIgnoringComments(true);
2021
2022 //allow includes in the xml file
2023 docBuilderFactory.setNamespaceAware(true);
2024 try {
2025 docBuilderFactory.setXIncludeAware(true);
2026 } catch (UnsupportedOperationException e) {
2027 LOG.error("Failed to set setXIncludeAware(true) for parser "
2028 + docBuilderFactory
2029 + ":" + e,
2030 e);
2031 }
2032 DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
2033 Document doc = null;
2034 Element root = null;
2035 boolean returnCachedProperties = false;
2036
2037 if (resource instanceof URL) { // an URL resource
2038 doc = parse(builder, (URL)resource);
2039 } else if (resource instanceof String) { // a CLASSPATH resource
2040 URL url = getResource((String)resource);
2041 doc = parse(builder, url);
2042 } else if (resource instanceof Path) { // a file resource
2043 // Can't use FileSystem API or we get an infinite loop
2044 // since FileSystem uses Configuration API. Use java.io.File instead.
2045 File file = new File(((Path)resource).toUri().getPath())
2046 .getAbsoluteFile();
2047 if (file.exists()) {
2048 if (!quiet) {
2049 LOG.info("parsing File " + file);
2050 }
2051 doc = parse(builder, new BufferedInputStream(
2052 new FileInputStream(file)), ((Path)resource).toString());
2053 }
2054 } else if (resource instanceof InputStream) {
2055 doc = parse(builder, (InputStream) resource, null);
2056 returnCachedProperties = true;
2057 } else if (resource instanceof Properties) {
2058 overlay(properties, (Properties)resource);
2059 } else if (resource instanceof Element) {
2060 root = (Element)resource;
2061 }
2062
2063 if (doc == null && root == null) {
2064 if (quiet)
2065 return null;
2066 throw new RuntimeException(resource + " not found");
2067 }
2068
2069 if (root == null) {
2070 root = doc.getDocumentElement();
2071 }
2072 Properties toAddTo = properties;
2073 if(returnCachedProperties) {
2074 toAddTo = new Properties();
2075 }
2076 if (!"configuration".equals(root.getTagName()))
2077 LOG.fatal("bad conf file: top-level element not <configuration>");
2078 NodeList props = root.getChildNodes();
2079 for (int i = 0; i < props.getLength(); i++) {
2080 Node propNode = props.item(i);
2081 if (!(propNode instanceof Element))
2082 continue;
2083 Element prop = (Element)propNode;
2084 if ("configuration".equals(prop.getTagName())) {
2085 loadResource(toAddTo, new Resource(prop, name), quiet);
2086 continue;
2087 }
2088 if (!"property".equals(prop.getTagName()))
2089 LOG.warn("bad conf file: element not <property>");
2090 NodeList fields = prop.getChildNodes();
2091 String attr = null;
2092 String value = null;
2093 boolean finalParameter = false;
2094 LinkedList<String> source = new LinkedList<String>();
2095 for (int j = 0; j < fields.getLength(); j++) {
2096 Node fieldNode = fields.item(j);
2097 if (!(fieldNode instanceof Element))
2098 continue;
2099 Element field = (Element)fieldNode;
2100 if ("name".equals(field.getTagName()) && field.hasChildNodes())
2101 attr = StringInterner.weakIntern(
2102 ((Text)field.getFirstChild()).getData().trim());
2103 if ("value".equals(field.getTagName()) && field.hasChildNodes())
2104 value = StringInterner.weakIntern(
2105 ((Text)field.getFirstChild()).getData());
2106 if ("final".equals(field.getTagName()) && field.hasChildNodes())
2107 finalParameter = "true".equals(((Text)field.getFirstChild()).getData());
2108 if ("source".equals(field.getTagName()) && field.hasChildNodes())
2109 source.add(StringInterner.weakIntern(
2110 ((Text)field.getFirstChild()).getData()));
2111 }
2112 source.add(name);
2113
2114 // Ignore this parameter if it has already been marked as 'final'
2115 if (attr != null) {
2116 if (deprecatedKeyMap.containsKey(attr)) {
2117 DeprecatedKeyInfo keyInfo = deprecatedKeyMap.get(attr);
2118 keyInfo.accessed = false;
2119 for (String key:keyInfo.newKeys) {
2120 // update new keys with deprecated key's value
2121 loadProperty(toAddTo, name, key, value, finalParameter,
2122 source.toArray(new String[source.size()]));
2123 }
2124 }
2125 else {
2126 loadProperty(toAddTo, name, attr, value, finalParameter,
2127 source.toArray(new String[source.size()]));
2128 }
2129 }
2130 }
2131
2132 if (returnCachedProperties) {
2133 overlay(properties, toAddTo);
2134 return new Resource(toAddTo, name);
2135 }
2136 return null;
2137 } catch (IOException e) {
2138 LOG.fatal("error parsing conf " + name, e);
2139 throw new RuntimeException(e);
2140 } catch (DOMException e) {
2141 LOG.fatal("error parsing conf " + name, e);
2142 throw new RuntimeException(e);
2143 } catch (SAXException e) {
2144 LOG.fatal("error parsing conf " + name, e);
2145 throw new RuntimeException(e);
2146 } catch (ParserConfigurationException e) {
2147 LOG.fatal("error parsing conf " + name , e);
2148 throw new RuntimeException(e);
2149 }
2150 }
2151
2152 private void overlay(Properties to, Properties from) {
2153 for (Entry<Object, Object> entry: from.entrySet()) {
2154 to.put(entry.getKey(), entry.getValue());
2155 }
2156 }
2157
2158 private void loadProperty(Properties properties, String name, String attr,
2159 String value, boolean finalParameter, String[] source) {
2160 if (value != null) {
2161 if (!finalParameters.contains(attr)) {
2162 properties.setProperty(attr, value);
2163 updatingResource.put(attr, source);
2164 } else {
2165 LOG.warn(name+":an attempt to override final parameter: "+attr
2166 +"; Ignoring.");
2167 }
2168 }
2169 if (finalParameter) {
2170 finalParameters.add(attr);
2171 }
2172 }
2173
2174 /**
2175 * Write out the non-default properties in this configuration to the given
2176 * {@link OutputStream}.
2177 *
2178 * @param out the output stream to write to.
2179 */
2180 public void writeXml(OutputStream out) throws IOException {
2181 writeXml(new OutputStreamWriter(out));
2182 }
2183
2184 /**
2185 * Write out the non-default properties in this configuration to the given
2186 * {@link Writer}.
2187 *
2188 * @param out the writer to write to.
2189 */
2190 public void writeXml(Writer out) throws IOException {
2191 Document doc = asXmlDocument();
2192
2193 try {
2194 DOMSource source = new DOMSource(doc);
2195 StreamResult result = new StreamResult(out);
2196 TransformerFactory transFactory = TransformerFactory.newInstance();
2197 Transformer transformer = transFactory.newTransformer();
2198
2199 // Important to not hold Configuration log while writing result, since
2200 // 'out' may be an HDFS stream which needs to lock this configuration
2201 // from another thread.
2202 transformer.transform(source, result);
2203 } catch (TransformerException te) {
2204 throw new IOException(te);
2205 }
2206 }
2207
2208 /**
2209 * Return the XML DOM corresponding to this Configuration.
2210 */
2211 private synchronized Document asXmlDocument() throws IOException {
2212 Document doc;
2213 try {
2214 doc =
2215 DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument();
2216 } catch (ParserConfigurationException pe) {
2217 throw new IOException(pe);
2218 }
2219 Element conf = doc.createElement("configuration");
2220 doc.appendChild(conf);
2221 conf.appendChild(doc.createTextNode("\n"));
2222 handleDeprecation(); //ensure properties is set and deprecation is handled
2223 for (Enumeration<Object> e = properties.keys(); e.hasMoreElements();) {
2224 String name = (String)e.nextElement();
2225 Object object = properties.get(name);
2226 String value = null;
2227 if (object instanceof String) {
2228 value = (String) object;
2229 }else {
2230 continue;
2231 }
2232 Element propNode = doc.createElement("property");
2233 conf.appendChild(propNode);
2234
2235 Element nameNode = doc.createElement("name");
2236 nameNode.appendChild(doc.createTextNode(name));
2237 propNode.appendChild(nameNode);
2238
2239 Element valueNode = doc.createElement("value");
2240 valueNode.appendChild(doc.createTextNode(value));
2241 propNode.appendChild(valueNode);
2242
2243 if (updatingResource != null) {
2244 String[] sources = updatingResource.get(name);
2245 if(sources != null) {
2246 for(String s : sources) {
2247 Element sourceNode = doc.createElement("source");
2248 sourceNode.appendChild(doc.createTextNode(s));
2249 propNode.appendChild(sourceNode);
2250 }
2251 }
2252 }
2253
2254 conf.appendChild(doc.createTextNode("\n"));
2255 }
2256 return doc;
2257 }
2258
2259 /**
2260 * Writes out all the parameters and their properties (final and resource) to
2261 * the given {@link Writer}
2262 * The format of the output would be
2263 * { "properties" : [ {key1,value1,key1.isFinal,key1.resource}, {key2,value2,
2264 * key2.isFinal,key2.resource}... ] }
2265 * It does not output the parameters of the configuration object which is
2266 * loaded from an input stream.
2267 * @param out the Writer to write to
2268 * @throws IOException
2269 */
2270 public static void dumpConfiguration(Configuration config,
2271 Writer out) throws IOException {
2272 JsonFactory dumpFactory = new JsonFactory();
2273 JsonGenerator dumpGenerator = dumpFactory.createJsonGenerator(out);
2274 dumpGenerator.writeStartObject();
2275 dumpGenerator.writeFieldName("properties");
2276 dumpGenerator.writeStartArray();
2277 dumpGenerator.flush();
2278 synchronized (config) {
2279 for (Map.Entry<Object,Object> item: config.getProps().entrySet()) {
2280 dumpGenerator.writeStartObject();
2281 dumpGenerator.writeStringField("key", (String) item.getKey());
2282 dumpGenerator.writeStringField("value",
2283 config.get((String) item.getKey()));
2284 dumpGenerator.writeBooleanField("isFinal",
2285 config.finalParameters.contains(item.getKey()));
2286 String[] resources = config.updatingResource.get(item.getKey());
2287 String resource = UNKNOWN_RESOURCE;
2288 if(resources != null && resources.length > 0) {
2289 resource = resources[0];
2290 }
2291 dumpGenerator.writeStringField("resource", resource);
2292 dumpGenerator.writeEndObject();
2293 }
2294 }
2295 dumpGenerator.writeEndArray();
2296 dumpGenerator.writeEndObject();
2297 dumpGenerator.flush();
2298 }
2299
2300 /**
2301 * Get the {@link ClassLoader} for this job.
2302 *
2303 * @return the correct class loader.
2304 */
2305 public ClassLoader getClassLoader() {
2306 return classLoader;
2307 }
2308
2309 /**
2310 * Set the class loader that will be used to load the various objects.
2311 *
2312 * @param classLoader the new class loader.
2313 */
2314 public void setClassLoader(ClassLoader classLoader) {
2315 this.classLoader = classLoader;
2316 }
2317
2318 @Override
2319 public String toString() {
2320 StringBuilder sb = new StringBuilder();
2321 sb.append("Configuration: ");
2322 if(loadDefaults) {
2323 toString(defaultResources, sb);
2324 if(resources.size()>0) {
2325 sb.append(", ");
2326 }
2327 }
2328 toString(resources, sb);
2329 return sb.toString();
2330 }
2331
2332 private <T> void toString(List<T> resources, StringBuilder sb) {
2333 ListIterator<T> i = resources.listIterator();
2334 while (i.hasNext()) {
2335 if (i.nextIndex() != 0) {
2336 sb.append(", ");
2337 }
2338 sb.append(i.next());
2339 }
2340 }
2341
2342 /**
2343 * Set the quietness-mode.
2344 *
2345 * In the quiet-mode, error and informational messages might not be logged.
2346 *
2347 * @param quietmode <code>true</code> to set quiet-mode on, <code>false</code>
2348 * to turn it off.
2349 */
2350 public synchronized void setQuietMode(boolean quietmode) {
2351 this.quietmode = quietmode;
2352 }
2353
2354 synchronized boolean getQuietMode() {
2355 return this.quietmode;
2356 }
2357
2358 /** For debugging. List non-default properties to the terminal and exit. */
2359 public static void main(String[] args) throws Exception {
2360 new Configuration().writeXml(System.out);
2361 }
2362
2363 @Override
2364 public void readFields(DataInput in) throws IOException {
2365 clear();
2366 int size = WritableUtils.readVInt(in);
2367 for(int i=0; i < size; ++i) {
2368 String key = org.apache.hadoop.io.Text.readString(in);
2369 String value = org.apache.hadoop.io.Text.readString(in);
2370 set(key, value);
2371 String sources[] = WritableUtils.readCompressedStringArray(in);
2372 updatingResource.put(key, sources);
2373 }
2374 }
2375
2376 //@Override
2377 @Override
2378 public void write(DataOutput out) throws IOException {
2379 Properties props = getProps();
2380 WritableUtils.writeVInt(out, props.size());
2381 for(Map.Entry<Object, Object> item: props.entrySet()) {
2382 org.apache.hadoop.io.Text.writeString(out, (String) item.getKey());
2383 org.apache.hadoop.io.Text.writeString(out, (String) item.getValue());
2384 WritableUtils.writeCompressedStringArray(out,
2385 updatingResource.get(item.getKey()));
2386 }
2387 }
2388
2389 /**
2390 * get keys matching the the regex
2391 * @param regex
2392 * @return Map<String,String> with matching keys
2393 */
2394 public Map<String,String> getValByRegex(String regex) {
2395 Pattern p = Pattern.compile(regex);
2396
2397 Map<String,String> result = new HashMap<String,String>();
2398 Matcher m;
2399
2400 for(Map.Entry<Object,Object> item: getProps().entrySet()) {
2401 if (item.getKey() instanceof String &&
2402 item.getValue() instanceof String) {
2403 m = p.matcher((String)item.getKey());
2404 if(m.find()) { // match
2405 result.put((String) item.getKey(), (String) item.getValue());
2406 }
2407 }
2408 }
2409 return result;
2410 }
2411
2412 //Load deprecated keys in common
2413 private static void addDeprecatedKeys() {
2414 Configuration.addDeprecation("topology.script.file.name",
2415 new String[]{CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY});
2416 Configuration.addDeprecation("topology.script.number.args",
2417 new String[]{CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_NUMBER_ARGS_KEY});
2418 Configuration.addDeprecation("hadoop.configured.node.mapping",
2419 new String[]{CommonConfigurationKeys.NET_TOPOLOGY_CONFIGURED_NODE_MAPPING_KEY});
2420 Configuration.addDeprecation("topology.node.switch.mapping.impl",
2421 new String[]{CommonConfigurationKeys.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY});
2422 Configuration.addDeprecation("dfs.df.interval",
2423 new String[]{CommonConfigurationKeys.FS_DF_INTERVAL_KEY});
2424 Configuration.addDeprecation("hadoop.native.lib",
2425 new String[]{CommonConfigurationKeys.IO_NATIVE_LIB_AVAILABLE_KEY});
2426 Configuration.addDeprecation("fs.default.name",
2427 new String[]{CommonConfigurationKeys.FS_DEFAULT_NAME_KEY});
2428 Configuration.addDeprecation("dfs.umaskmode",
2429 new String[]{CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY});
2430 }
2431
2432 /**
2433 * A unique class which is used as a sentinel value in the caching
2434 * for getClassByName. {@see Configuration#getClassByNameOrNull(String)}
2435 */
2436 private static abstract class NegativeCacheSentinel {}
2437
2438 public static void dumpDeprecatedKeys() {
2439 for (Map.Entry<String, DeprecatedKeyInfo> entry : deprecatedKeyMap.entrySet()) {
2440 String newKeys = "";
2441 for (String newKey : entry.getValue().newKeys) {
2442 newKeys += newKey + "\t";
2443 }
2444 System.out.println(entry.getKey() + "\t" + newKeys);
2445 }
2446 }
2447 }