001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.crypto.key;
020
021 import org.apache.commons.io.IOUtils;
022 import org.apache.hadoop.classification.InterfaceAudience;
023 import org.apache.hadoop.classification.InterfaceAudience.Private;
024 import org.apache.hadoop.conf.Configuration;
025 import org.apache.hadoop.fs.FSDataOutputStream;
026 import org.apache.hadoop.fs.FileStatus;
027 import org.apache.hadoop.fs.FileSystem;
028 import org.apache.hadoop.fs.Path;
029 import org.apache.hadoop.fs.permission.FsPermission;
030 import org.apache.hadoop.security.ProviderUtils;
031 import org.slf4j.Logger;
032 import org.slf4j.LoggerFactory;
033
034 import com.google.common.annotations.VisibleForTesting;
035
036 import javax.crypto.spec.SecretKeySpec;
037
038 import java.io.IOException;
039 import java.io.InputStream;
040 import java.io.ObjectInputStream;
041 import java.io.ObjectOutputStream;
042 import java.io.Serializable;
043 import java.net.URI;
044 import java.net.URL;
045 import java.security.Key;
046 import java.security.KeyStore;
047 import java.security.KeyStoreException;
048 import java.security.NoSuchAlgorithmException;
049 import java.security.UnrecoverableKeyException;
050 import java.security.cert.CertificateException;
051 import java.util.ArrayList;
052 import java.util.Date;
053 import java.util.Enumeration;
054 import java.util.HashMap;
055 import java.util.List;
056 import java.util.Map;
057 import java.util.concurrent.locks.Lock;
058 import java.util.concurrent.locks.ReadWriteLock;
059 import java.util.concurrent.locks.ReentrantReadWriteLock;
060
061 /**
062 * KeyProvider based on Java's KeyStore file format. The file may be stored in
063 * any Hadoop FileSystem using the following name mangling:
064 * jks://[email protected]/my/keys.jks -> hdfs://nn1.example.com/my/keys.jks
065 * jks://file/home/owen/keys.jks -> file:///home/owen/keys.jks
066 * <p/>
067 * If the <code>HADOOP_KEYSTORE_PASSWORD</code> environment variable is set,
068 * its value is used as the password for the keystore.
069 * <p/>
070 * If the <code>HADOOP_KEYSTORE_PASSWORD</code> environment variable is not set,
071 * the password for the keystore is read from file specified in the
072 * {@link #KEYSTORE_PASSWORD_FILE_KEY} configuration property. The password file
073 * is looked up in Hadoop's configuration directory via the classpath.
074 * <p/>
075 * <b>NOTE:</b> Make sure the password in the password file does not have an
076 * ENTER at the end, else it won't be valid for the Java KeyStore.
077 * <p/>
078 * If the environment variable, nor the property are not set, the password used
079 * is 'none'.
080 * <p/>
081 * It is expected for encrypted InputFormats and OutputFormats to copy the keys
082 * from the original provider into the job's Credentials object, which is
083 * accessed via the UserProvider. Therefore, this provider won't be used by
084 * MapReduce tasks.
085 */
086 @InterfaceAudience.Private
087 public class JavaKeyStoreProvider extends KeyProvider {
088 private static final String KEY_METADATA = "KeyMetadata";
089 private static Logger LOG =
090 LoggerFactory.getLogger(JavaKeyStoreProvider.class);
091
092 public static final String SCHEME_NAME = "jceks";
093
094 public static final String KEYSTORE_PASSWORD_FILE_KEY =
095 "hadoop.security.keystore.java-keystore-provider.password-file";
096
097 public static final String KEYSTORE_PASSWORD_ENV_VAR =
098 "HADOOP_KEYSTORE_PASSWORD";
099 public static final char[] KEYSTORE_PASSWORD_DEFAULT = "none".toCharArray();
100
101 private final URI uri;
102 private final Path path;
103 private final FileSystem fs;
104 private final FsPermission permissions;
105 private final KeyStore keyStore;
106 private char[] password;
107 private boolean changed = false;
108 private Lock readLock;
109 private Lock writeLock;
110
111 private final Map<String, Metadata> cache = new HashMap<String, Metadata>();
112
113 @VisibleForTesting
114 JavaKeyStoreProvider(JavaKeyStoreProvider other) {
115 super(new Configuration());
116 uri = other.uri;
117 path = other.path;
118 fs = other.fs;
119 permissions = other.permissions;
120 keyStore = other.keyStore;
121 password = other.password;
122 changed = other.changed;
123 readLock = other.readLock;
124 writeLock = other.writeLock;
125 }
126
127 private JavaKeyStoreProvider(URI uri, Configuration conf) throws IOException {
128 super(conf);
129 this.uri = uri;
130 path = ProviderUtils.unnestUri(uri);
131 fs = path.getFileSystem(conf);
132 // Get the password file from the conf, if not present from the user's
133 // environment var
134 if (System.getenv().containsKey(KEYSTORE_PASSWORD_ENV_VAR)) {
135 password = System.getenv(KEYSTORE_PASSWORD_ENV_VAR).toCharArray();
136 }
137 if (password == null) {
138 String pwFile = conf.get(KEYSTORE_PASSWORD_FILE_KEY);
139 if (pwFile != null) {
140 ClassLoader cl = Thread.currentThread().getContextClassLoader();
141 URL pwdFile = cl.getResource(pwFile);
142 if (pwdFile == null) {
143 // Provided Password file does not exist
144 throw new IOException("Password file does not exists");
145 }
146 if (pwdFile != null) {
147 InputStream is = pwdFile.openStream();
148 try {
149 password = IOUtils.toString(is).trim().toCharArray();
150 } finally {
151 is.close();
152 }
153 }
154 }
155 }
156 if (password == null) {
157 password = KEYSTORE_PASSWORD_DEFAULT;
158 }
159 try {
160 Path oldPath = constructOldPath(path);
161 Path newPath = constructNewPath(path);
162 keyStore = KeyStore.getInstance(SCHEME_NAME);
163 FsPermission perm = null;
164 if (fs.exists(path)) {
165 // flush did not proceed to completion
166 // _NEW should not exist
167 if (fs.exists(newPath)) {
168 throw new IOException(
169 String.format("Keystore not loaded due to some inconsistency "
170 + "('%s' and '%s' should not exist together)!!", path, newPath));
171 }
172 perm = tryLoadFromPath(path, oldPath);
173 } else {
174 perm = tryLoadIncompleteFlush(oldPath, newPath);
175 }
176 // Need to save off permissions in case we need to
177 // rewrite the keystore in flush()
178 permissions = perm;
179 } catch (KeyStoreException e) {
180 throw new IOException("Can't create keystore", e);
181 } catch (NoSuchAlgorithmException e) {
182 throw new IOException("Can't load keystore " + path, e);
183 } catch (CertificateException e) {
184 throw new IOException("Can't load keystore " + path, e);
185 }
186 ReadWriteLock lock = new ReentrantReadWriteLock(true);
187 readLock = lock.readLock();
188 writeLock = lock.writeLock();
189 }
190
191 /**
192 * Try loading from the user specified path, else load from the backup
193 * path in case Exception is not due to bad/wrong password
194 * @param path Actual path to load from
195 * @param backupPath Backup path (_OLD)
196 * @return The permissions of the loaded file
197 * @throws NoSuchAlgorithmException
198 * @throws CertificateException
199 * @throws IOException
200 */
201 private FsPermission tryLoadFromPath(Path path, Path backupPath)
202 throws NoSuchAlgorithmException, CertificateException,
203 IOException {
204 FsPermission perm = null;
205 try {
206 perm = loadFromPath(path, password);
207 // Remove _OLD if exists
208 if (fs.exists(backupPath)) {
209 fs.delete(backupPath, true);
210 }
211 LOG.debug("KeyStore loaded successfully !!");
212 } catch (IOException ioe) {
213 // If file is corrupted for some reason other than
214 // wrong password try the _OLD file if exits
215 if (!isBadorWrongPassword(ioe)) {
216 perm = loadFromPath(backupPath, password);
217 // Rename CURRENT to CORRUPTED
218 renameOrFail(path, new Path(path.toString() + "_CORRUPTED_"
219 + System.currentTimeMillis()));
220 renameOrFail(backupPath, path);
221 LOG.debug(String.format(
222 "KeyStore loaded successfully from '%s' since '%s'"
223 + "was corrupted !!", backupPath, path));
224 } else {
225 throw ioe;
226 }
227 }
228 return perm;
229 }
230
231 /**
232 * The KeyStore might have gone down during a flush, In which case either the
233 * _NEW or _OLD files might exists. This method tries to load the KeyStore
234 * from one of these intermediate files.
235 * @param oldPath the _OLD file created during flush
236 * @param newPath the _NEW file created during flush
237 * @return The permissions of the loaded file
238 * @throws IOException
239 * @throws NoSuchAlgorithmException
240 * @throws CertificateException
241 */
242 private FsPermission tryLoadIncompleteFlush(Path oldPath, Path newPath)
243 throws IOException, NoSuchAlgorithmException, CertificateException {
244 FsPermission perm = null;
245 // Check if _NEW exists (in case flush had finished writing but not
246 // completed the re-naming)
247 if (fs.exists(newPath)) {
248 perm = loadAndReturnPerm(newPath, oldPath);
249 }
250 // try loading from _OLD (An earlier Flushing MIGHT not have completed
251 // writing completely)
252 if ((perm == null) && fs.exists(oldPath)) {
253 perm = loadAndReturnPerm(oldPath, newPath);
254 }
255 // If not loaded yet,
256 // required to create an empty keystore. *sigh*
257 if (perm == null) {
258 keyStore.load(null, password);
259 LOG.debug("KeyStore initialized anew successfully !!");
260 perm = new FsPermission("700");
261 }
262 return perm;
263 }
264
265 private FsPermission loadAndReturnPerm(Path pathToLoad, Path pathToDelete)
266 throws NoSuchAlgorithmException, CertificateException,
267 IOException {
268 FsPermission perm = null;
269 try {
270 perm = loadFromPath(pathToLoad, password);
271 renameOrFail(pathToLoad, path);
272 LOG.debug(String.format("KeyStore loaded successfully from '%s'!!",
273 pathToLoad));
274 if (fs.exists(pathToDelete)) {
275 fs.delete(pathToDelete, true);
276 }
277 } catch (IOException e) {
278 // Check for password issue : don't want to trash file due
279 // to wrong password
280 if (isBadorWrongPassword(e)) {
281 throw e;
282 }
283 }
284 return perm;
285 }
286
287 private boolean isBadorWrongPassword(IOException ioe) {
288 // As per documentation this is supposed to be the way to figure
289 // if password was correct
290 if (ioe.getCause() instanceof UnrecoverableKeyException) {
291 return true;
292 }
293 // Unfortunately that doesn't seem to work..
294 // Workaround :
295 if ((ioe.getCause() == null)
296 && (ioe.getMessage() != null)
297 && ((ioe.getMessage().contains("Keystore was tampered")) || (ioe
298 .getMessage().contains("password was incorrect")))) {
299 return true;
300 }
301 return false;
302 }
303
304 private FsPermission loadFromPath(Path p, char[] password)
305 throws IOException, NoSuchAlgorithmException, CertificateException {
306 FileStatus s = fs.getFileStatus(p);
307 keyStore.load(fs.open(p), password);
308 return s.getPermission();
309 }
310
311 private Path constructNewPath(Path path) {
312 Path newPath = new Path(path.toString() + "_NEW");
313 return newPath;
314 }
315
316 private Path constructOldPath(Path path) {
317 Path oldPath = new Path(path.toString() + "_OLD");
318 return oldPath;
319 }
320
321 @Override
322 public KeyVersion getKeyVersion(String versionName) throws IOException {
323 readLock.lock();
324 try {
325 SecretKeySpec key = null;
326 try {
327 if (!keyStore.containsAlias(versionName)) {
328 return null;
329 }
330 key = (SecretKeySpec) keyStore.getKey(versionName, password);
331 } catch (KeyStoreException e) {
332 throw new IOException("Can't get key " + versionName + " from " +
333 path, e);
334 } catch (NoSuchAlgorithmException e) {
335 throw new IOException("Can't get algorithm for key " + key + " from " +
336 path, e);
337 } catch (UnrecoverableKeyException e) {
338 throw new IOException("Can't recover key " + key + " from " + path, e);
339 }
340 return new KeyVersion(getBaseName(versionName), versionName, key.getEncoded());
341 } finally {
342 readLock.unlock();
343 }
344 }
345
346 @Override
347 public List<String> getKeys() throws IOException {
348 readLock.lock();
349 try {
350 ArrayList<String> list = new ArrayList<String>();
351 String alias = null;
352 try {
353 Enumeration<String> e = keyStore.aliases();
354 while (e.hasMoreElements()) {
355 alias = e.nextElement();
356 // only include the metadata key names in the list of names
357 if (!alias.contains("@")) {
358 list.add(alias);
359 }
360 }
361 } catch (KeyStoreException e) {
362 throw new IOException("Can't get key " + alias + " from " + path, e);
363 }
364 return list;
365 } finally {
366 readLock.unlock();
367 }
368 }
369
370 @Override
371 public List<KeyVersion> getKeyVersions(String name) throws IOException {
372 readLock.lock();
373 try {
374 List<KeyVersion> list = new ArrayList<KeyVersion>();
375 Metadata km = getMetadata(name);
376 if (km != null) {
377 int latestVersion = km.getVersions();
378 KeyVersion v = null;
379 String versionName = null;
380 for (int i = 0; i < latestVersion; i++) {
381 versionName = buildVersionName(name, i);
382 v = getKeyVersion(versionName);
383 if (v != null) {
384 list.add(v);
385 }
386 }
387 }
388 return list;
389 } finally {
390 readLock.unlock();
391 }
392 }
393
394 @Override
395 public Metadata getMetadata(String name) throws IOException {
396 readLock.lock();
397 try {
398 if (cache.containsKey(name)) {
399 return cache.get(name);
400 }
401 try {
402 if (!keyStore.containsAlias(name)) {
403 return null;
404 }
405 Metadata meta = ((KeyMetadata) keyStore.getKey(name, password)).metadata;
406 cache.put(name, meta);
407 return meta;
408 } catch (KeyStoreException e) {
409 throw new IOException("Can't get metadata for " + name +
410 " from keystore " + path, e);
411 } catch (NoSuchAlgorithmException e) {
412 throw new IOException("Can't get algorithm for " + name +
413 " from keystore " + path, e);
414 } catch (UnrecoverableKeyException e) {
415 throw new IOException("Can't recover key for " + name +
416 " from keystore " + path, e);
417 }
418 } finally {
419 readLock.unlock();
420 }
421 }
422
423 @Override
424 public KeyVersion createKey(String name, byte[] material,
425 Options options) throws IOException {
426 writeLock.lock();
427 try {
428 try {
429 if (keyStore.containsAlias(name) || cache.containsKey(name)) {
430 throw new IOException("Key " + name + " already exists in " + this);
431 }
432 } catch (KeyStoreException e) {
433 throw new IOException("Problem looking up key " + name + " in " + this,
434 e);
435 }
436 Metadata meta = new Metadata(options.getCipher(), options.getBitLength(),
437 options.getDescription(), options.getAttributes(), new Date(), 1);
438 if (options.getBitLength() != 8 * material.length) {
439 throw new IOException("Wrong key length. Required " +
440 options.getBitLength() + ", but got " + (8 * material.length));
441 }
442 cache.put(name, meta);
443 String versionName = buildVersionName(name, 0);
444 return innerSetKeyVersion(name, versionName, material, meta.getCipher());
445 } finally {
446 writeLock.unlock();
447 }
448 }
449
450 @Override
451 public void deleteKey(String name) throws IOException {
452 writeLock.lock();
453 try {
454 Metadata meta = getMetadata(name);
455 if (meta == null) {
456 throw new IOException("Key " + name + " does not exist in " + this);
457 }
458 for(int v=0; v < meta.getVersions(); ++v) {
459 String versionName = buildVersionName(name, v);
460 try {
461 if (keyStore.containsAlias(versionName)) {
462 keyStore.deleteEntry(versionName);
463 }
464 } catch (KeyStoreException e) {
465 throw new IOException("Problem removing " + versionName + " from " +
466 this, e);
467 }
468 }
469 try {
470 if (keyStore.containsAlias(name)) {
471 keyStore.deleteEntry(name);
472 }
473 } catch (KeyStoreException e) {
474 throw new IOException("Problem removing " + name + " from " + this, e);
475 }
476 cache.remove(name);
477 changed = true;
478 } finally {
479 writeLock.unlock();
480 }
481 }
482
483 KeyVersion innerSetKeyVersion(String name, String versionName, byte[] material,
484 String cipher) throws IOException {
485 try {
486 keyStore.setKeyEntry(versionName, new SecretKeySpec(material, cipher),
487 password, null);
488 } catch (KeyStoreException e) {
489 throw new IOException("Can't store key " + versionName + " in " + this,
490 e);
491 }
492 changed = true;
493 return new KeyVersion(name, versionName, material);
494 }
495
496 @Override
497 public KeyVersion rollNewVersion(String name,
498 byte[] material) throws IOException {
499 writeLock.lock();
500 try {
501 Metadata meta = getMetadata(name);
502 if (meta == null) {
503 throw new IOException("Key " + name + " not found");
504 }
505 if (meta.getBitLength() != 8 * material.length) {
506 throw new IOException("Wrong key length. Required " +
507 meta.getBitLength() + ", but got " + (8 * material.length));
508 }
509 int nextVersion = meta.addVersion();
510 String versionName = buildVersionName(name, nextVersion);
511 return innerSetKeyVersion(name, versionName, material, meta.getCipher());
512 } finally {
513 writeLock.unlock();
514 }
515 }
516
517 @Override
518 public void flush() throws IOException {
519 Path newPath = constructNewPath(path);
520 Path oldPath = constructOldPath(path);
521 Path resetPath = path;
522 writeLock.lock();
523 try {
524 if (!changed) {
525 return;
526 }
527 // Might exist if a backup has been restored etc.
528 if (fs.exists(newPath)) {
529 renameOrFail(newPath, new Path(newPath.toString()
530 + "_ORPHANED_" + System.currentTimeMillis()));
531 }
532 if (fs.exists(oldPath)) {
533 renameOrFail(oldPath, new Path(oldPath.toString()
534 + "_ORPHANED_" + System.currentTimeMillis()));
535 }
536 // put all of the updates into the keystore
537 for(Map.Entry<String, Metadata> entry: cache.entrySet()) {
538 try {
539 keyStore.setKeyEntry(entry.getKey(), new KeyMetadata(entry.getValue()),
540 password, null);
541 } catch (KeyStoreException e) {
542 throw new IOException("Can't set metadata key " + entry.getKey(),e );
543 }
544 }
545
546 // Save old File first
547 boolean fileExisted = backupToOld(oldPath);
548 if (fileExisted) {
549 resetPath = oldPath;
550 }
551 // write out the keystore
552 // Write to _NEW path first :
553 try {
554 writeToNew(newPath);
555 } catch (IOException ioe) {
556 // rename _OLD back to curent and throw Exception
557 revertFromOld(oldPath, fileExisted);
558 resetPath = path;
559 throw ioe;
560 }
561 // Rename _NEW to CURRENT and delete _OLD
562 cleanupNewAndOld(newPath, oldPath);
563 changed = false;
564 } catch (IOException ioe) {
565 resetKeyStoreState(resetPath);
566 throw ioe;
567 } finally {
568 writeLock.unlock();
569 }
570 }
571
572 private void resetKeyStoreState(Path path) {
573 LOG.debug("Could not flush Keystore.."
574 + "attempting to reset to previous state !!");
575 // 1) flush cache
576 cache.clear();
577 // 2) load keyStore from previous path
578 try {
579 loadFromPath(path, password);
580 LOG.debug("KeyStore resetting to previously flushed state !!");
581 } catch (Exception e) {
582 LOG.debug("Could not reset Keystore to previous state", e);
583 }
584 }
585
586 private void cleanupNewAndOld(Path newPath, Path oldPath) throws IOException {
587 // Rename _NEW to CURRENT
588 renameOrFail(newPath, path);
589 // Delete _OLD
590 if (fs.exists(oldPath)) {
591 fs.delete(oldPath, true);
592 }
593 }
594
595 protected void writeToNew(Path newPath) throws IOException {
596 FSDataOutputStream out =
597 FileSystem.create(fs, newPath, permissions);
598 try {
599 keyStore.store(out, password);
600 } catch (KeyStoreException e) {
601 throw new IOException("Can't store keystore " + this, e);
602 } catch (NoSuchAlgorithmException e) {
603 throw new IOException(
604 "No such algorithm storing keystore " + this, e);
605 } catch (CertificateException e) {
606 throw new IOException(
607 "Certificate exception storing keystore " + this, e);
608 }
609 out.close();
610 }
611
612 protected boolean backupToOld(Path oldPath)
613 throws IOException {
614 boolean fileExisted = false;
615 if (fs.exists(path)) {
616 renameOrFail(path, oldPath);
617 fileExisted = true;
618 }
619 return fileExisted;
620 }
621
622 private void revertFromOld(Path oldPath, boolean fileExisted)
623 throws IOException {
624 if (fileExisted) {
625 renameOrFail(oldPath, path);
626 }
627 }
628
629
630 private void renameOrFail(Path src, Path dest)
631 throws IOException {
632 if (!fs.rename(src, dest)) {
633 throw new IOException("Rename unsuccessful : "
634 + String.format("'%s' to '%s'", src, dest));
635 }
636 }
637
638 @Override
639 public String toString() {
640 return uri.toString();
641 }
642
643 /**
644 * The factory to create JksProviders, which is used by the ServiceLoader.
645 */
646 public static class Factory extends KeyProviderFactory {
647 @Override
648 public KeyProvider createProvider(URI providerName,
649 Configuration conf) throws IOException {
650 if (SCHEME_NAME.equals(providerName.getScheme())) {
651 return new JavaKeyStoreProvider(providerName, conf);
652 }
653 return null;
654 }
655 }
656
657 /**
658 * An adapter between a KeyStore Key and our Metadata. This is used to store
659 * the metadata in a KeyStore even though isn't really a key.
660 */
661 public static class KeyMetadata implements Key, Serializable {
662 private Metadata metadata;
663 private final static long serialVersionUID = 8405872419967874451L;
664
665 private KeyMetadata(Metadata meta) {
666 this.metadata = meta;
667 }
668
669 @Override
670 public String getAlgorithm() {
671 return metadata.getCipher();
672 }
673
674 @Override
675 public String getFormat() {
676 return KEY_METADATA;
677 }
678
679 @Override
680 public byte[] getEncoded() {
681 return new byte[0];
682 }
683
684 private void writeObject(ObjectOutputStream out) throws IOException {
685 byte[] serialized = metadata.serialize();
686 out.writeInt(serialized.length);
687 out.write(serialized);
688 }
689
690 private void readObject(ObjectInputStream in
691 ) throws IOException, ClassNotFoundException {
692 byte[] buf = new byte[in.readInt()];
693 in.readFully(buf);
694 metadata = new Metadata(buf);
695 }
696
697 }
698 }