001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018 package org.apache.hadoop.security;
019
020 import java.io.BufferedReader;
021 import java.io.File;
022 import java.io.FileInputStream;
023 import java.io.IOException;
024 import java.io.InputStreamReader;
025 import java.util.HashMap;
026 import java.util.Map;
027 import java.util.regex.Matcher;
028 import java.util.regex.Pattern;
029
030 import org.apache.commons.logging.Log;
031 import org.apache.commons.logging.LogFactory;
032 import org.apache.hadoop.conf.Configuration;
033 import org.apache.hadoop.util.Time;
034
035 import com.google.common.annotations.VisibleForTesting;
036 import com.google.common.collect.BiMap;
037 import com.google.common.collect.HashBiMap;
038
039 /**
040 * A simple shell-based implementation of {@link IdMappingServiceProvider}
041 * Map id to user name or group name. It does update every 15 minutes. Only a
042 * single instance of this class is expected to be on the server.
043 */
044 public class ShellBasedIdMapping implements IdMappingServiceProvider {
045
046 private static final Log LOG =
047 LogFactory.getLog(ShellBasedIdMapping.class);
048
049 private final static String OS = System.getProperty("os.name");
050
051 /** Shell commands to get users and groups */
052 static final String GET_ALL_USERS_CMD = "getent passwd | cut -d: -f1,3";
053 static final String GET_ALL_GROUPS_CMD = "getent group | cut -d: -f1,3";
054 static final String MAC_GET_ALL_USERS_CMD = "dscl . -list /Users UniqueID";
055 static final String MAC_GET_ALL_GROUPS_CMD = "dscl . -list /Groups PrimaryGroupID";
056
057 private final File staticMappingFile;
058
059 // Used for parsing the static mapping file.
060 private static final Pattern EMPTY_LINE = Pattern.compile("^\\s*$");
061 private static final Pattern COMMENT_LINE = Pattern.compile("^\\s*#.*$");
062 private static final Pattern MAPPING_LINE =
063 Pattern.compile("^(uid|gid)\\s+(\\d+)\\s+(\\d+)\\s*(#.*)?$");
064
065 final private long timeout;
066
067 // Maps for id to name map. Guarded by this object monitor lock
068 private BiMap<Integer, String> uidNameMap = HashBiMap.create();
069 private BiMap<Integer, String> gidNameMap = HashBiMap.create();
070
071 private long lastUpdateTime = 0; // Last time maps were updated
072
073 public ShellBasedIdMapping(Configuration conf,
074 final String defaultStaticIdMappingFile) throws IOException {
075 long updateTime = conf.getLong(
076 IdMappingConstant.USERGROUPID_UPDATE_MILLIS_KEY,
077 IdMappingConstant.USERGROUPID_UPDATE_MILLIS_DEFAULT);
078 // Minimal interval is 1 minute
079 if (updateTime < IdMappingConstant.USERGROUPID_UPDATE_MILLIS_MIN) {
080 LOG.info("User configured user account update time is less"
081 + " than 1 minute. Use 1 minute instead.");
082 timeout = IdMappingConstant.USERGROUPID_UPDATE_MILLIS_MIN;
083 } else {
084 timeout = updateTime;
085 }
086
087 String staticFilePath = conf.get(IdMappingConstant.STATIC_ID_MAPPING_FILE_KEY,
088 defaultStaticIdMappingFile);
089 staticMappingFile = new File(staticFilePath);
090
091 updateMaps();
092 }
093
094 public ShellBasedIdMapping(Configuration conf) throws IOException {
095 this(conf, IdMappingConstant.STATIC_ID_MAPPING_FILE_DEFAULT);
096 }
097
098 @VisibleForTesting
099 public long getTimeout() {
100 return timeout;
101 }
102
103 synchronized private boolean isExpired() {
104 return Time.monotonicNow() - lastUpdateTime > timeout;
105 }
106
107 // If can't update the maps, will keep using the old ones
108 private void checkAndUpdateMaps() {
109 if (isExpired()) {
110 LOG.info("Update cache now");
111 try {
112 updateMaps();
113 } catch (IOException e) {
114 LOG.error("Can't update the maps. Will use the old ones,"
115 + " which can potentially cause problem.", e);
116 }
117 }
118 }
119
120 private static final String DUPLICATE_NAME_ID_DEBUG_INFO =
121 "NFS gateway could have problem starting with duplicate name or id on the host system.\n"
122 + "This is because HDFS (non-kerberos cluster) uses name as the only way to identify a user or group.\n"
123 + "The host system with duplicated user/group name or id might work fine most of the time by itself.\n"
124 + "However when NFS gateway talks to HDFS, HDFS accepts only user and group name.\n"
125 + "Therefore, same name means the same user or same group. To find the duplicated names/ids, one can do:\n"
126 + "<getent passwd | cut -d: -f1,3> and <getent group | cut -d: -f1,3> on Linux systems,\n"
127 + "<dscl . -list /Users UniqueID> and <dscl . -list /Groups PrimaryGroupID> on MacOS.";
128
129 private static void reportDuplicateEntry(final String header,
130 final Integer key, final String value,
131 final Integer ekey, final String evalue) {
132 LOG.warn("\n" + header + String.format(
133 "new entry (%d, %s), existing entry: (%d, %s).\n%s\n%s",
134 key, value, ekey, evalue,
135 "The new entry is to be ignored for the following reason.",
136 DUPLICATE_NAME_ID_DEBUG_INFO));
137 }
138
139 /**
140 * uid and gid are defined as uint32 in linux. Some systems create
141 * (intended or unintended) <nfsnobody, 4294967294> kind of <name,Id>
142 * mapping, where 4294967294 is 2**32-2 as unsigned int32. As an example,
143 * https://bugzilla.redhat.com/show_bug.cgi?id=511876.
144 * Because user or group id are treated as Integer (signed integer or int32)
145 * here, the number 4294967294 is out of range. The solution is to convert
146 * uint32 to int32, so to map the out-of-range ID to the negative side of
147 * Integer, e.g. 4294967294 maps to -2 and 4294967295 maps to -1.
148 */
149 private static Integer parseId(final String idStr) {
150 Long longVal = Long.parseLong(idStr);
151 int intVal = longVal.intValue();
152 return Integer.valueOf(intVal);
153 }
154
155 /**
156 * Get the whole list of users and groups and save them in the maps.
157 * @throws IOException
158 */
159 @VisibleForTesting
160 public static void updateMapInternal(BiMap<Integer, String> map, String mapName,
161 String command, String regex, Map<Integer, Integer> staticMapping)
162 throws IOException {
163 BufferedReader br = null;
164 try {
165 Process process = Runtime.getRuntime().exec(
166 new String[] { "bash", "-c", command });
167 br = new BufferedReader(new InputStreamReader(process.getInputStream()));
168 String line = null;
169 while ((line = br.readLine()) != null) {
170 String[] nameId = line.split(regex);
171 if ((nameId == null) || (nameId.length != 2)) {
172 throw new IOException("Can't parse " + mapName + " list entry:" + line);
173 }
174 LOG.debug("add to " + mapName + "map:" + nameId[0] + " id:" + nameId[1]);
175 // HDFS can't differentiate duplicate names with simple authentication
176 final Integer key = staticMapping.get(parseId(nameId[1]));
177 final String value = nameId[0];
178 if (map.containsKey(key)) {
179 final String prevValue = map.get(key);
180 if (value.equals(prevValue)) {
181 // silently ignore equivalent entries
182 continue;
183 }
184 reportDuplicateEntry(
185 "Got multiple names associated with the same id: ",
186 key, value, key, prevValue);
187 continue;
188 }
189 if (map.containsValue(value)) {
190 final Integer prevKey = map.inverse().get(value);
191 reportDuplicateEntry(
192 "Got multiple ids associated with the same name: ",
193 key, value, prevKey, value);
194 continue;
195 }
196 map.put(key, value);
197 }
198 LOG.info("Updated " + mapName + " map size: " + map.size());
199
200 } catch (IOException e) {
201 LOG.error("Can't update " + mapName + " map");
202 throw e;
203 } finally {
204 if (br != null) {
205 try {
206 br.close();
207 } catch (IOException e1) {
208 LOG.error("Can't close BufferedReader of command result", e1);
209 }
210 }
211 }
212 }
213
214 synchronized public void updateMaps() throws IOException {
215 BiMap<Integer, String> uMap = HashBiMap.create();
216 BiMap<Integer, String> gMap = HashBiMap.create();
217
218 if (!OS.startsWith("Linux") && !OS.startsWith("Mac")) {
219 LOG.error("Platform is not supported:" + OS
220 + ". Can't update user map and group map and"
221 + " 'nobody' will be used for any user and group.");
222 return;
223 }
224
225 StaticMapping staticMapping = new StaticMapping(
226 new HashMap<Integer, Integer>(), new HashMap<Integer, Integer>());
227 if (staticMappingFile.exists()) {
228 LOG.info("Using '" + staticMappingFile + "' for static UID/GID mapping...");
229 staticMapping = parseStaticMap(staticMappingFile);
230 } else {
231 LOG.info("Not doing static UID/GID mapping because '" + staticMappingFile
232 + "' does not exist.");
233 }
234
235 if (OS.startsWith("Mac")) {
236 updateMapInternal(uMap, "user", MAC_GET_ALL_USERS_CMD, "\\s+",
237 staticMapping.uidMapping);
238 updateMapInternal(gMap, "group", MAC_GET_ALL_GROUPS_CMD, "\\s+",
239 staticMapping.gidMapping);
240 } else {
241 updateMapInternal(uMap, "user", GET_ALL_USERS_CMD, ":",
242 staticMapping.uidMapping);
243 updateMapInternal(gMap, "group", GET_ALL_GROUPS_CMD, ":",
244 staticMapping.gidMapping);
245 }
246
247 uidNameMap = uMap;
248 gidNameMap = gMap;
249 lastUpdateTime = Time.monotonicNow();
250 }
251
252 @SuppressWarnings("serial")
253 static final class PassThroughMap<K> extends HashMap<K, K> {
254
255 public PassThroughMap() {
256 this(new HashMap<K, K>());
257 }
258
259 public PassThroughMap(Map<K, K> mapping) {
260 super();
261 for (Map.Entry<K, K> entry : mapping.entrySet()) {
262 super.put(entry.getKey(), entry.getValue());
263 }
264 }
265
266 @SuppressWarnings("unchecked")
267 @Override
268 public K get(Object key) {
269 if (super.containsKey(key)) {
270 return super.get(key);
271 } else {
272 return (K) key;
273 }
274 }
275 }
276
277 @VisibleForTesting
278 static final class StaticMapping {
279 final Map<Integer, Integer> uidMapping;
280 final Map<Integer, Integer> gidMapping;
281
282 public StaticMapping(Map<Integer, Integer> uidMapping,
283 Map<Integer, Integer> gidMapping) {
284 this.uidMapping = new PassThroughMap<Integer>(uidMapping);
285 this.gidMapping = new PassThroughMap<Integer>(gidMapping);
286 }
287 }
288
289 static StaticMapping parseStaticMap(File staticMapFile)
290 throws IOException {
291
292 Map<Integer, Integer> uidMapping = new HashMap<Integer, Integer>();
293 Map<Integer, Integer> gidMapping = new HashMap<Integer, Integer>();
294
295 BufferedReader in = new BufferedReader(new InputStreamReader(
296 new FileInputStream(staticMapFile)));
297
298 try {
299 String line = null;
300 while ((line = in.readLine()) != null) {
301 // Skip entirely empty and comment lines.
302 if (EMPTY_LINE.matcher(line).matches() ||
303 COMMENT_LINE.matcher(line).matches()) {
304 continue;
305 }
306
307 Matcher lineMatcher = MAPPING_LINE.matcher(line);
308 if (!lineMatcher.matches()) {
309 LOG.warn("Could not parse line '" + line + "'. Lines should be of " +
310 "the form '[uid|gid] [remote id] [local id]'. Blank lines and " +
311 "everything following a '#' on a line will be ignored.");
312 continue;
313 }
314
315 // We know the line is fine to parse without error checking like this
316 // since it matched the regex above.
317 String firstComponent = lineMatcher.group(1);
318 int remoteId = Integer.parseInt(lineMatcher.group(2));
319 int localId = Integer.parseInt(lineMatcher.group(3));
320 if (firstComponent.equals("uid")) {
321 uidMapping.put(localId, remoteId);
322 } else {
323 gidMapping.put(localId, remoteId);
324 }
325 }
326 } finally {
327 in.close();
328 }
329
330 return new StaticMapping(uidMapping, gidMapping);
331 }
332
333 synchronized public int getUid(String user) throws IOException {
334 checkAndUpdateMaps();
335
336 Integer id = uidNameMap.inverse().get(user);
337 if (id == null) {
338 throw new IOException("User just deleted?:" + user);
339 }
340 return id.intValue();
341 }
342
343 synchronized public int getGid(String group) throws IOException {
344 checkAndUpdateMaps();
345
346 Integer id = gidNameMap.inverse().get(group);
347 if (id == null) {
348 throw new IOException("No such group:" + group);
349
350 }
351 return id.intValue();
352 }
353
354 synchronized public String getUserName(int uid, String unknown) {
355 checkAndUpdateMaps();
356 String uname = uidNameMap.get(uid);
357 if (uname == null) {
358 LOG.warn("Can't find user name for uid " + uid
359 + ". Use default user name " + unknown);
360 uname = unknown;
361 }
362 return uname;
363 }
364
365 synchronized public String getGroupName(int gid, String unknown) {
366 checkAndUpdateMaps();
367 String gname = gidNameMap.get(gid);
368 if (gname == null) {
369 LOG.warn("Can't find group name for gid " + gid
370 + ". Use default group name " + unknown);
371 gname = unknown;
372 }
373 return gname;
374 }
375
376 // When can't map user, return user name's string hashcode
377 public int getUidAllowingUnknown(String user) {
378 checkAndUpdateMaps();
379 int uid;
380 try {
381 uid = getUid(user);
382 } catch (IOException e) {
383 uid = user.hashCode();
384 LOG.info("Can't map user " + user + ". Use its string hashcode:" + uid);
385 }
386 return uid;
387 }
388
389 // When can't map group, return group name's string hashcode
390 public int getGidAllowingUnknown(String group) {
391 checkAndUpdateMaps();
392 int gid;
393 try {
394 gid = getGid(group);
395 } catch (IOException e) {
396 gid = group.hashCode();
397 LOG.info("Can't map group " + group + ". Use its string hashcode:" + gid);
398 }
399 return gid;
400 }
401 }