[sudo-users] file ulimit not set correctly

[Todd C. Miller]

In message <OFB869D89B.BE1FC798-ONC12576BE.003CE270-C12576BE.003FBF69 at de.ibm.co
m>
	so spake Thomas Falkenberg (TFALKEN):

> I have a problem with the file ulimit setting after switching to another 
> user using sudo.
> If a user has a defined file limit, it will be reset to zero after using 
> sudo to open a shell as another user.

The problem is that the value for fsize in /etc/security/limits is
specified in blocks whereas the resource limit is specified in bytes.

When converting from blocks to bytes, 209715200 * 512 this overflows
the 32bit value in struct rlimit.  The fix is for sudo to use
setrlimit64() instead of setrlimit() if available.

I don't have access to an AIX machine these days but the following
(untested) diff should fix it.  Alternately, you could just set
fsize to a smaller value or -1 (unlimited) in /etc/security/limits.

 - todd

Index: aix.c
===================================================================
RCS file: /home/cvs/courtesan/sudo/aix.c,v
retrieving revision 1.7
diff -u -r1.7 aix.c
--- aix.c	6 Nov 2008 00:42:37 -0000	1.7
+++ aix.c	3 Feb 2010 19:45:27 -0000
@@ -39,7 +39,7 @@
 #ifdef HAVE_GETUSERATTR
 
 #ifndef RLIM_SAVED_MAX
-# define RLIM_SAVED_MAX	RLIM_INFINITY
+# define RLIM_SAVED_MAX	RLIM64_INFINITY
 #endif
 
 struct aix_limit {
@@ -74,12 +74,12 @@
 aix_setlimits(user)
     char *user;
 {
-    struct rlimit rlim;
+    struct rlimit64 rlim;
     int i, n;
 
     /*
      * For each resource limit, get the soft/hard values for the user
-     * and set those values via setrlimit().  Must be run as euid 0.
+     * and set those values via setrlimit64().  Must be run as euid 0.
      */
     for (n = 0; n < sizeof(aix_limits) / sizeof(aix_limits[0]); n++) {
 	/*
@@ -87,15 +87,15 @@
 	 * hard limit has been defined.
 	 */
 	if (aix_getlimit(user, aix_limits[n].hard, &i) == 0) {
-	    rlim.rlim_max = i == -1 ? RLIM_INFINITY : i * aix_limits[n].factor;
+	    rlim.rlim_max = i == -1 ? RLIM64_INFINITY : (rlim64_t)i * aix_limits[n].factor;
 	    if (aix_getlimit(user, aix_limits[n].soft, &i) == 0)
-		rlim.rlim_cur = i == -1 ? RLIM_INFINITY : i * aix_limits[n].factor;
+		rlim.rlim_cur = i == -1 ? RLIM64_INFINITY : (rlim64_t)i * aix_limits[n].factor;
 	    else
 		rlim.rlim_cur = rlim.rlim_max;	/* soft not specd, use hard */
 	} else {
 	    /* No hard limit set, try soft limit. */
 	    if (aix_getlimit(user, aix_limits[n].soft, &i) == 0)
-		rlim.rlim_cur = i == -1 ? RLIM_INFINITY : i * aix_limits[n].factor;
+		rlim.rlim_cur = i == -1 ? RLIM64_INFINITY : (rlim64_t)i * aix_limits[n].factor;
 
 	    /* Set hard limit per AIX /etc/security/limits documentation. */
 	    switch (aix_limits[n].resource) {
@@ -107,11 +107,11 @@
 		    rlim.rlim_max = RLIM_SAVED_MAX;
 		    break;
 		default:
-		    rlim.rlim_max = RLIM_INFINITY;
+		    rlim.rlim_max = RLIM64_INFINITY;
 		    break;
 	    }
 	}
-	(void)setrlimit(aix_limits[n].resource, &rlim);
+	(void)setrlimit64(aix_limits[n].resource, &rlim);
     }
 }