From: Andi Kleen <ak@suse.de> This fixes an incorrect sign extension in the compat layer that breaks 32bit shmget that are >2GB. sys_shmget has a signed size_t size argument, and the int size argument comming from 32bit user space would get sign extended to 64bit, which is wrong. I fixed it on all compat architectures, except PPC64 which was already ok. It was originally debugged and fixed by Karl Rister @ IBM for SLES9 on x86-64. Signed-off-by: Andi Kleen <ak@suse.de> Signed-off-by: Andrew Morton <akpm@osdl.org> --- 25-akpm/arch/ia64/ia32/sys_ia32.c | 2 +- 25-akpm/arch/mips/kernel/linux32.c | 2 +- 25-akpm/arch/s390/kernel/compat_linux.c | 2 +- 25-akpm/arch/sparc64/kernel/sys_sparc32.c | 2 +- 25-akpm/arch/x86_64/ia32/ipc32.c | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff -puN arch/ia64/ia32/sys_ia32.c~fix-compat-shmget-overflow arch/ia64/ia32/sys_ia32.c --- 25/arch/ia64/ia32/sys_ia32.c~fix-compat-shmget-overflow Wed Feb 9 14:45:24 2005 +++ 25-akpm/arch/ia64/ia32/sys_ia32.c Wed Feb 9 14:45:24 2005 @@ -1415,7 +1415,7 @@ sys32_ipc(u32 call, int first, int secon case SHMDT: return sys_shmdt(compat_ptr(ptr)); case SHMGET: - return sys_shmget(first, second, third); + return sys_shmget(first, (unsigned)second, third); case SHMCTL: return compat_sys_shmctl(first, second, compat_ptr(ptr)); diff -puN arch/mips/kernel/linux32.c~fix-compat-shmget-overflow arch/mips/kernel/linux32.c --- 25/arch/mips/kernel/linux32.c~fix-compat-shmget-overflow Wed Feb 9 14:45:24 2005 +++ 25-akpm/arch/mips/kernel/linux32.c Wed Feb 9 14:45:24 2005 @@ -1115,7 +1115,7 @@ sys32_ipc (u32 call, int first, int seco err = sys_shmdt ((char *)A(ptr)); break; case SHMGET: - err = sys_shmget (first, second, third); + err = sys_shmget (first, (unsigned)second, third); break; case SHMCTL: err = do_sys32_shmctl (first, second, (void *)AA(ptr)); diff -puN arch/s390/kernel/compat_linux.c~fix-compat-shmget-overflow arch/s390/kernel/compat_linux.c --- 25/arch/s390/kernel/compat_linux.c~fix-compat-shmget-overflow Wed Feb 9 14:45:24 2005 +++ 25-akpm/arch/s390/kernel/compat_linux.c Wed Feb 9 14:45:24 2005 @@ -331,7 +331,7 @@ asmlinkage long sys32_ipc(u32 call, int case SHMDT: return sys_shmdt(compat_ptr(ptr)); case SHMGET: - return sys_shmget(first, second, third); + return sys_shmget(first, (unsigned)second, third); case SHMCTL: return compat_sys_shmctl(first, second, compat_ptr(ptr)); } diff -puN arch/sparc64/kernel/sys_sparc32.c~fix-compat-shmget-overflow arch/sparc64/kernel/sys_sparc32.c --- 25/arch/sparc64/kernel/sys_sparc32.c~fix-compat-shmget-overflow Wed Feb 9 14:45:24 2005 +++ 25-akpm/arch/sparc64/kernel/sys_sparc32.c Wed Feb 9 14:45:24 2005 @@ -835,7 +835,7 @@ asmlinkage long compat_sys_ipc(u32 call, err = sys_shmdt(ptr); goto out; case SHMGET: - err = sys_shmget(first, second, third); + err = sys_shmget(first, (unsigned)second, third); goto out; case SHMCTL: err = do_sys32_shmctl(first, second, ptr); diff -puN arch/x86_64/ia32/ipc32.c~fix-compat-shmget-overflow arch/x86_64/ia32/ipc32.c --- 25/arch/x86_64/ia32/ipc32.c~fix-compat-shmget-overflow Wed Feb 9 14:45:24 2005 +++ 25-akpm/arch/x86_64/ia32/ipc32.c Wed Feb 9 14:45:24 2005 @@ -49,7 +49,7 @@ sys32_ipc(u32 call, int first, int secon case SHMDT: return sys_shmdt(compat_ptr(ptr)); case SHMGET: - return sys_shmget(first, second, third); + return sys_shmget(first, (unsigned)second, third); case SHMCTL: return compat_sys_shmctl(first, second, compat_ptr(ptr)); } _