[gentoo-hardened] vmware broken on amd64 hardened

[Alex Efros]

Hi!

I've just converted my system from x86 to amd64 (Core i7), and one of
things which become broken because of this is vmware. When I start any
guest my host immediately reset, and after booting I didn't see anything
in logs - neither in kernel nor in vmware's logs.

I've experimented with different kernels, and here is what I found:

- hardened-sources-3.2.2-r1 work ok on x86
- gentoo-sources-3.2.1-r2 work ok on amd64
- no one hardened-sources since 2.6.39-r8 work on amd64 (I didn't tried
  older versions)

Disabling both GRSEC and PAX in hardened kernels doesn't solve this issue,
so this bug probably in that part of hardened patches which is active even
with disabled GRSEC and PAX config options.

I can't try gentoo-sources and hardened-sources with exactly same
vmware-modules, because of extra patches needed for vmware-modules to make
it compatible with hardened, and these patches incompatible with non-hardened.
So, gentoo-sources work ok with vmware-modules from main portage, while
hardened-sources work on x86 and doesn't work on amd64 with vmware-modules
patched using these 3 patches:
    https://384739.bugs.gentoo.org/attachment.cgi?id=295017
    https://384739.bugs.gentoo.org/attachment.cgi?id=295019
    https://384739.bugs.gentoo.org/attachment.cgi?id=295021

I've also tried hardened-sources-3.2.1, both x86 and amd64 - vmware work
on x86 and didn't work on amd64. I've tried to keep .config same, but
there are a lot of differences anyway (I suppose they all should be
related to 32/64-bit).

So, here is diff between -gentoo and -hardened on amd64:

--- /tmp/config-amd64-gentoo	2012-02-14 20:33:31.579285488 +0200
+++ /tmp/config-amd64-hardened	2012-02-14 20:33:40.383285603 +0200
@@ -179,6 +179,7 @@
 CONFIG_X86_L1_CACHE_SHIFT=6
 CONFIG_X86_XADD=y
 CONFIG_X86_WP_WORKS_OK=y
+CONFIG_X86_ALIGNMENT_16=y
 CONFIG_X86_INTEL_USERCOPY=y
 CONFIG_X86_USE_PPRO_CHECKSUM=y
 CONFIG_X86_P6_NOP=y
@@ -599,7 +600,6 @@
 CONFIG_NTFS_FS=y
 CONFIG_PROC_FS=y
 CONFIG_PROC_SYSCTL=y
-CONFIG_PROC_PAGE_MONITOR=y
 CONFIG_SYSFS=y
 CONFIG_TMPFS=y
 CONFIG_CONFIGFS_FS=y
@@ -647,6 +647,7 @@
 CONFIG_IO_DELAY_TYPE_NONE=3
 CONFIG_IO_DELAY_0X80=y
 CONFIG_DEFAULT_IO_DELAY_TYPE=0
+CONFIG_TASK_SIZE_MAX_SHIFT=47
 CONFIG_SECURITY_DMESG_RESTRICT=y
 CONFIG_SECURITY=y
 CONFIG_DEFAULT_SECURITY_DAC=y

And here is diff between -hardened x86 and -hardened amd64:

--- /tmp/config-x86	2012-02-14 20:31:08.183283609 +0200
+++ /tmp/config-amd64	2012-02-14 20:30:53.192283412 +0200
@@ -1,26 +1,31 @@
-CONFIG_X86_32=y
+CONFIG_64BIT=y
+CONFIG_X86_64=y
 CONFIG_X86=y
 CONFIG_INSTRUCTION_DECODER=y
-CONFIG_OUTPUT_FORMAT="elf32-i386"
-CONFIG_ARCH_DEFCONFIG="arch/x86/configs/i386_defconfig"
+CONFIG_OUTPUT_FORMAT="elf64-x86-64"
+CONFIG_ARCH_DEFCONFIG="arch/x86/configs/x86_64_defconfig"
 CONFIG_GENERIC_CMOS_UPDATE=y
 CONFIG_CLOCKSOURCE_WATCHDOG=y
 CONFIG_GENERIC_CLOCKEVENTS=y
+CONFIG_ARCH_CLOCKSOURCE_DATA=y
 CONFIG_GENERIC_CLOCKEVENTS_BROADCAST=y
 CONFIG_LOCKDEP_SUPPORT=y
 CONFIG_STACKTRACE_SUPPORT=y
 CONFIG_HAVE_LATENCYTOP_SUPPORT=y
 CONFIG_MMU=y
 CONFIG_ZONE_DMA=y
+CONFIG_NEED_DMA_MAP_STATE=y
 CONFIG_NEED_SG_DMA_LENGTH=y
 CONFIG_GENERIC_ISA_DMA=y
 CONFIG_GENERIC_IOMAP=y
 CONFIG_GENERIC_BUG=y
+CONFIG_GENERIC_BUG_RELATIVE_POINTERS=y
 CONFIG_GENERIC_HWEIGHT=y
 CONFIG_ARCH_MAY_HAVE_PC_FDC=y
 CONFIG_RWSEM_XCHGADD_ALGORITHM=y
 CONFIG_ARCH_HAS_CPU_IDLE_WAIT=y
 CONFIG_GENERIC_CALIBRATE_DELAY=y
+CONFIG_GENERIC_TIME_VSYSCALL=y
 CONFIG_ARCH_HAS_CPU_RELAX=y
 CONFIG_ARCH_HAS_DEFAULT_IDLE=y
 CONFIG_ARCH_HAS_CACHE_LINE_SIZE=y
@@ -29,13 +34,14 @@
 CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK=y
 CONFIG_ARCH_HIBERNATION_POSSIBLE=y
 CONFIG_ARCH_SUSPEND_POSSIBLE=y
+CONFIG_ZONE_DMA32=y
 CONFIG_ARCH_POPULATES_NODE_MAP=y
+CONFIG_AUDIT_ARCH=y
 CONFIG_ARCH_SUPPORTS_OPTIMIZED_INLINING=y
 CONFIG_ARCH_SUPPORTS_DEBUG_PAGEALLOC=y
-CONFIG_X86_32_SMP=y
+CONFIG_X86_64_SMP=y
 CONFIG_X86_HT=y
-CONFIG_ARCH_HWEIGHT_CFLAGS="-fcall-saved-ecx -fcall-saved-edx"
-CONFIG_KTIME_SCALAR=y
+CONFIG_ARCH_HWEIGHT_CFLAGS="-fcall-saved-rdi -fcall-saved-rsi -fcall-saved-rdx -fcall-saved-rcx -fcall-saved-r8 -fcall-saved-r9 -fcall-saved-r10 -fcall-saved-r11"
 CONFIG_DEFCONFIG_LIST="/lib/modules/$UNAME_RELEASE/.config"
 CONFIG_HAVE_IRQ_WORK=y
 CONFIG_IRQ_WORK=y
@@ -131,7 +137,6 @@
 CONFIG_HAVE_PERF_EVENTS_NMI=y
 CONFIG_HAVE_ARCH_JUMP_LABEL=y
 CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG=y
-CONFIG_HAVE_GENERIC_DMA_COHERENT=y
 CONFIG_SLABINFO=y
 CONFIG_RT_MUTEXES=y
 CONFIG_BASE_SMALL=0
@@ -140,9 +145,9 @@
 CONFIG_MODULE_FORCE_UNLOAD=y
 CONFIG_STOP_MACHINE=y
 CONFIG_BLOCK=y
-CONFIG_LBDAF=y
 CONFIG_BLK_DEV_BSG=y
 CONFIG_BLK_DEV_THROTTLING=y
+CONFIG_BLOCK_COMPAT=y
 CONFIG_IOSCHED_NOOP=y
 CONFIG_IOSCHED_DEADLINE=y
 CONFIG_IOSCHED_CFQ=y
@@ -174,26 +179,24 @@
 CONFIG_X86_L1_CACHE_SHIFT=6
 CONFIG_X86_XADD=y
 CONFIG_X86_WP_WORKS_OK=y
-CONFIG_X86_INVLPG=y
-CONFIG_X86_BSWAP=y
-CONFIG_X86_POPAD_OK=y
 CONFIG_X86_ALIGNMENT_16=y
 CONFIG_X86_INTEL_USERCOPY=y
 CONFIG_X86_USE_PPRO_CHECKSUM=y
+CONFIG_X86_P6_NOP=y
 CONFIG_X86_TSC=y
 CONFIG_X86_CMPXCHG64=y
 CONFIG_X86_CMOV=y
-CONFIG_X86_MINIMUM_CPU_FAMILY=5
+CONFIG_X86_MINIMUM_CPU_FAMILY=64
 CONFIG_X86_DEBUGCTLMSR=y
 CONFIG_CPU_SUP_INTEL=y
-CONFIG_CPU_SUP_CYRIX_32=y
 CONFIG_CPU_SUP_AMD=y
 CONFIG_CPU_SUP_CENTAUR=y
-CONFIG_CPU_SUP_TRANSMETA_32=y
-CONFIG_CPU_SUP_UMC_32=y
 CONFIG_HPET_TIMER=y
 CONFIG_HPET_EMULATE_RTC=y
 CONFIG_DMI=y
+CONFIG_GART_IOMMU=y
+CONFIG_SWIOTLB=y
+CONFIG_IOMMU_HELPER=y
 CONFIG_NR_CPUS=8
 CONFIG_SCHED_MC=y
 CONFIG_PREEMPT_VOLUNTARY=y
@@ -201,27 +204,25 @@
 CONFIG_X86_IO_APIC=y
 CONFIG_X86_MCE=y
 CONFIG_X86_MCE_INTEL=y
-CONFIG_X86_MCE_AMD=y
 CONFIG_X86_MCE_THRESHOLD=y
 CONFIG_X86_THERMAL_VECTOR=y
-CONFIG_VM86=y
 CONFIG_X86_MSR=y
 CONFIG_X86_CPUID=y
-CONFIG_HIGHMEM64G=y
-CONFIG_PAGE_OFFSET=0xC0000000
-CONFIG_HIGHMEM=y
-CONFIG_X86_PAE=y
 CONFIG_ARCH_PHYS_ADDR_T_64BIT=y
 CONFIG_ARCH_DMA_ADDR_T_64BIT=y
-CONFIG_ARCH_FLATMEM_ENABLE=y
+CONFIG_DIRECT_GBPAGES=y
 CONFIG_ARCH_SPARSEMEM_ENABLE=y
+CONFIG_ARCH_SPARSEMEM_DEFAULT=y
 CONFIG_ARCH_SELECT_MEMORY_MODEL=y
-CONFIG_ILLEGAL_POINTER_VALUE=0
+CONFIG_ILLEGAL_POINTER_VALUE=0xdead000000000000
 CONFIG_SELECT_MEMORY_MODEL=y
-CONFIG_FLATMEM_MANUAL=y
-CONFIG_FLATMEM=y
-CONFIG_FLAT_NODE_MEM_MAP=y
-CONFIG_SPARSEMEM_STATIC=y
+CONFIG_SPARSEMEM_MANUAL=y
+CONFIG_SPARSEMEM=y
+CONFIG_HAVE_MEMORY_PRESENT=y
+CONFIG_SPARSEMEM_EXTREME=y
+CONFIG_SPARSEMEM_VMEMMAP_ENABLE=y
+CONFIG_SPARSEMEM_ALLOC_MEM_MAP_TOGETHER=y
+CONFIG_SPARSEMEM_VMEMMAP=y
 CONFIG_HAVE_MEMBLOCK=y
 CONFIG_PAGEFLAGS_EXTENDED=y
 CONFIG_SPLIT_PTLOCK_CPUS=4
@@ -247,7 +248,7 @@
 CONFIG_HZ=1000
 CONFIG_SCHED_HRTICK=y
 CONFIG_PHYSICAL_START=0x1000000
-CONFIG_PHYSICAL_ALIGN=0x400000
+CONFIG_PHYSICAL_ALIGN=0x1000000
 CONFIG_ARCH_ENABLE_MEMORY_HOTPLUG=y
 CONFIG_PM_RUNTIME=y
 CONFIG_PM=y
@@ -266,8 +267,6 @@
 CONFIG_CPU_IDLE_GOV_LADDER=y
 CONFIG_INTEL_IDLE=y
 CONFIG_PCI=y
-CONFIG_PCI_GOANY=y
-CONFIG_PCI_BIOS=y
 CONFIG_PCI_DIRECT=y
 CONFIG_PCI_MMCONFIG=y
 CONFIG_PCI_DOMAINS=y
@@ -282,8 +281,12 @@
 CONFIG_ISA_DMA_API=y
 CONFIG_AMD_NB=y
 CONFIG_BINFMT_ELF=y
-CONFIG_HAVE_AOUT=y
-CONFIG_HAVE_ATOMIC_IOMAP=y
+CONFIG_COMPAT_BINFMT_ELF=y
+CONFIG_IA32_EMULATION=y
+CONFIG_IA32_AOUT=y
+CONFIG_COMPAT=y
+CONFIG_COMPAT_FOR_U64_ALIGNMENT=y
+CONFIG_SYSVIPC_COMPAT=y
 CONFIG_HAVE_TEXT_POKE_SMP=y
 CONFIG_NET=y
 CONFIG_PACKET=y
@@ -351,6 +354,7 @@
 CONFIG_RPS=y
 CONFIG_RFS_ACCEL=y
 CONFIG_XPS=y
+CONFIG_HAVE_BPF_JIT=y
 CONFIG_FIB_RULES=y
 CONFIG_NET_9P=y
 CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
@@ -547,7 +551,6 @@
 CONFIG_USB_STORAGE=y
 CONFIG_USB_UAS=y
 CONFIG_EDAC=y
-CONFIG_EDAC_DECODE_MCE=y
 CONFIG_EDAC_MM_EDAC=y
 CONFIG_RTC_LIB=y
 CONFIG_RTC_CLASS=y
@@ -559,7 +562,6 @@
 CONFIG_RTC_INTF_DEV_UIE_EMUL=y
 CONFIG_RTC_DRV_CMOS=y
 CONFIG_DMADEVICES=y
-CONFIG_CLKSRC_I8253=y
 CONFIG_CLKEVT_I8253=y
 CONFIG_I8253_LOCK=y
 CONFIG_CLKBLD_I8253=y
@@ -638,7 +640,6 @@
 CONFIG_STRICT_DEVMEM=y
 CONFIG_X86_VERBOSE_BOOTUP=y
 CONFIG_EARLY_PRINTK=y
-CONFIG_DOUBLEFAULT=y
 CONFIG_HAVE_MMIOTRACE_SUPPORT=y
 CONFIG_IO_DELAY_TYPE_0X80=0
 CONFIG_IO_DELAY_TYPE_0XED=1
@@ -646,7 +647,7 @@
 CONFIG_IO_DELAY_TYPE_NONE=3
 CONFIG_IO_DELAY_0X80=y
 CONFIG_DEFAULT_IO_DELAY_TYPE=0
-CONFIG_PAX_ENABLE_PAE=y
+CONFIG_TASK_SIZE_MAX_SHIFT=47
 CONFIG_SECURITY_DMESG_RESTRICT=y
 CONFIG_SECURITY=y
 CONFIG_DEFAULT_SECURITY_DAC=y
@@ -687,7 +688,6 @@
 CONFIG_CRC_ITU_T=y
 CONFIG_CRC32=y
 CONFIG_LIBCRC32C=y
-CONFIG_AUDIT_GENERIC=y
 CONFIG_ZLIB_INFLATE=y
 CONFIG_ZLIB_DEFLATE=y
 CONFIG_HAS_IOMEM=y


Maybe this is same bug as https://bugs.gentoo.org/show_bug.cgi?id=382793

-- 
			WBR, Alex.