Software renderer crashes on Linux

[_mental_]

---

Code: Select all

Dump of assembler code for function _ZN19OpenGLSWFrameBuffer9OpenGLPal6UpdateEv:
   0x00000000009cbdb0 <+0>:	push   r13
   0x00000000009cbdb2 <+2>:	push   r12
   0x00000000009cbdb4 <+4>:	push   rbp
   0x00000000009cbdb5 <+5>:	push   rbx
   0x00000000009cbdb6 <+6>:	mov    rbp,rdi
   0x00000000009cbdb9 <+9>:	sub    rsp,0x18
   0x00000000009cbdbd <+13>:	mov    rax,QWORD PTR fs:0x28
   0x00000000009cbdc6 <+22>:	mov    QWORD PTR [rsp+0x8],rax
   0x00000000009cbdcb <+27>:	xor    eax,eax
   0x00000000009cbdcd <+29>:	mov    rax,QWORD PTR [rdi+0x18]
   0x00000000009cbdd1 <+33>:	mov    edx,DWORD PTR [rax+0x4]
   0x00000000009cbdd4 <+36>:	test   edx,edx
   0x00000000009cbdd6 <+38>:	jne    0x9cc5c8 <OpenGLSWFrameBuffer::OpenGLPal::Update()+2072>
   0x00000000009cbddc <+44>:	lea    rsi,[rax+0x4]
   0x00000000009cbde0 <+48>:	mov    edi,0x2
   0x00000000009cbde5 <+53>:	call   QWORD PTR [rip+0x70994d]        # 0x10d5738 <_ptrc_glGenBuffers>
   0x00000000009cbdeb <+59>:	mov    rax,QWORD PTR [rbp+0x18]
   0x00000000009cbdef <+63>:	mov    edi,0x88ec
   0x00000000009cbdf4 <+68>:	mov    esi,DWORD PTR [rax+0x4]
   0x00000000009cbdf7 <+71>:	call   QWORD PTR [rip+0x70996b]        # 0x10d5768 <_ptrc_glBindBuffer>
   0x00000000009cbdfd <+77>:	mov    eax,DWORD PTR [rbp+0x30]
   0x00000000009cbe00 <+80>:	mov    ecx,0x88e0
   0x00000000009cbe05 <+85>:	xor    edx,edx
   0x00000000009cbe07 <+87>:	mov    edi,0x88ec
   0x00000000009cbe0c <+92>:	lea    esi,[rax*4+0x0]
   0x00000000009cbe13 <+99>:	movsxd rsi,esi
   0x00000000009cbe16 <+102>:	call   QWORD PTR [rip+0x709944]        # 0x10d5760 <_ptrc_glBufferData>
   0x00000000009cbe1c <+108>:	mov    rax,QWORD PTR [rbp+0x18]
   0x00000000009cbe20 <+112>:	mov    edi,0x88ec
   0x00000000009cbe25 <+117>:	mov    esi,DWORD PTR [rax+0x8]
   0x00000000009cbe28 <+120>:	call   QWORD PTR [rip+0x70993a]        # 0x10d5768 <_ptrc_glBindBuffer>
   0x00000000009cbe2e <+126>:	mov    eax,DWORD PTR [rbp+0x30]
   0x00000000009cbe31 <+129>:	mov    ecx,0x88e0
   0x00000000009cbe36 <+134>:	xor    edx,edx
   0x00000000009cbe38 <+136>:	mov    edi,0x88ec
   0x00000000009cbe3d <+141>:	lea    esi,[rax*4+0x0]
   0x00000000009cbe44 <+148>:	movsxd rsi,esi
   0x00000000009cbe47 <+151>:	call   QWORD PTR [rip+0x709913]        # 0x10d5760 <_ptrc_glBufferData>
   0x00000000009cbe4d <+157>:	mov    r12,QWORD PTR [rbp+0x18]
   0x00000000009cbe51 <+161>:	mov    rax,QWORD PTR [rbp+0x28]
   0x00000000009cbe55 <+165>:	mov    ebx,DWORD PTR [rbp+0x30]
   0x00000000009cbe58 <+168>:	cmp    DWORD PTR [rax+0x18],ebx
   0x00000000009cbe5b <+171>:	cmovle ebx,DWORD PTR [rax+0x18]
   0x00000000009cbe5f <+175>:	cmp    BYTE PTR [rip+0xbb8e63],0x0        # 0x1584cc9 <gl+41>
   0x00000000009cbe66 <+182>:	lea    eax,[rbx*4+0x0]
   0x00000000009cbe6d <+189>:	movsxd r13,eax
   0x00000000009cbe70 <+192>:	jne    0x9cc640 <OpenGLSWFrameBuffer::OpenGLPal::Update()+2192>
   0x00000000009cbe76 <+198>:	mov    rax,QWORD PTR [rip+0x7093bb]        # 0x10d5238 <_ptrc_glMapBufferRange>
   0x00000000009cbe7d <+205>:	test   rax,rax
   0x00000000009cbe80 <+208>:	je     0x9cc680 <OpenGLSWFrameBuffer::OpenGLPal::Update()+2256>
   0x00000000009cbe86 <+214>:	mov    ecx,0xe
   0x00000000009cbe8b <+219>:	mov    rdx,r13
   0x00000000009cbe8e <+222>:	xor    esi,esi
   0x00000000009cbe90 <+224>:	mov    edi,0x88ec
   0x00000000009cbe95 <+229>:	call   rax
   0x00000000009cbe97 <+231>:	mov    rcx,rax
   0x00000000009cbe9a <+234>:	test   rcx,rcx
   0x00000000009cbe9d <+237>:	je     0x9cc670 <OpenGLSWFrameBuffer::OpenGLPal::Update()+2240>
   0x00000000009cbea3 <+243>:	cmp    BYTE PTR [rbp+0x24],0x1
   0x00000000009cbea7 <+247>:	mov    rax,QWORD PTR [rbp+0x28]
   0x00000000009cbeab <+251>:	mov    rdi,QWORD PTR [rax+0x8]
   0x00000000009cbeaf <+255>:	sbb    eax,eax
   0x00000000009cbeb1 <+257>:	and    eax,0x8
   0x00000000009cbeb4 <+260>:	add    eax,0xf8
   0x00000000009cbeb9 <+265>:	cmp    eax,ebx
   0x00000000009cbebb <+267>:	cmovg  eax,ebx
   0x00000000009cbebe <+270>:	test   eax,eax
   0x00000000009cbec0 <+272>:	jle    0x9cc6f0 <OpenGLSWFrameBuffer::OpenGLPal::Update()+2368>
   0x00000000009cbec6 <+278>:	lea    rdx,[rdi+0x10]
   0x00000000009cbeca <+282>:	cmp    rcx,rdx
   0x00000000009cbecd <+285>:	lea    rdx,[rcx+0x10]
   0x00000000009cbed1 <+289>:	setae  sil
   0x00000000009cbed5 <+293>:	cmp    rdi,rdx
   0x00000000009cbed8 <+296>:	setae  dl
   0x00000000009cbedb <+299>:	or     sil,dl
   0x00000000009cbede <+302>:	je     0x9cc6b0 <OpenGLSWFrameBuffer::OpenGLPal::Update()+2304>
   0x00000000009cbee4 <+308>:	cmp    eax,0xc
   0x00000000009cbee7 <+311>:	jbe    0x9cc6b0 <OpenGLSWFrameBuffer::OpenGLPal::Update()+2304>
   0x00000000009cbeed <+317>:	mov    rdx,rdi
   0x00000000009cbef0 <+320>:	xor    esi,esi
   0x00000000009cbef2 <+322>:	shr    rdx,0x2
   0x00000000009cbef6 <+326>:	neg    rdx
   0x00000000009cbef9 <+329>:	and    edx,0x3
   0x00000000009cbefc <+332>:	je     0x9cbf27 <OpenGLSWFrameBuffer::OpenGLPal::Update()+375>
   0x00000000009cbefe <+334>:	mov    esi,DWORD PTR [rdi]
   0x00000000009cbf00 <+336>:	cmp    edx,0x1
   0x00000000009cbf03 <+339>:	mov    DWORD PTR [rcx],esi
   0x00000000009cbf05 <+341>:	mov    esi,0x1
   0x00000000009cbf0a <+346>:	je     0x9cbf27 <OpenGLSWFrameBuffer::OpenGLPal::Update()+375>
   0x00000000009cbf0c <+348>:	mov    esi,DWORD PTR [rdi+0x4]
   0x00000000009cbf0f <+351>:	cmp    edx,0x2
   0x00000000009cbf12 <+354>:	mov    DWORD PTR [rcx+0x4],esi
   0x00000000009cbf15 <+357>:	mov    esi,0x2
   0x00000000009cbf1a <+362>:	je     0x9cbf27 <OpenGLSWFrameBuffer::OpenGLPal::Update()+375>
   0x00000000009cbf1c <+364>:	mov    esi,DWORD PTR [rdi+0x8]
   0x00000000009cbf1f <+367>:	mov    DWORD PTR [rcx+0x8],esi
   0x00000000009cbf22 <+370>:	mov    esi,0x3
   0x00000000009cbf27 <+375>:	mov    r10d,eax
   0x00000000009cbf2a <+378>:	xor    r9d,r9d
   0x00000000009cbf2d <+381>:	xor    r12d,r12d
   0x00000000009cbf30 <+384>:	sub    r10d,edx
   0x00000000009cbf33 <+387>:	mov    edx,edx
   0x00000000009cbf35 <+389>:	lea    r8d,[r10-0x4]
   0x00000000009cbf39 <+393>:	shl    rdx,0x2
   0x00000000009cbf3d <+397>:	lea    r13,[rdi+rdx*1]
   0x00000000009cbf41 <+401>:	add    rdx,rcx
   0x00000000009cbf44 <+404>:	shr    r8d,0x2
   0x00000000009cbf48 <+408>:	add    r8d,0x1
   0x00000000009cbf4c <+412>:	lea    r11d,[r8*4+0x0]
=> 0x00000000009cbf54 <+420>:	movdqa xmm0,XMMWORD PTR [r13+r9*1+0x0]
   0x00000000009cbf5b <+427>:	add    r12d,0x1
   0x00000000009cbf5f <+431>:	movups XMMWORD PTR [rdx+r9*1],xmm0
   0x00000000009cbf64 <+436>:	add    r9,0x10
   0x00000000009cbf68 <+440>:	cmp    r8d,r12d
   0x00000000009cbf6b <+443>:	ja     0x9cbf54 <OpenGLSWFrameBuffer::OpenGLPal::Update()+420>
   0x00000000009cbf6d <+445>:	add    esi,r11d
   0x00000000009cbf70 <+448>:	cmp    r10d,r11d
   0x00000000009cbf73 <+451>:	je     0x9cbfa2 <OpenGLSWFrameBuffer::OpenGLPal::Update()+498>
   0x00000000009cbf75 <+453>:	movsxd rdx,esi
   0x00000000009cbf78 <+456>:	mov    r8d,DWORD PTR [rdi+rdx*4]
   0x00000000009cbf7c <+460>:	mov    DWORD PTR [rcx+rdx*4],r8d
   0x00000000009cbf80 <+464>:	lea    edx,[rsi+0x1]
   0x00000000009cbf83 <+467>:	cmp    eax,edx
   0x00000000009cbf85 <+469>:	jle    0x9cbfa2 <OpenGLSWFrameBuffer::OpenGLPal::Update()+498>
   0x00000000009cbf87 <+471>:	movsxd rdx,edx
   0x00000000009cbf8a <+474>:	add    esi,0x2
   0x00000000009cbf8d <+477>:	mov    r8d,DWORD PTR [rdi+rdx*4]
   0x00000000009cbf91 <+481>:	cmp    eax,esi
   0x00000000009cbf93 <+483>:	mov    DWORD PTR [rcx+rdx*4],r8d
   0x00000000009cbf97 <+487>:	jle    0x9cbfa2 <OpenGLSWFrameBuffer::OpenGLPal::Update()+498>
   0x00000000009cbf99 <+489>:	movsxd rsi,esi
   0x00000000009cbf9c <+492>:	mov    edx,DWORD PTR [rdi+rsi*4]
   0x00000000009cbf9f <+495>:	mov    DWORD PTR [rcx+rsi*4],edx
   0x00000000009cbfa2 <+498>:	add    eax,0x1
   0x00000000009cbfa5 <+501>:	cmp    ebx,eax
   0x00000000009cbfa7 <+503>:	jle    0x9cc6e0 <OpenGLSWFrameBuffer::OpenGLPal::Update()+2352>
   0x00000000009cbfad <+509>:	movsxd rsi,eax
   ...

Crash location is marked with =>

The result of r13+r9*1+0x0 (or simply r13+r9) is 0x1848193.
I believe it must be 16-bytes aligned to work.

[Edward-san]

Can you recompile in Release mode and adding '-DNO_STRIP=ON -DCMAKE_CXX_FLAGS=-g3 -DCMAKE_C_FLAGS=-g3'?

[dpJudas]

The result of r13+r9*1+0x0 (or simply r13+r9) is 0x1848193.
I believe it must be 16-bytes aligned to work.

Hmm, yes, that instruction must use 16-byte aligned addresses to work.

Here's another guy having a similar type of crash: http://pzemtsov.github.io/2016/11/06/bu ... n-x86.html. The compiler assumes that any uint32_t is 4 byte aligned. When it generates the SSE optimized code, it makes sure it is 16-byte aligned, but only if the original assumption that it was already 4 byte aligned is correct.

Could it be that one of our pointers are not properly dword aligned?

Edit: I found this information about glMapBuffer: "Alignment of the returned pointer is guaranteed only if the version of the GL version is 4.2 or greater." So we could very well have an alignment issue at this spot.

[_mental_]

I had several issues with GCC code generation involving SSE instructions: broken auto-vectorization, broken alignment even for l-values, invalid transformation of intrinsics to instructions, ... Their Bugzilla is full of similar "success stories".

By the way, I tested in VM without hardware acceleration, it was Mesa3D LLVMpipe driver.
AFAIK it's quite close to spec so I hope it was the same crash as in OP.

[Rachael]

Should we pass -DNO_SSE if CMake detects it is using GCC by default, for now, until GCC gets a chance to fix those issues? And then recommend officially switching to Clang if someone wants a proper executable with the intrinsics enabled?

[dpJudas]

It would probably be better to specify the flags disabling the optimization passes that is causing the problem rather than try disable SSE completely. The software renderer runs at half its normal speed if you define NO_SSE.

[Rachael]

Can that be done on specific files like r_all.cpp and poly_all.cpp rather than the entire project?

[dpJudas]

As far as I know, the broken code isn't part of those files. But I was more thinking of specifying something like -fno-tree-vectorize globally, or whatever other pass it is that causes it to crash.

[Rachael]

Ah. >_> My mistake.

[Graf Zahl]

Can that be done on specific files like r_all.cpp and poly_all.cpp rather than the entire project?

Those two files contain all relevant SSE code, so it'd be a bad idea.

[_mental_]

Well, SSE support isn't completely broken with GCC, it simply has some issues.
Most of Linux distros use GCC by default and disabling SSE would be an overkill.

Moreover I doubt that it's possible to disable SSE for x64 builds.
At least floats/double are passed to functions using XMM registers.

Actually, this can be the only problem we have with this instruction set in GCC.
Need to think what will do with it though.

[dpJudas]

If this one function is the only crash we have, then I can fix it by rewriting it. Its current version looks visually terrible anyway.

[dpJudas]

I commited a version of OpenGLSWFrameBuffer::OpenGLPal::Update that does some SSE using intrinsics to prevent GCC from optimizing it using the auto-vectorizer. Hopefully that should make it stop crashing at this location.

[Chris]

Another option may be to replace -O3 with -O2. I often hear "-O3 could do bad stuff in the past because of bugs, but it's fine now!" while also continuing to hear stories like this. It's also been said that -O3 may not produce more efficient code than -O2 anyway, due to over-optimizing and not accounting for things like code size effecting cache locality (and similar stories of -Os sometimes being faster than -O2/3).

[Rachael]

I'd like to do some speed tests with Clang before we make the official switch since Clang does not seem to have -O3 issues - I'd probably suggest only doing it for GCC.