Towards Beautiful Test Cases for Compiler Bugs

Towards Beautiful Test Cases for Compiler Bugs

John RegehrUniversity of Utah

2

template< class A0 , class A1> __attribute__((always_inline)) typename boost::dispatch::meta::call<tag::shift_right_( A0 const& , A1 const& )>::type shift_right ( A0 const& a0 , A1 const& a1 ) { typename boost::dispatch::make_functor<tag::shift_right_, A0>::type callee; return callee( a0 , a1);; } template< class A0 , class A1> __attribute__((always_inline)) typename boost::dispatch::meta::call<tag::shift_right_( A0 const& , A1 const& )>::type shr ( A0 const& a0 , A1 const& a1 ) { typename boost::dispatch::make_functor<tag::shift_right_, A0>::type callee; return callee( a0 , a1);; }} }# 5 "/home/gaunard/build/may_alias/include/boost/simd/toolbox/operator/include/functions/shift_right.hpp" 2# 1 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/scalar/shift_right.hpp" 1# 14 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/scalar/shift_right.hpp"namespace boost { namespace simd { namespace ext{ } } } namespace boost { namespace dispatch { namespace meta { template< class A0 , class A1> __attribute__((always_inline)) boost :: simd :: ext :: implement< boost::simd::tag::shift_right_( scalar_< floating_<A0> > , scalar_< integer_<A1> >) , tag::cpu_ > dispatching( boost::simd::tag::shift_right_, tag::cpu_ , scalar_< floating_<A0> > const , scalar_< integer_<A1> > const , adl_helper = adl_helper() ) { boost :: simd :: ext :: implement< boost::simd::tag::shift_right_( scalar_< floating_<A0> > , scalar_< integer_<A1> > ) , tag::cpu_ > that; return that; } } } } namespace boost { namespace simd { namespace ext { template< class A0 , class A1 , class Dummy > struct implement < boost::simd::tag::shift_right_( scalar_< floating_<A0> > , scalar_< integer_<A1> > ) , tag::cpu_, Dummy > { typedef A0 result_type; inline result_type operator()( A0 const& a0 , A1 const& a1 ) const { typedef typename dispatch::meta::as_integer<A0, unsigned>::type itype; return bitwise_cast<result_type>(shift_right(bitwise_cast<itype>(a0),a1)); } };} } }namespace boost { namespace simd { namespace ext{ } } } namespace boost { namespace dispatch { namespace meta { template< class A0 , class A1> __attribute__((always_inline)) boost :: simd :: ext :: implement< boost::simd::tag::shift_right_( scalar_< integer_<A0> > , scalar_< integer_<A1> >) , tag::cpu_ > dispatching( boost::simd::tag::shift_right_, tag::cpu_ , scalar_< integer_<A0> > const , scalar_< integer_<A1> > const , adl_helper = adl_helper() ) { boost :: simd :: ext :: implement< boost::simd::tag::shift_right_( scalar_< integer_<A0> > , scalar_< integer_<A1> > ) , tag::cpu_ > that; return that; } } } } namespace boost { namespace simd { namespace ext { template< class A0 , class A1 , class Dummy > struct implement < boost::simd::tag::shift_right_( scalar_< integer_<A0> > , scalar_< integer_<A1> > ) , tag::cpu_, Dummy > { typedef A0 result_type; inline result_type operator()( A0 const& a0 , A1 const& a1 ) const { return a0 >> a1; } };} } }# 6 "/home/gaunard/build/may_alias/include/boost/simd/toolbox/operator/include/functions/shift_right.hpp" 2# 1 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/simd/common/shift_right.hpp" 1# 20 "/home/gaunard/dev/may_alias/modules/boost/simd/operator/include/boost/simd/toolbox/operator/functions/simd/common/shift_right.hpp"namespace boost { namespace simd { namespace ext{

3


From GCC PR 50800:“Testcase is [here] (couldn't attach it due to bugzilla size restrictions)”

4



Next comment:“That you couldn't attach it should tell you something…”

5



Next comment:“That you couldn't attach it should tell you something…”Next comment:

203 KB reduced test case attached

6

(safe_47 (((g_1161 ^ 0x2893L), (((safe_9 ((!l_1747),

((l_1749 = func_37 (func_58

((l_1748, 4294967293UL), g_110, l_1168, p_7, l_1747), g_381, p_9)),

g_894))) != l_1747) <= p_10)), p_8))),g_11) <= p_7)), g_1315, p_8), 10))) > 4UL),

0x93L)) ^ g_1161))) <= l_1750), p_8), g_458); g_1122 = (l_1749

&& func_37 (l_1750, (g_1161 = ((safe_17 (((safe_60 (l_1755, 4)) |

(g_703, (safe_55 ((p_10, 3UL), p_7)))),((safe_51 ((safe_42 (0UL, ((l_1747 != 0x3AL) < g_28))))) | 0xF862D3BEL))) != 0xC7L)),

g_1487)); g_146 = 0xBC806FC8L; l_1788 =

func_24 (((safe_13 ((~ (((g_458 !=

(safe_54 (((~ (l_1749 = (safe_43 ((safe_14

((((safe_62 ((safe_4 ((safe_63

(((l_1750 = 1L), (safe_59 (l_1168, 1))), (((p_10 = func_46 (func_46

(l_1749, g_894, (safe_23 (l_1749,

(l_1167 = g_1161))), ((l_1469 =

7

(safe_47 (((g_1161 ^ 0x2893L), (((safe_9 ((!l_1747),

((l_1749 = func_37 (func_58

((l_1748, 4294967293UL), g_110, l_1168, p_7, l_1747), g_381, p_9)),

g_894))) != l_1747) <= p_10)), p_8))),g_11) <= p_7)), g_1315, p_8), 10))) > 4UL),

0x93L)) ^ g_1161))) <= l_1750), p_8), g_458); g_1122 = (l_1749



g_1487)); g_146 = 0xBC806FC8L; l_1788 =

func_24 (((safe_13 ((~ (((g_458 !=

(safe_54 (((~ (l_1749 = (safe_43 ((safe_14



(l_1749, g_894, (safe_23 (l_1749,

(l_1167 = g_1161))), ((l_1469 =

Exposes an LLVM miscompilation bug

Part of an 89 KB program randomly generated by Csmith

8

(safe_47 (((g_1161 ^ 0x2893L), (((safe_9 ((!l_1747),

((l_1749 = func_37 (func_58

((l_1748, 4294967293UL), g_110, l_1168, p_7, l_1747), g_381, p_9)),

g_894))) != l_1747) <= p_10)), p_8))),g_11) <= p_7)), g_1315, p_8), 10))) > 4UL),

0x93L)) ^ g_1161))) <= l_1750), p_8), g_458); g_1122 = (l_1749



g_1487)); g_146 = 0xBC806FC8L; l_1788 =

func_24 (((safe_13 ((~ (((g_458 !=

(safe_54 (((~ (l_1749 = (safe_43 ((safe_14



(l_1749, g_894, (safe_23 (l_1749,

(l_1167 = g_1161))), ((l_1469 =

Part of an 89 KB program randomly generated by Csmith

Exposes an LLVM miscompilation bug

Csmith is our C compiler fuzzer

~425 compiler bugs discovered and reported so far

http://embed.cs.utah.edu/csmith/

9

short a;int b, d, k, l, m;

int fn1 (short p1, int p2) { return p1 - p2;}

int fn2 (short p1) { return p1 == 0 || a == 0 ? a : a % p1;}

int fn3 (int p1, unsigned short p2) { return p2 == 0 ? 0 : p1 % p2;}

void fn4 (unsigned char p1) { if (p1) b = 1;}

void fn5 (p1) { short n = 60018; k = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m);}

int main () { fn5 (2); printf ("%d\n", b); return 0;}

$ clang -O1 foo.c ; ./a.out 0$ clang -O2 foo.c ; ./a.out 1

10








$ clang -O1 foo.c ; ./a.out 0$ clang -O2 foo.c ; ./a.out 1

Reported as LLVM PR 12189 on 3/5/2012, fixed same day

11

int printf (const char *, …);

char f[] = { -9L };

int main (void) { printf ("%d\n", 255 | f[0]); }

Intel CC 12.0.5 for x86-64 is wrong at “-Ofast -ipo”

12

int printf (const char *, ...);

const union { short f1; int f2 : 13;} a = { 30155 };

int main (void) { printf ("%d\n", a.f1); printf ("%d\n", a.f2); return 0;}

GCC 4.4.3 from Ubuntu 10.04 LTS for x86-64 is wrong at -O1

13

int printf (const char *, ...);

int b;

static struct S0 { int f2 : 1;} c;

int main (void) { b = -1 ^ c.f2; printf ("%d\n", b);}

GCC from mid-Feb 2012 for x86-64 is wrong at -O0(PR 52209)

14

Beautiful test case ==1. Small (but not obfuscated)2. Well-defined3. Unambiguously elicits bad behavior4. Makes the compiler bug obvious

Not all bugs have beautiful test cases

15








Delta Debugging algorithm “ddmin” ==• Greedy search• Delete chunks• Test results for

“interestingness”• Keep reducing

granularity

16








ddmin ==• Greedy search• Delete chunks• Test for


granularity

Is this still interesting?

17

#!/bin/bashclang -O1 test.c &&\./a.out > out1.txt &&\clang -O2 test.c &&\./a.out > out2.txt &&\! diff out1.txt out2.txt

18










granularity

Assume not interesting, so we backtrack

19










granularity

20










granularity

21










granularity

22










granularity

23










granularity

24










granularity

25

• Problem: ddmin gets stuck–Output is often not beautiful

• Our premise: –The Delta Debugging concept is sound –But ddmin is too hard-coded, inflexible

26

Generalized Delta Debugging

1. One or more transformations– E.g. “remove chunks of text”

2. Interestingness test3. Fitness function– E.g. “prefer smaller test cases”

4. Search algorithm– E.g. “greedy”

27

Generalized Delta Debugging

1. One or more transformations– E.g. “remove chunks of text”

2. Interestingness test3. Fitness function– E.g. “prefer smaller test cases”

4. Search algorithm– E.g. “greedy”

28

C-Reduce: A Generalized Delta Debugger• Fixpoint computation• Parameterized by– Interestingness test– Plugins that perform transformations

Plugins are iterators implementing:• reset()• advance()• transform(file)

29



30

void fn5 (p1) { short 0 = 60018; k = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m);}


31



32

void fn5 (p1) { short n = 60018; 0 = fn2 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m);}


33

void fn5 (p1) { short n = 60018; k = 0 (p1); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m);}


34

void fn5 (p1) { short n = 60018; k = fn2 (0); d = k != 0; l = fn1 (d, n); m = fn3 (l, p1); fn4 (m);}


35

Example transformations:• Remove chunks of text (like ddmin)• C-specific peephole transformations–0xfeedbeefULL 1– x ^= y x = y – (x + 1) x + 1–while (…) if (…)– x ? y : z y

C-Reduce has 49 plugins so far

36

• Some transformations make coordinated changes across different parts of the program– Inline a function call– Scalar replacement of aggregates–Un-nest nested function calls–Remove dead arguments–Make function return void–Reduce array dimension or pointer level– Shorten identifier name

• These use Clang as a rewriter

37

transformationtransformationtransformationtransformationtransformation

current test case

interesting?

yes

no

38

Beautiful test case ==1. Small (but not obfuscated)2. Well-defined3. Unambiguously elicits bad behavior4. Makes the compiler bug obvious

39

#include <iostream>using namespace std;int r[3], x[3], y[3];int main() { int xa=2,ya=5,xb=4,yb=2,n=3; x[0] = 3; x[1] = 5; x[2] = 1; y[0] = 1; y[1] = 3; y[2] = 3; r[0] = 2; r[1] = 1; r[2] = 2; int tcount = 0; for (int k=min(xa,xb); k<=max(xa,xb); k++) { bool found1,found2 = false; for (int j=0; j<n; j++) { if (((k-x[j])*(k-x[j])+(y[j]-ya)*(y[j]-ya))<=r[j]*r[j]) { found1 = true; } if (((k-x[j])*(k-x[j])+(y[j]-yb)*(y[j]-yb))<=r[j]*r[j]) { found2 = true; } if (found1 && found2) break; } if (!found1) tcount++; if (!found2) tcount++; } cout << tcount << endl; return 0; }

GCC PR 51962

40


GCC PR 51962

Bug report says:

“Compile the following simple code without -O3, and run.

Now compile it with -O3 option (for optimization), run again.

Surprisingly 2 different outputs appear.”

41


GCC PR 51962

Bug report says:

“Compile the following simple code without -O3, and run.

Now compile it with -O3 option (for optimization), run again.

Surprisingly 2 different outputs appear.”

GCC developer responds:

“You do not initialise found1.”

PR 51962 is RESOLVED INVALID

And this person may have a hard time getting someone to read his next bug report

42


GCC PR 51962

43

• C99 has –191 kinds of undefined behavior–52 kinds of unspecified behavior

• Code in a bug report must not execute these behaviors

• ddmin and C-Reduce tend to introduce these behaviors

44

#!/bin/bashis_valid_c99 test.c &&\clang -O1 test.c &&\./a.out > out1.txt &&\clang -O2 test.c &&\./a.out > out2.txt &&\! diff out1.txt out2.txt

45

Validity checkers:

KCC: executable semantics for C99http://code.google.com/p/c-semantics/

Frama-C: static analyzer that supports an interpreter modehttp://frama-c.com/

46

Median size output from reducers

• For 57 compiler-crash bugs– ddmin*: 8.6 KB– C-Reduce: 0.33 KB

• For 43 wrong-code bugs– ddmin*: 6.5 KB– C-Reduce: 0.51 KB

*http://delta.tigris.org/

47

• Reduction fails when compiler is non-deterministic–Usually due to memory unsafety + ASLR

• Reduction works poorly when compiler bug stems from a resource overflow– Infinite loop, memory leak, register spill

48

Can C-Reduce create reportable test cases from proprietary codes?• We hope so• Clearly “declassification” has to be

done on a case-by-case basis• We would love to get feedback

about successes and failures

49

• Does C-Reduce work for C++ code?–Yes, but not (yet) very well

• C++-specific transformations are needed–Template instantiation–Namespace and class hierarchy flattening

• Problem: No validity checker for C++

50

• Factoring the transformations out of a delta debugger is a good idea

• C-Reduce can (sometimes) produce beautiful test cases

• Ensuring validity of test cases is hard

51

C-Reduce is here:https://github.com/csmith-project/creduce

Documents

Towards Beautiful Test Cases for Compiler Bugs