Skip to main content

Common Errors

In [1]:
%run prelude.ipy
In [17]:
def common_errors(base, version=None, threshold=2):
    ts = util.filter_program(trials, base, version)
    ts = ts[-ts.grade_correct]
    clusters = util.grading.cluster_responses(ts.pred_output.values)
    return [c for c in clusters if c[1] > threshold]
In [46]:
x = programs[["base", "version"]]
x["output"] = x.apply(lambda r: data.hansen_2012.program_output(r["base"], r["version"]), axis=1)
x["count"] = x.apply(lambda r: len(util.filter_program(trials, r["base"], r["version"])), axis=1)
x["base_count"] = x.apply(lambda r: len(util.filter_program(trials, r["base"])), axis=1)
x
Out[46]:
base version output count base_count
0 between functions [[8, 7, 9]\n, [1, 0, 8, 1]\n, [8, 9, 0]\n] 69 159
1 between inline [[8, 7, 9]\n, [1, 0, 8, 1]\n, [8, 9, 0]\n] 90 159
2 counting nospace [The count is 1\n, Done counting\n, The count ... 88 161
3 counting twospaces [The count is 1\n, Done counting\n, The count ... 73 161
4 funcall nospace [60\n] 52 162
5 funcall space [60\n] 58 162
6 funcall vars [60\n] 52 162
7 initvar bothbad [0\n, 11\n] 52 160
8 initvar good [24\n, 10\n] 54 160
9 initvar onebad [24\n, 11\n] 54 160
10 order inorder [5 2 7\n] 72 161
11 order shuffled [5 2 7\n] 89 161
12 overload multmixed [12\n, 14\n, 53\n] 59 161
13 overload plusmixed [7\n, 9\n, 53\n] 47 161
14 overload strings [hibye\n, streetpenny\n, 53\n] 55 161
15 partition balanced [1 low\n, 2 low\n, 4 high\n, 5 high\n] 50 160
16 partition unbalanced [1 low\n, 2 low\n, 4 high\n] 55 160
17 partition unbalanced_pivot [1 low\n, 2 low\n, 4 high\n] 55 160
18 rectangle basic [100\n, 25\n] 47 159
19 rectangle class [100\n, 25\n] 58 159
20 rectangle tuples [100\n, 25\n] 54 159
21 scope diffname [4\n] 83 159
22 scope samename [4\n] 76 159
23 whitespace linedup [0 1\n, 1 6\n, 2 11\n] 79 160
24 whitespace zigzag [0 1\n, 1 6\n, 2 11\n] 81 160
In [18]:
common_errors("between")
Out[18]:
[({u'879\n1081\n8',
   u'87910818',
   u'[8,7,9]\n[1,0,8,1]\n[8]',
   '[8,7,9]\n[1,0,8,1]\n[8]\n'},
  35),
 ({u'879\n108\n890', u'879108890', '[8,7,9]\n[1,0,8]\n[8,9,0]'}, 3),
 ({u'879\n1081\n089', u'8791081089', '[8,7,9]\n[1,0,8,1]\n[0,8,9]'}, 3),
 ({u'879\n1081\n', u'8791081', '[8, 7, 9]\n[1, 0, 8, 1]\n[]'}, 3)]
In [21]:
common_errors("counting")
Out[21]:
[({u'The count is,1\nThe count is,2\nThe count is,3\nThe count is,4\nDone counting',
   'The count is,1\nThe count is,2\nThe count is,3\nThe count is,4\nDone counting\n',
   u'thecountis1\nthecountis2\nthecountis3\nthecountis4\ndonecounting',
   u'thecountis1thecountis2thecountis3thecountis4donecounting'},
  56)]
In [22]:
common_errors("funcall")
Out[22]:
[({'0'}, 4), ({'-60'}, 4), ({'-80'}, 3)]
In [26]:
for v in programs[programs.base == "initvar"].version:
    print v, common_errors("initvar", v)
bothbad [(set([u'000024711', '0\n0\n0\n0\n2\n4\n7\n11']), 3), (set([u'010', '0\n10']), 3)]
good [(set([u'12341234', '1\n2\n3\n4\n1\n2\n3\n4']), 3)]
onebad [(set([u'12342345', '[1, 2, 3, 4]\n[2, 3, 4, 5]\n', u'1234\n2345', u'[1, 2, 3, 4]\n[2, 3, 4, 5]']), 3), (set([u'2410', '24\n10']), 3)]

In [27]:
common_errors("order")
Out[27]:
[({'5 2 10', u'5210'}, 3)]
In [28]:
for v in programs[programs.base == "overload"].version:
    print v, common_errors("overload", v)
multmixed [(set(['12\n14\n8', u'12148']), 6)]
plusmixed [(set([u'798', '7\n9\n8']), 4)]
strings [(set([u'hibyestreetpenny8', 'hibye\nstreetpenny\n8']), 8)]

In [30]:
for v in programs[programs.base == "partition"].version:
    print v, common_errors("partition", v)
balanced [(set([u'lowlowhighhigh', 'low\nlow\nhigh\nhigh']), 12), (set([u'1low2low3high4high5high', u'1low\n2low\n3high\n4high\n5high', '1 low\n2 low\n3 high\n4 high\n5 high']), 3)]
unbalanced [(set(['low low high', u'lowlowhigh']), 8)]
unbalanced_pivot [(set(['low low high', u'lowlowhigh']), 11)]

In [33]:
common_errors("rectangle")
Out[33]:
[]
In [36]:
trials[(trials.base == "rectangle") & -trials.grade_correct].pred_output
Out[36]:
320     100\n2500
393     100\n2500
450         0\n25
689       100\n50
854       1\n\n25
1026        error
1036        Error
1233        0\n25
Name: pred_output, dtype: object
In [38]:
common_errors("scope")
Out[38]:
[({u'22', '22\n'}, 69)]
In [39]:
common_errors("whitespace")
Out[39]:
[({'0,5\n1,10\n2,15', u'05\n110\n215', u'05110215'}, 3)]