[ASAN] Use struct instead of array in sancov.py

[slthakur]

Summary

When using 32-bit python with 64-bit asan the pc array in sancov.py cannot fit in 64-bit pc's because the type-code 'L' for arrays in python corresponds to the C type long which is only of 4 bytes. Because of this some of the coverage tool tests fail on mips. To fix these test possible solutions are to use 64-bit python or use struct.unpack with the 'Q' type-code. We have used struct.unpack with 'Q' type code since it is not appropriate to have a 64-bit python on all hosts.

[slthakur]

Added similar change to MergeAndPrint and ReadOneFIle

[slthakur]

Gentle reminder.

[kcc]

I'd like to eventually deprecate the use of sancov.py in favor of the C++-based sancov.
Does this tool work for you?

[slthakur]

Yes, sancov.py works properly for me.

[kcc]

In D18817#402243, @Sagar wrote:

Yes, sancov.py works properly for me.

That wasn't my question.
I asked whether the new C++-based sancov (no .py) works for you.
http://clang.llvm.org/docs/SanitizerCoverage.html#sancov-tool

[aizatsky]

Yes, making sancov.cc working for you would be ideal.

As for this change, it does make existing code better in terms of bitness flexibility.

[slthakur]

Committed in revision 267126

lib/sanitizer_common/scripts/sancov.py

 def CheckBits(bits):
   if bits != 32 and bits != 64:
     raise Exception("Wrong bitness: %d" % bits)
 
 def TypeCodeForBits(bits):
   CheckBits(bits)
   return 'L' if bits == 64 else 'I'
 
+def TypeCodeForStruct(bits):
+  CheckBits(bits)
+  return 'Q' if bits == 64 else 'I'
+
 kMagic32SecondHalf = 0xFFFFFF32;
 kMagic64SecondHalf = 0xFFFFFF64;
 kMagicFirstHalf    = 0xC0BFFFFF;
 
 def MagicForBits(bits):
   CheckBits(bits)
   if sys.byteorder == 'little':
     return [kMagic64SecondHalf if bits == 64 else kMagic32SecondHalf, kMagicFirstHalf]
def ReadOneFile(path):
   with open(path, mode="rb") as f:
     f.seek(0, 2)
     size = f.tell()
     f.seek(0, 0)
     if size < 8:
       raise Exception('File %s is short (< 8 bytes)' % path)
     bits = ReadMagicAndReturnBitness(f, path)
     size -= 8
-    s = array.array(TypeCodeForBits(bits), f.read(size))
+    s = struct.unpack_from(TypeCodeForStruct(bits) * (size * 8 / bits), f.read(size))
   print >>sys.stderr, "%s: read %d %d-bit PCs from %s" % (prog_name, size * 8 / bits, bits, path)
   return s
 
 def Merge(files):
   s = set()
   for f in files:
     s = s.union(set(ReadOneFile(f)))
   print >> sys.stderr, "%s: %d files merged; %d PCs total" % \

 def MergeAndPrint(files):
   if sys.stdout.isatty():
     Usage()
   s = Merge(files)
   bits = 32
   if max(s) > 0xFFFFFFFF:
     bits = 64
   array.array('I', MagicForBits(bits)).tofile(sys.stdout)
-  a = array.array(TypeCodeForBits(bits), s)
-  a.tofile(sys.stdout)
+  a = struct.pack(TypeCodeForStruct(bits) * len(s), *s)
+  sys.stdout.write(a)
 
 
 def UnpackOneFile(path):
   with open(path, mode="rb") as f:
     print >> sys.stderr, "%s: unpacking %s" % (prog_name, path)
     while True:
       header = f.read(12)
       if not header: return
def UnpackOneRawFile(path, map_path):
   mem_map_keys = [m[0] for m in mem_map]
 
   with open(path, mode="rb") as f:
     print >> sys.stderr, "%s: unpacking %s" % (prog_name, path)
 
     f.seek(0, 2)
     size = f.tell()
     f.seek(0, 0)
-    pcs = array.array(TypeCodeForBits(bits), f.read(size))
+    pcs = struct.unpack_from(TypeCodeForStruct(bits) * (size * 8 / bits), f.read(size))
     mem_map_pcs = [[] for i in range(0, len(mem_map))]
 
     for pc in pcs:
       if pc == 0: continue
       map_idx = bisect.bisect(mem_map_keys, pc) - 1
       (start, end, base, module_path) = mem_map[map_idx]
       assert pc >= start
       if pc >= end:
         print >> sys.stderr, "warning: %s: pc %x outside of any known mapping" % (prog_name, pc)
         continue
       mem_map_pcs[map_idx].append(pc - base)
 
     for ((start, end, base, module_path), pc_list) in zip(mem_map, mem_map_pcs):
       if len(pc_list) == 0: continue
       assert path.endswith('.sancov.raw')
       dst_path = module_path + '.' + os.path.basename(path)[:-4]
       print >> sys.stderr, "%s: writing %d PCs to %s" % (prog_name, len(pc_list), dst_path)
-      arr = array.array(TypeCodeForBits(bits))
-      arr.fromlist(sorted(pc_list))
-      with open(dst_path, 'ab') as f2:
+      sorted_pc_list = sorted(pc_list)
+      pc_buffer = struct.pack(TypeCodeForStruct(bits) * len(pc_list), *sorted_pc_list)
+      with open(dst_path, 'ab+') as f2:
         array.array('I', MagicForBits(bits)).tofile(f2)
-        arr.tofile(f2)
+        f2.seek(0, 2)
+        f2.write(pc_buffer)
 
 def RawUnpack(files):
   for f in files:
     if not f.endswith('.sancov.raw'):
       raise Exception('Unexpected raw file name %s' % f)
     f_map = f[:-3] + 'map'
     UnpackOneRawFile(f, f_map)