Merge pull request #520 from google/google_sync

Google sync

pytype/pyi/lexer.lex

@@ -119,6 +119,18 @@ typedef pytype::parser::token t;
   yylval->obj=PyInt_FromString(yytext, NULL, 10);
   return t::NUMBER;
 }
+[-+]?0b[01]+  {
+  yylval->obj=PyInt_FromString(yytext, NULL, 2);
+  return t::NUMBER;
+}
+[-+]?0o[0-7]+  {
+  yylval->obj=PyInt_FromString(yytext, NULL, 8);
+  return t::NUMBER;
+}
+[-+]?0x[0-9a-fA-F]+  {
+  yylval->obj=PyInt_FromString(yytext, NULL, 16);
+  return t::NUMBER;
+}
 [-+]?[0-9]*\.[0-9]+  {
   yylval->obj=PyFloat_FromDouble(atof(yytext));
   return t::NUMBER;

pytype/pyi/lexer_test.py

@@ -6,6 +6,9 @@
 
 import unittest
 
+# We use '\' to make test code more readable:
+# pylint: disable=g-backslash-continuation
+
 # Map from token code to name.
 TOKEN_NAMES = {code: name for name, code in parser_ext.TOKENS.items()}
 
@@ -130,6 +133,23 @@ def test_number(self):
     self.check([0.5], "+.5")
     self.check([-0.5], "-.5")
 
+  def test_number_base(self):
+    self.check([0], "0b0")
+    self.check([1], "0b1")
+    self.check([42], "0b101010")
+    self.check([-8], "-0b1000")
+
+    self.check([1], "0o1")
+    self.check([8], "0o10")
+    self.check([42], "0o52")
+    self.check([-7], "-0o7")
+
+    self.check([1], "0x1")
+    self.check([240], "0xF0")
+    self.check([-240], "-0xF0")
+    self.check([15], "0x0f")
+    self.check([-15], "-0x0f")
+
   def test_line_numbers(self):
     self.check([("NAME", "a", 1), ("NAME", "b", 2)], "a\nb")
 

pytype/tools/traces/CMakeLists.txt

@@ -44,6 +44,8 @@ py_library(
     source
   SRCS
     source.py
+  DEPS
+    pytype.utils
 )
 
 py_test(

pytype/tools/traces/source.py

@@ -4,6 +4,9 @@
 from __future__ import print_function
 import collections
 
+from pytype import compat
+
+
 Location = collections.namedtuple("Location", ("line", "column"))
 
 
@@ -50,7 +53,9 @@ def _init_byte_offsets(self):
     offset = 0
     for line in self._lines:
       self._offsets.append(offset)
-      offset += len(line) + 1  # account for the \n
+      # convert line to bytes
+      bytes_ = compat.bytestring(line)
+      offset += len(bytes_) + 1  # account for the \n
 
   def get_offset(self, location):
     """Gets the utf-8 byte offset of a source.Location from start of source."""

pytype/tools/traces/source_test.py

@@ -1,3 +1,4 @@
+# coding=utf-8
 # Lint as: python2, python3
 """Tests for traces.source."""
 
@@ -52,6 +53,23 @@ def test_get_offset(self):
     src = source.Code("line1\nline2", [], _FakeTrace, "")
     self.assertEqual(src.get_offset(source.Location(2, 3)), 9)
 
+  def test_get_offset_multibyte(self):
+    # With single-byte characters
+    src = source.Code("""\
+      # coding=utf-8
+      line1 # a
+      line2
+    """, [], _FakeTrace, "")
+    self.assertEqual(src.get_offset(source.Location(3, 3)), 40)
+
+    # With a multibyte character the byte offset should change
+    src = source.Code("""\
+      # coding=utf-8
+      line1 # ツ
+      line2
+    """, [], _FakeTrace, "")
+    self.assertEqual(src.get_offset(source.Location(3, 3)), 42)
+
   def test_line(self):
     src = source.Code("line1\nline2", [], _FakeTrace, "")
     self.assertEqual(src.line(2), "line2")