All patches

setup.py

@@ -11,7 +11,7 @@
     url='https://github.com/Khan/tinyquery',
     keywords=['bigquery'],
     packages=['tinyquery'],
-    install_requires=['arrow==0.12.1', 'ply==3.10'],
+    install_requires=['arrow==0.12.1', 'ply==3.10', 'six==1.11.0'],
     classifiers=[
         'License :: OSI Approved :: MIT License',
         'Programming Language :: Python :: 2',

tinyquery/api_client.py

@@ -4,9 +4,13 @@
 
 This can be used in place of the value returned by apiclient.discovery.build().
 """
+from __future__ import absolute_import
+
 import functools
 import json
 
+import six
+
 
 class TinyQueryApiClient(object):
     def __init__(self, tq_service):
@@ -36,7 +40,7 @@ def __init__(self, func, args, kwargs):
         self.args = args
         self.kwargs = kwargs
 
-    def execute(self):
+    def execute(self, **kwargs):
         return self.func(*self.args, **self.kwargs)
 
 
@@ -153,7 +157,7 @@ def insert(self, projectId, body):
                 create_disposition, write_disposition)
         else:
             assert False, 'Unknown job type: {}'.format(
-                body['configuration'].keys())
+                list(body['configuration'].keys()))
 
     @staticmethod
     def _get_config_table(config, key):
@@ -225,16 +229,16 @@ def schema_from_table(table):
     """Given a tinyquery.Table, build an API-compatible schema."""
     return {'fields': [
         {'name': name, 'type': col.type}
-        for name, col in table.columns.iteritems()
+        for name, col in table.columns.items()
     ]}
 
 
 def rows_from_table(table):
     """Given a tinyquery.Table, build an API-compatible rows object."""
     result_rows = []
-    for i in xrange(table.num_rows):
+    for i in six.moves.xrange(table.num_rows):
         field_values = [{'v': str(col.values[i])}
-                        for col in table.columns.itervalues()]
+                        for col in table.columns.values()]
         result_rows.append({
             'f': field_values
         })

tinyquery/api_client_test.py

@@ -1,8 +1,10 @@
+from __future__ import absolute_import
+
 import unittest
 
-import api_client
-import tq_types
-import tinyquery
+from tinyquery import api_client
+from tinyquery import tq_types
+from tinyquery import tinyquery
 
 
 class ApiClientTest(unittest.TestCase):
@@ -144,7 +146,7 @@ def test_table_copy(self):
             }
         ).execute()
 
-        for _ in xrange(5):
+        for _ in range(5):
             self.tq_service.jobs().insert(
                 projectId='test_project',
                 body={

tinyquery/compiler.py

@@ -4,19 +4,20 @@
 -Validate that the expression is well-typed.
 -Resolve all select fields to their aliases and types.
 """
+from __future__ import absolute_import
+
 import collections
 import itertools
 
-import parser
-import runtime
-import tq_ast
-import typed_ast
-import type_context
-import tq_types
-
+import six
 
-class CompileError(Exception):
-    pass
+from tinyquery import exceptions
+from tinyquery import parser
+from tinyquery import runtime
+from tinyquery import tq_ast
+from tinyquery import typed_ast
+from tinyquery import type_context
+from tinyquery import tq_types
 
 
 def compile_text(text, tables_by_name):
@@ -93,7 +94,7 @@ def expand_select_fields(self, select_fields, table_expr):
         """
         table_ctx = table_expr.type_ctx
         star_select_fields = []
-        for table_name, col_name in table_ctx.columns.iterkeys():
+        for table_name, col_name in table_ctx.columns:
             if table_name is not None:
                 col_ref = table_name + '.' + col_name
             else:
@@ -112,9 +113,9 @@ def expand_select_fields(self, select_fields, table_expr):
             elif (field.expr and isinstance(field.expr, tq_ast.ColumnId) and
                   field.expr.name.endswith('.*')):
                 prefix = field.expr.name[:-len('.*')]
-                record_star_fields = filter(
-                    lambda f: f.alias.startswith(prefix),
-                    star_select_fields)
+                record_star_fields = [f
+                                      for f in star_select_fields
+                                      if f.alias.startswith(prefix)]
                 result_fields.extend(record_star_fields)
             else:
                 result_fields.append(field)
@@ -213,7 +214,7 @@ def compile_table_expr(self, table_expr):
             return method(table_expr)
 
     def compile_table_expr_TableId(self, table_expr):
-        import tinyquery  # TODO(colin): fix circular import
+        from tinyquery import tinyquery  # TODO(colin): fix circular import
         table = self.tables_by_name[table_expr.name]
         if isinstance(table, tinyquery.Table):
             return self.compile_table_ref(table_expr, table)
@@ -225,7 +226,7 @@ def compile_table_expr_TableId(self, table_expr):
     def compile_table_ref(self, table_expr, table):
         alias = table_expr.alias or table_expr.name
         columns = collections.OrderedDict([
-            (name, column.type) for name, column in table.columns.iteritems()
+            (name, column.type) for name, column in table.columns.items()
         ])
         type_ctx = type_context.TypeContext.from_table_and_columns(
             alias, columns, None)
@@ -265,8 +266,7 @@ def compile_table_expr_Join(self, table_expr):
             [table_expr.base],
             (join_part.table_expr for join_part in table_expr.join_parts)
         )
-        compiled_result = map(self.compile_joined_table,
-                              table_expressions)
+        compiled_result = [self.compile_joined_table(x) for x in table_expressions]
         compiled_table_exprs, compiled_aliases = zip(*compiled_result)
         type_contexts = [compiled_table.type_ctx
                          for compiled_table in compiled_table_exprs]
@@ -280,9 +280,11 @@ def compile_table_expr_Join(self, table_expr):
             type_contexts)
         return typed_ast.Join(
             base=compiled_table_exprs[0],
-            tables=zip(compiled_table_exprs[1:],
-                       (join_part.join_type
-                        for join_part in table_expr.join_parts)),
+            # wrapping in list() for python 3 support (shouldn't be a large number
+            # of items so performance impact should be minimal)
+            tables=list(zip(compiled_table_exprs[1:],
+                            (join_part.join_type
+                             for join_part in table_expr.join_parts))),
             conditions=result_fields,
             type_ctx=result_type_ctx)
 
@@ -294,7 +296,7 @@ def compile_joined_table(self, table_expr):
         elif isinstance(table_expr, tq_ast.TableId):
             alias = table_expr.name
         else:
-            raise CompileError('Table expression must have an alias name.')
+            raise exceptions.CompileError('Table expression must have an alias name.')
         result_ctx = compiled_table.type_ctx.context_with_full_alias(alias)
         compiled_table = compiled_table.with_type_ctx(result_ctx)
         return compiled_table, alias
@@ -366,7 +368,7 @@ def compile_join_field(expr, join_type):
                                                      left_column_id)]
                     # Fall through to the error case if the aliases are the
                     # same for both sides.
-            raise CompileError('JOIN conditions must consist of an AND of = '
+            raise exceptions.CompileError('JOIN conditions must consist of an AND of = '
                                'comparisons between two field on distinct '
                                'tables. Got expression %s' % expr)
         return [compile_join_field(expr, join_type)
@@ -434,7 +436,7 @@ def compile_groups(self, groups, select_fields, aliases, table_ctx):
     def compile_select_field(self, expr, alias, within_clause, type_ctx):
         if within_clause is not None and within_clause != 'RECORD' and (
                     expr.args[0].name.split('.')[0] != within_clause):
-            raise CompileError('WITHIN clause syntax error')
+            raise exceptions.CompileError('WITHIN clause syntax error')
         else:
             compiled_expr = self.compile_expr(expr, type_ctx)
             return typed_ast.SelectField(compiled_expr, alias, within_clause)
@@ -467,7 +469,7 @@ def compile_Literal(self, expr, type_ctx):
             return typed_ast.Literal(expr.value, tq_types.INT)
         if isinstance(expr.value, float):
             return typed_ast.Literal(expr.value, tq_types.FLOAT)
-        elif isinstance(expr.value, basestring):
+        elif isinstance(expr.value, six.string_types):
             return typed_ast.Literal(expr.value, tq_types.STRING)
         elif expr.value is None:
             return typed_ast.Literal(expr.value, tq_types.NONETYPE)
@@ -485,7 +487,7 @@ def compile_UnaryOperator(self, expr, type_ctx):
         try:
             result_type = func.check_types(compiled_val.type)
         except TypeError:
-            raise CompileError('Invalid type for operator {}: {}'.format(
+            raise exceptions.CompileError('Invalid type for operator {}: {}'.format(
                 expr.operator, [compiled_val.type]))
         return typed_ast.FunctionCall(func, [compiled_val], result_type)
 
@@ -501,7 +503,7 @@ def compile_BinaryOperator(self, expr, type_ctx):
             result_type = func.check_types(compiled_left.type,
                                            compiled_right.type)
         except TypeError:
-            raise CompileError('Invalid types for operator {}: {}'.format(
+            raise exceptions.CompileError('Invalid types for operator {}: {}'.format(
                 expr.operator, [arg.type for arg in [compiled_left,
                                                      compiled_right]]))
 
@@ -516,7 +518,7 @@ def compile_FunctionCall(self, expr, type_ctx):
         # that the evaluator knows to change the context.
         if self.is_innermost_aggregate(expr):
             if type_ctx.aggregate_context is None:
-                raise CompileError('Unexpected aggregate function.')
+                raise exceptions.CompileError('Unexpected aggregate function.')
             sub_expr_ctx = type_ctx.aggregate_context
             ast_type = typed_ast.AggregateFunctionCall
         else:
@@ -530,7 +532,7 @@ def compile_FunctionCall(self, expr, type_ctx):
             result_type = func.check_types(
                 *(arg.type for arg in compiled_args))
         except TypeError:
-            raise CompileError('Invalid types for function {}: {}'.format(
+            raise exceptions.CompileError('Invalid types for function {}: {}'.format(
                 expr.name, [arg.type for arg in compiled_args]))
         return ast_type(func, compiled_args, result_type)
 
@@ -557,7 +559,7 @@ def get_aliases(cls, select_field_list):
         for alias in proposed_aliases:
             if alias is not None:
                 if alias in used_aliases:
-                    raise CompileError(
+                    raise exceptions.CompileError(
                         'Ambiguous column name {}.'.format(alias))
                 used_aliases.add(alias)
 

tinyquery/compiler_test.py

@@ -1,18 +1,21 @@
 # TODO(colin): fix these lint errors (http://pep8.readthedocs.io/en/release-1.7.x/intro.html#error-codes)
 # pep8-disable:E122
+from __future__ import absolute_import
+
 import collections
 import datetime
 import unittest
 
-import compiler
-import context
-import runtime
-import tinyquery
-import tq_ast
-import tq_modes
-import tq_types
-import type_context
-import typed_ast
+from tinyquery import exceptions
+from tinyquery import compiler
+from tinyquery import context
+from tinyquery import runtime
+from tinyquery import tinyquery
+from tinyquery import tq_ast
+from tinyquery import tq_modes
+from tinyquery import tq_types
+from tinyquery import type_context
+from tinyquery import typed_ast
 
 
 class CompilerTest(unittest.TestCase):
@@ -124,7 +127,7 @@ def assert_compiled_select(self, text, expected_ast):
         self.assertEqual(expected_ast, ast)
 
     def assert_compile_error(self, text):
-        self.assertRaises(compiler.CompileError, compiler.compile_text,
+        self.assertRaises(exceptions.CompileError, compiler.compile_text,
                           text, self.tables_by_name)
 
     def make_type_context(self, table_column_type_triples,
@@ -178,7 +181,7 @@ def test_unary_operator(self):
         )
 
     def test_mistyped_unary_operator(self):
-        with self.assertRaises(compiler.CompileError) as context:
+        with self.assertRaises(exceptions.CompileError) as context:
             compiler.compile_text('SELECT -strings FROM rainbow_table',
                                   self.tables_by_name)
         self.assertTrue('Invalid type for operator' in str(context.exception))
@@ -187,12 +190,12 @@ def test_strange_arithmetic(self):
         try:
             compiler.compile_text('SELECT times + ints + floats + bools FROM '
                                   'rainbow_table', self.tables_by_name)
-        except compiler.CompileError:
+        except exceptions.CompileError:
             self.fail('Compiler exception on arithmetic across all numeric '
                       'types.')
 
     def test_mistyped_binary_operator(self):
-        with self.assertRaises(compiler.CompileError) as context:
+        with self.assertRaises(exceptions.CompileError) as context:
             compiler.compile_text('SELECT ints CONTAINS strings FROM '
                                   'rainbow_table',
                                   self.tables_by_name)
@@ -241,7 +244,7 @@ def test_function_calls(self):
         )
 
     def test_mistyped_function_call(self):
-        with self.assertRaises(compiler.CompileError) as context:
+        with self.assertRaises(exceptions.CompileError) as context:
             compiler.compile_text('SELECT SUM(strings) FROM rainbow_table',
                                   self.tables_by_name)
         self.assertTrue('Invalid types for function' in str(context.exception))
@@ -1001,7 +1004,7 @@ def test_within_clause(self):
                     self.make_type_context([]))))
 
     def test_within_clause_error(self):
-        with self.assertRaises(compiler.CompileError) as context:
+        with self.assertRaises(exceptions.CompileError) as context:
             compiler.compile_text(
                 'SELECT r1.s, COUNT(r1.s) WITHIN r2 AS '
                 'num_s_in_r1 FROM record_table',