From 05f6781c4b501a6a9244ac527bb1dcf3dd064770 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 9 Apr 2021 12:21:48 +0100 Subject: [PATCH 001/180] [vec] Basic vec support --- mypy/plugins/default.py | 1 + mypyc/codegen/emit.py | 9 + mypyc/codegen/emitmodule.py | 13 + mypyc/ir/rtypes.py | 87 ++++ mypyc/irbuild/builder.py | 19 +- mypyc/irbuild/expression.py | 101 +++- mypyc/irbuild/for_helpers.py | 21 +- mypyc/irbuild/ll_builder.py | 48 ++ mypyc/irbuild/mapper.py | 3 + mypyc/irbuild/specialize.py | 29 +- mypyc/irbuild/vec.py | 412 +++++++++++++++++ mypyc/lib-rt/CPy.h | 1 + mypyc/lib-rt/vecs.h | 1 + mypyc/primitives/list_ops.py | 2 +- mypyc/rt_subtype.py | 7 + mypyc/sametype.py | 4 + mypyc/subtype.py | 5 + mypyc/test-data/irbuild-vecs.test | 737 ++++++++++++++++++++++++++++++ mypyc/test-data/refcount.test | 152 ++++++ mypyc/test/test_irbuild.py | 1 + test-data/unit/fixtures/dict.pyi | 7 +- test-data/unit/lib-stub/vecs.pyi | 21 + vecs.pyi | 24 + 23 files changed, 1689 insertions(+), 16 deletions(-) create mode 100644 mypyc/irbuild/vec.py create mode 120000 mypyc/lib-rt/vecs.h create mode 100644 mypyc/test-data/irbuild-vecs.test create mode 100644 test-data/unit/lib-stub/vecs.pyi create mode 100644 vecs.pyi diff --git a/mypy/plugins/default.py b/mypy/plugins/default.py index 57909ce7c7309..71cd6fd14efa5 100644 --- a/mypy/plugins/default.py +++ b/mypy/plugins/default.py @@ -227,6 +227,7 @@ def len_callback(ctx: FunctionContext) -> Type: return ctx.default_return_type + def typed_dict_get_signature_callback(ctx: MethodSigContext) -> CallableType: """Try to infer a better signature type for TypedDict.get. diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index 409018d284108..354c2f0e798f3 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -31,6 +31,7 @@ RTuple, RType, RUnion, + RVec, int_rprimitive, is_bool_or_bit_rprimitive, is_bytearray_rprimitive, @@ -751,6 +752,14 @@ def emit_cast( elif isinstance(typ, RTuple): assert not optional self.emit_tuple_cast(src, dest, typ, declare_dest, error, src_type) + elif isinstance(typ, RVec): + # TODO: Actually perform the type check, this is a no-op + if declare_dest: + self.emit_line("PyObject *{};".format(dest)) + self.emit_arg_check(src, dest, typ, "", optional) + self.emit_line("{} = {};".format(dest, src)) + if optional: + self.emit_line("}") else: assert False, "Cast not implemented: %s" % typ diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 2b0693ea2f2dd..1bd5246906861 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -583,6 +583,11 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]: self.generate_literal_tables() + self.declare_global("VecCapsule *", "VecApi") + self.declare_global("VecI64Features ", "VecI64Api") + self.declare_global("VecTFeatures ", "VecTApi") + self.declare_global("VecTExtFeatures ", "VecTExtApi") + for module_name, module in self.modules.items(): if multi_file: emitter = Emitter(self.context, filepath=self.source_paths[module_name]) @@ -964,6 +969,14 @@ def generate_globals_init(self, emitter: Emitter) -> None: f"if (CPyStatics_Initialize(CPyStatics, {values}) < 0) {{", "return -1;", "}" ) + emitter.emit_lines( + 'VecApi = PyCapsule_Import("vecs._C_API", 0);', + "if (!VecApi) return -1;", + "VecI64Api = *VecApi->i64;", + "VecTApi = *VecApi->t;", + "VecTExtApi = *VecApi->t_ext;", + ) + emitter.emit_lines("is_initialized = 1;", "return 0;", "}") def generate_module_def(self, emitter: Emitter, module_name: str, module: ModuleIR) -> None: diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 96f6f7c676f1a..5c241e641cb29 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -150,6 +150,10 @@ def visit_rprimitive(self, typ: RPrimitive, /) -> T: def visit_rinstance(self, typ: RInstance, /) -> T: raise NotImplementedError + @abstractmethod + def visit_rvec(self, typ: RVec, /) -> T: + raise NotImplementedError + @abstractmethod def visit_runion(self, typ: RUnion, /) -> T: raise NotImplementedError @@ -542,6 +546,10 @@ def is_tagged(rtype: RType) -> TypeGuard[RPrimitive]: return rtype is int_rprimitive or rtype is short_int_rprimitive +def is_any_int(rtype: RType) -> bool: + return is_tagged(rtype) or is_int32_rprimitive(rtype) or is_int64_rprimitive(rtype) + + def is_int_rprimitive(rtype: RType) -> TypeGuard[RPrimitive]: return rtype is int_rprimitive @@ -680,6 +688,9 @@ class TupleNameVisitor(RTypeVisitor[str]): def visit_rinstance(self, t: RInstance) -> str: return "O" + def visit_rvec(self, t: RVec) -> str: + return "O" + def visit_runion(self, t: RUnion) -> str: return "O" @@ -980,6 +991,49 @@ def serialize(self) -> str: return self.name +@final +class RVec(RType): + """vecs.vec[T]""" + + is_unboxed = False + + def __init__(self, item_type: RType) -> None: + self.name = "vec[%s]" % item_type + self.item_type = item_type + self._ctype = "PyObject *" + + def accept(self, visitor: "RTypeVisitor[T]") -> T: + return visitor.visit_rvec(self) + + def __repr__(self) -> str: + return "" % self.item_type + + def __eq__(self, other: object) -> bool: + return isinstance(other, RVec) and other.item_type == self.item_type + + def __hash__(self) -> int: + return hash(self.item_type) ^ 1 + + def serialize(self) -> str: + return self.name + + +def vec_depth(t: RVec) -> int: + """Return nesting depth of a RVec type (0 == no nested vecs).""" + it = t.item_type + if isinstance(it, RUnion): + non_opt = optional_value_type(it) + it = non_opt + if isinstance(it, (RPrimitive, RInstance)): + return 0 + elif isinstance(it, RVec): + if is_int64_rprimitive(it.item_type): + return 0 + else: + return 1 + vec_depth(vec.item_type) + assert False, t + + @final class RUnion(RType): """union[x, ..., y]""" @@ -1175,3 +1229,36 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: else: limit = 1 << (rtype.size * 8 - 1) return -limit <= n < limit + + +# vec (common fields) +VecObject = RStruct( + name="VecObject", names=["ob_base", "len"], types=[PyObject, c_pyssize_t_rprimitive] +) + +# vec[i64] +VecI64Object = RStruct( + name="VecI64Object", + names=["ob_base", "len", "items"], + types=[PyObject, c_pyssize_t_rprimitive, int64_rprimitive], +) + +# vec[t] +VecTObject = RStruct( + name="VecTObject", + names=["ob_base", "len", "items"], + types=[PyObject, c_pyssize_t_rprimitive, object_rprimitive], +) + +# vec[t], for nested vec or optional t +VecTExtObject = RStruct( + name="VecTExtObject", + names=["ob_base", "len", "depth", "optionals", "items"], + types=[ + PyObject, + c_pyssize_t_rprimitive, + int32_rprimitive, + int32_rprimitive, + object_rprimitive, + ], +) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 164765dd8db42..2998b90598a7a 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -95,6 +95,8 @@ bitmap_rprimitive, bool_rprimitive, bytes_rprimitive, + RVec, + c_int_rprimitive, c_pyssize_t_rprimitive, dict_rprimitive, int_rprimitive, @@ -128,6 +130,8 @@ AssignmentTargetTuple, ) from mypyc.irbuild.util import bytes_from_str, is_constant +from mypyc.irbuild.util import is_constant +from mypyc.irbuild.vec import vec_set_item from mypyc.options import CompilerOptions from mypyc.primitives.dict_ops import dict_get_item_op, dict_set_item_op from mypyc.primitives.generic_ops import iter_op, next_op, py_setattr_op @@ -420,8 +424,8 @@ def int_op(self, type: RType, lhs: Value, rhs: Value, op: int, line: int) -> Val def compare_tuples(self, lhs: Value, rhs: Value, op: str, line: int) -> Value: return self.builder.compare_tuples(lhs, rhs, op, line) - def builtin_len(self, val: Value, line: int) -> Value: - return self.builder.builtin_len(val, line) + def builtin_len(self, val: Value, line: int, use_pyssize_t: bool = False) -> Value: + return self.builder.builtin_len(val, line, use_pyssize_t) def new_tuple(self, items: list[Value], line: int) -> Value: return self.builder.new_tuple(items, line) @@ -769,10 +773,13 @@ def assign(self, target: Register | AssignmentTarget, rvalue_reg: Value, line: i boxed_reg = self.builder.box(rvalue_reg) self.primitive_op(py_setattr_op, [target.obj, key, boxed_reg], line) elif isinstance(target, AssignmentTargetIndex): - target_reg2 = self.gen_method_call( - target.base, "__setitem__", [target.index, rvalue_reg], None, line - ) - assert target_reg2 is not None, target.base.type + if isinstance(target.base.type, RVec): + vec_set_item(self.builder, target.base, target.index, rvalue_reg, line) + else: + target_reg2 = self.gen_method_call( + target.base, "__setitem__", [target.index, rvalue_reg], None, line + ) + assert target_reg2 is not None, target.base.type elif isinstance(target, AssignmentTargetTuple): if isinstance(rvalue_reg.type, RTuple) and target.star_idx is None: rtypes = rvalue_reg.type.types diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index e99542138ed79..f9f91644b2d3c 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -79,8 +79,12 @@ RInstance, RTuple, bool_rprimitive, + RVec, + c_pyssize_t_rprimitive, int_rprimitive, + is_any_int, is_fixed_width_rtype, + is_int64_rprimitive, is_int_rprimitive, is_list_rprimitive, is_none_rprimitive, @@ -96,6 +100,7 @@ raise_error_if_contains_unreachable_names, translate_list_comprehension, translate_set_comprehension, + translate_vec_comprehension, ) from mypyc.irbuild.format_str_tokenizer import ( convert_format_expr_to_bytes, @@ -111,6 +116,14 @@ translate_object_new, translate_object_setattr, ) +from mypyc.irbuild.vec import ( + vec_create, + vec_create_from_values, + vec_create_initialized, + vec_pop, + vec_remove, + vec_slice, +) from mypyc.primitives.bytes_ops import bytes_slice_op from mypyc.primitives.dict_ops import dict_get_item_op, dict_new_op, exact_dict_set_item_op from mypyc.primitives.generic_ops import iter_op, name_op @@ -334,7 +347,19 @@ def transform_call_expr(builder: IRBuilder, expr: CallExpr) -> Value: return builder.accept(expr.args[0]) if isinstance(callee, IndexExpr) and isinstance(callee.analyzed, TypeApplication): - callee = callee.analyzed.expr # Unwrap type application + analyzed = callee.analyzed + if ( + isinstance(analyzed.expr, RefExpr) + and analyzed.expr.fullname == "vecs.vec" + and len(analyzed.types) == 1 + ): + item_type = builder.type_to_rtype(analyzed.types[0]) + vec_type = RVec(item_type) + if len(expr.args) == 0: + return vec_create(builder.builder, vec_type, 0, expr.line) + elif len(expr.args) == 1 and expr.arg_kinds == [ARG_POS]: + return translate_vec_create_from_iterable(builder, vec_type, expr.args[0]) + callee = expr # Unwrap type application if isinstance(callee, MemberExpr): if isinstance(callee.expr, RefExpr) and isinstance(callee.expr.node, MypyFile): @@ -417,6 +442,13 @@ def translate_method_call(builder: IRBuilder, expr: CallExpr, callee: MemberExpr if val is not None: return val + if isinstance(receiver_typ, RVec) and all(k == ARG_POS for k in expr.arg_kinds): + # Vec methods are specialized with a custom mechanism, + # since normal specializations don't support vec receivers. + val = translate_vec_method_call(builder, callee.expr, callee.name, expr.args) + if val is not None: + return val + obj = builder.accept(callee.expr) args = [builder.accept(arg) for arg in expr.args] return builder.gen_method_call( @@ -527,6 +559,57 @@ def translate_super_method_call(builder: IRBuilder, expr: CallExpr, callee: Supe return builder.builder.call(decl, arg_values, arg_kinds, arg_names, expr.line) +def translate_vec_create_from_iterable( + builder: IRBuilder, vec_type: RVec, arg: Expression +) -> Value: + line = arg.line + item_type = vec_type.item_type + if isinstance(arg, OpExpr) and arg.op == "*": + if isinstance(arg.left, ListExpr): + lst = arg.left + other = arg.right + elif isinstance(arg.right, ListExpr): + lst = arg.right + other = arg.left + else: + assert False + assert len(lst.items) == 1 + other_type = builder.node_type(other) + # TODO: is_any_int(...) + if is_int64_rprimitive(other_type) or is_int_rprimitive(other_type): + length = builder.accept(other) + init = builder.accept(lst.items[0]) + return vec_create_initialized(builder.builder, vec_type, length, init, line) + assert False, other_type + if isinstance(arg, ListExpr): + items = [] + for item in arg.items: + value = builder.accept(item) + items.append(builder.coerce(value, item_type, line)) + return vec_create_from_values(builder.builder, vec_type, items, line) + if isinstance(arg, ListComprehension): + return translate_vec_comprehension(builder, vec_type, arg.generator) + + assert False, (vec_type, arg) + + +def translate_vec_method_call( + builder: IRBuilder, vec_expr: Expression, method: str, args: List[Expression] +) -> Optional[Value]: + if method == "pop" and 0 <= len(args) <= 1: + vec_val = builder.accept(vec_expr) + if args: + index = builder.accept(args[0]) + else: + index = Integer(-1, c_pyssize_t_rprimitive) + return vec_pop(builder.builder, vec_val, index, vec_expr.line) + if method == "remove" and len(args) == 1: + vec_val = builder.accept(vec_expr) + item = builder.accept(args[0]) + return vec_remove(builder.builder, vec_val, item, vec_expr.line) + return None + + def translate_cast_expr(builder: IRBuilder, expr: CastExpr) -> Value: src = builder.accept(expr.expr) target_type = builder.type_to_rtype(expr.type) @@ -620,9 +703,7 @@ def transform_index_expr(builder: IRBuilder, expr: IndexExpr) -> Value: return value index_reg = builder.accept(expr.index, can_borrow=is_list) - return builder.gen_method_call( - base, "__getitem__", [index_reg], builder.node_type(expr), expr.line - ) + return builder.builder.get_item(base, index_reg, builder.node_type(expr), expr.line) def try_constant_fold(builder: IRBuilder, expr: Expression) -> Value | None: @@ -657,7 +738,7 @@ def try_gen_slice_op(builder: IRBuilder, base: Value, index: SliceExpr) -> Value end_type = int_rprimitive # Both begin and end index must be int (or missing). - if is_int_rprimitive(begin_type) and is_int_rprimitive(end_type): + if is_any_int(begin_type) and is_any_int(end_type): if index.begin_index: begin = builder.accept(index.begin_index) else: @@ -668,6 +749,8 @@ def try_gen_slice_op(builder: IRBuilder, base: Value, index: SliceExpr) -> Value # Replace missing end index with the largest short integer # (a sequence can't be longer). end = builder.load_int(MAX_SHORT_INT) + if isinstance(base.type, RVec): + return vec_slice(builder.builder, base, begin, end, index.line) candidates = [list_slice_op, tuple_slice_op, str_slice_op, bytes_slice_op] return builder.builder.matching_call_c(candidates, [base, begin, end], index.line) @@ -882,6 +965,14 @@ def translate_is_none(builder: IRBuilder, expr: Expression, negated: bool) -> Va def transform_basic_comparison( builder: IRBuilder, op: str, left: Value, right: Value, line: int ) -> Value: + """Transform single comparison. + + 'op' must be one of these: + + '==', '!=', '<', '<=', '>', '>=' + 'in', 'not in' + 'is', 'is not' + """ if is_fixed_width_rtype(left.type) and op in ComparisonOp.signed_ops: if right.type == left.type: if left.type.is_signed: diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index 5c7589f99dfc1..ce939906de0ff 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -50,6 +50,7 @@ RTuple, RType, bool_rprimitive, + RVec, c_pyssize_t_rprimitive, int_rprimitive, is_dict_rprimitive, @@ -69,6 +70,7 @@ from mypyc.irbuild.constant_fold import constant_fold_expr from mypyc.irbuild.prepare import GENERATOR_HELPER_NAME from mypyc.irbuild.targets import AssignmentTarget, AssignmentTargetTuple +from mypyc.irbuild.vec import vec_append, vec_create, vec_get_item_unsafe from mypyc.primitives.dict_ops import ( dict_check_size_op, dict_item_iter_op, @@ -355,6 +357,19 @@ def gen_inner_stmts() -> None: return builder.read(set_ops, gen.line) +def translate_vec_comprehension(builder: IRBuilder, vec_type: RVec, gen: GeneratorExpr) -> Value: + vec = Register(vec_type) + builder.assign(vec, vec_create(builder.builder, vec_type, 0, gen.line), gen.line) + loop_params = list(zip(gen.indices, gen.sequences, gen.condlists)) + + def gen_inner_stmts() -> None: + e = builder.accept(gen.left_expr) + builder.assign(vec, vec_append(builder.builder, vec, e, gen.line), gen.line) + + comprehension_helper(builder, loop_params, gen_inner_stmts, gen.line) + return vec + + def comprehension_helper( builder: IRBuilder, loop_params: list[tuple[Lvalue, Expression, list[Expression], bool]], @@ -455,8 +470,8 @@ def make_for_loop_generator( return async_obj rtyp = builder.node_type(expr) - if is_sequence_rprimitive(rtyp): - # Special case "for x in ". + if is_sequence_rprimitive(rtyp) or isinstance(rtyp, RVec): + # Special case "for x in " for concrete sequence types. expr_reg = builder.accept(expr) target_type = builder.get_sequence_type(expr) @@ -831,6 +846,8 @@ def unsafe_index(builder: IRBuilder, target: Value, index: Value, line: int) -> return builder.call_c(tuple_get_item_unsafe_op, [target, index], line) elif is_str_rprimitive(target.type): return builder.call_c(str_get_item_unsafe_op, [target, index], line) + elif isinstance(target.type, RVec): + return vec_get_item_unsafe(builder.builder, target, index, line) else: return builder.gen_method_call(target, "__getitem__", [index], None, line) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 99d62adca9702..3700744c45e59 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -87,6 +87,7 @@ RTuple, RType, RUnion, + RVec, bit_rprimitive, bitmap_rprimitive, bool_rprimitive, @@ -128,6 +129,7 @@ str_rprimitive, ) from mypyc.irbuild.util import concrete_arg_kind +from mypyc.irbuild.vec import vec_contains, vec_get_item, vec_len from mypyc.options import CompilerOptions from mypyc.primitives.bytes_ops import bytes_compare from mypyc.primitives.dict_ops import ( @@ -1653,6 +1655,9 @@ def binary_op(self, lreg: Value, rreg: Value, op: str, line: int) -> Value: if dunder_op: return dunder_op + if isinstance(rtype, RVec) and op == "in": + return vec_contains(self, rreg, lreg, line) + primitive_ops_candidates = binary_ops.get(op, []) target = self.matching_primitive_op(primitive_ops_candidates, [lreg, rreg], line) assert target, "Unsupported binary operation: %s" % op @@ -1977,6 +1982,17 @@ def unary_op(self, value: Value, op: str, line: int) -> Value: return self.unary_invert(value, line) raise RuntimeError("Unsupported unary operation: %s" % op) + def get_item(self, base: Value, item: Value, result_type: Optional[RType], line: int) -> Value: + """Generate base[item].""" + if isinstance(base.type, RVec): + if is_int_rprimitive(item.type) or is_short_int_rprimitive(item.type): + item = self.coerce(item, int64_rprimitive, line) + if is_int64_rprimitive(item.type): + return vec_get_item(self, base, item, line) + # TODO: Move special casing for RTuple here from transform_index_expr + return self.gen_method_call(base, "__getitem__", [item], result_type, line) + + def make_dict(self, key_value_pairs: Sequence[DictEntry], line: int) -> Value: result: Value | None = None keys: list[Value] = [] @@ -2655,6 +2671,13 @@ def builtin_len(self, val: Value, line: int, use_pyssize_t: bool = False) -> Val self.activate_block(ok) return length + elif isinstance(typ, RVec): + len_value = vec_len(self, val) + if use_pyssize_t: + return len_value + count = Integer(1, int64_rprimitive, line) + return self.int_op(short_int_rprimitive, len_value, count, IntOp.LEFT_SHIFT, line) + op = self.matching_primitive_op(function_ops["builtins.len"], [val], line) if op is not None: return op @@ -2694,6 +2717,31 @@ def set_immortal_if_free_threaded(self, v: Value, line: int) -> None: if IS_FREE_THREADED and sys.version_info >= (3, 14): self.primitive_op(set_immortal_op, [v], line) + # Loop helpers + + def begin_for(self, start: Value, end: Value, step: Value, *, signed: bool) -> "ForBuilder": + """Generate for loop over a pointer or native int range. + + Roughly corresponds to "for i in range(start, end, step): ....". Only + positive values for step are supported. + + The loop index register is generated automatically and is available + via the return value. Do not modify it! + + Example usage that clears a memory area: + + for_loop = builder.begin_for(start_ptr, end_ptr, int32_rprimitive.size) + + # For loop body + builder.set_mem(for_loop.index, int32_rprimitive, Integer(0, int32_rprimitive)) + + for_loop.finish() + """ + b = ForBuilder(self, start, end, step, signed=signed) + b.begin() + return b + + # Internal helpers def decompose_union_helper( diff --git a/mypyc/irbuild/mapper.py b/mypyc/irbuild/mapper.py index 550dc6e42c9e2..70869a2c16676 100644 --- a/mypyc/irbuild/mapper.py +++ b/mypyc/irbuild/mapper.py @@ -30,6 +30,7 @@ RTuple, RType, RUnion, + RVec, bool_rprimitive, bytearray_rprimitive, bytes_rprimitive, @@ -123,6 +124,8 @@ def type_to_rtype(self, typ: Type | None) -> RType: return int16_rprimitive elif typ.type.fullname == "mypy_extensions.u8": return uint8_rprimitive + elif typ.type.fullname == "vecs.vec": + return RVec(self.type_to_rtype(typ.args[0])) elif typ.type.fullname in KNOWN_NATIVE_TYPES: return KNOWN_NATIVE_TYPES[typ.type.fullname] else: diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index ca7e5a004f3ea..2157edbdde397 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -48,12 +48,16 @@ Truncate, Unreachable, Value, + ERR_MAGIC, + CallC, ) from mypyc.ir.rtypes import ( RInstance, RPrimitive, RTuple, RType, + RUnion, + RVec, bool_rprimitive, bytes_rprimitive, bytes_writer_rprimitive, @@ -82,6 +86,8 @@ str_rprimitive, string_writer_rprimitive, uint8_rprimitive, + is_float_rprimitive, + vec_depth, ) from mypyc.irbuild.builder import IRBuilder from mypyc.irbuild.constant_fold import constant_fold_expr @@ -98,6 +104,7 @@ join_formatted_strings, tokenizer_format_call, ) +from mypyc.irbuild.vec import vec_append from mypyc.primitives.bytearray_ops import isinstance_bytearray from mypyc.primitives.bytes_ops import ( bytes_adjust_index_op, @@ -145,6 +152,7 @@ str_range_check_op, ) from mypyc.primitives.tuple_ops import isinstance_tuple, new_tuple_set_item_op +from mypyc.rt_subtype import is_runtime_subtype # Specializers are attempted before compiling the arguments to the # function. Specializers can return None to indicate that they failed @@ -317,7 +325,11 @@ def translate_len(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value if is_sequence_rprimitive(expr_rtype) or isinstance(expr_rtype, RTuple): return get_expr_length_value(builder, arg, obj, expr.line, use_pyssize_t=False) else: - return builder.builtin_len(obj, expr.line) + # TODO: Decide type of result based on context somehow? + if isinstance(obj.type, RVec): + return builder.builtin_len(obj, expr.line, use_pyssize_t=True) + else: + return builder.builtin_len(obj, expr.line) return None @@ -1177,6 +1189,7 @@ def translate_ord(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value return None + def is_object(callee: RefExpr) -> bool: """Returns True for object. calls.""" return ( @@ -1487,3 +1500,17 @@ def translate_bytes_get_item( bytes_range_check_op, bytes_get_item_unsafe_op, ) + + +@specialize_function("vecs.append") +def translate_vec_append(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Optional[Value]: + if len(expr.args) == 2 and expr.arg_kinds == [ARG_POS, ARG_POS]: + vec_arg = expr.args[0] + item_arg = expr.args[1] + vec_type = builder.node_type(vec_arg) + item_type = builder.node_type(item_arg) + if isinstance(vec_type, RVec): + vec_value = builder.accept(vec_arg) + arg_value = builder.accept(item_arg) + return vec_append(builder.builder, vec_value, arg_value, item_arg.line) + return None diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py new file mode 100644 index 0000000000000..76d836b99d9b6 --- /dev/null +++ b/mypyc/irbuild/vec.py @@ -0,0 +1,412 @@ +"""Generate IR for vecs.vec operations""" + +from typing import TYPE_CHECKING, List, Optional, Tuple, Union, cast + +from mypyc.common import PLATFORM_SIZE +from mypyc.ir.ops import ( + ERR_FALSE, + ERR_MAGIC, + BasicBlock, + Branch, + CallC, + ComparisonOp, + GetElementPtr, + Integer, + IntOp, + KeepAlive, + LoadAddress, + RaiseStandardError, + Register, + Unreachable, + Value, +) +from mypyc.ir.rtypes import ( + RInstance, + RPrimitive, + RType, + RUnion, + RVec, + VecI64Object, + VecObject, + VecTExtObject, + VecTObject, + bool_rprimitive, + c_int_rprimitive, + c_pyssize_t_rprimitive, + int32_rprimitive, + int64_rprimitive, + is_c_py_ssize_t_rprimitive, + is_int32_rprimitive, + is_int64_rprimitive, + is_none_rprimitive, + object_pointer_rprimitive, + object_rprimitive, + optional_value_type, + pointer_rprimitive, + vec_depth, +) +from mypyc.primitives.registry import builtin_names + +if TYPE_CHECKING: + from mypyc.irbuild.ll_builder import LowLevelIRBuilder + + +def as_platform_int(builder: "LowLevelIRBuilder", v: Value, line: int) -> Value: + rtype = v.type + if is_c_py_ssize_t_rprimitive(rtype): + return v + if isinstance(rtype, Integer): + if is_short_int_rprimitive(rtype) or is_int_rprimitive(rtype): + return Integer(v.value // 2, c_pyssize_t_rprimitive) + return Integer(v.value, c_pyssize_t_rprimitive) + if isinstance(rtype, RPrimitive): + if PLATFORM_SIZE == 8 and is_int64_rprimitive(rtype): + return v + if PLATFORM_SIZE == 4 and is_int32_rprimitive(rtype): + return v + return builder.coerce(v, c_pyssize_t_rprimitive, line) + + +def vec_create( + builder: "LowLevelIRBuilder", vtype: RVec, length: Union[int, Value], line: int +) -> Value: + if isinstance(length, int): + length = Integer(length, c_pyssize_t_rprimitive) + length = as_platform_int(builder, length, line) + + item_type = vtype.item_type + if is_int64_rprimitive(item_type): + call = CallC( + "VecI64Api.alloc", [length], vtype, False, False, error_kind=ERR_MAGIC, line=line + ) + return builder.add(call) + + typeobj, optionals, depth = vec_item_type_info(builder, item_type, line) + if typeobj is not None: + if optionals == 0 and depth == 0: + call = CallC( + "VecTApi.alloc", + [length, typeobj], + vtype, + False, + False, + error_kind=ERR_MAGIC, + line=line, + ) + return builder.add(call) + else: + call = CallC( + "VecTExtApi.alloc", + [ + length, + typeobj, + Integer(optionals, int32_rprimitive), + Integer(depth, int32_rprimitive), + ], + vtype, + False, + False, + error_kind=ERR_MAGIC, + line=line, + ) + return builder.add(call) + + assert False, "unsupported: %s" % vtype + + +def vec_create_initialized( + builder: "LowLevelIRBuilder", vtype: RVec, length: Union[int, Value], init: Value, line: int +) -> Value: + """Create vec with items initialized to the given value.""" + if isinstance(length, int): + length = Integer(length, c_pyssize_t_rprimitive) + length = as_platform_int(builder, length, line) + + item_type = vtype.item_type + init = builder.coerce(init, item_type, line) + vec = vec_create(builder, vtype, length, line) + + items_start = vec_items(builder, vec) + step = step_size(item_type) + items_end = builder.int_add(items_start, builder.int_mul(length, step)) + + for_loop = builder.begin_for(items_start, items_end, step, signed=False) + builder.set_mem(for_loop.index, item_type, init) + for_loop.finish() + + builder.keep_alive([vec]) + return vec + + +def vec_create_from_values( + builder: "LowLevelIRBuilder", vtype: RVec, values: List[Value], line: int +) -> Value: + vec = vec_create(builder, vtype, len(values), line) + ptr = vec_items(builder, vec) + item_type = vtype.item_type + step = step_size(item_type) + for value in values: + builder.set_mem(ptr, item_type, value) + ptr = builder.int_add(ptr, step) + builder.keep_alive([vec]) + return vec + + +def step_size(item_type: RType) -> int: + if isinstance(item_type, RPrimitive): + return item_type.size + else: + return PLATFORM_SIZE + + +def vec_item_type_info( + builder: "LowLevelIRBuilder", typ: RType, line: int +) -> Tuple[Optional[Value], int, int]: + if isinstance(typ, RPrimitive) and typ.is_refcounted: + typ, src = builtin_names[typ.name] + return builder.load_address(src, typ), 0, 0 + elif isinstance(typ, RInstance): + return builder.load_native_type_object(typ.name), 0, 0 + elif isinstance(typ, RUnion): + non_opt = optional_value_type(typ) + if non_opt is not None: + typeval, optionals, depth = vec_item_type_info(builder, non_opt, line) + if typeval is not None: + return typeval, (optionals << 1) | 1, depth + elif isinstance(typ, RVec): + if is_int64_rprimitive(typ.item_type): + addr = builder.load_address("VecI64Api.type", pointer_rprimitive) + load = builder.load_mem(addr, object_rprimitive) + return load, 0, 0 + typeval, optionals, depth = vec_item_type_info(builder, typ.item_type, line) + if typeval is not None: + return typeval, optionals, depth + 1 + return None, 0, 0 + + +def vec_len(builder: "LowLevelIRBuilder", val: Value) -> Value: + address = builder.add(GetElementPtr(val, VecObject, "len")) + # TODO: what about 32-bit archs? + len_value = builder.load_mem(address, int64_rprimitive) + builder.keep_alive([val]) + return len_value + + +def vec_len_native(builder: "LowLevelIRBuilder", val: Value) -> Value: + return builder.load_struct_field(val, VecObject, "len") + + +def vec_items(builder: "LowLevelIRBuilder", vecobj: Value) -> Value: + vtype = cast(RVec, vecobj.type) + if is_int64_rprimitive(vtype.item_type): + vec_struct = VecI64Object + elif vec_depth(vtype) == 0 and not isinstance(vtype.item_type, RUnion): + vec_struct = VecTObject + else: + vec_struct = VecTExtObject + return builder.add(GetElementPtr(vecobj, vec_struct, "items")) + + +def vec_item_ptr(builder: "LowLevelIRBuilder", vecobj: Value, index: Value) -> Value: + items_addr = vec_items(builder, vecobj) + delta = builder.int_mul(index, 8) + return builder.int_add(items_addr, delta) + + +def vec_check_index(builder: "LowLevelIRBuilder", lenv: Value, index: Value, line: int) -> None: + ok, fail = BasicBlock(), BasicBlock() + is_less = builder.comparison_op(index, lenv, ComparisonOp.ULT, line) + builder.add_bool_branch(is_less, ok, fail) + builder.activate_block(fail) + # TODO: Include index in exception + builder.add(RaiseStandardError(RaiseStandardError.INDEX_ERROR, None, line)) + builder.add(Unreachable()) + builder.activate_block(ok) + + +def vec_get_item(builder: "LowLevelIRBuilder", base: Value, index: Value, line: int) -> Value: + """Generate inlined vec __getitem__ call. + + We inline this, since it's fairly simple but very + performance-critical. + """ + assert isinstance(base.type, RVec) + vtype = base.type + # TODO: Support more item types + # TODO: Support more index types + len_val = vec_len_native(builder, base) + vec_check_index(builder, len_val, index, line) + item_addr = vec_item_ptr(builder, base, index) + result = builder.load_mem(item_addr, vtype.item_type) + builder.keep_alive([base]) + return result + + +def vec_get_item_unsafe( + builder: "LowLevelIRBuilder", base: Value, index: Value, line: int +) -> Value: + """Get vec item, assuming index is non-negative and within bounds.""" + assert isinstance(base.type, RVec) + index = as_platform_int(builder, index, line) + vtype = base.type + item_addr = vec_item_ptr(builder, base, index) + result = builder.load_mem(item_addr, vtype.item_type) + builder.keep_alive([base]) + return result + + +def vec_set_item( + builder: "LowLevelIRBuilder", base: Value, index: Value, item: Value, line: int +) -> None: + assert isinstance(base.type, RVec) + vtype = base.type + len_val = vec_len_native(builder, base) + vec_check_index(builder, len_val, index, line) + item_addr = vec_item_ptr(builder, base, index) + item_type = vtype.item_type + item = builder.coerce(item, item_type, line) + if item_type.is_refcounted: + # Read an unborrowed reference to cause a decref to be + # generated for the old item. + old_item = builder.load_mem(item_addr, item_type) + old_item.is_borrowed = False + builder.set_mem(item_addr, item_type, item) + builder.keep_alive([base]) + + +def vec_append(builder: "LowLevelIRBuilder", vec: Value, item: Value, line: int) -> Value: + vec_type = vec.type + assert isinstance(vec_type, RVec) + item_type = vec_type.item_type + item = builder.coerce(item, item_type, line) + if is_int64_rprimitive(item_type): + name = "VecI64Api.append" + elif vec_depth(vec_type) == 0 and not isinstance(item_type, RUnion): + name = "VecTApi.append" + else: + name = "VecTExtApi.append" + call = CallC( + name, + [vec, item], + vec_type, + steals=[True, False], + is_borrowed=False, + error_kind=ERR_MAGIC, + line=line, + ) + return builder.add(call) + + +def vec_pop(builder: "LowLevelIRBuilder", base: Value, index: Value, line: int) -> Value: + assert isinstance(base.type, RVec) + vec_type = base.type + item_type = vec_type.item_type + index = as_platform_int(builder, index, line) + + if isinstance(index, Integer) and index.value == -1: + # Inline simple case + len_val = vec_len_native(builder, base) + index = builder.int_sub(len_val, 1) + item_addr = vec_item_ptr(builder, base, index) + item = builder.load_mem(item_addr, item_type) + builder.set_struct_field(base, VecObject, "len", index, line) + return item + + if is_int64_rprimitive(item_type): + name = "VecI64Api.pop" + elif vec_depth(vec_type) == 0 and not isinstance(item_type, RUnion): + name = "VecTApi.pop" + else: + name = "VecTExtApi.pop" + call = CallC( + name, + [base, index], + item_type, + steals=[False, False], + is_borrowed=False, + error_kind=ERR_MAGIC, + line=line, + ) + return builder.add(call) + + +def vec_remove(builder: "LowLevelIRBuilder", vec: Value, item: Value, line: int) -> Value: + assert isinstance(vec.type, RVec) + vec_type = vec.type + item_type = vec_type.item_type + item = builder.coerce(item, item_type, line) + + if is_int64_rprimitive(item_type): + name = "VecI64Api.remove" + elif vec_depth(vec_type) == 0 and not isinstance(item_type, RUnion): + name = "VecTApi.remove" + else: + name = "VecTExtApi.remove" + call = CallC( + name, + [vec, item], + c_int_rprimitive, + steals=[False, False], + is_borrowed=False, + error_kind=ERR_FALSE, + line=line, + ) + return builder.add(call) + + +def vec_contains(builder: "LowLevelIRBuilder", vec: Value, target: Value, line: int) -> Value: + assert isinstance(vec.type, RVec) + vec_type = vec.type + item_type = vec_type.item_type + target = builder.coerce(target, item_type, line) + + step = step_size(item_type) + len_val = vec_len_native(builder, vec) + items_start = vec_items(builder, vec) + items_end = builder.int_add(items_start, builder.int_mul(len_val, step)) + + true, end = BasicBlock(), BasicBlock() + + for_loop = builder.begin_for(items_start, items_end, step, signed=False) + item = builder.load_mem(for_loop.index, item_type) + comp = builder.binary_op(item, target, "==", line) + false = BasicBlock() + builder.add(Branch(comp, true, false, Branch.BOOL)) + builder.activate_block(false) + for_loop.finish() + + builder.keep_alive([vec]) + + res = Register(bool_rprimitive) + builder.assign(res, Integer(0, bool_rprimitive)) + builder.goto(end) + builder.activate_block(true) + builder.assign(res, Integer(1, bool_rprimitive)) + builder.goto_and_activate(end) + return res + + +def vec_slice( + builder: "LowLevelIRBuilder", vec: Value, begin: Value, end: Value, line: int +) -> Value: + assert isinstance(vec.type, RVec) + vec_type = vec.type + item_type = vec_type.item_type + begin = builder.coerce(begin, int64_rprimitive, line) + end = builder.coerce(end, int64_rprimitive, line) + if is_int64_rprimitive(item_type): + name = "VecI64Api.slice" + elif vec_depth(vec_type) == 0 and not isinstance(item_type, RUnion): + name = "VecTApi.slice" + else: + name = "VecTExtApi.slice" + call = CallC( + name, + [vec, begin, end], + vec_type, + steals=[False, False, False], + is_borrowed=False, + error_kind=ERR_MAGIC, + line=line, + ) + return builder.add(call) diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index b2a48bb29ecbd..2ca247be454a9 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -9,6 +9,7 @@ #include #include #include +#include "vecs.h" #include "pythonsupport.h" #include "mypyc_util.h" diff --git a/mypyc/lib-rt/vecs.h b/mypyc/lib-rt/vecs.h new file mode 120000 index 0000000000000..03666343e12f4 --- /dev/null +++ b/mypyc/lib-rt/vecs.h @@ -0,0 +1 @@ +/home/jukka/src/vecs/vecs.h \ No newline at end of file diff --git a/mypyc/primitives/list_ops.py b/mypyc/primitives/list_ops.py index b9d20a25bea35..efd40469606f3 100644 --- a/mypyc/primitives/list_ops.py +++ b/mypyc/primitives/list_ops.py @@ -151,7 +151,7 @@ priority=6, ) -# This is unsafe because it assumes that the index is a non-negative short integer +# This is unsafe because it assumes that the index is a non-negative integer # that is in-bounds for the list. list_get_item_unsafe_op = custom_primitive_op( name="list_get_item_unsafe", diff --git a/mypyc/rt_subtype.py b/mypyc/rt_subtype.py index 004e56ed75bc9..54205a1de6984 100644 --- a/mypyc/rt_subtype.py +++ b/mypyc/rt_subtype.py @@ -24,6 +24,7 @@ RType, RTypeVisitor, RUnion, + RVec, RVoid, is_bit_rprimitive, is_bool_rprimitive, @@ -34,6 +35,8 @@ def is_runtime_subtype(left: RType, right: RType) -> bool: + if isinstance(right, RUnion) and not isinstance(left, RUnion): + return any(not item.is_unboxed and is_runtime_subtype(left, item) for item in right.items) return left.accept(RTSubtypeVisitor(right)) @@ -50,6 +53,10 @@ def __init__(self, right: RType) -> None: def visit_rinstance(self, left: RInstance) -> bool: return is_subtype(left, self.right) + def visit_rvec(self, left: RVec) -> bool: + # TODO: Better implementation + return left == self.right + def visit_runion(self, left: RUnion) -> bool: return not self.right.is_unboxed and is_subtype(left, self.right) diff --git a/mypyc/sametype.py b/mypyc/sametype.py index 1b811d4e9041e..d6e33f81cccdd 100644 --- a/mypyc/sametype.py +++ b/mypyc/sametype.py @@ -12,6 +12,7 @@ RType, RTypeVisitor, RUnion, + RVec, RVoid, ) @@ -50,6 +51,9 @@ def __init__(self, right: RType) -> None: def visit_rinstance(self, left: RInstance) -> bool: return isinstance(self.right, RInstance) and left.name == self.right.name + def visit_rvec(self, left: RVec) -> bool: + return isinstance(self.right, RVec) and left.item_type == self.right.item_type + def visit_runion(self, left: RUnion) -> bool: if isinstance(self.right, RUnion): items = list(self.right.items) diff --git a/mypyc/subtype.py b/mypyc/subtype.py index 3d5fa1fb4fca1..085cfc4f60a4b 100644 --- a/mypyc/subtype.py +++ b/mypyc/subtype.py @@ -11,6 +11,7 @@ RType, RTypeVisitor, RUnion, + RVec, RVoid, is_bit_rprimitive, is_bool_rprimitive, @@ -54,6 +55,10 @@ def __init__(self, right: RType, *, relaxed: bool = False) -> None: def visit_rinstance(self, left: RInstance) -> bool: return isinstance(self.right, RInstance) and self.right.class_ir in left.class_ir.mro + def visit_rvec(self, left: RVec) -> bool: + # TODO: Better implementation + return left == self.right + def visit_runion(self, left: RUnion) -> bool: return all(is_subtype(item, self.right, relaxed=self.relaxed) for item in left.items) diff --git a/mypyc/test-data/irbuild-vecs.test b/mypyc/test-data/irbuild-vecs.test new file mode 100644 index 0000000000000..1aa243eb3aa39 --- /dev/null +++ b/mypyc/test-data/irbuild-vecs.test @@ -0,0 +1,737 @@ +[case testVecI64CreateEmpty] +from vecs import vec, append +from mypy_extensions import i64 + +def f() -> vec[i64]: + return vec[i64]() +[out] +def f(): + r0 :: vec[int64] +L0: + r0 = VecI64Api.alloc(0) + return r0 + +[case testVecI64Len] +from vecs import vec +from mypy_extensions import i64 + +def f(v: vec[i64]) -> i64: + l = len(v) + return l +[out] +def f(v): + v :: vec[int64] + r0 :: ptr + r1, l :: int64 +L0: + r0 = get_element_ptr v len :: VecObject + r1 = load_mem r0 :: int64* + keep_alive v + l = r1 + return l + +[case testVecI64GetItem] +from vecs import vec +from mypy_extensions import i64 + +def f(v: vec[i64], i: i64) -> i64: + return v[i] +[out] +def f(v, i): + v :: vec[int64] + i :: int64 + r0 :: ptr + r1 :: native_int + r2 :: bit + r3 :: bool + r4 :: ptr + r5 :: int64 + r6 :: ptr + r7 :: int64 +L0: + r0 = get_element_ptr v len :: VecObject + r1 = load_mem r0 :: native_int* + r2 = i < r1 :: unsigned + if r2 goto L2 else goto L1 :: bool +L1: + r3 = raise IndexError + unreachable +L2: + r4 = get_element_ptr v items :: VecI64Object + r5 = i * 8 + r6 = r4 + r5 + r7 = load_mem r6 :: int64* + keep_alive v + return r7 + +[case testVecI64Append] +from vecs import vec, append +from mypy_extensions import i64 + +def f(v: vec[i64], i: i64) -> vec[i64]: + return append(v, i) +[out] +def f(v, i): + v :: vec[int64] + i :: int64 + r0 :: vec[int64] +L0: + r0 = VecI64Api.append(v, i) + return r0 + +[case testVecTCreateEmpty] +from vecs import vec, append + +class C: pass + +def primitive() -> vec[str]: + return vec[str]() + +def native_class() -> vec[C]: + return vec[C]() +[out] +def primitive(): + r0 :: object + r1 :: vec[str] +L0: + r0 = load_address PyUnicode_Type + r1 = VecTApi.alloc(0, r0) + return r1 +def native_class(): + r0 :: object + r1 :: vec[__main__.C] +L0: + r0 = __main__.C :: type + r1 = VecTApi.alloc(0, r0) + return r1 + +[case testVecTAppend] +from vecs import vec, append + +def f(v: vec[str]) -> vec[str]: + return append(v, 'x') +[out] +def f(v): + v :: vec[str] + r0 :: str + r1 :: vec[str] +L0: + r0 = 'x' + r1 = VecTApi.append(v, r0) + return r1 + +[case testVecOptionalCreateEmpty] +from vecs import vec, append +from typing import Optional + +class C: pass + +def primitive() -> vec[Optional[str]]: + return vec[Optional[str]]() + +def native_class() -> vec[Optional[C]]: + return vec[Optional[C]]() +[out] +def primitive(): + r0 :: object + r1 :: vec[union[str, None]] +L0: + r0 = load_address PyUnicode_Type + r1 = VecTExtApi.alloc(0, r0, 1, 0) + return r1 +def native_class(): + r0 :: object + r1 :: vec[union[__main__.C, None]] +L0: + r0 = __main__.C :: type + r1 = VecTExtApi.alloc(0, r0, 1, 0) + return r1 + +[case testVecNestedCreateEmpty] +from vecs import vec, append +from mypy_extensions import i64 + +def f() -> vec[vec[str]]: + return vec[vec[str]]() + +def g() -> vec[vec[i64]]: + return vec[vec[i64]]() +[out] +def f(): + r0 :: object + r1 :: vec[vec[str]] +L0: + r0 = load_address PyUnicode_Type + r1 = VecTExtApi.alloc(0, r0, 0, 1) + return r1 +def g(): + r0 :: ptr + r1 :: object + r2 :: vec[vec[int64]] +L0: + r0 = load_address VecI64Api.type + r1 = load_mem r0 :: builtins.object* + r2 = VecTApi.alloc(0, r1) + return r2 + +[case testVecOptionalAppend] +from vecs import vec, append +from typing import Optional + +def f(v: vec[Optional[str]]) -> vec[Optional[str]]: + v = append(v, 'x') + return append(v, None) +[out] +def f(v): + v :: vec[union[str, None]] + r0 :: str + r1 :: vec[union[str, None]] + r2 :: object + r3 :: vec[union[str, None]] +L0: + r0 = 'x' + r1 = VecTExtApi.append(v, r0) + v = r1 + r2 = box(None, 1) + r3 = VecTExtApi.append(v, r2) + return r3 + +[case testVecGenericLen] +from vecs import vec +from mypy_extensions import i64 +from typing import Optional + +def f(v: vec[str]) -> i64: + return len(v) + +def g(v: vec[Optional[str]]) -> i64: + return len(v) +[out] +def f(v): + v :: vec[str] + r0 :: ptr + r1 :: int64 +L0: + r0 = get_element_ptr v len :: VecObject + r1 = load_mem r0 :: int64* + keep_alive v + return r1 +def g(v): + v :: vec[union[str, None]] + r0 :: ptr + r1 :: int64 +L0: + r0 = get_element_ptr v len :: VecObject + r1 = load_mem r0 :: int64* + keep_alive v + return r1 + +[case testVecTGetItem] +from vecs import vec +from mypy_extensions import i64 +from typing import Optional + +def f(v: vec[str], n: i64) -> str: + return v[n] +[out] +def f(v, n): + v :: vec[str] + n :: int64 + r0 :: ptr + r1 :: native_int + r2 :: bit + r3 :: bool + r4 :: ptr + r5 :: int64 + r6 :: ptr + r7 :: str +L0: + r0 = get_element_ptr v len :: VecObject + r1 = load_mem r0 :: native_int* + r2 = n < r1 :: unsigned + if r2 goto L2 else goto L1 :: bool +L1: + r3 = raise IndexError + unreachable +L2: + r4 = get_element_ptr v items :: VecTObject + r5 = n * 8 + r6 = r4 + r5 + r7 = load_mem r6 :: builtins.str* + keep_alive v + return r7 + +[case testVecTExtGetItem] +from vecs import vec +from mypy_extensions import i64 +from typing import Optional + +def f(v: vec[Optional[str]], n: i64) -> Optional[str]: + return v[n] +[out] +def f(v, n): + v :: vec[union[str, None]] + n :: int64 + r0 :: ptr + r1 :: native_int + r2 :: bit + r3 :: bool + r4 :: ptr + r5 :: int64 + r6 :: ptr + r7 :: union[str, None] +L0: + r0 = get_element_ptr v len :: VecObject + r1 = load_mem r0 :: native_int* + r2 = n < r1 :: unsigned + if r2 goto L2 else goto L1 :: bool +L1: + r3 = raise IndexError + unreachable +L2: + r4 = get_element_ptr v items :: VecTExtObject + r5 = n * 8 + r6 = r4 + r5 + r7 = load_mem r6 :: union* + keep_alive v + return r7 + +[case testVecNestedVecI64Append] +from vecs import vec, append +from mypy_extensions import i64 + +def f(v: vec[vec[i64]], vv: vec[i64]) -> vec[vec[i64]]: + return append(v, vv) +[out] +def f(v, vv): + v :: vec[vec[int64]] + vv :: vec[int64] + r0 :: vec[vec[int64]] +L0: + r0 = VecTApi.append(v, vv) + return r0 + +[case testVecNestedVecI64GetItem] +from vecs import vec +from mypy_extensions import i64 + +def f(v: vec[vec[i64]], n: i64) -> vec[i64]: + return v[n] +[out] +def f(v, n): + v :: vec[vec[int64]] + n :: int64 + r0 :: ptr + r1 :: native_int + r2 :: bit + r3 :: bool + r4 :: ptr + r5 :: int64 + r6 :: ptr + r7 :: vec[int64] +L0: + r0 = get_element_ptr v len :: VecObject + r1 = load_mem r0 :: native_int* + r2 = n < r1 :: unsigned + if r2 goto L2 else goto L1 :: bool +L1: + r3 = raise IndexError + unreachable +L2: + r4 = get_element_ptr v items :: VecTObject + r5 = n * 8 + r6 = r4 + r5 + r7 = load_mem r6 :: vec[int64]* + keep_alive v + return r7 + +[case testVecI64SetItem] +from vecs import vec +from mypy_extensions import i64 + +def f(v: vec[i64], i: i64, x: i64) -> None: + v[i] = x +[out] +def f(v, i, x): + v :: vec[int64] + i, x :: int64 + r0 :: ptr + r1 :: native_int + r2 :: bit + r3 :: bool + r4 :: ptr + r5 :: int64 + r6 :: ptr +L0: + r0 = get_element_ptr v len :: VecObject + r1 = load_mem r0 :: native_int* + r2 = i < r1 :: unsigned + if r2 goto L2 else goto L1 :: bool +L1: + r3 = raise IndexError + unreachable +L2: + r4 = get_element_ptr v items :: VecI64Object + r5 = i * 8 + r6 = r4 + r5 + set_mem r6, x :: int64* + keep_alive v + return 1 + +[case testVecI64ConstructFromListExpr] +from vecs import vec +from mypy_extensions import i64 + +def f() -> vec[i64]: + return vec[i64]([1, 5, 14]) +[out] +def f(n): + r0 :: vec[int64] + r1, r2, r3, r4 :: ptr +L0: + r0 = VecI64Api.alloc(3) + r1 = get_element_ptr r0 items :: VecI64Object + set_mem r1, 1 :: int64* + r2 = r1 + 8 + set_mem r2, 5 :: int64* + r3 = r2 + 8 + set_mem r3, 14 :: int64* + r4 = r3 + 8 + keep_alive r0 + return r0 + +[case testVecI64ConstructFromListMultiply] +from vecs import vec +from mypy_extensions import i64 + +def f(n: i64) -> vec[i64]: + return vec[i64]([3] * n) +[out] +def f(n): + n :: int64 + r0 :: vec[int64] + r1 :: ptr + r2 :: int64 + r3, r4 :: ptr + r5 :: bit + r6 :: ptr +L0: + r0 = VecI64Api.alloc(n) + r1 = get_element_ptr r0 items :: VecI64Object + r2 = n * 8 + r3 = r1 + r2 + r4 = r1 +L1: + r5 = r4 < r3 :: unsigned + if r5 goto L2 else goto L3 :: bool +L2: + set_mem r4, 3 :: int64* + r6 = r4 + 8 + r4 = r6 + goto L1 +L3: + keep_alive r0 + return r0 + +[case testVecI64ConstructFromListMultiply2] +from vecs import vec +from mypy_extensions import i64 + +def f(n: i64, x: i64) -> vec[i64]: + return vec[i64]([x] * 3) +[out] +def f(n, x): + n, x :: int64 + r0 :: vec[int64] + r1 :: ptr + r2 :: int64 + r3, r4 :: ptr + r5 :: bit + r6 :: ptr +L0: + r0 = VecI64Api.alloc(3) + r1 = get_element_ptr r0 items :: VecI64Object + r2 = 3 * 8 + r3 = r1 + r2 + r4 = r1 +L1: + r5 = r4 < r3 :: unsigned + if r5 goto L2 else goto L3 :: bool +L2: + set_mem r4, x :: int64* + r6 = r4 + 8 + r4 = r6 + goto L1 +L3: + keep_alive r0 + return r0 + +[case testVecI64ConstructFromListComprehension] +from vecs import vec +from mypy_extensions import i64 + +def f(n: i64) -> vec[i64]: + return vec[i64]([x + 1 for x in range(5)]) +[out] +def f(n): + n :: int64 + r0, r1 :: vec[int64] + r2 :: short_int + x :: int + r3 :: bit + r4 :: int + r5 :: native_int + r6 :: bit + r7, r8 :: int64 + r9 :: ptr + r10 :: c_ptr + r11 :: int64 + r12 :: vec[int64] + r13 :: short_int +L0: + r0 = VecI64Api.alloc(0) + r1 = r0 + r2 = 0 + x = r2 +L1: + r3 = r2 < 10 :: signed + if r3 goto L2 else goto L7 :: bool +L2: + r4 = CPyTagged_Add(x, 2) + r5 = r4 & 1 + r6 = r5 == 0 + if r6 goto L3 else goto L4 :: bool +L3: + r7 = r4 >> 1 + r8 = r7 + goto L5 +L4: + r9 = r4 ^ 1 + r10 = r9 + r11 = CPyLong_AsInt64(r10) + r8 = r11 + keep_alive r4 +L5: + r12 = VecI64Api.append(r1, r8) + r1 = r12 +L6: + r13 = r2 + 2 + r2 = r13 + x = r13 + goto L1 +L7: + return r1 + + +[case testVecI64ForLoop] +from vecs import vec +from mypy_extensions import i64 + +def f(v: vec[i64]) -> i64: + t: i64 = 0 + for x in v: + t += 1 + return t +[out] +def f(v): + v :: vec[int64] + t :: int64 + r0 :: native_int + r1 :: ptr + r2 :: int64 + r3 :: bit + r4 :: ptr + r5 :: native_int + r6 :: ptr + r7, x, r8 :: int64 + r9 :: native_int +L0: + t = 0 + r0 = 0 +L1: + r1 = get_element_ptr v len :: VecObject + r2 = load_mem r1 :: int64* + keep_alive v + r3 = r0 < r2 :: signed + if r3 goto L2 else goto L4 :: bool +L2: + r4 = get_element_ptr v items :: VecI64Object + r5 = r0 * 8 + r6 = r4 + r5 + r7 = load_mem r6 :: int64* + keep_alive v + x = r7 + r8 = t + 1 + t = r8 +L3: + r9 = r0 + 1 + r0 = r9 + goto L1 +L4: + return t + +[case testVecI64PopLast] +from vecs import vec +from mypy_extensions import i64 + +def pop_last(v: vec[i64]) -> i64: + return v.pop() +[out] +def pop_last(v): + v :: vec[int64] + r0 :: ptr + r1, r2 :: native_int + r3 :: ptr + r4 :: native_int + r5 :: ptr + r6 :: int64 + r7 :: ptr +L0: + r0 = get_element_ptr v len :: VecObject + r1 = load_mem r0 :: native_int* + r2 = r1 - 1 + r3 = get_element_ptr v items :: VecI64Object + r4 = r2 * 8 + r5 = r3 + r4 + r6 = load_mem r5 :: int64* + r7 = get_element_ptr v len :: VecObject + set_mem r7, r2 :: native_int* + keep_alive v + return r6 + +[case testVecI64PopNth] +from vecs import vec +from mypy_extensions import i64 + +def pop_nth(v: vec[i64], n: i64) -> i64: + return v.pop(n) +[out] +def pop_nth(v, n): + v :: vec[int64] + n, r0 :: int64 +L0: + r0 = VecI64Api.pop(v, n) + return r0 + +[case testVecI64Remove] +from vecs import vec +from mypy_extensions import i64 + +def remove(v: vec[i64], n: i64) -> None: + v.remove(n) +[out] +def remove(v, n): + v :: vec[int64] + n :: int64 + r0 :: int32 +L0: + r0 = VecI64Api.remove(v, n) + return 1 + +[case testVecI64Contains] +from vecs import vec +from mypy_extensions import i64 + +def contains(v: vec[i64], n: i64) -> bool: + return n in v +[out] +def contains(v, n): + v :: vec[int64] + n :: int64 + r0 :: ptr + r1 :: native_int + r2 :: ptr + r3 :: native_int + r4, r5 :: ptr + r6 :: bit + r7 :: int64 + r8 :: bit + r9 :: ptr + r10 :: bool +L0: + r0 = get_element_ptr v len :: VecObject + r1 = load_mem r0 :: native_int* + r2 = get_element_ptr v items :: VecI64Object + r3 = r1 * 8 + r4 = r2 + r3 + r5 = r2 +L1: + r6 = r5 < r4 :: unsigned + if r6 goto L2 else goto L4 :: bool +L2: + r7 = load_mem r5 :: int64* + r8 = r7 == n + if r8 goto L5 else goto L3 :: bool +L3: + r9 = r5 + 8 + r5 = r9 + goto L1 +L4: + keep_alive v + r10 = 0 + goto L6 +L5: + r10 = 1 +L6: + return r10 + +[case testVecI64GetItemWithInt] +from vecs import vec +from mypy_extensions import i64 + +def f(v: vec[i64]) -> i64: + return v[0] +[out] +def f(v): + v :: vec[int64] + r0 :: ptr + r1 :: native_int + r2 :: bit + r3 :: bool + r4 :: ptr + r5 :: int64 + r6 :: ptr + r7 :: int64 +L0: + r0 = get_element_ptr v len :: VecObject + r1 = load_mem r0 :: native_int* + r2 = 0 < r1 :: unsigned + if r2 goto L2 else goto L1 :: bool +L1: + r3 = raise IndexError + unreachable +L2: + r4 = get_element_ptr v items :: VecI64Object + r5 = 0 * 8 + r6 = r4 + r5 + r7 = load_mem r6 :: int64* + keep_alive v + return r7 + +[case testVecI64Slicing] +from vecs import vec +from mypy_extensions import i64 + +def f(v: vec[i64], n: i64, m: i64) -> None: + a = v[:] + b = v[n:] + c = v[n:m] + d = v[:m] + e = v[1:-2] +[out] +def f(v, n, m): + v :: vec[int64] + n, m :: int64 + r0, a, r1, b, r2, c, r3, d, r4, e :: vec[int64] +L0: + r0 = VecI64Api.slice(v, 0, 4611686018427387903) + a = r0 + r1 = VecI64Api.slice(v, n, 4611686018427387903) + b = r1 + r2 = VecI64Api.slice(v, n, m) + c = r2 + r3 = VecI64Api.slice(v, 0, m) + d = r3 + r4 = VecI64Api.slice(v, 1, -2) + e = r4 + return 1 diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index a71c53041cf7e..515e383a3556c 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -1499,3 +1499,155 @@ L2: dec_ref r0 :: int L3: return r4 + +[case testVecTSetItem] +from vecs import vec +from mypy_extensions import i64 + +def f(v: vec[str], i: i64, x: str) -> None: + v[i] = x +[out] +def f(v, i, x): + v :: vec[str] + i :: int64 + x :: str + r0 :: ptr + r1 :: native_int + r2 :: bit + r3 :: bool + r4 :: ptr + r5 :: int64 + r6 :: ptr + r7 :: str +L0: + r0 = get_element_ptr v len :: VecObject + r1 = load_mem r0 :: native_int* + r2 = i < r1 :: unsigned + if r2 goto L2 else goto L1 :: bool +L1: + r3 = raise IndexError + unreachable +L2: + r4 = get_element_ptr v items :: VecTObject + r5 = i * 8 + r6 = r4 + r5 + r7 = load_mem r6 :: builtins.str* + dec_ref r7 + inc_ref x + set_mem r6, x :: builtins.str* + return 1 + +[case testVecTConstructFromListMultiply] +from typing import Optional +from vecs import vec +from mypy_extensions import i64 + +def f(n: i64) -> vec[Optional[str]]: + return vec[Optional[str]]([None] * n) +[out] +def f(n): + n :: int64 + r0, r1 :: object + r2 :: vec[union[str, None]] + r3 :: ptr + r4 :: int64 + r5, r6 :: ptr + r7 :: bit + r8 :: ptr +L0: + r0 = box(None, 1) + r1 = load_address PyUnicode_Type + r2 = VecTExtApi.alloc(n, r1, 1, 0) + r3 = get_element_ptr r2 items :: VecTExtObject + r4 = n * 8 + r5 = r3 + r4 + r6 = r3 +L1: + r7 = r6 < r5 :: unsigned + if r7 goto L2 else goto L3 :: bool +L2: + inc_ref r0 + set_mem r6, r0 :: union* + r8 = r6 + 8 + r6 = r8 + goto L1 +L3: + return r2 + +[case testVecTForLoop] +from vecs import vec +from mypy_extensions import i64 + +def f(v: vec[str]) -> i64: + t: i64 = 0 + for s in v: + g(s) + return t + +def g(s: str) -> None: pass +[out] +def f(v): + v :: vec[str] + t :: int64 + r0 :: native_int + r1 :: ptr + r2 :: int64 + r3 :: bit + r4 :: ptr + r5 :: native_int + r6 :: ptr + r7, s :: str + r8 :: None + r9 :: native_int +L0: + t = 0 + r0 = 0 +L1: + r1 = get_element_ptr v len :: VecObject + r2 = load_mem r1 :: int64* + r3 = r0 < r2 :: signed + if r3 goto L2 else goto L4 :: bool +L2: + r4 = get_element_ptr v items :: VecTObject + r5 = r0 * 8 + r6 = r4 + r5 + r7 = load_mem r6 :: builtins.str* + inc_ref r7 + s = r7 + r8 = g(s) + dec_ref s +L3: + r9 = r0 + 1 + r0 = r9 + goto L1 +L4: + return t +def g(s): + s :: str +L0: + return 1 + +[case testVecTConstructFromListExpr] +from vecs import vec + +def f() -> vec[str]: + return vec[str](['x', 'y']) +[out] +def f(): + r0, r1 :: str + r2 :: object + r3 :: vec[str] + r4, r5, r6 :: ptr +L0: + r0 = 'x' + r1 = 'y' + r2 = load_address PyUnicode_Type + r3 = VecTApi.alloc(2, r2) + r4 = get_element_ptr r3 items :: VecTObject + inc_ref r0 + set_mem r4, r0 :: builtins.str* + r5 = r4 + 8 + inc_ref r1 + set_mem r5, r1 :: builtins.str* + r6 = r5 + 8 + return r3 diff --git a/mypyc/test/test_irbuild.py b/mypyc/test/test_irbuild.py index 707ee11428626..9f706ec7bc426 100644 --- a/mypyc/test/test_irbuild.py +++ b/mypyc/test/test_irbuild.py @@ -44,6 +44,7 @@ "irbuild-i32.test", "irbuild-i16.test", "irbuild-u8.test", + "irbuild-vecs.test", "irbuild-vectorcall.test", "irbuild-unreachable.test", "irbuild-isinstance.test", diff --git a/test-data/unit/fixtures/dict.pyi b/test-data/unit/fixtures/dict.pyi index ed2287511161b..012d6c7eb2842 100644 --- a/test-data/unit/fixtures/dict.pyi +++ b/test-data/unit/fixtures/dict.pyi @@ -7,7 +7,7 @@ from _typeshed import SupportsKeysAndGetItem import _typeshed from typing import ( TypeVar, Generic, Iterable, Iterator, Mapping, Tuple, overload, Optional, Union, Sequence, - Self, + Self, Protocol ) T = TypeVar('T') @@ -62,3 +62,8 @@ class BaseException: pass def isinstance(x: object, t: Union[type, Tuple[type, ...]]) -> bool: pass def iter(__iterable: Iterable[T]) -> Iterator[T]: pass + +class Sized(Protocol): + def __len__(self) -> int: ... + +def len(s: Sized) -> int: ... diff --git a/test-data/unit/lib-stub/vecs.pyi b/test-data/unit/lib-stub/vecs.pyi new file mode 100644 index 0000000000000..7a34567d3fb2e --- /dev/null +++ b/test-data/unit/lib-stub/vecs.pyi @@ -0,0 +1,21 @@ +from typing import TypeVar, Generic, Type, Iterable, Iterator, overload +from mypy_extensions import i64 + +T = TypeVar("T") + +class vec(Generic[T]): + @overload + def __init__(self) -> None: ... + @overload + def __init__(self, __items: Iterable[T]) -> None: ... + def __len__(self) -> i64: ... + @overload + def __getitem__(self, i: i64) -> T: ... + @overload + def __getitem__(self, i: slice) -> vec[T]: ... + def __setitem__(self, i: i64, o: T) -> None: ... + def __iter__(self) -> Iterator[T]: ... + def pop(self, __i: i64 = -1) -> T: ... + def remove(self, __x: T) -> None: ... + +def append(v: vec[T], o: T) -> vec[T]: ... diff --git a/vecs.pyi b/vecs.pyi new file mode 100644 index 0000000000000..85ebfc433523c --- /dev/null +++ b/vecs.pyi @@ -0,0 +1,24 @@ +from typing import TypeVar, Generic, Type, Iterable, Iterator, overload +from mypy_extensions import i64 + +T = TypeVar("T") +S = TypeVar("S") + +class vec(Generic[T]): + @overload + def __init__(self) -> None: ... + @overload + def __init__(self, __init: Iterable[T]) -> None: ... + + def __len__(self) -> i64: ... + @overload + def __getitem__(self, i: i64) -> T: ... + @overload + def __getitem__(self, i: slice) -> vec[T]: ... + def __setitem__(self, i: i64, o: T) -> None: ... + def __iter__(self) -> Iterator[T]: ... + + def pop(self, __i: i64 = -1) -> T: ... + def remove(self, __x: T) -> None: ... + +def append(v: vec[T], o: T) -> vec[T]: ... From 398b48347632ed2b1fe0016e5af86bcdf41875c1 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 16 Sep 2022 22:18:11 +0100 Subject: [PATCH 002/180] Various fixes --- mypyc/ir/rtypes.py | 2 +- mypyc/irbuild/expression.py | 7 ++++--- mypyc/irbuild/ll_builder.py | 6 ++++-- mypyc/irbuild/vec.py | 6 ++++-- mypyc/test-data/irbuild-vecs.test | 2 +- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 5c241e641cb29..0493520594031 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -1030,7 +1030,7 @@ def vec_depth(t: RVec) -> int: if is_int64_rprimitive(it.item_type): return 0 else: - return 1 + vec_depth(vec.item_type) + return 1 + vec_depth(it.item_type) assert False, t diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index f9f91644b2d3c..9a491f255c053 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -594,8 +594,8 @@ def translate_vec_create_from_iterable( def translate_vec_method_call( - builder: IRBuilder, vec_expr: Expression, method: str, args: List[Expression] -) -> Optional[Value]: + builder: IRBuilder, vec_expr: Expression, method: str, args: list[Expression] +) -> Value | None: if method == "pop" and 0 <= len(args) <= 1: vec_val = builder.accept(vec_expr) if args: @@ -703,7 +703,8 @@ def transform_index_expr(builder: IRBuilder, expr: IndexExpr) -> Value: return value index_reg = builder.accept(expr.index, can_borrow=is_list) - return builder.builder.get_item(base, index_reg, builder.node_type(expr), expr.line) + return builder.builder.get_item(base, index_reg, builder.node_type(expr), expr.line, + can_borrow=builder.can_borrow) def try_constant_fold(builder: IRBuilder, expr: Expression) -> Value | None: diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 3700744c45e59..9f8892ea86b7a 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -109,6 +109,7 @@ is_int16_rprimitive, is_int32_rprimitive, is_int64_rprimitive, + int64_rprimitive, is_int_rprimitive, is_list_rprimitive, is_none_rprimitive, @@ -1982,7 +1983,8 @@ def unary_op(self, value: Value, op: str, line: int) -> Value: return self.unary_invert(value, line) raise RuntimeError("Unsupported unary operation: %s" % op) - def get_item(self, base: Value, item: Value, result_type: Optional[RType], line: int) -> Value: + def get_item(self, base: Value, item: Value, result_type: Optional[RType], line: int, + *, can_borrow: bool = False) -> Value: """Generate base[item].""" if isinstance(base.type, RVec): if is_int_rprimitive(item.type) or is_short_int_rprimitive(item.type): @@ -1990,7 +1992,7 @@ def get_item(self, base: Value, item: Value, result_type: Optional[RType], line: if is_int64_rprimitive(item.type): return vec_get_item(self, base, item, line) # TODO: Move special casing for RTuple here from transform_index_expr - return self.gen_method_call(base, "__getitem__", [item], result_type, line) + return self.gen_method_call(base, "__getitem__", [item], result_type, line, can_borrow=can_borrow) def make_dict(self, key_value_pairs: Sequence[DictEntry], line: int) -> Value: diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 76d836b99d9b6..93d7903486f9d 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -1,5 +1,6 @@ """Generate IR for vecs.vec operations""" +from __future__ import annotations from typing import TYPE_CHECKING, List, Optional, Tuple, Union, cast from mypyc.common import PLATFORM_SIZE @@ -44,6 +45,7 @@ optional_value_type, pointer_rprimitive, vec_depth, + is_short_int_rprimitive, is_int_rprimitive, ) from mypyc.primitives.registry import builtin_names @@ -51,11 +53,11 @@ from mypyc.irbuild.ll_builder import LowLevelIRBuilder -def as_platform_int(builder: "LowLevelIRBuilder", v: Value, line: int) -> Value: +def as_platform_int(builder: LowLevelIRBuilder, v: Value, line: int) -> Value: rtype = v.type if is_c_py_ssize_t_rprimitive(rtype): return v - if isinstance(rtype, Integer): + if isinstance(v, Integer): if is_short_int_rprimitive(rtype) or is_int_rprimitive(rtype): return Integer(v.value // 2, c_pyssize_t_rprimitive) return Integer(v.value, c_pyssize_t_rprimitive) diff --git a/mypyc/test-data/irbuild-vecs.test b/mypyc/test-data/irbuild-vecs.test index 1aa243eb3aa39..f2bfed6df35ab 100644 --- a/mypyc/test-data/irbuild-vecs.test +++ b/mypyc/test-data/irbuild-vecs.test @@ -444,7 +444,7 @@ def f(n, x): n, x :: int64 r0 :: vec[int64] r1 :: ptr - r2 :: int64 + r2 :: native_int r3, r4 :: ptr r5 :: bit r6 :: ptr From 33c41678fae9ad4ef6e74bf26cc76ed70950ce85 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 17 Sep 2022 09:52:52 +0100 Subject: [PATCH 003/180] Fix vec test case --- mypyc/test-data/irbuild-vecs.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/test-data/irbuild-vecs.test b/mypyc/test-data/irbuild-vecs.test index f2bfed6df35ab..b9d16e479f55f 100644 --- a/mypyc/test-data/irbuild-vecs.test +++ b/mypyc/test-data/irbuild-vecs.test @@ -385,7 +385,7 @@ from mypy_extensions import i64 def f() -> vec[i64]: return vec[i64]([1, 5, 14]) [out] -def f(n): +def f(): r0 :: vec[int64] r1, r2, r3, r4 :: ptr L0: From 93d4deb942680142c216d18dc570d0300573e1fa Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 17 Sep 2022 14:02:10 +0100 Subject: [PATCH 004/180] Fix vec comprehensions --- mypyc/irbuild/for_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index ce939906de0ff..583ab61eef66e 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -360,7 +360,7 @@ def gen_inner_stmts() -> None: def translate_vec_comprehension(builder: IRBuilder, vec_type: RVec, gen: GeneratorExpr) -> Value: vec = Register(vec_type) builder.assign(vec, vec_create(builder.builder, vec_type, 0, gen.line), gen.line) - loop_params = list(zip(gen.indices, gen.sequences, gen.condlists)) + loop_params = list(zip(gen.indices, gen.sequences, gen.condlists, gen.is_async)) def gen_inner_stmts() -> None: e = builder.accept(gen.left_expr) From ad81180a36a2cd9b93019b7fe0a22446be966c9e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 17 Sep 2022 14:04:57 +0100 Subject: [PATCH 005/180] Fix type applications --- mypyc/irbuild/expression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index 9a491f255c053..d0632792900bc 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -359,7 +359,7 @@ def transform_call_expr(builder: IRBuilder, expr: CallExpr) -> Value: return vec_create(builder.builder, vec_type, 0, expr.line) elif len(expr.args) == 1 and expr.arg_kinds == [ARG_POS]: return translate_vec_create_from_iterable(builder, vec_type, expr.args[0]) - callee = expr # Unwrap type application + callee = analyzed.expr # Unwrap type application if isinstance(callee, MemberExpr): if isinstance(callee.expr, RefExpr) and isinstance(callee.expr.node, MypyFile): From f2be756d708b0dccb6e92d94335e75b607fdd91c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 17 Sep 2022 17:32:15 +0100 Subject: [PATCH 006/180] Support some borrowing with vecs --- mypyc/irbuild/expression.py | 6 ++-- mypyc/irbuild/specialize.py | 2 +- mypyc/test-data/refcount.test | 60 +++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 4 deletions(-) diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index d0632792900bc..42fd9437d5ba5 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -679,8 +679,8 @@ def try_optimize_int_floor_divide(builder: IRBuilder, expr: OpExpr) -> OpExpr: def transform_index_expr(builder: IRBuilder, expr: IndexExpr) -> Value: index = expr.index base_type = builder.node_type(expr.base) - is_list = is_list_rprimitive(base_type) - can_borrow_base = is_list and is_borrow_friendly_expr(builder, index) + can_borrow = is_list_rprimitive(base_type) or isinstance(base_type, RVec) + can_borrow_base = can_borrow and is_borrow_friendly_expr(builder, index) # Check for dunder specialization for non-slice indexing if not isinstance(index, SliceExpr): @@ -702,7 +702,7 @@ def transform_index_expr(builder: IRBuilder, expr: IndexExpr) -> Value: if value: return value - index_reg = builder.accept(expr.index, can_borrow=is_list) + index_reg = builder.accept(expr.index, can_borrow=can_borrow) return builder.builder.get_item(base, index_reg, builder.node_type(expr), expr.line, can_borrow=builder.can_borrow) diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index 2157edbdde397..6a8f135dd4102 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -321,7 +321,7 @@ def translate_len(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value arg = expr.args[0] expr_rtype = builder.node_type(arg) # NOTE (?) I'm not sure if my handling of can_borrow is correct here - obj = builder.accept(arg, can_borrow=is_list_rprimitive(expr_rtype)) + obj = builder.accept(arg, can_borrow=is_list_rprimitive(expr_rtype) or isinstance(expr_rtype, RVec)) if is_sequence_rprimitive(expr_rtype) or isinstance(expr_rtype, RTuple): return get_expr_length_value(builder, arg, obj, expr.line, use_pyssize_t=False) else: diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index 515e383a3556c..6d6a77b0b4647 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -1651,3 +1651,63 @@ L0: set_mem r5, r1 :: builtins.str* r6 = r5 + 8 return r3 + +[case testVecI64GetItemBorrowVec] +from vecs import vec +from mypy_extensions import i64 + +class C: + v: vec[i64] + + def f(self, x: i64) -> i64: + return self.v[x] +[out] +def C.f(self, x): + self :: __main__.C + x :: int64 + r0 :: vec[int64] + r1 :: ptr + r2 :: native_int + r3 :: bit + r4 :: bool + r5 :: ptr + r6 :: int64 + r7 :: ptr + r8 :: int64 +L0: + r0 = borrow self.v + r1 = get_element_ptr r0 len :: VecObject + r2 = load_mem r1 :: native_int* + r3 = x < r2 :: unsigned + if r3 goto L2 else goto L1 :: bool +L1: + r4 = raise IndexError + unreachable +L2: + r5 = get_element_ptr r0 items :: VecI64Object + r6 = x * 8 + r7 = r5 + r6 + r8 = load_mem r7 :: int64* + return r8 + +[case testVecI64LenBorrowVec] +from vecs import vec +from mypy_extensions import i64 + +class C: + v: vec[i64] + + def f(self, x: i64) -> i64: + return len(self.v) +[out] +def C.f(self, x): + self :: __main__.C + x :: int64 + r0 :: vec[int64] + r1 :: ptr + r2 :: int64 +L0: + r0 = borrow self.v + r1 = get_element_ptr r0 len :: VecObject + r2 = load_mem r1 :: int64* + return r2 From 5205ae789f4cf0c8b249fcecce190f3dbc90c83e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 10 Apr 2023 10:19:11 +0100 Subject: [PATCH 007/180] Update a few test cases --- mypyc/test-data/irbuild-dict.test | 94 ++-- mypyc/test-data/irbuild-lists.test | 825 +++++++---------------------- 2 files changed, 245 insertions(+), 674 deletions(-) diff --git a/mypyc/test-data/irbuild-dict.test b/mypyc/test-data/irbuild-dict.test index e7a330951ab0d..dd39c7f3977de 100644 --- a/mypyc/test-data/irbuild-dict.test +++ b/mypyc/test-data/irbuild-dict.test @@ -159,7 +159,7 @@ def increment(d): r6 :: object r7, k :: str r8, r9, r10 :: object - r11 :: i32 + r11 :: int32 r12, r13, r14 :: bit L0: r0 = 0 @@ -184,7 +184,7 @@ L3: r13 = CPyDict_CheckSize(d, r1) goto L1 L4: - r14 = CPy_NoErrOccurred() + r14 = CPy_NoErrOccured() L5: return d @@ -216,7 +216,8 @@ L0: return r2 [case testDictIterationMethods] -from typing import Dict, TypedDict, Union +from typing import Dict, Union +from typing_extensions import TypedDict class Person(TypedDict): name: str @@ -236,7 +237,6 @@ def typeddict(d: Person) -> None: for k, v in d.items(): if k == "name": name = v -[typing fixtures/typing-full.pyi] [out] def print_dict_methods(d1, d2): d1, d2 :: dict @@ -249,7 +249,7 @@ def print_dict_methods(d1, d2): r6 :: object r7, v :: int r8 :: object - r9 :: i32 + r9 :: int32 r10 :: bit r11 :: bool r12, r13 :: bit @@ -262,7 +262,7 @@ def print_dict_methods(d1, d2): r20, r21 :: object r22, r23, k :: int r24, r25, r26, r27, r28 :: object - r29 :: i32 + r29 :: int32 r30, r31, r32 :: bit L0: r0 = 0 @@ -281,7 +281,7 @@ L2: r8 = box(int, v) r9 = PyDict_Contains(d2, r8) r10 = r9 >= 0 :: signed - r11 = truncate r9: i32 to builtins.bool + r11 = truncate r9: int32 to builtins.bool if r11 goto L3 else goto L4 :: bool L3: return 1 @@ -290,7 +290,7 @@ L5: r12 = CPyDict_CheckSize(d1, r1) goto L1 L6: - r13 = CPy_NoErrOccurred() + r13 = CPy_NoErrOccured() L7: r14 = 0 r15 = PyDict_Size(d2) @@ -319,7 +319,7 @@ L10: r31 = CPyDict_CheckSize(d2, r15) goto L8 L11: - r32 = CPy_NoErrOccurred() + r32 = CPy_NoErrOccured() L12: return 1 def union_of_dicts(d): @@ -335,14 +335,11 @@ def union_of_dicts(d): r10 :: union[int, str] k :: str v :: union[int, str] - r11 :: object - r12 :: object[1] - r13 :: object_ptr + r11, r12 :: object + r13 :: int r14 :: object - r15 :: int - r16 :: object - r17 :: i32 - r18, r19, r20 :: bit + r15 :: i32 + r16, r17, r18 :: bit L0: r0 = PyDict_New() new = r0 @@ -363,19 +360,16 @@ L2: k = r9 v = r10 r11 = load_address PyLong_Type - r12 = [v] - r13 = load_address r12 - r14 = PyObject_Vectorcall(r11, r13, 1, 0) - keep_alive v - r15 = unbox(int, r14) - r16 = box(int, r15) - r17 = CPyDict_SetItem(new, k, r16) - r18 = r17 >= 0 :: signed + r12 = PyObject_CallFunctionObjArgs(r11, v, 0) + r13 = unbox(int, r12) + r14 = box(int, r13) + r15 = CPyDict_SetItem(new, k, r14) + r16 = r15 >= 0 :: signed L3: - r19 = CPyDict_CheckSize(d, r2) + r17 = CPyDict_CheckSize(d, r2) goto L1 L4: - r20 = CPy_NoErrOccurred() + r18 = CPy_NoErrOccured() L5: return 1 def typeddict(d): @@ -390,9 +384,12 @@ def typeddict(d): r8, k :: str v :: object r9 :: str - r10 :: bool + r10 :: int32 + r11 :: bit + r12 :: object + r13, r14, r15 :: bit name :: object - r11, r12 :: bit + r16, r17 :: bit L0: r0 = 0 r1 = PyDict_Size(d) @@ -402,7 +399,7 @@ L1: r4 = r3[1] r0 = r4 r5 = r3[0] - if r5 goto L2 else goto L6 :: bool + if r5 goto L2 else goto L9 :: bool L2: r6 = r3[2] r7 = r3[3] @@ -410,17 +407,27 @@ L2: k = r8 v = r7 r9 = 'name' - r10 = CPyStr_EqualLiteral(k, r9, 4) - if r10 goto L3 else goto L4 :: bool + r10 = PyUnicode_Compare(k, r9) + r11 = r10 == -1 + if r11 goto L3 else goto L5 :: bool L3: - name = v + r12 = PyErr_Occurred() + r13 = r12 != 0 + if r13 goto L4 else goto L5 :: bool L4: + r14 = CPy_KeepPropagating() L5: - r11 = CPyDict_CheckSize(d, r1) - goto L1 + r15 = r10 == 0 + if r15 goto L6 else goto L7 :: bool L6: - r12 = CPy_NoErrOccurred() + name = v L7: +L8: + r16 = CPyDict_CheckSize(d, r1) + goto L1 +L9: + r17 = CPy_NoErrOccured() +L10: return 1 [case testDictLoadAddress] @@ -520,8 +527,8 @@ def f3(d, flag): r2 :: str r3 :: list r4 :: object - r5 :: ptr - r6, r7 :: object + r5, r6 :: ptr + r7, r8 :: object L0: if flag goto L1 else goto L2 :: bool L1: @@ -532,14 +539,15 @@ L2: r2 = 'a' r3 = PyList_New(1) r4 = object 1 - r5 = list_items r3 - buf_init_item r5, 0, r4 + r5 = get_element_ptr r3 ob_item :: PyListObject + r6 = load_mem r5 :: ptr* + set_mem r6, r4 :: builtins.object* keep_alive r3 - r6 = CPyDict_SetDefault(d, r2, r3) - return r6 -L3: - r7 = box(None, 1) + r7 = CPyDict_SetDefault(d, r2, r3) return r7 +L3: + r8 = box(None, 1) + return r8 def f4(d, flag): d :: dict flag :: bool diff --git a/mypyc/test-data/irbuild-lists.test b/mypyc/test-data/irbuild-lists.test index d864bfd19df26..2e5f8553fe8c8 100644 --- a/mypyc/test-data/irbuild-lists.test +++ b/mypyc/test-data/irbuild-lists.test @@ -108,15 +108,17 @@ def f() -> None: def f(): r0 :: list r1, r2 :: object - r3 :: ptr + r3, r4, r5 :: ptr x :: list L0: r0 = PyList_New(2) r1 = object 1 r2 = object 2 - r3 = list_items r0 - buf_init_item r3, 0, r1 - buf_init_item r3, 1, r2 + r3 = get_element_ptr r0 ob_item :: PyListObject + r4 = load_mem r3 :: ptr* + set_mem r4, r1 :: builtins.object* + r5 = r4 + WORD_SIZE*1 + set_mem r5, r2 :: builtins.object* keep_alive r0 x = r0 return 1 @@ -145,32 +147,6 @@ L0: x = r10 return 1 -[case testListAdd] -from typing import List -def f(a: List[int], b: List[int]) -> None: - c = a + b -[out] -def f(a, b): - a, b, r0, c :: list -L0: - r0 = PySequence_Concat(a, b) - c = r0 - return 1 - -[case testListIAdd] -from typing import List, Any -def f(a: List[int], b: Any) -> None: - a += b -[out] -def f(a, b): - a :: list - b :: object - r0 :: list -L0: - r0 = PySequence_InPlaceConcat(a, b) - a = r0 - return 1 - [case testListMultiply] from typing import List def f(a: List[int]) -> None: @@ -180,30 +156,19 @@ def f(a: List[int]) -> None: def f(a): a, r0, b, r1 :: list r2 :: object - r3 :: ptr - r4 :: list + r3, r4 :: ptr + r5 :: list L0: r0 = CPySequence_Multiply(a, 4) b = r0 r1 = PyList_New(1) r2 = object 4 - r3 = list_items r1 - buf_init_item r3, 0, r2 + r3 = get_element_ptr r1 ob_item :: PyListObject + r4 = load_mem r3 :: ptr* + set_mem r4, r2 :: builtins.object* keep_alive r1 - r4 = CPySequence_RMultiply(6, r1) - b = r4 - return 1 - -[case testListIMultiply] -from typing import List -def f(a: List[int]) -> None: - a *= 2 -[out] -def f(a): - a, r0 :: list -L0: - r0 = CPySequence_InPlaceMultiply(a, 4) - a = r0 + r5 = CPySequence_RMultiply(6, r1) + b = r5 return 1 [case testListLen] @@ -213,36 +178,15 @@ def f(a: List[int]) -> int: [out] def f(a): a :: list - r0 :: native_int - r1 :: short_int -L0: - r0 = var_object_size a - r1 = r0 << 1 - return r1 - -[case testListClear] -from typing import List -def f(l: List[int]) -> None: - return l.clear() -[out] -def f(l): - l :: list - r0 :: bit -L0: - r0 = CPyList_Clear(l) - return 1 - -[case testListCopy] -from typing import List -from typing import Any -def f(a: List[Any]) -> List[Any]: - return a.copy() -[out] -def f(a): - a, r0 :: list + r0 :: ptr + r1 :: native_int + r2 :: short_int L0: - r0 = CPyList_Copy(a) - return r0 + r0 = get_element_ptr a ob_size :: PyVarObject + r1 = load_mem r0 :: native_int* + keep_alive a + r2 = r1 << 1 + return r2 [case testListAppend] from typing import List @@ -270,30 +214,33 @@ def increment(l: List[int]) -> List[int]: [out] def increment(l): l :: list - r0 :: native_int - r1, r2 :: short_int + r0 :: ptr + r1 :: native_int + r2, r3 :: short_int i :: int - r3 :: bit - r4, r5, r6 :: object - r7 :: bit - r8 :: short_int + r4 :: bit + r5, r6, r7 :: object + r8 :: bit + r9 :: short_int L0: - r0 = var_object_size l - r1 = r0 << 1 - r2 = 0 - i = r2 + r0 = get_element_ptr l ob_size :: PyVarObject + r1 = load_mem r0 :: native_int* + keep_alive l + r2 = r1 << 1 + r3 = 0 + i = r3 L1: - r3 = int_lt r2, r1 - if r3 goto L2 else goto L4 :: bool + r4 = r3 < r2 :: signed + if r4 goto L2 else goto L4 :: bool L2: - r4 = CPyList_GetItem(l, i) - r5 = object 1 - r6 = PyNumber_InPlaceAdd(r4, r5) - r7 = CPyList_SetItem(l, i, r6) + r5 = CPyList_GetItem(l, i) + r6 = object 1 + r7 = PyNumber_InPlaceAdd(r5, r6) + r8 = CPyList_SetItem(l, i, r7) L3: - r8 = r2 + 2 - r2 = r8 - i = r8 + r9 = r3 + 2 + r3 = r9 + i = r9 goto L1 L4: return l @@ -306,23 +253,25 @@ def f(x: List[int], y: List[int]) -> List[int]: def f(x, y): x, y, r0 :: list r1, r2 :: object - r3 :: ptr - r4, r5, r6 :: object - r7 :: i32 - r8 :: bit + r3, r4, r5 :: ptr + r6, r7, r8 :: object + r9 :: i32 + r10 :: bit L0: r0 = PyList_New(2) r1 = object 1 r2 = object 2 - r3 = list_items r0 - buf_init_item r3, 0, r1 - buf_init_item r3, 1, r2 + r3 = get_element_ptr r0 ob_item :: PyListObject + r4 = load_mem r3 :: ptr* + set_mem r4, r1 :: builtins.object* + r5 = r4 + WORD_SIZE*1 + set_mem r5, r2 :: builtins.object* keep_alive r0 - r4 = CPyList_Extend(r0, x) - r5 = CPyList_Extend(r0, y) - r6 = object 3 - r7 = PyList_Append(r0, r6) - r8 = r7 >= 0 :: signed + r6 = CPyList_Extend(r0, x) + r7 = CPyList_Extend(r0, y) + r8 = object 3 + r9 = PyList_Append(r0, r8) + r10 = r9 >= 0 :: signed return r0 [case testListIn] @@ -369,65 +318,79 @@ def f(source: List[int]) -> None: [out] def f(source): source :: list - r0 :: native_int - r1 :: list - r2, r3 :: native_int - r4 :: bit - r5 :: object - r6, x, r7 :: int - r8 :: object - r9 :: native_int + r0 :: ptr + r1 :: native_int + r2 :: list + r3 :: native_int + r4 :: ptr + r5 :: native_int + r6 :: bit + r7 :: object + r8, x, r9 :: int + r10 :: object + r11 :: native_int a :: list - r10 :: native_int - r11 :: list - r12, r13 :: native_int - r14 :: bit - r15 :: object - r16, x_2, r17 :: int - r18 :: object - r19 :: native_int + r12 :: ptr + r13 :: native_int + r14 :: list + r15 :: native_int + r16 :: ptr + r17 :: native_int + r18 :: bit + r19 :: object + r20, x_2, r21 :: int + r22 :: object + r23 :: native_int b :: list L0: - r0 = var_object_size source - r1 = PyList_New(r0) - r2 = 0 + r0 = get_element_ptr source ob_size :: PyVarObject + r1 = load_mem r0 :: native_int* + keep_alive source + r2 = PyList_New(r1) + r3 = 0 L1: - r3 = var_object_size source - r4 = r2 < r3 :: signed - if r4 goto L2 else goto L4 :: bool + r4 = get_element_ptr source ob_size :: PyVarObject + r5 = load_mem r4 :: native_int* + keep_alive source + r6 = r3 < r5 :: signed + if r6 goto L2 else goto L4 :: bool L2: - r5 = list_get_item_unsafe source, r2 - r6 = unbox(int, r5) - x = r6 - r7 = CPyTagged_Add(x, 2) - r8 = box(int, r7) - CPyList_SetItemUnsafe(r1, r2, r8) + r7 = CPyList_GetItemUnsafe(source, r3) + r8 = unbox(int, r7) + x = r8 + r9 = CPyTagged_Add(x, 2) + r10 = box(int, r9) + CPyList_SetItemUnsafe(r2, r3, r10) L3: - r9 = r2 + 1 - r2 = r9 + r11 = r3 + 1 + r3 = r11 goto L1 L4: - a = r1 - r10 = var_object_size source - r11 = PyList_New(r10) - r12 = 0 + a = r2 + r12 = get_element_ptr source ob_size :: PyVarObject + r13 = load_mem r12 :: native_int* + keep_alive source + r14 = PyList_New(r13) + r15 = 0 L5: - r13 = var_object_size source - r14 = r12 < r13 :: signed - if r14 goto L6 else goto L8 :: bool + r16 = get_element_ptr source ob_size :: PyVarObject + r17 = load_mem r16 :: native_int* + keep_alive source + r18 = r15 < r17 :: signed + if r18 goto L6 else goto L8 :: bool L6: - r15 = list_get_item_unsafe source, r12 - r16 = unbox(int, r15) - x_2 = r16 - r17 = CPyTagged_Add(x_2, 2) - r18 = box(int, r17) - CPyList_SetItemUnsafe(r11, r12, r18) + r19 = CPyList_GetItemUnsafe(source, r15) + r20 = unbox(int, r19) + x_2 = r20 + r21 = CPyTagged_Add(x_2, 2) + r22 = box(int, r21) + CPyList_SetItemUnsafe(r14, r15, r22) L7: - r19 = r12 + 1 - r12 = r19 + r23 = r15 + 1 + r15 = r23 goto L5 L8: - b = r11 + b = r14 return 1 [case testGeneratorNext] @@ -438,37 +401,41 @@ def test(x: List[int]) -> None: [out] def test(x): x :: list - r0, r1 :: native_int - r2 :: bit - r3 :: object - r4, i :: int - r5 :: object - r6 :: union[int, None] - r7 :: native_int - r8 :: object + r0 :: native_int + r1 :: ptr + r2 :: native_int + r3 :: bit + r4 :: object + r5, i :: int + r6 :: object + r7 :: union[int, None] + r8 :: native_int + r9 :: object res :: union[int, None] L0: r0 = 0 L1: - r1 = var_object_size x - r2 = r0 < r1 :: signed - if r2 goto L2 else goto L4 :: bool + r1 = get_element_ptr x ob_size :: PyVarObject + r2 = load_mem r1 :: native_int* + keep_alive x + r3 = r0 < r2 :: signed + if r3 goto L2 else goto L4 :: bool L2: - r3 = list_get_item_unsafe x, r0 - r4 = unbox(int, r3) - i = r4 - r5 = box(int, i) - r6 = r5 + r4 = CPyList_GetItemUnsafe(x, r0) + r5 = unbox(int, r4) + i = r5 + r6 = box(int, i) + r7 = r6 goto L5 L3: - r7 = r0 + 1 - r0 = r7 + r8 = r0 + 1 + r0 = r8 goto L1 L4: - r8 = box(None, 1) - r6 = r8 + r9 = box(None, 1) + r7 = r9 L5: - res = r6 + res = r7 return 1 [case testSimplifyListUnion] @@ -487,488 +454,84 @@ def nested_union(a: Union[List[str], List[Optional[str]]]) -> None: [out] def narrow(a): a :: union[list, int] - r0 :: bit - r1 :: list - r2 :: native_int - r3 :: short_int - r4 :: int + r0 :: object + r1 :: i32 + r2 :: bit + r3 :: bool + r4 :: list + r5 :: ptr + r6 :: native_int + r7 :: short_int + r8 :: int L0: - r0 = PyList_Check(a) - if r0 goto L1 else goto L2 :: bool + r0 = load_address PyList_Type + r1 = PyObject_IsInstance(a, r0) + r2 = r1 >= 0 :: signed + r3 = truncate r1: i32 to builtins.bool + if r3 goto L1 else goto L2 :: bool L1: - r1 = borrow cast(list, a) - r2 = var_object_size r1 - r3 = r2 << 1 + r4 = borrow cast(list, a) + r5 = get_element_ptr r4 ob_size :: PyVarObject + r6 = load_mem r5 :: native_int* + keep_alive r4 + r7 = r6 << 1 keep_alive a - return r3 + return r7 L2: - r4 = unbox(int, a) - return r4 + r8 = unbox(int, a) + return r8 def loop(a): a :: list - r0, r1 :: native_int - r2 :: bit - r3 :: object - r4, x :: union[str, bytes] - r5 :: native_int -L0: - r0 = 0 -L1: - r1 = var_object_size a - r2 = r0 < r1 :: signed - if r2 goto L2 else goto L4 :: bool -L2: - r3 = list_get_item_unsafe a, r0 - r4 = cast(union[str, bytes], r3) - x = r4 -L3: - r5 = r0 + 1 - r0 = r5 - goto L1 -L4: - return 1 -def nested_union(a): - a :: list - r0, r1 :: native_int - r2 :: bit - r3 :: object - r4, x :: union[str, None] - r5 :: native_int -L0: - r0 = 0 -L1: - r1 = var_object_size a - r2 = r0 < r1 :: signed - if r2 goto L2 else goto L4 :: bool -L2: - r3 = list_get_item_unsafe a, r0 - r4 = cast(union[str, None], r3) - x = r4 -L3: - r5 = r0 + 1 - r0 = r5 - goto L1 -L4: - return 1 - -[case testSorted] -from typing import List, Any -def list_sort(a: List[int]) -> None: - a.sort() -def sort_iterable(a: Any) -> None: - sorted(a) -[out] -def list_sort(a): - a :: list - r0 :: i32 - r1 :: bit -L0: - r0 = PyList_Sort(a) - r1 = r0 >= 0 :: signed - return 1 -def sort_iterable(a): - a :: object - r0 :: list -L0: - r0 = CPySequence_Sort(a) - return 1 - -[case testListBuiltFromStr] -def f2(val: str) -> str: - return val + "f2" - -def test() -> None: - source = "abc" - a = [f2(x) for x in source] -[out] -def f2(val): - val, r0, r1 :: str -L0: - r0 = 'f2' - r1 = PyUnicode_Concat(val, r0) - return r1 -def test(): - r0, source :: str - r1 :: native_int - r2 :: bit - r3 :: list - r4 :: native_int - r5 :: bit - r6, x, r7 :: str - r8 :: native_int - a :: list -L0: - r0 = 'abc' - source = r0 - r1 = CPyStr_Size_size_t(source) - r2 = r1 >= 0 :: signed - r3 = PyList_New(r1) - r4 = 0 -L1: - r5 = r4 < r1 :: signed - if r5 goto L2 else goto L4 :: bool -L2: - r6 = CPyStr_GetItemUnsafe(source, r4) - x = r6 - r7 = f2(x) - CPyList_SetItemUnsafe(r3, r4, r7) -L3: - r8 = r4 + 1 - r4 = r8 - goto L1 -L4: - a = r3 - return 1 - -[case testListBuiltFromStrExpr] -def f2(val: str) -> str: - return val + "f2" - -def test() -> None: - a = [f2(x) for x in "abc"] -[out] -def f2(val): - val, r0, r1 :: str -L0: - r0 = 'f2' - r1 = PyUnicode_Concat(val, r0) - return r1 -def test(): - r0 :: str - r1 :: list + r0 :: native_int + r1 :: ptr r2 :: native_int r3 :: bit - r4, x, r5 :: str + r4 :: object + r5, x :: union[str, bytes] r6 :: native_int - a :: list L0: - r0 = 'abc' - r1 = PyList_New(3) - r2 = 0 - goto L2 + r0 = 0 L1: - r3 = r2 < 3 :: signed + r1 = get_element_ptr a ob_size :: PyVarObject + r2 = load_mem r1 :: native_int* + keep_alive a + r3 = r0 < r2 :: signed if r3 goto L2 else goto L4 :: bool L2: - r4 = CPyStr_GetItemUnsafe(r0, r2) - x = r4 - r5 = f2(x) - CPyList_SetItemUnsafe(r1, r2, r5) + r4 = CPyList_GetItemUnsafe(a, r0) + r5 = cast(union[str, bytes], r4) + x = r5 L3: - r6 = r2 + 1 - r2 = r6 + r6 = r0 + 1 + r0 = r6 goto L1 L4: - a = r1 return 1 - -[case testListBuiltFromFinalStr] -from typing import Final - -source: Final = "abc" - -def f2(val: str) -> str: - return val + "f2" - -def test() -> None: - a = [f2(x) for x in source] -[out] -def f2(val): - val, r0, r1 :: str -L0: - r0 = 'f2' - r1 = PyUnicode_Concat(val, r0) - return r1 -def test(): - r0 :: str - r1 :: list +def nested_union(a): + a :: list + r0 :: native_int + r1 :: ptr r2 :: native_int r3 :: bit - r4, x, r5 :: str + r4 :: object + r5, x :: union[str, None] r6 :: native_int - a :: list L0: - r0 = 'abc' - r1 = PyList_New(3) - r2 = 0 - goto L2 + r0 = 0 L1: - r3 = r2 < 3 :: signed + r1 = get_element_ptr a ob_size :: PyVarObject + r2 = load_mem r1 :: native_int* + keep_alive a + r3 = r0 < r2 :: signed if r3 goto L2 else goto L4 :: bool L2: - r4 = CPyStr_GetItemUnsafe(r0, r2) - x = r4 - r5 = f2(x) - CPyList_SetItemUnsafe(r1, r2, r5) -L3: - r6 = r2 + 1 - r2 = r6 - goto L1 -L4: - a = r1 - return 1 - -[case testListBuiltFromBytes_64bit] -def f2(val: int) -> int: - return val + 2 - -def test() -> None: - source = b"abc" - a = [f2(x) for x in source] - -[out] -def f2(val): - val, r0 :: int -L0: - r0 = CPyTagged_Add(val, 4) - return r0 -def test(): - r0, source :: bytes - r1 :: native_int - r2 :: list - r3 :: native_int - r4, r5, r6 :: bit - r7, r8, r9, r10 :: int - r11 :: object - r12, x, r13 :: int - r14 :: object - r15 :: native_int - a :: list -L0: - r0 = b'abc' - source = r0 - r1 = var_object_size source - r2 = PyList_New(r1) - r3 = 0 -L1: - r4 = r3 < r1 :: signed - if r4 goto L2 else goto L8 :: bool -L2: - r5 = r3 <= 4611686018427387903 :: signed - if r5 goto L3 else goto L4 :: bool -L3: - r6 = r3 >= -4611686018427387904 :: signed - if r6 goto L5 else goto L4 :: bool -L4: - r7 = CPyTagged_FromInt64(r3) - r8 = r7 - goto L6 -L5: - r9 = r3 << 1 - r8 = r9 -L6: - r10 = CPyBytes_GetItem(source, r8) - r11 = box(int, r10) - r12 = unbox(int, r11) - x = r12 - r13 = f2(x) - r14 = box(int, r13) - CPyList_SetItemUnsafe(r2, r3, r14) -L7: - r15 = r3 + 1 - r3 = r15 - goto L1 -L8: - a = r2 - return 1 - -[case testListBuiltFromBytesExpr_64bit] -def f2(val: int) -> int: - return val + 2 - -def test() -> None: - a = [f2(x) for x in b"abc"] - -[out] -def f2(val): - val, r0 :: int -L0: - r0 = CPyTagged_Add(val, 4) - return r0 -def test(): - r0 :: bytes - r1 :: list - r2 :: native_int - r3, r4, r5 :: bit - r6, r7, r8, r9 :: int - r10 :: object - r11, x, r12 :: int - r13 :: object - r14 :: native_int - a :: list -L0: - r0 = b'abc' - r1 = PyList_New(3) - r2 = 0 - goto L2 -L1: - r3 = r2 < 3 :: signed - if r3 goto L2 else goto L8 :: bool -L2: - r4 = r2 <= 4611686018427387903 :: signed - if r4 goto L3 else goto L4 :: bool + r4 = CPyList_GetItemUnsafe(a, r0) + r5 = cast(union[str, None], r4) + x = r5 L3: - r5 = r2 >= -4611686018427387904 :: signed - if r5 goto L5 else goto L4 :: bool -L4: - r6 = CPyTagged_FromInt64(r2) - r7 = r6 - goto L6 -L5: - r8 = r2 << 1 - r7 = r8 -L6: - r9 = CPyBytes_GetItem(r0, r7) - r10 = box(int, r9) - r11 = unbox(int, r10) - x = r11 - r12 = f2(x) - r13 = box(int, r12) - CPyList_SetItemUnsafe(r1, r2, r13) -L7: - r14 = r2 + 1 - r2 = r14 + r6 = r0 + 1 + r0 = r6 goto L1 -L8: - a = r1 - return 1 - -[case testListBuiltFromFinalBytes_64bit] -from typing import Final - -source: Final = b"abc" - -def f2(val: int) -> int: - return val + 2 - -def test() -> None: - a = [f2(x) for x in source] - -[out] -def f2(val): - val, r0 :: int -L0: - r0 = CPyTagged_Add(val, 4) - return r0 -def test(): - r0 :: bytes - r1 :: bool - r2 :: native_int - r3 :: list - r4 :: native_int - r5, r6, r7 :: bit - r8, r9, r10, r11 :: int - r12 :: object - r13, x, r14 :: int - r15 :: object - r16 :: native_int - a :: list -L0: - r0 = __main__.source :: static - if is_error(r0) goto L1 else goto L2 -L1: - r1 = raise NameError('value for final name "source" was not set') - unreachable -L2: - r2 = var_object_size r0 - r3 = PyList_New(r2) - r4 = 0 -L3: - r5 = r4 < r2 :: signed - if r5 goto L4 else goto L10 :: bool L4: - r6 = r4 <= 4611686018427387903 :: signed - if r6 goto L5 else goto L6 :: bool -L5: - r7 = r4 >= -4611686018427387904 :: signed - if r7 goto L7 else goto L6 :: bool -L6: - r8 = CPyTagged_FromInt64(r4) - r9 = r8 - goto L8 -L7: - r10 = r4 << 1 - r9 = r10 -L8: - r11 = CPyBytes_GetItem(r0, r9) - r12 = box(int, r11) - r13 = unbox(int, r12) - x = r13 - r14 = f2(x) - r15 = box(int, r14) - CPyList_SetItemUnsafe(r3, r4, r15) -L9: - r16 = r4 + 1 - r4 = r16 - goto L3 -L10: - a = r3 return 1 -[case testListBuiltFromStars] -from typing import Final - -abc: Final = "abc" - -def test() -> None: - a = [str(x) for x in [*abc, *"def", *b"ghi", ("j", "k"), *("l", "m", "n")]] - -[out] -def test(): - r0, r1 :: str - r2 :: bytes - r3, r4 :: str - r5 :: tuple[str, str] - r6, r7, r8 :: str - r9 :: tuple[str, str, str] - r10 :: list - r11, r12, r13, r14 :: object - r15 :: i32 - r16 :: bit - r17, r18 :: object - r19 :: list - r20, r21 :: native_int - r22 :: bit - r23, x :: object - r24 :: str - r25 :: native_int - a :: list -L0: - r0 = 'abc' - r1 = 'def' - r2 = b'ghi' - r3 = 'j' - r4 = 'k' - r5 = (r3, r4) - r6 = 'l' - r7 = 'm' - r8 = 'n' - r9 = (r6, r7, r8) - r10 = PyList_New(0) - r11 = CPyList_Extend(r10, r0) - r12 = CPyList_Extend(r10, r1) - r13 = CPyList_Extend(r10, r2) - r14 = box(tuple[str, str], r5) - r15 = PyList_Append(r10, r14) - r16 = r15 >= 0 :: signed - r17 = box(tuple[str, str, str], r9) - r18 = CPyList_Extend(r10, r17) - r19 = PyList_New(13) - r20 = 0 - goto L2 -L1: - r21 = var_object_size r10 - r22 = r20 < r21 :: signed - if r22 goto L2 else goto L4 :: bool -L2: - r23 = list_get_item_unsafe r10, r20 - x = r23 - r24 = PyObject_Str(x) - CPyList_SetItemUnsafe(r19, r20, r24) -L3: - r25 = r20 + 1 - r20 = r25 - goto L1 -L4: - a = r19 - return 1 From e7a79d7fe7ee839e8d6b6683aeb9d602982a7e58 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 30 Apr 2023 20:04:19 +0100 Subject: [PATCH 008/180] Add TODO --- mypyc/irbuild/expression.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index 42fd9437d5ba5..ffcb49e8a2ea7 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -590,6 +590,7 @@ def translate_vec_create_from_iterable( if isinstance(arg, ListComprehension): return translate_vec_comprehension(builder, vec_type, arg.generator) + # TODO: Construct vec from arbitrary iterable assert False, (vec_type, arg) From a77c41a3873dc84b4442e80030854576e5a0a5cd Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 8 May 2023 10:34:16 +0100 Subject: [PATCH 009/180] [WIP] Start work on new representation for vecs Some vec[i64] features are supported at the moment, but things are still very incomplete. --- mypyc/codegen/emit.py | 25 +- mypyc/codegen/emitfunc.py | 11 +- mypyc/ir/ops.py | 3 +- mypyc/ir/rtypes.py | 69 ++++-- mypyc/irbuild/ll_builder.py | 4 + mypyc/irbuild/vec.py | 24 +- mypyc/lib-rt/CPy.h | 4 +- mypyc/test-data/irbuild-vecs.test | 399 ++++++++++++++++++++++++++++++ 8 files changed, 501 insertions(+), 38 deletions(-) diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index 354c2f0e798f3..32f21a20271b2 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -519,6 +519,9 @@ def emit_inc_ref(self, dest: str, rtype: RType, *, rare: bool = False) -> None: elif isinstance(rtype, RTuple): for i, item_type in enumerate(rtype.types): self.emit_inc_ref(f"{dest}.f{i}", item_type) + elif isinstance(rtype, RVec): + # TODO: Only use the X variant if buf can be NULL + self.emit_line(f"CPy_XINCREF({dest}.buf);") elif not rtype.is_unboxed: # Always inline, since this is a simple but very hot op if rtype.may_be_immortal or not HAVE_IMMORTAL: @@ -547,6 +550,12 @@ def emit_dec_ref( elif isinstance(rtype, RTuple): for i, item_type in enumerate(rtype.types): self.emit_dec_ref(f"{dest}.f{i}", item_type, is_xdec=is_xdec, rare=rare) + elif isinstance(rtype, RVec): + # TODO: Only use the X variant if buf can be NULL + if rare: + self.emit_line(f"CPy_XDecRef({dest}.buf);") + else: + self.emit_line(f"CPy_XDECREF({dest}.buf);") elif not rtype.is_unboxed: if rare: self.emit_line(f"CPy_{x}DecRef({dest});") @@ -556,6 +565,8 @@ def emit_dec_ref( self.emit_line(f"CPy_{x}DECREF({dest});") else: self.emit_line(f"CPy_{x}DECREF_NO_IMM({dest});") + elif rtype.is_refcounted: + assert False, f"dec_ref not implemented for {rtype}" # Otherwise assume it's an unboxed, pointerless value and do nothing. def pretty_name(self, typ: RType) -> str: @@ -1034,7 +1045,13 @@ def emit_unbox( self.emit_line("}") if optional: self.emit_line("}") - + elif isinstance(typ, RVec): + assert is_int64_rprimitive(typ.item_type) # TODO: Support more item types + if declare_dest: + self.emit_line(f"{self.ctype(typ)} {dest};") + # TODO: Handle 'optional' + # TODO: Handle 'failure' + self.emit_line(f"{dest} = VecI64Api.unbox({src});") else: assert False, "Unboxing not implemented: %s" % typ @@ -1092,6 +1109,10 @@ def emit_box( inner_name = self.temp_name() self.emit_box(f"{src}.f{i}", inner_name, typ.types[i], declare_dest=True) self.emit_line(f"PyTuple_SET_ITEM({dest}, {i}, {inner_name});") + elif isinstance(typ, RVec): + assert is_int64_rprimitive(typ.item_type) # TODO: Support more item types + self.emit_line(f"{declaration}{dest} = VecI64Api.box({src});") + else: assert not typ.is_unboxed # Type is boxed -- trivially just assign. @@ -1105,6 +1126,8 @@ def emit_error_check(self, value: str, rtype: RType, failure: str) -> None: else: cond = self.tuple_undefined_check_cond(rtype, value, self.c_error_value, "==") self.emit_line(f"if ({cond}) {{") + elif isinstance(rtype, RVec): + self.emit_line(f"if ({value}.len < 0) {{") elif rtype.error_overlap: # The error value is also valid as a normal value, so we need to also check # for a raised exception. diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index 3f1bbab58895f..984aa588c0b51 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py @@ -82,6 +82,7 @@ RStruct, RTuple, RType, + RVec, is_bool_or_bit_rprimitive, is_int32_rprimitive, is_int64_rprimitive, @@ -221,6 +222,10 @@ def error_value_check(self, value: Value, compare: str) -> str: return self.emitter.tuple_undefined_check_cond( typ, self.reg(value), self.c_error_value, compare ) + elif isinstance(typ, RVec): + # Error values for vecs are represented by a negative length. + vec_compare = ">=" if compare == "!=" else "<" + return f"{self.reg(value)}.len {vec_compare} 0" else: return f"{self.reg(value)} {compare} {self.c_error_value(typ)}" @@ -312,11 +317,15 @@ def visit_assign_multi(self, op: AssignMulti) -> None: ) def visit_load_error_value(self, op: LoadErrorValue) -> None: + reg = self.reg(op) if isinstance(op.type, RTuple): values = [self.c_undefined_value(item) for item in op.type.types] tmp = self.temp_name() self.emit_line("{} {} = {{ {} }};".format(self.ctype(op.type), tmp, ", ".join(values))) - self.emit_line(f"{self.reg(op)} = {tmp};") + self.emit_line(f"{reg} = {tmp};") + elif isinstance(op.type, RVec): + self.emit_line(f"{reg}.len = -1;") + self.emit_line(f"{reg}.buf = NULL;") else: self.emit_line(f"{self.reg(op)} = {self.c_error_value(op.type)};") diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py index 8140660361acd..19849dffa2742 100644 --- a/mypyc/ir/ops.py +++ b/mypyc/ir/ops.py @@ -39,6 +39,7 @@ class to enable the new behavior. Sometimes adding a new abstract RTuple, RType, RUnion, + RVec, RVoid, bit_rprimitive, bool_rprimitive, @@ -1690,7 +1691,7 @@ class GetElement(RegisterOp): def __init__(self, src: Value, field: str, line: int = -1) -> None: super().__init__(line) - assert isinstance(src.type, RStruct) + assert isinstance(src.type, (RStruct, RVec)) self.type = src.type.field_type(field) self.src = src self.src_type = src.type diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 0493520594031..5009ef97d62ed 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -38,7 +38,7 @@ class to enable the new behavior. In rare cases, adding a new from __future__ import annotations from abc import abstractmethod -from typing import TYPE_CHECKING, ClassVar, Final, Generic, TypeGuard, TypeVar, final +from typing import TYPE_CHECKING, ClassVar, Final, Generic, TypeGuard, TypeVar, Union, final from mypyc.common import HAVE_IMMORTAL, IS_32_BIT_PLATFORM, PLATFORM_SIZE, JsonDict, short_name from mypyc.ir.deps import LIBRT_STRINGS, Dependency @@ -175,6 +175,10 @@ def visit_rvoid(self, typ: RVoid, /) -> T: raise NotImplementedError +# These types are implemented as C structs. +RStructLike = Union["RStruct", "RVec"] + + @final class RVoid(RType): """The void type (no value). @@ -995,12 +999,31 @@ def serialize(self) -> str: class RVec(RType): """vecs.vec[T]""" - is_unboxed = False + is_unboxed = True def __init__(self, item_type: RType) -> None: self.name = "vec[%s]" % item_type self.item_type = item_type - self._ctype = "PyObject *" + if isinstance(item_type, RUnion): + non_opt = optional_value_type(item_type) + else: + non_opt = item_type + if is_int64_rprimitive(item_type): + self._ctype = "VecI64" + self.buf_type = VecbufI64Object + elif isinstance(non_opt, RVec): + self._ctype = "VecTExt" + # TODO: buf_type + else: + self._ctype = "VecT" + # TODO: buf_type + + def field_type(self, name: str) -> RType: + if name == "len": + return c_pyssize_t_rprimitive + elif name == "buf": + return object_rprimitive + assert False, f"RVec has no field '{name}'" def accept(self, visitor: "RTypeVisitor[T]") -> T: return visitor.visit_rvec(self) @@ -1023,14 +1046,12 @@ def vec_depth(t: RVec) -> int: it = t.item_type if isinstance(it, RUnion): non_opt = optional_value_type(it) + assert non_opt is not None it = non_opt if isinstance(it, (RPrimitive, RInstance)): return 0 elif isinstance(it, RVec): - if is_int64_rprimitive(it.item_type): - return 0 - else: - return 1 + vec_depth(it.item_type) + return 1 + vec_depth(it) assert False, t @@ -1231,25 +1252,42 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: return -limit <= n < limit +""" + # vec (common fields) VecObject = RStruct( name="VecObject", names=["ob_base", "len"], types=[PyObject, c_pyssize_t_rprimitive] ) +""" -# vec[i64] -VecI64Object = RStruct( - name="VecI64Object", + +# Buffer for vec[i64] +VecbufI64Object = RStruct( + name="VecbufI64Object", names=["ob_base", "len", "items"], - types=[PyObject, c_pyssize_t_rprimitive, int64_rprimitive], + types=[PyVarObject, int64_rprimitive], ) -# vec[t] -VecTObject = RStruct( +#vecbuf_i64_rprimitive: Final = RPrimitive( +# "VecbufI64Object", is_unboxed=False, is_refcounted=True, ctype="VecbufI64Object *" +#) + +# vec[i64] +VecI64 = RStruct( + name="VecI64", + names=["len", "buf"], + types=[c_pyssize_t_rprimitive, object_rprimitive], +) + + +# Buffer for vec[t] +VecbufTObject = RStruct( name="VecTObject", - names=["ob_base", "len", "items"], - types=[PyObject, c_pyssize_t_rprimitive, object_rprimitive], + names=["ob_base", "item_type", "items"], + types=[PyVarObject, c_pyssize_t_rprimitive, object_rprimitive], ) +""" # vec[t], for nested vec or optional t VecTExtObject = RStruct( name="VecTExtObject", @@ -1262,3 +1300,4 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: object_rprimitive, ], ) +""" diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 9f8892ea86b7a..ba0131dd376f9 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -49,6 +49,7 @@ FloatNeg, FloatOp, GetAttr, + GetElement, GetElementPtr, Goto, Integer, @@ -319,6 +320,9 @@ def load_mem(self, ptr: Value, value_type: RType, *, borrow: bool = False) -> Va def set_mem(self, ptr: Value, value_type: RType, value: Value) -> None: self.add(SetMem(value_type, ptr, value, line=-1)) + def get_element(self, reg: Value, field: str) -> Value: + return self.add(GetElement(reg, field)) + def load_address(self, name: str, rtype: RType) -> Value: return self.add(LoadAddress(rtype, name)) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 93d7903486f9d..34140ce5e156d 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -27,10 +27,6 @@ RType, RUnion, RVec, - VecI64Object, - VecObject, - VecTExtObject, - VecTObject, bool_rprimitive, c_int_rprimitive, c_pyssize_t_rprimitive, @@ -187,26 +183,19 @@ def vec_item_type_info( def vec_len(builder: "LowLevelIRBuilder", val: Value) -> Value: - address = builder.add(GetElementPtr(val, VecObject, "len")) # TODO: what about 32-bit archs? - len_value = builder.load_mem(address, int64_rprimitive) - builder.keep_alive([val]) - return len_value + # TODO: merge vec_len and vec_len_native + return vec_len_native(builder, val) def vec_len_native(builder: "LowLevelIRBuilder", val: Value) -> Value: - return builder.load_struct_field(val, VecObject, "len") + return builder.get_element(val, "len") def vec_items(builder: "LowLevelIRBuilder", vecobj: Value) -> Value: vtype = cast(RVec, vecobj.type) - if is_int64_rprimitive(vtype.item_type): - vec_struct = VecI64Object - elif vec_depth(vtype) == 0 and not isinstance(vtype.item_type, RUnion): - vec_struct = VecTObject - else: - vec_struct = VecTExtObject - return builder.add(GetElementPtr(vecobj, vec_struct, "items")) + buf = builder.get_element(vecobj, "buf") + return builder.add(GetElementPtr(buf, vtype.buf_type, "items")) def vec_item_ptr(builder: "LowLevelIRBuilder", vecobj: Value, index: Value) -> Value: @@ -229,8 +218,7 @@ def vec_check_index(builder: "LowLevelIRBuilder", lenv: Value, index: Value, lin def vec_get_item(builder: "LowLevelIRBuilder", base: Value, index: Value, line: int) -> Value: """Generate inlined vec __getitem__ call. - We inline this, since it's fairly simple but very - performance-critical. + We inline this, since it's simple but performance-critical. """ assert isinstance(base.type, RVec) vtype = base.type diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index 2ca247be454a9..f77c8e663ef48 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -609,8 +609,8 @@ static void CPy_DecRef(PyObject *p) { } CPy_NOINLINE -static void CPy_XDecRef(PyObject *p) { - CPy_XDECREF(p); +static void CPy_XDecRef(void *p) { + CPy_XDECREF((PyObject *)p); } static inline CPyTagged CPyObject_Size(PyObject *obj) { diff --git a/mypyc/test-data/irbuild-vecs.test b/mypyc/test-data/irbuild-vecs.test index b9d16e479f55f..3c9de92179b09 100644 --- a/mypyc/test-data/irbuild-vecs.test +++ b/mypyc/test-data/irbuild-vecs.test @@ -735,3 +735,402 @@ L0: r4 = VecI64Api.slice(v, 1, -2) e = r4 return 1 + +-- +-- New repr +-- + +[case testNewVecI64CreateEmpty] +from vecs import vec, append +from mypy_extensions import i64 + +def f() -> vec[i64]: + return vec[i64]() +[out] +def f(): + r0 :: vec[int64] +L0: + r0 = VecI64Api.alloc(0) + return r0 + +[case testNewVecI64Len] +from vecs import vec +from mypy_extensions import i64 + +def f(v: vec[i64]) -> i64: + l = len(v) + return l +[out] +def f(v): + v :: vec[int64] + r0 :: native_int + l :: int64 +L0: + r0 = v.len + l = r0 + return l + +[case testNewVecI64GetItem] +from vecs import vec +from mypy_extensions import i64 + +def f(v: vec[i64], i: i64) -> i64: + return v[i] +[out] +def f(v, i): + v :: vec[int64] + i :: int64 + r0 :: native_int + r1 :: bit + r2 :: bool + r3 :: object + r4 :: ptr + r5 :: int64 + r6 :: ptr + r7 :: int64 +L0: + r0 = v.len + r1 = i < r0 :: unsigned + if r1 goto L2 else goto L1 :: bool +L1: + r2 = raise IndexError + unreachable +L2: + r3 = v.buf + r4 = get_element_ptr r3 items :: VecbufI64Object + r5 = i * 8 + r6 = r4 + r5 + r7 = load_mem r6 :: int64* + keep_alive v + return r7 + +[case testNewVecI64Append] +from vecs import vec, append +from mypy_extensions import i64 + +def f(v: vec[i64], i: i64) -> vec[i64]: + return append(v, i) +[out] +def f(v, i): + v :: vec[int64] + i :: int64 + r0 :: vec[int64] +L0: + r0 = VecI64Api.append(v, i) + return r0 + +[case testNewVecI64SetItem] +from vecs import vec +from mypy_extensions import i64 + +def f(v: vec[i64], i: i64, x: i64) -> None: + v[i] = x +[out] +def f(v, i, x): + v :: vec[int64] + i, x :: int64 + r0 :: native_int + r1 :: bit + r2 :: bool + r3 :: object + r4 :: ptr + r5 :: int64 + r6 :: ptr +L0: + r0 = v.len + r1 = i < r0 :: unsigned + if r1 goto L2 else goto L1 :: bool +L1: + r2 = raise IndexError + unreachable +L2: + r3 = v.buf + r4 = get_element_ptr r3 items :: VecbufI64Object + r5 = i * 8 + r6 = r4 + r5 + set_mem r6, x :: int64* + keep_alive v + return 1 + + +[case testNewVecI64ConstructFromListExpr] +from vecs import vec +from mypy_extensions import i64 + +def f() -> vec[i64]: + return vec[i64]([1, 5, 14]) +[out] +def f(): + r0 :: vec[int64] + r1 :: object + r2, r3, r4, r5 :: ptr +L0: + r0 = VecI64Api.alloc(3) + r1 = r0.buf + r2 = get_element_ptr r1 items :: VecbufI64Object + set_mem r2, 1 :: int64* + r3 = r2 + 8 + set_mem r3, 5 :: int64* + r4 = r3 + 8 + set_mem r4, 14 :: int64* + r5 = r4 + 8 + keep_alive r0 + return r0 + +[case testNewVecI64ConstructFromListMultiply] +from vecs import vec +from mypy_extensions import i64 + +def f(n: i64) -> vec[i64]: + return vec[i64]([3] * n) +[out] +def f(n): + n :: int64 + r0 :: vec[int64] + r1 :: object + r2 :: ptr + r3 :: int64 + r4, r5 :: ptr + r6 :: bit + r7 :: ptr +L0: + r0 = VecI64Api.alloc(n) + r1 = r0.buf + r2 = get_element_ptr r1 items :: VecbufI64Object + r3 = n * 8 + r4 = r2 + r3 + r5 = r2 +L1: + r6 = r5 < r4 :: unsigned + if r6 goto L2 else goto L3 :: bool +L2: + set_mem r5, 3 :: int64* + r7 = r5 + 8 + r5 = r7 + goto L1 +L3: + keep_alive r0 + return r0 + +[case testNewVecI64ConstructFromListMultiply2] +from vecs import vec +from mypy_extensions import i64 + +def f(n: i64, x: i64) -> vec[i64]: + return vec[i64]([x] * 3) +[out] +def f(n, x): + n, x :: int64 + r0 :: vec[int64] + r1 :: object + r2 :: ptr + r3 :: native_int + r4, r5 :: ptr + r6 :: bit + r7 :: ptr +L0: + r0 = VecI64Api.alloc(3) + r1 = r0.buf + r2 = get_element_ptr r1 items :: VecbufI64Object + r3 = 3 * 8 + r4 = r2 + r3 + r5 = r2 +L1: + r6 = r5 < r4 :: unsigned + if r6 goto L2 else goto L3 :: bool +L2: + set_mem r5, x :: int64* + r7 = r5 + 8 + r5 = r7 + goto L1 +L3: + keep_alive r0 + return r0 + +[case testNewVecI64ConstructFromListComprehension] +from vecs import vec +from mypy_extensions import i64 + +def f(n: i64) -> vec[i64]: + return vec[i64]([x + 1 for x in range(i64(5))]) +[out] +def f(n): + n :: int64 + r0, r1 :: vec[int64] + r2, x :: int64 + r3 :: bit + r4 :: int64 + r5 :: vec[int64] + r6 :: int64 +L0: + r0 = VecI64Api.alloc(0) + r1 = r0 + r2 = 0 + x = r2 +L1: + r3 = r2 < 5 :: signed + if r3 goto L2 else goto L4 :: bool +L2: + r4 = x + 1 + r5 = VecI64Api.append(r1, r4) + r1 = r5 +L3: + r6 = r2 + 1 + r2 = r6 + x = r6 + goto L1 +L4: + return r1 + +[case testNewVecI64ForLoop] +from vecs import vec +from mypy_extensions import i64 + +def f(v: vec[i64]) -> i64: + t: i64 = 0 + for x in v: + t += 1 + return t +[out] +def f(v): + v :: vec[int64] + t :: int64 + r0, r1 :: native_int + r2 :: bit + r3 :: object + r4 :: ptr + r5 :: native_int + r6 :: ptr + r7, x, r8 :: int64 + r9 :: native_int +L0: + t = 0 + r0 = 0 +L1: + r1 = v.len + r2 = r0 < r1 :: signed + if r2 goto L2 else goto L4 :: bool +L2: + r3 = v.buf + r4 = get_element_ptr r3 items :: VecbufI64Object + r5 = r0 * 8 + r6 = r4 + r5 + r7 = load_mem r6 :: int64* + keep_alive v + x = r7 + r8 = t + 1 + t = r8 +L3: + r9 = r0 + 1 + r0 = r9 + goto L1 +L4: + return t + +[case testNewVecI64Contains] +from vecs import vec +from mypy_extensions import i64 + +def contains(v: vec[i64], n: i64) -> bool: + return n in v +[out] +def contains(v, n): + v :: vec[int64] + n :: int64 + r0 :: native_int + r1 :: object + r2 :: ptr + r3 :: native_int + r4, r5 :: ptr + r6 :: bit + r7 :: int64 + r8 :: bit + r9 :: ptr + r10 :: bool +L0: + r0 = v.len + r1 = v.buf + r2 = get_element_ptr r1 items :: VecbufI64Object + r3 = r0 * 8 + r4 = r2 + r3 + r5 = r2 +L1: + r6 = r5 < r4 :: unsigned + if r6 goto L2 else goto L4 :: bool +L2: + r7 = load_mem r5 :: int64* + r8 = r7 == n + if r8 goto L5 else goto L3 :: bool +L3: + r9 = r5 + 8 + r5 = r9 + goto L1 +L4: + keep_alive v + r10 = 0 + goto L6 +L5: + r10 = 1 +L6: + return r10 + +[case testNewVecI64GetItemWithInt] +from vecs import vec +from mypy_extensions import i64 + +def f(v: vec[i64]) -> i64: + return v[0] +[out] +def f(v): + v :: vec[int64] + r0 :: native_int + r1 :: bit + r2 :: bool + r3 :: object + r4 :: ptr + r5 :: int64 + r6 :: ptr + r7 :: int64 +L0: + r0 = v.len + r1 = 0 < r0 :: unsigned + if r1 goto L2 else goto L1 :: bool +L1: + r2 = raise IndexError + unreachable +L2: + r3 = v.buf + r4 = get_element_ptr r3 items :: VecbufI64Object + r5 = 0 * 8 + r6 = r4 + r5 + r7 = load_mem r6 :: int64* + keep_alive v + return r7 + +[case testNewVecI64Slicing] +from vecs import vec +from mypy_extensions import i64 + +def f(v: vec[i64], n: i64, m: i64) -> None: + a = v[:] + b = v[n:] + c = v[n:m] + d = v[:m] + e = v[1:-2] +[out] +def f(v, n, m): + v :: vec[int64] + n, m :: int64 + r0, a, r1, b, r2, c, r3, d, r4, e :: vec[int64] +L0: + r0 = VecI64Api.slice(v, 0, 4611686018427387903) + a = r0 + r1 = VecI64Api.slice(v, n, 4611686018427387903) + b = r1 + r2 = VecI64Api.slice(v, n, m) + c = r2 + r3 = VecI64Api.slice(v, 0, m) + d = r3 + r4 = VecI64Api.slice(v, 1, -2) + e = r4 + return 1 From ee2153429b9fa2b0af8e27a461fc63b0cdd0e21c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 8 May 2023 16:08:31 +0100 Subject: [PATCH 010/180] Support vec[i64] remove() --- mypyc/irbuild/specialize.py | 16 +++++++++++++++- mypyc/irbuild/vec.py | 2 +- mypyc/test-data/irbuild-vecs.test | 16 ++++++++++++++++ test-data/unit/lib-stub/vecs.pyi | 2 +- 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index 6a8f135dd4102..40d7311901788 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -104,7 +104,7 @@ join_formatted_strings, tokenizer_format_call, ) -from mypyc.irbuild.vec import vec_append +from mypyc.irbuild.vec import vec_append, vec_remove from mypyc.primitives.bytearray_ops import isinstance_bytearray from mypyc.primitives.bytes_ops import ( bytes_adjust_index_op, @@ -1514,3 +1514,17 @@ def translate_vec_append(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> arg_value = builder.accept(item_arg) return vec_append(builder.builder, vec_value, arg_value, item_arg.line) return None + + +@specialize_function("vecs.remove") +def translate_vec_append(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Optional[Value]: + if len(expr.args) == 2 and expr.arg_kinds == [ARG_POS, ARG_POS]: + vec_arg = expr.args[0] + item_arg = expr.args[1] + vec_type = builder.node_type(vec_arg) + item_type = builder.node_type(item_arg) + if isinstance(vec_type, RVec): + vec_value = builder.accept(vec_arg) + arg_value = builder.accept(item_arg) + return vec_remove(builder.builder, vec_value, arg_value, item_arg.line) + return None diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 34140ce5e156d..708225e85ead3 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -335,7 +335,7 @@ def vec_remove(builder: "LowLevelIRBuilder", vec: Value, item: Value, line: int) call = CallC( name, [vec, item], - c_int_rprimitive, + vec_type, steals=[False, False], is_borrowed=False, error_kind=ERR_FALSE, diff --git a/mypyc/test-data/irbuild-vecs.test b/mypyc/test-data/irbuild-vecs.test index 3c9de92179b09..18734ccb6d665 100644 --- a/mypyc/test-data/irbuild-vecs.test +++ b/mypyc/test-data/irbuild-vecs.test @@ -1134,3 +1134,19 @@ L0: r4 = VecI64Api.slice(v, 1, -2) e = r4 return 1 + +[case testNewVecI64Remove] +from vecs import vec, remove +from mypy_extensions import i64 + +def rem(v: vec[i64], n: i64) -> None: + v = remove(v, n) +[out] +def rem(v, n): + v :: vec[int64] + n :: int64 + r0 :: vec[int64] +L0: + r0 = VecI64Api.remove(v, n) + v = r0 + return 1 diff --git a/test-data/unit/lib-stub/vecs.pyi b/test-data/unit/lib-stub/vecs.pyi index 7a34567d3fb2e..a9e8dcb03227e 100644 --- a/test-data/unit/lib-stub/vecs.pyi +++ b/test-data/unit/lib-stub/vecs.pyi @@ -16,6 +16,6 @@ class vec(Generic[T]): def __setitem__(self, i: i64, o: T) -> None: ... def __iter__(self) -> Iterator[T]: ... def pop(self, __i: i64 = -1) -> T: ... - def remove(self, __x: T) -> None: ... def append(v: vec[T], o: T) -> vec[T]: ... +def remove(v: vec[T], o: T) -> vec[T]: ... From c3138ba54892cd90c1b163b69bb6a4f21d4610cb Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 8 May 2023 20:08:11 +0100 Subject: [PATCH 011/180] Support vec instances in attributes --- mypyc/codegen/emit.py | 19 ++++++++++++++++++- mypyc/codegen/emitclass.py | 4 ++-- mypyc/codegen/emitfunc.py | 3 +-- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index 32f21a20271b2..e3695866bce0c 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -327,6 +327,13 @@ def ctype_spaced(self, rtype: RType) -> str: else: return ctype + " " + def set_undefined_value(self, target: str, rtype: RType) -> None: + if isinstance(rtype, RVec): + self.emit_line(f"{target}.len = -1;") + self.emit_line(f"{target}.buf = NULL;") + else: + self.emit_line(f"{target} = {self.c_undefined_value(rtype)};") + def c_undefined_value(self, rtype: RType) -> str: if not rtype.is_unboxed: return "NULL" @@ -436,6 +443,12 @@ def error_value_check(self, rtype: RType, value: str, compare: str) -> str: return self.tuple_undefined_check_cond( rtype, value, self.c_error_value, compare, check_exception=False ) + elif isinstance(rtype, RVec): + if compare == "==": + return f"{value}.len < 0" + elif compare == "!=": + return f"{value}.len >= 0" + assert False, compare else: return f"{value} {compare} {self.c_error_value(rtype)}" @@ -521,7 +534,7 @@ def emit_inc_ref(self, dest: str, rtype: RType, *, rare: bool = False) -> None: self.emit_inc_ref(f"{dest}.f{i}", item_type) elif isinstance(rtype, RVec): # TODO: Only use the X variant if buf can be NULL - self.emit_line(f"CPy_XINCREF({dest}.buf);") + self.emit_line(f"Py_XINCREF({dest}.buf);") elif not rtype.is_unboxed: # Always inline, since this is a simple but very hot op if rtype.may_be_immortal or not HAVE_IMMORTAL: @@ -1152,6 +1165,8 @@ def emit_gc_visit(self, target: str, rtype: RType) -> None: elif isinstance(rtype, RTuple): for i, item_type in enumerate(rtype.types): self.emit_gc_visit(f"{target}.f{i}", item_type) + elif isinstance(rtype, RVec): + self.emit_line(f"Py_VISIT({target}.buf);") elif self.ctype(rtype) == "PyObject *": # The simplest case. self.emit_line(f"Py_VISIT({target});") @@ -1176,6 +1191,8 @@ def emit_gc_clear(self, target: str, rtype: RType) -> None: elif isinstance(rtype, RTuple): for i, item_type in enumerate(rtype.types): self.emit_gc_clear(f"{target}.f{i}", item_type) + elif isinstance(rtype, RVec): + self.emit_line(f"Py_CLEAR({target}.buf);") elif self.ctype(rtype) == "PyObject *" and self.c_undefined_value(rtype) == "NULL": # The simplest case. self.emit_line(f"Py_CLEAR({target});") diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 8c3fa5de98f85..387065d93bcfe 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -653,7 +653,7 @@ def generate_setup_for_class( # We don't need to set this field to NULL since tp_alloc() already # zero-initializes `self`. if value != "NULL": - emitter.emit_line(rf"self->{emitter.attr(attr)} = {value};") + emitter.set_undefined_value(f"self->{emitter.attr(attr)}", rtype) # Initialize attributes to default values, if necessary if defaults_fn is not None: @@ -1194,7 +1194,7 @@ def generate_setter(cl: ClassIR, attr: str, rtype: RType, emitter: Emitter) -> N if deletable: emitter.emit_line("} else") - emitter.emit_line(f" self->{attr_field} = {emitter.c_undefined_value(rtype)};") + emitter.set_undefined_value(f" self->{attr_field}", rtype) if rtype.error_overlap: emitter.emit_attr_bitmap_clear("self", rtype, cl, attr) emitter.emit_line("return 0;") diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index 984aa588c0b51..42fb1e7c0f647 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py @@ -324,8 +324,7 @@ def visit_load_error_value(self, op: LoadErrorValue) -> None: self.emit_line("{} {} = {{ {} }};".format(self.ctype(op.type), tmp, ", ".join(values))) self.emit_line(f"{reg} = {tmp};") elif isinstance(op.type, RVec): - self.emit_line(f"{reg}.len = -1;") - self.emit_line(f"{reg}.buf = NULL;") + self.emitter.set_undefined_value(reg, op.type); else: self.emit_line(f"{self.reg(op)} = {self.c_error_value(op.type)};") From a3d62676c002a6323e0df3b679e8ea3df0f7f0da Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 29 May 2023 13:45:26 +0100 Subject: [PATCH 012/180] Support vec[i64] pop with new repr --- mypyc/codegen/emit.py | 4 + mypyc/ir/rtypes.py | 5 +- mypyc/irbuild/expression.py | 24 -- mypyc/irbuild/specialize.py | 19 +- mypyc/irbuild/vec.py | 12 +- mypyc/test-data/irbuild-vecs.test | 504 ++---------------------------- test-data/unit/lib-stub/vecs.pyi | 8 +- vecs.pyi | 8 +- 8 files changed, 66 insertions(+), 518 deletions(-) diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index e3695866bce0c..43290dac787c8 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -480,6 +480,8 @@ def tuple_undefined_check_cond( return self.tuple_undefined_check_cond( item_type, tuple_expr_in_c + f".f{i}", c_type_compare_val, compare ) + elif isinstance(item_type, RVec): + return f"{tuple_expr_in_c}.f{i}.len {compare} -1" else: check = f"{tuple_expr_in_c}.f{i} {compare} {c_type_compare_val(item_type)}" if rtuple.error_overlap and check_exception: @@ -499,6 +501,8 @@ def c_initializer_undefined_value(self, rtype: RType) -> str: return f"{{ {int_rprimitive.c_undefined} }}" items = ", ".join([self.c_initializer_undefined_value(t) for t in rtype.types]) return f"{{ {items} }}" + elif isinstance(rtype, RVec): + res.append("{ -1, NULL }") else: return self.c_undefined_value(rtype) diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 5009ef97d62ed..e69c4816be35e 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -693,7 +693,10 @@ def visit_rinstance(self, t: RInstance) -> str: return "O" def visit_rvec(self, t: RVec) -> str: - return "O" + if isinstance(t.item_type, RVec): + # All nested vecs share a representation + return "Vv" + return "V" + t.item_type.accept(self) def visit_runion(self, t: RUnion) -> str: return "O" diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index ffcb49e8a2ea7..e351bbb7d6dc9 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -442,13 +442,6 @@ def translate_method_call(builder: IRBuilder, expr: CallExpr, callee: MemberExpr if val is not None: return val - if isinstance(receiver_typ, RVec) and all(k == ARG_POS for k in expr.arg_kinds): - # Vec methods are specialized with a custom mechanism, - # since normal specializations don't support vec receivers. - val = translate_vec_method_call(builder, callee.expr, callee.name, expr.args) - if val is not None: - return val - obj = builder.accept(callee.expr) args = [builder.accept(arg) for arg in expr.args] return builder.gen_method_call( @@ -594,23 +587,6 @@ def translate_vec_create_from_iterable( assert False, (vec_type, arg) -def translate_vec_method_call( - builder: IRBuilder, vec_expr: Expression, method: str, args: list[Expression] -) -> Value | None: - if method == "pop" and 0 <= len(args) <= 1: - vec_val = builder.accept(vec_expr) - if args: - index = builder.accept(args[0]) - else: - index = Integer(-1, c_pyssize_t_rprimitive) - return vec_pop(builder.builder, vec_val, index, vec_expr.line) - if method == "remove" and len(args) == 1: - vec_val = builder.accept(vec_expr) - item = builder.accept(args[0]) - return vec_remove(builder.builder, vec_val, item, vec_expr.line) - return None - - def translate_cast_expr(builder: IRBuilder, expr: CastExpr) -> Value: src = builder.accept(expr.expr) target_type = builder.type_to_rtype(expr.type) diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index 40d7311901788..78fd08d864ca7 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -104,7 +104,7 @@ join_formatted_strings, tokenizer_format_call, ) -from mypyc.irbuild.vec import vec_append, vec_remove +from mypyc.irbuild.vec import vec_append, vec_pop, vec_remove from mypyc.primitives.bytearray_ops import isinstance_bytearray from mypyc.primitives.bytes_ops import ( bytes_adjust_index_op, @@ -1517,7 +1517,7 @@ def translate_vec_append(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> @specialize_function("vecs.remove") -def translate_vec_append(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Optional[Value]: +def translate_vec_remove(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Optional[Value]: if len(expr.args) == 2 and expr.arg_kinds == [ARG_POS, ARG_POS]: vec_arg = expr.args[0] item_arg = expr.args[1] @@ -1528,3 +1528,18 @@ def translate_vec_append(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> arg_value = builder.accept(item_arg) return vec_remove(builder.builder, vec_value, arg_value, item_arg.line) return None + + +@specialize_function("vecs.pop") +def translate_vec_pop(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Optional[Value]: + if 1 <= len(expr.args) <= 2 and all(kind == ARG_POS for kind in expr.arg_kinds): + vec_arg = expr.args[0] + vec_type = builder.node_type(vec_arg) + if isinstance(vec_type, RVec): + vec_value = builder.accept(vec_arg) + if len(expr.args) == 2: + index_value = builder.accept(expr.args[1]) + else: + index_value = Integer(-1, int64_rprimitive) + return vec_pop(builder.builder, vec_value, index_value, vec_arg.line) + return None diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 708225e85ead3..1545e58835b3c 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -26,6 +26,7 @@ RPrimitive, RType, RUnion, + RTuple, RVec, bool_rprimitive, c_int_rprimitive, @@ -293,15 +294,6 @@ def vec_pop(builder: "LowLevelIRBuilder", base: Value, index: Value, line: int) item_type = vec_type.item_type index = as_platform_int(builder, index, line) - if isinstance(index, Integer) and index.value == -1: - # Inline simple case - len_val = vec_len_native(builder, base) - index = builder.int_sub(len_val, 1) - item_addr = vec_item_ptr(builder, base, index) - item = builder.load_mem(item_addr, item_type) - builder.set_struct_field(base, VecObject, "len", index, line) - return item - if is_int64_rprimitive(item_type): name = "VecI64Api.pop" elif vec_depth(vec_type) == 0 and not isinstance(item_type, RUnion): @@ -311,7 +303,7 @@ def vec_pop(builder: "LowLevelIRBuilder", base: Value, index: Value, line: int) call = CallC( name, [base, index], - item_type, + RTuple([vec_type, item_type]), steals=[False, False], is_borrowed=False, error_kind=ERR_MAGIC, diff --git a/mypyc/test-data/irbuild-vecs.test b/mypyc/test-data/irbuild-vecs.test index 18734ccb6d665..2ee81422031ae 100644 --- a/mypyc/test-data/irbuild-vecs.test +++ b/mypyc/test-data/irbuild-vecs.test @@ -1,84 +1,3 @@ -[case testVecI64CreateEmpty] -from vecs import vec, append -from mypy_extensions import i64 - -def f() -> vec[i64]: - return vec[i64]() -[out] -def f(): - r0 :: vec[int64] -L0: - r0 = VecI64Api.alloc(0) - return r0 - -[case testVecI64Len] -from vecs import vec -from mypy_extensions import i64 - -def f(v: vec[i64]) -> i64: - l = len(v) - return l -[out] -def f(v): - v :: vec[int64] - r0 :: ptr - r1, l :: int64 -L0: - r0 = get_element_ptr v len :: VecObject - r1 = load_mem r0 :: int64* - keep_alive v - l = r1 - return l - -[case testVecI64GetItem] -from vecs import vec -from mypy_extensions import i64 - -def f(v: vec[i64], i: i64) -> i64: - return v[i] -[out] -def f(v, i): - v :: vec[int64] - i :: int64 - r0 :: ptr - r1 :: native_int - r2 :: bit - r3 :: bool - r4 :: ptr - r5 :: int64 - r6 :: ptr - r7 :: int64 -L0: - r0 = get_element_ptr v len :: VecObject - r1 = load_mem r0 :: native_int* - r2 = i < r1 :: unsigned - if r2 goto L2 else goto L1 :: bool -L1: - r3 = raise IndexError - unreachable -L2: - r4 = get_element_ptr v items :: VecI64Object - r5 = i * 8 - r6 = r4 + r5 - r7 = load_mem r6 :: int64* - keep_alive v - return r7 - -[case testVecI64Append] -from vecs import vec, append -from mypy_extensions import i64 - -def f(v: vec[i64], i: i64) -> vec[i64]: - return append(v, i) -[out] -def f(v, i): - v :: vec[int64] - i :: int64 - r0 :: vec[int64] -L0: - r0 = VecI64Api.append(v, i) - return r0 - [case testVecTCreateEmpty] from vecs import vec, append @@ -345,397 +264,6 @@ L2: keep_alive v return r7 -[case testVecI64SetItem] -from vecs import vec -from mypy_extensions import i64 - -def f(v: vec[i64], i: i64, x: i64) -> None: - v[i] = x -[out] -def f(v, i, x): - v :: vec[int64] - i, x :: int64 - r0 :: ptr - r1 :: native_int - r2 :: bit - r3 :: bool - r4 :: ptr - r5 :: int64 - r6 :: ptr -L0: - r0 = get_element_ptr v len :: VecObject - r1 = load_mem r0 :: native_int* - r2 = i < r1 :: unsigned - if r2 goto L2 else goto L1 :: bool -L1: - r3 = raise IndexError - unreachable -L2: - r4 = get_element_ptr v items :: VecI64Object - r5 = i * 8 - r6 = r4 + r5 - set_mem r6, x :: int64* - keep_alive v - return 1 - -[case testVecI64ConstructFromListExpr] -from vecs import vec -from mypy_extensions import i64 - -def f() -> vec[i64]: - return vec[i64]([1, 5, 14]) -[out] -def f(): - r0 :: vec[int64] - r1, r2, r3, r4 :: ptr -L0: - r0 = VecI64Api.alloc(3) - r1 = get_element_ptr r0 items :: VecI64Object - set_mem r1, 1 :: int64* - r2 = r1 + 8 - set_mem r2, 5 :: int64* - r3 = r2 + 8 - set_mem r3, 14 :: int64* - r4 = r3 + 8 - keep_alive r0 - return r0 - -[case testVecI64ConstructFromListMultiply] -from vecs import vec -from mypy_extensions import i64 - -def f(n: i64) -> vec[i64]: - return vec[i64]([3] * n) -[out] -def f(n): - n :: int64 - r0 :: vec[int64] - r1 :: ptr - r2 :: int64 - r3, r4 :: ptr - r5 :: bit - r6 :: ptr -L0: - r0 = VecI64Api.alloc(n) - r1 = get_element_ptr r0 items :: VecI64Object - r2 = n * 8 - r3 = r1 + r2 - r4 = r1 -L1: - r5 = r4 < r3 :: unsigned - if r5 goto L2 else goto L3 :: bool -L2: - set_mem r4, 3 :: int64* - r6 = r4 + 8 - r4 = r6 - goto L1 -L3: - keep_alive r0 - return r0 - -[case testVecI64ConstructFromListMultiply2] -from vecs import vec -from mypy_extensions import i64 - -def f(n: i64, x: i64) -> vec[i64]: - return vec[i64]([x] * 3) -[out] -def f(n, x): - n, x :: int64 - r0 :: vec[int64] - r1 :: ptr - r2 :: native_int - r3, r4 :: ptr - r5 :: bit - r6 :: ptr -L0: - r0 = VecI64Api.alloc(3) - r1 = get_element_ptr r0 items :: VecI64Object - r2 = 3 * 8 - r3 = r1 + r2 - r4 = r1 -L1: - r5 = r4 < r3 :: unsigned - if r5 goto L2 else goto L3 :: bool -L2: - set_mem r4, x :: int64* - r6 = r4 + 8 - r4 = r6 - goto L1 -L3: - keep_alive r0 - return r0 - -[case testVecI64ConstructFromListComprehension] -from vecs import vec -from mypy_extensions import i64 - -def f(n: i64) -> vec[i64]: - return vec[i64]([x + 1 for x in range(5)]) -[out] -def f(n): - n :: int64 - r0, r1 :: vec[int64] - r2 :: short_int - x :: int - r3 :: bit - r4 :: int - r5 :: native_int - r6 :: bit - r7, r8 :: int64 - r9 :: ptr - r10 :: c_ptr - r11 :: int64 - r12 :: vec[int64] - r13 :: short_int -L0: - r0 = VecI64Api.alloc(0) - r1 = r0 - r2 = 0 - x = r2 -L1: - r3 = r2 < 10 :: signed - if r3 goto L2 else goto L7 :: bool -L2: - r4 = CPyTagged_Add(x, 2) - r5 = r4 & 1 - r6 = r5 == 0 - if r6 goto L3 else goto L4 :: bool -L3: - r7 = r4 >> 1 - r8 = r7 - goto L5 -L4: - r9 = r4 ^ 1 - r10 = r9 - r11 = CPyLong_AsInt64(r10) - r8 = r11 - keep_alive r4 -L5: - r12 = VecI64Api.append(r1, r8) - r1 = r12 -L6: - r13 = r2 + 2 - r2 = r13 - x = r13 - goto L1 -L7: - return r1 - - -[case testVecI64ForLoop] -from vecs import vec -from mypy_extensions import i64 - -def f(v: vec[i64]) -> i64: - t: i64 = 0 - for x in v: - t += 1 - return t -[out] -def f(v): - v :: vec[int64] - t :: int64 - r0 :: native_int - r1 :: ptr - r2 :: int64 - r3 :: bit - r4 :: ptr - r5 :: native_int - r6 :: ptr - r7, x, r8 :: int64 - r9 :: native_int -L0: - t = 0 - r0 = 0 -L1: - r1 = get_element_ptr v len :: VecObject - r2 = load_mem r1 :: int64* - keep_alive v - r3 = r0 < r2 :: signed - if r3 goto L2 else goto L4 :: bool -L2: - r4 = get_element_ptr v items :: VecI64Object - r5 = r0 * 8 - r6 = r4 + r5 - r7 = load_mem r6 :: int64* - keep_alive v - x = r7 - r8 = t + 1 - t = r8 -L3: - r9 = r0 + 1 - r0 = r9 - goto L1 -L4: - return t - -[case testVecI64PopLast] -from vecs import vec -from mypy_extensions import i64 - -def pop_last(v: vec[i64]) -> i64: - return v.pop() -[out] -def pop_last(v): - v :: vec[int64] - r0 :: ptr - r1, r2 :: native_int - r3 :: ptr - r4 :: native_int - r5 :: ptr - r6 :: int64 - r7 :: ptr -L0: - r0 = get_element_ptr v len :: VecObject - r1 = load_mem r0 :: native_int* - r2 = r1 - 1 - r3 = get_element_ptr v items :: VecI64Object - r4 = r2 * 8 - r5 = r3 + r4 - r6 = load_mem r5 :: int64* - r7 = get_element_ptr v len :: VecObject - set_mem r7, r2 :: native_int* - keep_alive v - return r6 - -[case testVecI64PopNth] -from vecs import vec -from mypy_extensions import i64 - -def pop_nth(v: vec[i64], n: i64) -> i64: - return v.pop(n) -[out] -def pop_nth(v, n): - v :: vec[int64] - n, r0 :: int64 -L0: - r0 = VecI64Api.pop(v, n) - return r0 - -[case testVecI64Remove] -from vecs import vec -from mypy_extensions import i64 - -def remove(v: vec[i64], n: i64) -> None: - v.remove(n) -[out] -def remove(v, n): - v :: vec[int64] - n :: int64 - r0 :: int32 -L0: - r0 = VecI64Api.remove(v, n) - return 1 - -[case testVecI64Contains] -from vecs import vec -from mypy_extensions import i64 - -def contains(v: vec[i64], n: i64) -> bool: - return n in v -[out] -def contains(v, n): - v :: vec[int64] - n :: int64 - r0 :: ptr - r1 :: native_int - r2 :: ptr - r3 :: native_int - r4, r5 :: ptr - r6 :: bit - r7 :: int64 - r8 :: bit - r9 :: ptr - r10 :: bool -L0: - r0 = get_element_ptr v len :: VecObject - r1 = load_mem r0 :: native_int* - r2 = get_element_ptr v items :: VecI64Object - r3 = r1 * 8 - r4 = r2 + r3 - r5 = r2 -L1: - r6 = r5 < r4 :: unsigned - if r6 goto L2 else goto L4 :: bool -L2: - r7 = load_mem r5 :: int64* - r8 = r7 == n - if r8 goto L5 else goto L3 :: bool -L3: - r9 = r5 + 8 - r5 = r9 - goto L1 -L4: - keep_alive v - r10 = 0 - goto L6 -L5: - r10 = 1 -L6: - return r10 - -[case testVecI64GetItemWithInt] -from vecs import vec -from mypy_extensions import i64 - -def f(v: vec[i64]) -> i64: - return v[0] -[out] -def f(v): - v :: vec[int64] - r0 :: ptr - r1 :: native_int - r2 :: bit - r3 :: bool - r4 :: ptr - r5 :: int64 - r6 :: ptr - r7 :: int64 -L0: - r0 = get_element_ptr v len :: VecObject - r1 = load_mem r0 :: native_int* - r2 = 0 < r1 :: unsigned - if r2 goto L2 else goto L1 :: bool -L1: - r3 = raise IndexError - unreachable -L2: - r4 = get_element_ptr v items :: VecI64Object - r5 = 0 * 8 - r6 = r4 + r5 - r7 = load_mem r6 :: int64* - keep_alive v - return r7 - -[case testVecI64Slicing] -from vecs import vec -from mypy_extensions import i64 - -def f(v: vec[i64], n: i64, m: i64) -> None: - a = v[:] - b = v[n:] - c = v[n:m] - d = v[:m] - e = v[1:-2] -[out] -def f(v, n, m): - v :: vec[int64] - n, m :: int64 - r0, a, r1, b, r2, c, r3, d, r4, e :: vec[int64] -L0: - r0 = VecI64Api.slice(v, 0, 4611686018427387903) - a = r0 - r1 = VecI64Api.slice(v, n, 4611686018427387903) - b = r1 - r2 = VecI64Api.slice(v, n, m) - c = r2 - r3 = VecI64Api.slice(v, 0, m) - d = r3 - r4 = VecI64Api.slice(v, 1, -2) - e = r4 - return 1 - -- -- New repr -- @@ -852,7 +380,6 @@ L2: keep_alive v return 1 - [case testNewVecI64ConstructFromListExpr] from vecs import vec from mypy_extensions import i64 @@ -1150,3 +677,34 @@ L0: r0 = VecI64Api.remove(v, n) v = r0 return 1 + +[case testNewVecI64PopLast] +from typing import Tuple +from vecs import vec, pop +from mypy_extensions import i64 + +def pop_last(v: vec[i64]) -> Tuple[vec[i64], i64]: + return pop(v) +[out] +def pop_last(v): + v :: vec[int64] + r0 :: tuple[vec[int64], int64] +L0: + r0 = VecI64Api.pop(v, -1) + return r0 + +[case testNewVecI64PopNth] +from typing import Tuple +from vecs import vec, pop +from mypy_extensions import i64 + +def pop_nth(v: vec[i64], n: i64) -> Tuple[vec[i64], i64]: + return pop(v, n) +[out] +def pop_nth(v, n): + v :: vec[int64] + n :: int64 + r0 :: tuple[vec[int64], int64] +L0: + r0 = VecI64Api.pop(v, n) + return r0 diff --git a/test-data/unit/lib-stub/vecs.pyi b/test-data/unit/lib-stub/vecs.pyi index a9e8dcb03227e..c181da627d0ff 100644 --- a/test-data/unit/lib-stub/vecs.pyi +++ b/test-data/unit/lib-stub/vecs.pyi @@ -1,4 +1,4 @@ -from typing import TypeVar, Generic, Type, Iterable, Iterator, overload +from typing import TypeVar, Generic, Type, Iterable, Iterator, overload, Tuple from mypy_extensions import i64 T = TypeVar("T") @@ -15,7 +15,7 @@ class vec(Generic[T]): def __getitem__(self, i: slice) -> vec[T]: ... def __setitem__(self, i: i64, o: T) -> None: ... def __iter__(self) -> Iterator[T]: ... - def pop(self, __i: i64 = -1) -> T: ... -def append(v: vec[T], o: T) -> vec[T]: ... -def remove(v: vec[T], o: T) -> vec[T]: ... +def append(__v: vec[T], __o: T) -> vec[T]: ... +def remove(__v: vec[T], __o: T) -> vec[T]: ... +def pop(__v: vec[T], __i: i64 = -1) -> Tuple[vec[T], T]: ... diff --git a/vecs.pyi b/vecs.pyi index 85ebfc433523c..2298145ebdd70 100644 --- a/vecs.pyi +++ b/vecs.pyi @@ -1,4 +1,4 @@ -from typing import TypeVar, Generic, Type, Iterable, Iterator, overload +from typing import TypeVar, Generic, Type, Iterable, Iterator, overload, Tuple from mypy_extensions import i64 T = TypeVar("T") @@ -18,7 +18,7 @@ class vec(Generic[T]): def __setitem__(self, i: i64, o: T) -> None: ... def __iter__(self) -> Iterator[T]: ... - def pop(self, __i: i64 = -1) -> T: ... - def remove(self, __x: T) -> None: ... -def append(v: vec[T], o: T) -> vec[T]: ... +def append(__v: vec[T], __o: T) -> vec[T]: ... +def remove(__v: vec[T], __x: T) -> vec[T]: ... +def pop(__v: vec[T], __i: i64 = -1) -> Tuple[vec[T], T]: ... From 64213846d9bae9deb2a013bb90d36cfd2d60b854 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 29 May 2023 14:20:00 +0100 Subject: [PATCH 013/180] Some partial support for vec[t] --- mypyc/codegen/emit.py | 11 ++++++++--- mypyc/codegen/emitfunc.py | 12 +++++++++--- mypyc/irbuild/vec.py | 5 ++++- mypyc/test-data/irbuild-vecs.test | 18 +++++++++++------- 4 files changed, 32 insertions(+), 14 deletions(-) diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index 43290dac787c8..f360f321353c4 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -1127,9 +1127,14 @@ def emit_box( self.emit_box(f"{src}.f{i}", inner_name, typ.types[i], declare_dest=True) self.emit_line(f"PyTuple_SET_ITEM({dest}, {i}, {inner_name});") elif isinstance(typ, RVec): - assert is_int64_rprimitive(typ.item_type) # TODO: Support more item types - self.emit_line(f"{declaration}{dest} = VecI64Api.box({src});") - + if is_int64_rprimitive(typ.item_type): + api = "VecI64Api" + elif not isinstance(typ.item_type, RVec): + api = "VecTApi" + else: + # TODO: Support more item types + assert False, typ + self.emit_line(f"{declaration}{dest} = {api}.box({src});") else: assert not typ.is_unboxed # Type is boxed -- trivially just assign. diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index 42fb1e7c0f647..1067e64a23f2a 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py @@ -294,10 +294,16 @@ def visit_assign(self, op: Assign) -> None: # clang whines about self assignment (which we might generate # for some casts), so don't emit it. if dest != src: - # We sometimes assign from an integer prepresentation of a pointer - # to a real pointer, and C compilers insist on a cast. - if op.src.type.is_unboxed and not op.dest.type.is_unboxed: + src_type = op.src.type + dest_type = op.dest.type + if src_type.is_unboxed and not dest_type.is_unboxed: + # We sometimes assign from an integer prepresentation of a pointer + # to a real pointer, and C compilers insist on a cast. src = f"(void *){src}" + elif not src_type.is_unboxed and dest_type.is_unboxed: + # We sometimes assign a pointer to an integer type (e.g. to create + # tagged pointers), and here we need an explicit cast. + src = f"({self.emitter.ctype(dest_type)}){src}" self.emit_line(f"{dest} = {src};") def visit_assign_multi(self, op: AssignMulti) -> None: diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 1545e58835b3c..1c8febd9331a3 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -7,6 +7,7 @@ from mypyc.ir.ops import ( ERR_FALSE, ERR_MAGIC, + Assign, BasicBlock, Branch, CallC, @@ -83,9 +84,11 @@ def vec_create( typeobj, optionals, depth = vec_item_type_info(builder, item_type, line) if typeobj is not None: if optionals == 0 and depth == 0: + typeval = Register(pointer_rprimitive) + builder.add(Assign(typeval, typeobj)) call = CallC( "VecTApi.alloc", - [length, typeobj], + [length, typeval], vtype, False, False, diff --git a/mypyc/test-data/irbuild-vecs.test b/mypyc/test-data/irbuild-vecs.test index 2ee81422031ae..bc19fd6ee36f5 100644 --- a/mypyc/test-data/irbuild-vecs.test +++ b/mypyc/test-data/irbuild-vecs.test @@ -11,18 +11,22 @@ def native_class() -> vec[C]: [out] def primitive(): r0 :: object - r1 :: vec[str] + r1 :: ptr + r2 :: vec[str] L0: r0 = load_address PyUnicode_Type - r1 = VecTApi.alloc(0, r0) - return r1 + r1 = r0 + r2 = VecTApi.alloc(0, r1) + return r2 def native_class(): r0 :: object - r1 :: vec[__main__.C] + r1 :: ptr + r2 :: vec[__main__.C] L0: r0 = __main__.C :: type - r1 = VecTApi.alloc(0, r0) - return r1 + r1 = r0 + r2 = VecTApi.alloc(0, r1) + return r2 [case testVecTAppend] from vecs import vec, append @@ -39,7 +43,7 @@ L0: r1 = VecTApi.append(v, r0) return r1 -[case testVecOptionalCreateEmpty] +[case testVecTOptionalCreateEmpty] from vecs import vec, append from typing import Optional From 1c30edab803c3666c446884a141ecb012da4197d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 29 May 2023 14:27:10 +0100 Subject: [PATCH 014/180] Some partial support for vec[t | None] --- mypyc/irbuild/vec.py | 7 +++++-- mypyc/test-data/irbuild-vecs.test | 32 ++++++++++++++++++++++++++----- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 1c8febd9331a3..4a73189f114a1 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -83,9 +83,12 @@ def vec_create( typeobj, optionals, depth = vec_item_type_info(builder, item_type, line) if typeobj is not None: - if optionals == 0 and depth == 0: + if depth == 0: typeval = Register(pointer_rprimitive) builder.add(Assign(typeval, typeobj)) + if optionals: + typeval = builder.add( + IntOp(pointer_rprimitive, typeval, Integer(1, pointer_rprimitive), IntOp.OR)) call = CallC( "VecTApi.alloc", [length, typeval], @@ -275,7 +278,7 @@ def vec_append(builder: "LowLevelIRBuilder", vec: Value, item: Value, line: int) item = builder.coerce(item, item_type, line) if is_int64_rprimitive(item_type): name = "VecI64Api.append" - elif vec_depth(vec_type) == 0 and not isinstance(item_type, RUnion): + elif vec_depth(vec_type) == 0: name = "VecTApi.append" else: name = "VecTExtApi.append" diff --git a/mypyc/test-data/irbuild-vecs.test b/mypyc/test-data/irbuild-vecs.test index bc19fd6ee36f5..d7ea14c080efe 100644 --- a/mypyc/test-data/irbuild-vecs.test +++ b/mypyc/test-data/irbuild-vecs.test @@ -57,17 +57,39 @@ def native_class() -> vec[Optional[C]]: [out] def primitive(): r0 :: object - r1 :: vec[union[str, None]] + r1, r2 :: ptr + r3 :: vec[union[str, None]] L0: r0 = load_address PyUnicode_Type - r1 = VecTExtApi.alloc(0, r0, 1, 0) - return r1 + r1 = r0 + r2 = r1 | 1 + r3 = VecTApi.alloc(0, r2) + return r3 def native_class(): r0 :: object - r1 :: vec[union[__main__.C, None]] + r1, r2 :: ptr + r3 :: vec[union[__main__.C, None]] L0: r0 = __main__.C :: type - r1 = VecTExtApi.alloc(0, r0, 1, 0) + r1 = r0 + r2 = r1 | 1 + r3 = VecTApi.alloc(0, r2) + return r3 + +[case testVecTOptionalAppend] +from vecs import vec, append +from typing import Optional + +def f(v: vec[Optional[str]]) -> vec[Optional[str]]: + return append(v, 'x') +[out] +def f(v): + v :: vec[union[str, None]] + r0 :: str + r1 :: vec[union[str, None]] +L0: + r0 = 'x' + r1 = VecTApi.append(v, r0) return r1 [case testVecNestedCreateEmpty] From a70dbfe4ed58df18a532e7a80b2d4c5f840a4e50 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 29 May 2023 15:03:54 +0100 Subject: [PATCH 015/180] Test updates --- mypyc/test-data/irbuild-vecs.test | 32 ++++++++----------------------- 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/mypyc/test-data/irbuild-vecs.test b/mypyc/test-data/irbuild-vecs.test index d7ea14c080efe..33a43f3ac950a 100644 --- a/mypyc/test-data/irbuild-vecs.test +++ b/mypyc/test-data/irbuild-vecs.test @@ -81,16 +81,22 @@ from vecs import vec, append from typing import Optional def f(v: vec[Optional[str]]) -> vec[Optional[str]]: - return append(v, 'x') + v = append(v, 'x') + return append(v, None) [out] def f(v): v :: vec[union[str, None]] r0 :: str r1 :: vec[union[str, None]] + r2 :: object + r3 :: vec[union[str, None]] L0: r0 = 'x' r1 = VecTApi.append(v, r0) - return r1 + v = r1 + r2 = box(None, 1) + r3 = VecTApi.append(v, r2) + return r3 [case testVecNestedCreateEmpty] from vecs import vec, append @@ -119,28 +125,6 @@ L0: r2 = VecTApi.alloc(0, r1) return r2 -[case testVecOptionalAppend] -from vecs import vec, append -from typing import Optional - -def f(v: vec[Optional[str]]) -> vec[Optional[str]]: - v = append(v, 'x') - return append(v, None) -[out] -def f(v): - v :: vec[union[str, None]] - r0 :: str - r1 :: vec[union[str, None]] - r2 :: object - r3 :: vec[union[str, None]] -L0: - r0 = 'x' - r1 = VecTExtApi.append(v, r0) - v = r1 - r2 = box(None, 1) - r3 = VecTExtApi.append(v, r2) - return r3 - [case testVecGenericLen] from vecs import vec from mypy_extensions import i64 From 466cc855acc2f663dae6fa8b0a3ed1003489c48d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 29 May 2023 15:12:15 +0100 Subject: [PATCH 016/180] More vec[t] operations --- mypyc/ir/rtypes.py | 4 ++-- mypyc/test-data/irbuild-vecs.test | 40 +++++++++++++------------------ 2 files changed, 19 insertions(+), 25 deletions(-) diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index e69c4816be35e..4d095fcf4fb26 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -1019,7 +1019,7 @@ def __init__(self, item_type: RType) -> None: # TODO: buf_type else: self._ctype = "VecT" - # TODO: buf_type + self.buf_type = VecbufTObject def field_type(self, name: str) -> RType: if name == "len": @@ -1285,7 +1285,7 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: # Buffer for vec[t] VecbufTObject = RStruct( - name="VecTObject", + name="VecbufTObject", names=["ob_base", "item_type", "items"], types=[PyVarObject, c_pyssize_t_rprimitive, object_rprimitive], ) diff --git a/mypyc/test-data/irbuild-vecs.test b/mypyc/test-data/irbuild-vecs.test index 33a43f3ac950a..6635e7a9575b2 100644 --- a/mypyc/test-data/irbuild-vecs.test +++ b/mypyc/test-data/irbuild-vecs.test @@ -125,7 +125,7 @@ L0: r2 = VecTApi.alloc(0, r1) return r2 -[case testVecGenericLen] +[case testVecTLen] from vecs import vec from mypy_extensions import i64 from typing import Optional @@ -138,22 +138,16 @@ def g(v: vec[Optional[str]]) -> i64: [out] def f(v): v :: vec[str] - r0 :: ptr - r1 :: int64 + r0 :: native_int L0: - r0 = get_element_ptr v len :: VecObject - r1 = load_mem r0 :: int64* - keep_alive v - return r1 + r0 = v.len + return r0 def g(v): v :: vec[union[str, None]] - r0 :: ptr - r1 :: int64 + r0 :: native_int L0: - r0 = get_element_ptr v len :: VecObject - r1 = load_mem r0 :: int64* - keep_alive v - return r1 + r0 = v.len + return r0 [case testVecTGetItem] from vecs import vec @@ -166,24 +160,24 @@ def f(v: vec[str], n: i64) -> str: def f(v, n): v :: vec[str] n :: int64 - r0 :: ptr - r1 :: native_int - r2 :: bit - r3 :: bool + r0 :: native_int + r1 :: bit + r2 :: bool + r3 :: object r4 :: ptr r5 :: int64 r6 :: ptr r7 :: str L0: - r0 = get_element_ptr v len :: VecObject - r1 = load_mem r0 :: native_int* - r2 = n < r1 :: unsigned - if r2 goto L2 else goto L1 :: bool + r0 = v.len + r1 = n < r0 :: unsigned + if r1 goto L2 else goto L1 :: bool L1: - r3 = raise IndexError + r2 = raise IndexError unreachable L2: - r4 = get_element_ptr v items :: VecTObject + r3 = v.buf + r4 = get_element_ptr r3 items :: VecbufTObject r5 = n * 8 r6 = r4 + r5 r7 = load_mem r6 :: builtins.str* From 6bdadc7064b64dac3bea4d4e8aaec6e829ed1db8 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 10 Jun 2023 15:12:29 +0100 Subject: [PATCH 017/180] Format code --- mypyc/codegen/emitfunc.py | 2 +- mypyc/ir/rtypes.py | 8 +++----- mypyc/irbuild/builder.py | 3 +-- mypyc/irbuild/expression.py | 7 ++++--- mypyc/irbuild/for_helpers.py | 2 +- mypyc/irbuild/ll_builder.py | 17 +++++++++++++---- mypyc/irbuild/specialize.py | 4 ++-- mypyc/irbuild/vec.py | 9 ++++++--- 8 files changed, 31 insertions(+), 21 deletions(-) diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index 1067e64a23f2a..07d34e0d5289f 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py @@ -330,7 +330,7 @@ def visit_load_error_value(self, op: LoadErrorValue) -> None: self.emit_line("{} {} = {{ {} }};".format(self.ctype(op.type), tmp, ", ".join(values))) self.emit_line(f"{reg} = {tmp};") elif isinstance(op.type, RVec): - self.emitter.set_undefined_value(reg, op.type); + self.emitter.set_undefined_value(reg, op.type) else: self.emit_line(f"{self.reg(op)} = {self.c_error_value(op.type)};") diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 4d095fcf4fb26..c4c19f60d449d 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -1271,15 +1271,13 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: types=[PyVarObject, int64_rprimitive], ) -#vecbuf_i64_rprimitive: Final = RPrimitive( +# vecbuf_i64_rprimitive: Final = RPrimitive( # "VecbufI64Object", is_unboxed=False, is_refcounted=True, ctype="VecbufI64Object *" -#) +# ) # vec[i64] VecI64 = RStruct( - name="VecI64", - names=["len", "buf"], - types=[c_pyssize_t_rprimitive, object_rprimitive], + name="VecI64", names=["len", "buf"], types=[c_pyssize_t_rprimitive, object_rprimitive] ) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index 2998b90598a7a..ef127a1f3c2ee 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -92,10 +92,10 @@ RTuple, RType, RUnion, + RVec, bitmap_rprimitive, bool_rprimitive, bytes_rprimitive, - RVec, c_int_rprimitive, c_pyssize_t_rprimitive, dict_rprimitive, @@ -130,7 +130,6 @@ AssignmentTargetTuple, ) from mypyc.irbuild.util import bytes_from_str, is_constant -from mypyc.irbuild.util import is_constant from mypyc.irbuild.vec import vec_set_item from mypyc.options import CompilerOptions from mypyc.primitives.dict_ops import dict_get_item_op, dict_set_item_op diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index e351bbb7d6dc9..ff21a477d5131 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -78,8 +78,8 @@ from mypyc.ir.rtypes import ( RInstance, RTuple, - bool_rprimitive, RVec, + bool_rprimitive, c_pyssize_t_rprimitive, int_rprimitive, is_any_int, @@ -680,8 +680,9 @@ def transform_index_expr(builder: IRBuilder, expr: IndexExpr) -> Value: return value index_reg = builder.accept(expr.index, can_borrow=can_borrow) - return builder.builder.get_item(base, index_reg, builder.node_type(expr), expr.line, - can_borrow=builder.can_borrow) + return builder.builder.get_item( + base, index_reg, builder.node_type(expr), expr.line, can_borrow=builder.can_borrow + ) def try_constant_fold(builder: IRBuilder, expr: Expression) -> Value | None: diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index 583ab61eef66e..2666345818158 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -49,8 +49,8 @@ RInstance, RTuple, RType, - bool_rprimitive, RVec, + bool_rprimitive, c_pyssize_t_rprimitive, int_rprimitive, is_dict_rprimitive, diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index ba0131dd376f9..abc1737f19000 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -99,6 +99,7 @@ c_size_t_rprimitive, check_native_int_range, float_rprimitive, + int64_rprimitive, int_rprimitive, is_bool_or_bit_rprimitive, is_bytes_rprimitive, @@ -110,7 +111,6 @@ is_int16_rprimitive, is_int32_rprimitive, is_int64_rprimitive, - int64_rprimitive, is_int_rprimitive, is_list_rprimitive, is_none_rprimitive, @@ -1987,8 +1987,15 @@ def unary_op(self, value: Value, op: str, line: int) -> Value: return self.unary_invert(value, line) raise RuntimeError("Unsupported unary operation: %s" % op) - def get_item(self, base: Value, item: Value, result_type: Optional[RType], line: int, - *, can_borrow: bool = False) -> Value: + def get_item( + self, + base: Value, + item: Value, + result_type: Optional[RType], + line: int, + *, + can_borrow: bool = False, + ) -> Value: """Generate base[item].""" if isinstance(base.type, RVec): if is_int_rprimitive(item.type) or is_short_int_rprimitive(item.type): @@ -1996,7 +2003,9 @@ def get_item(self, base: Value, item: Value, result_type: Optional[RType], line: if is_int64_rprimitive(item.type): return vec_get_item(self, base, item, line) # TODO: Move special casing for RTuple here from transform_index_expr - return self.gen_method_call(base, "__getitem__", [item], result_type, line, can_borrow=can_borrow) + return self.gen_method_call( + base, "__getitem__", [item], result_type, line, can_borrow=can_borrow + ) def make_dict(self, key_value_pairs: Sequence[DictEntry], line: int) -> Value: diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index 78fd08d864ca7..6ff6700c55144 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -37,8 +37,10 @@ ) from mypy.types import AnyType, TypeOfAny from mypyc.ir.ops import ( + ERR_MAGIC, BasicBlock, Call, + CallC, Extend, Integer, PrimitiveDescription, @@ -48,8 +50,6 @@ Truncate, Unreachable, Value, - ERR_MAGIC, - CallC, ) from mypyc.ir.rtypes import ( RInstance, diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 4a73189f114a1..ba862d37e8379 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -1,6 +1,7 @@ """Generate IR for vecs.vec operations""" from __future__ import annotations + from typing import TYPE_CHECKING, List, Optional, Tuple, Union, cast from mypyc.common import PLATFORM_SIZE @@ -25,9 +26,9 @@ from mypyc.ir.rtypes import ( RInstance, RPrimitive, + RTuple, RType, RUnion, - RTuple, RVec, bool_rprimitive, c_int_rprimitive, @@ -37,13 +38,14 @@ is_c_py_ssize_t_rprimitive, is_int32_rprimitive, is_int64_rprimitive, + is_int_rprimitive, is_none_rprimitive, + is_short_int_rprimitive, object_pointer_rprimitive, object_rprimitive, optional_value_type, pointer_rprimitive, vec_depth, - is_short_int_rprimitive, is_int_rprimitive, ) from mypyc.primitives.registry import builtin_names @@ -88,7 +90,8 @@ def vec_create( builder.add(Assign(typeval, typeobj)) if optionals: typeval = builder.add( - IntOp(pointer_rprimitive, typeval, Integer(1, pointer_rprimitive), IntOp.OR)) + IntOp(pointer_rprimitive, typeval, Integer(1, pointer_rprimitive), IntOp.OR) + ) call = CallC( "VecTApi.alloc", [length, typeval], From c22a74447b9b3124aa5759e08eae3f8915106095 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 10 Jun 2023 15:18:02 +0100 Subject: [PATCH 018/180] [mypyc] Add SetElement op for initializing struct values Also add Undef value type that can currently only used as the operand for SetElement to signify that we are creating a new value instead of modifying an existing value. --- mypyc/analysis/dataflow.py | 3 +++ mypyc/analysis/ircheck.py | 3 +++ mypyc/analysis/selfleaks.py | 3 +++ mypyc/codegen/emitfunc.py | 20 ++++++++++++++++++++ mypyc/ir/ops.py | 7 ++++++- mypyc/ir/pprint.py | 3 +++ mypyc/test/test_emitfunc.py | 1 + 7 files changed, 39 insertions(+), 1 deletion(-) diff --git a/mypyc/analysis/dataflow.py b/mypyc/analysis/dataflow.py index 8b6b1d52c988f..779454e22fd7d 100644 --- a/mypyc/analysis/dataflow.py +++ b/mypyc/analysis/dataflow.py @@ -275,6 +275,9 @@ def visit_load_mem(self, op: LoadMem) -> GenAndKill[T]: def visit_get_element(self, op: GetElement) -> GenAndKill[T]: return self.visit_register_op(op) + def visit_set_element(self, op: SetElement) -> GenAndKill[T]: + return self.visit_register_op(op) + def visit_get_element_ptr(self, op: GetElementPtr) -> GenAndKill[T]: return self.visit_register_op(op) diff --git a/mypyc/analysis/ircheck.py b/mypyc/analysis/ircheck.py index 238d416729241..920ba40f8be1f 100644 --- a/mypyc/analysis/ircheck.py +++ b/mypyc/analysis/ircheck.py @@ -453,6 +453,9 @@ def visit_set_mem(self, op: SetMem) -> None: def visit_get_element(self, op: GetElement) -> None: pass + def visit_set_element(self, op: SetElement) -> None: + pass + def visit_get_element_ptr(self, op: GetElementPtr) -> None: pass diff --git a/mypyc/analysis/selfleaks.py b/mypyc/analysis/selfleaks.py index b2e837565307f..6b886a83b44b9 100644 --- a/mypyc/analysis/selfleaks.py +++ b/mypyc/analysis/selfleaks.py @@ -183,6 +183,9 @@ def visit_load_mem(self, op: LoadMem) -> GenAndKill: def visit_get_element(self, op: GetElement) -> GenAndKill: return CLEAN + def visit_set_element(self, op: SetElement) -> GenAndKill: + return CLEAN + def visit_get_element_ptr(self, op: GetElementPtr) -> GenAndKill: return CLEAN diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index 07d34e0d5289f..1aa7b7202f055 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py @@ -816,6 +816,26 @@ def visit_get_element(self, op: GetElement) -> None: dest_type = self.ctype(op.type) self.emit_line(f"{dest} = ({dest_type}){src}.{op.field};") + def visit_set_element(self, op: SetElement) -> None: + # TODO: do properly + dest = self.reg(op) + item = self.reg(op.item) + field = op.field + if isinstance(op.src, Undef): + self.emit_line(f"{dest}.{field} = {item};") + else: + src = self.reg(op.src) + # TODO: Support tuples (or use RStruct for tuples) + src_type = op.src.type + assert isinstance(src_type, RStruct), src_type + init_items = [] + for n in src_type.names: + if n != field: + init_items.append(f"{src}.{n}") + else: + init_items.append(item) + self.emit_line(f"{dest} = ({self.ctype(src_type)}) {{ {', '.join(init_items)} }};") + def visit_get_element_ptr(self, op: GetElementPtr) -> None: dest = self.reg(op) src = self.reg(op.src) diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py index 19849dffa2742..17a703c13ed14 100644 --- a/mypyc/ir/ops.py +++ b/mypyc/ir/ops.py @@ -269,6 +269,7 @@ def __init__(self, rtype: RType) -> None: self.type = rtype + class Op(Value): """Abstract base class for all IR operations. @@ -1751,7 +1752,7 @@ class SetElement(RegisterOp): def __init__(self, src: Value, field: str, item: Value, line: int = -1) -> None: super().__init__(line) - assert isinstance(src.type, RStruct), src.type + assert isinstance(src.type, (RStruct, RVec)), src.type self.type = src.type self.src = src self.item = item @@ -2043,6 +2044,10 @@ def visit_set_mem(self, op: SetMem) -> T: def visit_get_element(self, op: GetElement) -> T: raise NotImplementedError + @abstractmethod + def visit_set_element(self, op: SetElement) -> T: + raise NotImplementedError + @abstractmethod def visit_get_element_ptr(self, op: GetElementPtr) -> T: raise NotImplementedError diff --git a/mypyc/ir/pprint.py b/mypyc/ir/pprint.py index d0db9f2460a1d..ee4ffcaad6f86 100644 --- a/mypyc/ir/pprint.py +++ b/mypyc/ir/pprint.py @@ -284,6 +284,9 @@ def visit_set_mem(self, op: SetMem) -> str: def visit_get_element(self, op: GetElement) -> str: return self.format("%r = %r.%s", op, op.src, op.field) + def visit_set_element(self, op: SetElement) -> str: + return self.format("%r = set_element %r, %s, %r", op, op.src, op.field, op.item) + def visit_get_element_ptr(self, op: GetElementPtr) -> str: return self.format("%r = get_element_ptr %r %s :: %t", op, op.src, op.field, op.src_type) diff --git a/mypyc/test/test_emitfunc.py b/mypyc/test/test_emitfunc.py index 642e1cd5c3cc3..11e1ab0369b0e 100644 --- a/mypyc/test/test_emitfunc.py +++ b/mypyc/test/test_emitfunc.py @@ -692,6 +692,7 @@ def test_set_element(self) -> None: """cpy_r_r0 = (Foo) { cpy_r_st.b, cpy_r_i32, cpy_r_st.y };""", ) + def test_load_address(self) -> None: self.assert_emit( LoadAddress(object_rprimitive, "PyDict_Type"), From 2a902c56eec025771330e71dca1d238947fcf415 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 10 Jun 2023 20:01:20 +0100 Subject: [PATCH 019/180] Add initial nested vec support --- mypyc/codegen/emit.py | 3 +- mypyc/ir/rtypes.py | 75 +++++++++++++++++++++++-------- mypyc/irbuild/vec.py | 73 ++++++++++++++++++++---------- mypyc/test-data/irbuild-vecs.test | 62 ++++++++++++++++--------- mypyc/test-data/refcount.test | 22 +++++++++ 5 files changed, 170 insertions(+), 65 deletions(-) diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index f360f321353c4..14822046c0559 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -1132,8 +1132,7 @@ def emit_box( elif not isinstance(typ.item_type, RVec): api = "VecTApi" else: - # TODO: Support more item types - assert False, typ + api = "VecTExtApi" self.emit_line(f"{declaration}{dest} = {api}.box({src});") else: assert not typ.is_unboxed diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index c4c19f60d449d..b9d70a6b4e704 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -238,6 +238,7 @@ def __init__( is_refcounted: bool, is_native_int: bool = False, is_signed: bool = False, + is_pointer: bool = False, ctype: str = "PyObject *", size: int = PLATFORM_SIZE, error_overlap: bool = False, @@ -313,11 +314,19 @@ def __hash__(self) -> int: # little as possible, as generic ops are typically slow. Other types, # including other primitive types and RInstance, are usually much # faster. -object_rprimitive: Final = RPrimitive("builtins.object", is_unboxed=False, is_refcounted=True) +object_rprimitive: Final = RPrimitive( + "builtins.object", is_unboxed=False, is_refcounted=True, is_pointer=True +) # represents a low level pointer of an object object_pointer_rprimitive: Final = RPrimitive( - "object_ptr", is_unboxed=False, is_refcounted=False, ctype="PyObject **" + "object_ptr", is_unboxed=False, is_refcounted=False, ctype="PyObject **", is_pointer=True +) + +# Similar to object_primitive, but doesn not use automatic reference +# counting. Useful for temporaries. +object_non_refcounted_rprimitive: Final = RPrimitive( + "builtins.object", is_unboxed=False, is_refcounted=False, is_pointer=True ) # Arbitrary-precision integer (corresponds to Python 'int'). Small @@ -451,7 +460,7 @@ def __hash__(self) -> int: # Untyped pointer, represented as void * in the C backend c_pointer_rprimitive: Final = RPrimitive( - "c_ptr", is_unboxed=False, is_refcounted=False, ctype="void *" + "c_ptr", is_unboxed=False, is_refcounted=False, ctype="void *", is_pointer=True ) cstring_rprimitive: Final = RPrimitive( @@ -495,14 +504,18 @@ def __hash__(self) -> int: # immortal, but since this is expected to be very rare, and the immortality checks # can be pretty expensive for lists, we treat lists as non-immortal. list_rprimitive: Final = RPrimitive( - "builtins.list", is_unboxed=False, is_refcounted=True, may_be_immortal=False + "builtins.list", is_unboxed=False, is_refcounted=True, is_pointer=True, may_be_immortal=False ) # Python dict object (or an instance of a subclass of dict). -dict_rprimitive: Final = RPrimitive("builtins.dict", is_unboxed=False, is_refcounted=True) +dict_rprimitive: Final = RPrimitive( + "builtins.dict", is_unboxed=False, is_refcounted=True, is_pointer=True +) # Python set object (or an instance of a subclass of set). -set_rprimitive: Final = RPrimitive("builtins.set", is_unboxed=False, is_refcounted=True) +set_rprimitive: Final = RPrimitive( + "builtins.set", is_unboxed=False, is_refcounted=True, is_pointer=True +) # Python frozenset object (or an instance of a subclass of frozenset). frozenset_rprimitive: Final = RPrimitive( @@ -511,10 +524,14 @@ def __hash__(self) -> int: # Python str object. At the C layer, str is referred to as unicode # (PyUnicode). -str_rprimitive: Final = RPrimitive("builtins.str", is_unboxed=False, is_refcounted=True) +str_rprimitive: Final = RPrimitive( + "builtins.str", is_unboxed=False, is_refcounted=True, is_pointer=True +) # Python bytes object. -bytes_rprimitive: Final = RPrimitive("builtins.bytes", is_unboxed=False, is_refcounted=True) +bytes_rprimitive: Final = RPrimitive( + "builtins.bytes", is_unboxed=False, is_refcounted=True, is_pointer=True +) # Python bytearray object. bytearray_rprimitive: Final = RPrimitive( @@ -523,10 +540,14 @@ def __hash__(self) -> int: # Tuple of an arbitrary length (corresponds to Tuple[t, ...], with # explicit '...'). -tuple_rprimitive: Final = RPrimitive("builtins.tuple", is_unboxed=False, is_refcounted=True) +tuple_rprimitive: Final = RPrimitive( + "builtins.tuple", is_unboxed=False, is_refcounted=True, is_pointer=True +) # Python range object. -range_rprimitive: Final = RPrimitive("builtins.range", is_unboxed=False, is_refcounted=True) +range_rprimitive: Final = RPrimitive( + "builtins.range", is_unboxed=False, is_refcounted=True, is_pointer=True +) KNOWN_NATIVE_TYPES: Final = { name: RPrimitive(name, is_unboxed=False, is_refcounted=True, dependencies=(LIBRT_STRINGS,)) @@ -1007,18 +1028,22 @@ class RVec(RType): def __init__(self, item_type: RType) -> None: self.name = "vec[%s]" % item_type self.item_type = item_type + self.names = ["len", "buf"] if isinstance(item_type, RUnion): non_opt = optional_value_type(item_type) else: non_opt = item_type if is_int64_rprimitive(item_type): self._ctype = "VecI64" + self.types = [c_pyssize_t_rprimitive, VecbufI64Object] self.buf_type = VecbufI64Object elif isinstance(non_opt, RVec): self._ctype = "VecTExt" - # TODO: buf_type + self.types = [c_pyssize_t_rprimitive, VecbufTObject] + self.buf_type = VecbufTExtObject else: self._ctype = "VecT" + self.types = [c_pyssize_t_rprimitive, VecbufTObject] self.buf_type = VecbufTObject def field_type(self, name: str) -> RType: @@ -1288,17 +1313,29 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: types=[PyVarObject, c_pyssize_t_rprimitive, object_rprimitive], ) -""" -# vec[t], for nested vec or optional t -VecTExtObject = RStruct( - name="VecTExtObject", - names=["ob_base", "len", "depth", "optionals", "items"], +VecbufTExtItem = RStruct( + name="VecbufTExtItem", + names=["len", "buf"], + types=[c_pyssize_t_rprimitive, object_non_refcounted_rprimitive], +) + +# Buffer for vec[vec[t]] +VecbufTExtObject = RStruct( + name="VecbufTExtObject", + names=["ob_base", "item_type", "depth", "optionals", "items"], types=[ - PyObject, + PyVarObject, c_pyssize_t_rprimitive, int32_rprimitive, int32_rprimitive, - object_rprimitive, + VecbufTExtItem, ], ) -""" + +VecbufTExtObject_rprimitive = RPrimitive( + "VecbufTExtObject_ptr", + is_unboxed=False, + is_pointer=True, + is_refcounted=True, + ctype="VecbufTExtObject *", +) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index ba862d37e8379..31008176242d9 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -13,6 +13,7 @@ Branch, CallC, ComparisonOp, + GetElement, GetElementPtr, Integer, IntOp, @@ -20,6 +21,8 @@ LoadAddress, RaiseStandardError, Register, + SetElement, + Undef, Unreachable, Value, ) @@ -30,6 +33,8 @@ RType, RUnion, RVec, + c_size_t_rprimitive, + VecbufTExtItem, bool_rprimitive, c_int_rprimitive, c_pyssize_t_rprimitive, @@ -85,13 +90,16 @@ def vec_create( typeobj, optionals, depth = vec_item_type_info(builder, item_type, line) if typeobj is not None: - if depth == 0: + typeval: Value + if not (optionals & 1): + typeval = typeobj + else: typeval = Register(pointer_rprimitive) builder.add(Assign(typeval, typeobj)) - if optionals: - typeval = builder.add( - IntOp(pointer_rprimitive, typeval, Integer(1, pointer_rprimitive), IntOp.OR) - ) + typeval = builder.add( + IntOp(pointer_rprimitive, typeval, Integer(1, pointer_rprimitive), IntOp.OR) + ) + if depth == 0: call = CallC( "VecTApi.alloc", [length, typeval], @@ -107,7 +115,7 @@ def vec_create( "VecTExtApi.alloc", [ length, - typeobj, + typeval, Integer(optionals, int32_rprimitive), Integer(depth, int32_rprimitive), ], @@ -138,7 +146,9 @@ def vec_create_initialized( step = step_size(item_type) items_end = builder.int_add(items_start, builder.int_mul(length, step)) - for_loop = builder.begin_for(items_start, items_end, step, signed=False) + for_loop = builder.begin_for( + items_start, items_end, Integer(step, c_pyssize_t_rprimitive), signed=False + ) builder.set_mem(for_loop.index, item_type, init) for_loop.finish() @@ -167,6 +177,9 @@ def step_size(item_type: RType) -> int: return PLATFORM_SIZE +VEC_TYPE_INFO_I64: Final = 2 + + def vec_item_type_info( builder: "LowLevelIRBuilder", typ: RType, line: int ) -> Tuple[Optional[Value], int, int]: @@ -175,20 +188,18 @@ def vec_item_type_info( return builder.load_address(src, typ), 0, 0 elif isinstance(typ, RInstance): return builder.load_native_type_object(typ.name), 0, 0 + elif is_int64_rprimitive(typ): + return Integer(VEC_TYPE_INFO_I64, c_size_t_rprimitive), 0, 0 elif isinstance(typ, RUnion): non_opt = optional_value_type(typ) if non_opt is not None: typeval, optionals, depth = vec_item_type_info(builder, non_opt, line) if typeval is not None: - return typeval, (optionals << 1) | 1, depth + return typeval, optionals | 1, depth elif isinstance(typ, RVec): - if is_int64_rprimitive(typ.item_type): - addr = builder.load_address("VecI64Api.type", pointer_rprimitive) - load = builder.load_mem(addr, object_rprimitive) - return load, 0, 0 typeval, optionals, depth = vec_item_type_info(builder, typ.item_type, line) if typeval is not None: - return typeval, optionals, depth + 1 + return typeval, optionals << 1, depth + 1 return None, 0, 0 @@ -274,27 +285,39 @@ def vec_set_item( builder.keep_alive([base]) +def convert_to_t_ext_item(builder: LowLevelIRBuilder, item: Value) -> Value: + vec_len = builder.add(GetElement(item, "len")) + vec_buf = builder.add(GetElement(item, "buf")) + temp = builder.add(SetElement(Undef(VecbufTExtItem), "len", vec_len)) + return builder.add(SetElement(temp, "buf", vec_buf)) + + def vec_append(builder: "LowLevelIRBuilder", vec: Value, item: Value, line: int) -> Value: vec_type = vec.type assert isinstance(vec_type, RVec) item_type = vec_type.item_type - item = builder.coerce(item, item_type, line) + coerced_item = builder.coerce(item, item_type, line) if is_int64_rprimitive(item_type): name = "VecI64Api.append" elif vec_depth(vec_type) == 0: name = "VecTApi.append" else: + coerced_item = convert_to_t_ext_item(builder, coerced_item) name = "VecTExtApi.append" - call = CallC( - name, - [vec, item], - vec_type, - steals=[True, False], - is_borrowed=False, - error_kind=ERR_MAGIC, - line=line, + call = builder.add( + CallC( + name, + [vec, coerced_item], + vec_type, + steals=[True, False], + is_borrowed=False, + error_kind=ERR_MAGIC, + line=line, + ) ) - return builder.add(call) + if vec_depth(vec_type) > 0: + builder.keep_alive([item]) + return call def vec_pop(builder: "LowLevelIRBuilder", base: Value, index: Value, line: int) -> Value: @@ -358,7 +381,9 @@ def vec_contains(builder: "LowLevelIRBuilder", vec: Value, target: Value, line: true, end = BasicBlock(), BasicBlock() - for_loop = builder.begin_for(items_start, items_end, step, signed=False) + for_loop = builder.begin_for( + items_start, items_end, Integer(step, c_pyssize_t_rprimitive), signed=False + ) item = builder.load_mem(for_loop.index, item_type) comp = builder.binary_op(item, target, "==", line) false = BasicBlock() diff --git a/mypyc/test-data/irbuild-vecs.test b/mypyc/test-data/irbuild-vecs.test index 6635e7a9575b2..d9de012c368de 100644 --- a/mypyc/test-data/irbuild-vecs.test +++ b/mypyc/test-data/irbuild-vecs.test @@ -11,22 +11,18 @@ def native_class() -> vec[C]: [out] def primitive(): r0 :: object - r1 :: ptr - r2 :: vec[str] + r1 :: vec[str] L0: r0 = load_address PyUnicode_Type - r1 = r0 - r2 = VecTApi.alloc(0, r1) - return r2 + r1 = VecTApi.alloc(0, r0) + return r1 def native_class(): r0 :: object - r1 :: ptr - r2 :: vec[__main__.C] + r1 :: vec[__main__.C] L0: r0 = __main__.C :: type - r1 = r0 - r2 = VecTApi.alloc(0, r1) - return r2 + r1 = VecTApi.alloc(0, r0) + return r1 [case testVecTAppend] from vecs import vec, append @@ -116,14 +112,10 @@ L0: r1 = VecTExtApi.alloc(0, r0, 0, 1) return r1 def g(): - r0 :: ptr - r1 :: object - r2 :: vec[vec[int64]] + r0 :: vec[vec[int64]] L0: - r0 = load_address VecI64Api.type - r1 = load_mem r0 :: builtins.object* - r2 = VecTApi.alloc(0, r1) - return r2 + r0 = VecTExtApi.alloc(0, 2, 0, 1) + return r0 [case testVecTLen] from vecs import vec @@ -219,6 +211,28 @@ L2: keep_alive v return r7 +[case testVecNestedAppend] +from vecs import vec, append + +def f(v: vec[vec[str]], vv: vec[str]) -> vec[vec[str]]: + return append(v, vv) +[out] +def f(v, vv): + v :: vec[vec[str]] + vv :: vec[str] + r0 :: native_int + r1 :: object + r2, r3 :: VecbufTExtItem{len:native_int, buf:object} + r4 :: vec[vec[str]] +L0: + r0 = vv.len + r1 = vv.buf + r2 = set_element undef, len, r0 + r3 = set_element r2, buf, r1 + r4 = VecTExtApi.append(v, r3) + keep_alive vv + return r4 + [case testVecNestedVecI64Append] from vecs import vec, append from mypy_extensions import i64 @@ -229,10 +243,18 @@ def f(v: vec[vec[i64]], vv: vec[i64]) -> vec[vec[i64]]: def f(v, vv): v :: vec[vec[int64]] vv :: vec[int64] - r0 :: vec[vec[int64]] + r0 :: native_int + r1 :: object + r2, r3 :: VecbufTExtItem{len:native_int, buf:object} + r4 :: vec[vec[int64]] L0: - r0 = VecTApi.append(v, vv) - return r0 + r0 = vv.len + r1 = vv.buf + r2 = set_element undef, len, r0 + r3 = set_element r2, buf, r1 + r4 = VecTExtApi.append(v, r3) + keep_alive vv + return r4 [case testVecNestedVecI64GetItem] from vecs import vec diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index 6d6a77b0b4647..ecf29b35031fc 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -1711,3 +1711,25 @@ L0: r1 = get_element_ptr r0 len :: VecObject r2 = load_mem r1 :: int64* return r2 + +[case testVecNestedAppend] +from vecs import vec, append + +def f(vv: vec[vec[str]], v: vec[str]) -> vec[vec[str]]: + return append(vv, v) +[out] +def f(vv, v): + vv :: vec[vec[str]] + v :: vec[str] + r0 :: native_int + r1 :: object + r2, r3 :: VecbufTExtItem{len:native_int, buf:object} + r4 :: vec[vec[str]] +L0: + r0 = v.len + r1 = v.buf + r2 = set_element undef, len, r0 + r3 = set_element r2, buf, r1 + inc_ref vv + r4 = VecTExtApi.append(vv, r3) + return r4 From 3da9b663c181a6e1c466b125d8c152fd1a64508b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 11 Jun 2023 11:03:37 +0100 Subject: [PATCH 020/180] Add vec tests --- mypyc/test-data/irbuild-vecs.test | 25 ++++++++++++++++++++++- mypyc/test-data/refcount.test | 34 +++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/mypyc/test-data/irbuild-vecs.test b/mypyc/test-data/irbuild-vecs.test index d9de012c368de..380ff3e3eac02 100644 --- a/mypyc/test-data/irbuild-vecs.test +++ b/mypyc/test-data/irbuild-vecs.test @@ -144,7 +144,6 @@ L0: [case testVecTGetItem] from vecs import vec from mypy_extensions import i64 -from typing import Optional def f(v: vec[str], n: i64) -> str: return v[n] @@ -256,6 +255,30 @@ L0: keep_alive vv return r4 +[case testVecNestedLen] +from vecs import vec +from mypy_extensions import i64 +from typing import Optional + +def f(v: vec[vec[str]]) -> i64: + return len(v) + +def g(v: vec[vec[i64]]) -> i64: + return len(v) +[out] +def f(v): + v :: vec[vec[str]] + r0 :: native_int +L0: + r0 = v.len + return r0 +def g(v): + v :: vec[vec[int64]] + r0 :: native_int +L0: + r0 = v.len + return r0 + [case testVecNestedVecI64GetItem] from vecs import vec from mypy_extensions import i64 diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index ecf29b35031fc..0155fa844cb9f 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -1733,3 +1733,37 @@ L0: inc_ref vv r4 = VecTExtApi.append(vv, r3) return r4 + +[case testVecTGetItem] +from vecs import vec +from mypy_extensions import i64 + +def f(v: vec[str], n: i64) -> str: + return v[n] +[out] +def f(v, n): + v :: vec[str] + n :: int64 + r0 :: native_int + r1 :: bit + r2 :: bool + r3 :: object + r4 :: ptr + r5 :: int64 + r6 :: ptr + r7 :: str +L0: + r0 = v.len + r1 = n < r0 :: unsigned + if r1 goto L2 else goto L1 :: bool +L1: + r2 = raise IndexError + unreachable +L2: + r3 = v.buf + r4 = get_element_ptr r3 items :: VecbufTObject + r5 = n * 8 + r6 = r4 + r5 + r7 = load_mem r6 :: builtins.str* + inc_ref r7 + return r7 From 8be74b6a479246a617341b1b75e1166897025f99 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 11 Jun 2023 11:17:48 +0100 Subject: [PATCH 021/180] Fix compile errors during vec construction --- mypyc/irbuild/vec.py | 11 +++++++---- mypyc/test-data/irbuild-vecs.test | 24 +++++++++++++++--------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 31008176242d9..9ded0702957cc 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -91,14 +91,17 @@ def vec_create( typeobj, optionals, depth = vec_item_type_info(builder, item_type, line) if typeobj is not None: typeval: Value - if not (optionals & 1): + if isinstance(typeobj, Integer): typeval = typeobj else: + # Create an integer which will hold the type object * as an integral value. + # Assign implicitly coerces between pointer/integer types. typeval = Register(pointer_rprimitive) builder.add(Assign(typeval, typeobj)) - typeval = builder.add( - IntOp(pointer_rprimitive, typeval, Integer(1, pointer_rprimitive), IntOp.OR) - ) + if optionals & 1: + typeval = builder.add( + IntOp(pointer_rprimitive, typeval, Integer(1, pointer_rprimitive), IntOp.OR) + ) if depth == 0: call = CallC( "VecTApi.alloc", diff --git a/mypyc/test-data/irbuild-vecs.test b/mypyc/test-data/irbuild-vecs.test index 380ff3e3eac02..b16d903bd1bf8 100644 --- a/mypyc/test-data/irbuild-vecs.test +++ b/mypyc/test-data/irbuild-vecs.test @@ -11,18 +11,22 @@ def native_class() -> vec[C]: [out] def primitive(): r0 :: object - r1 :: vec[str] + r1 :: ptr + r2 :: vec[str] L0: r0 = load_address PyUnicode_Type - r1 = VecTApi.alloc(0, r0) - return r1 + r1 = r0 + r2 = VecTApi.alloc(0, r1) + return r2 def native_class(): r0 :: object - r1 :: vec[__main__.C] + r1 :: ptr + r2 :: vec[__main__.C] L0: r0 = __main__.C :: type - r1 = VecTApi.alloc(0, r0) - return r1 + r1 = r0 + r2 = VecTApi.alloc(0, r1) + return r2 [case testVecTAppend] from vecs import vec, append @@ -106,11 +110,13 @@ def g() -> vec[vec[i64]]: [out] def f(): r0 :: object - r1 :: vec[vec[str]] + r1 :: ptr + r2 :: vec[vec[str]] L0: r0 = load_address PyUnicode_Type - r1 = VecTExtApi.alloc(0, r0, 0, 1) - return r1 + r1 = r0 + r2 = VecTExtApi.alloc(0, r1, 0, 1) + return r2 def g(): r0 :: vec[vec[int64]] L0: From 3724b5ad2fa6bab7d00c18a912ddd5ff7fbf2f0e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 11 Jun 2023 13:02:14 +0100 Subject: [PATCH 022/180] Update test case --- mypyc/test-data/irbuild-vecs.test | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/mypyc/test-data/irbuild-vecs.test b/mypyc/test-data/irbuild-vecs.test index b16d903bd1bf8..717212c96de19 100644 --- a/mypyc/test-data/irbuild-vecs.test +++ b/mypyc/test-data/irbuild-vecs.test @@ -181,7 +181,7 @@ L2: keep_alive v return r7 -[case testVecTExtGetItem] +[case testVecTOptionalGetItem] from vecs import vec from mypy_extensions import i64 from typing import Optional @@ -192,24 +192,24 @@ def f(v: vec[Optional[str]], n: i64) -> Optional[str]: def f(v, n): v :: vec[union[str, None]] n :: int64 - r0 :: ptr - r1 :: native_int - r2 :: bit - r3 :: bool + r0 :: native_int + r1 :: bit + r2 :: bool + r3 :: object r4 :: ptr r5 :: int64 r6 :: ptr r7 :: union[str, None] L0: - r0 = get_element_ptr v len :: VecObject - r1 = load_mem r0 :: native_int* - r2 = n < r1 :: unsigned - if r2 goto L2 else goto L1 :: bool + r0 = v.len + r1 = n < r0 :: unsigned + if r1 goto L2 else goto L1 :: bool L1: - r3 = raise IndexError + r2 = raise IndexError unreachable L2: - r4 = get_element_ptr v items :: VecTExtObject + r3 = v.buf + r4 = get_element_ptr r3 items :: VecbufTObject r5 = n * 8 r6 = r4 + r5 r7 = load_mem r6 :: union* From e2c04a6a204d0e20183229e526c9ceee2bc96619 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 11 Jun 2023 15:05:04 +0100 Subject: [PATCH 023/180] Support vec get item in more cases --- mypyc/irbuild/ll_builder.py | 2 +- mypyc/irbuild/vec.py | 13 +-- mypyc/test-data/irbuild-vecs.test | 144 +++++++++++++++++++++++++++--- mypyc/test-data/refcount.test | 67 +++++++++++--- 4 files changed, 197 insertions(+), 29 deletions(-) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index abc1737f19000..81bc4699e7406 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -2001,7 +2001,7 @@ def get_item( if is_int_rprimitive(item.type) or is_short_int_rprimitive(item.type): item = self.coerce(item, int64_rprimitive, line) if is_int64_rprimitive(item.type): - return vec_get_item(self, base, item, line) + return vec_get_item(self, base, item, line, can_borrow=can_borrow) # TODO: Move special casing for RTuple here from transform_index_expr return self.gen_method_call( base, "__getitem__", [item], result_type, line, can_borrow=can_borrow diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 9ded0702957cc..6897e55101401 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -3,6 +3,7 @@ from __future__ import annotations from typing import TYPE_CHECKING, List, Optional, Tuple, Union, cast +from typing_extensions import Final from mypyc.common import PLATFORM_SIZE from mypyc.ir.ops import ( @@ -239,7 +240,8 @@ def vec_check_index(builder: "LowLevelIRBuilder", lenv: Value, index: Value, lin builder.activate_block(ok) -def vec_get_item(builder: "LowLevelIRBuilder", base: Value, index: Value, line: int) -> Value: +def vec_get_item(builder: "LowLevelIRBuilder", base: Value, index: Value, line: int, *, + can_borrow: bool = False) -> Value: """Generate inlined vec __getitem__ call. We inline this, since it's simple but performance-critical. @@ -251,8 +253,8 @@ def vec_get_item(builder: "LowLevelIRBuilder", base: Value, index: Value, line: len_val = vec_len_native(builder, base) vec_check_index(builder, len_val, index, line) item_addr = vec_item_ptr(builder, base, index) - result = builder.load_mem(item_addr, vtype.item_type) - builder.keep_alive([base]) + result = builder.load_mem(item_addr, vtype.item_type, borrow=can_borrow) + builder.keep_alives.append(base) return result @@ -282,8 +284,7 @@ def vec_set_item( if item_type.is_refcounted: # Read an unborrowed reference to cause a decref to be # generated for the old item. - old_item = builder.load_mem(item_addr, item_type) - old_item.is_borrowed = False + old_item = builder.load_mem(item_addr, item_type, borrow=False) builder.set_mem(item_addr, item_type, item) builder.keep_alive([base]) @@ -387,7 +388,7 @@ def vec_contains(builder: "LowLevelIRBuilder", vec: Value, target: Value, line: for_loop = builder.begin_for( items_start, items_end, Integer(step, c_pyssize_t_rprimitive), signed=False ) - item = builder.load_mem(for_loop.index, item_type) + item = builder.load_mem(for_loop.index, item_type, borrow=True) comp = builder.binary_op(item, target, "==", line) false = BasicBlock() builder.add(Branch(comp, true, false, Branch.BOOL)) diff --git a/mypyc/test-data/irbuild-vecs.test b/mypyc/test-data/irbuild-vecs.test index 717212c96de19..b56c1081cb7c6 100644 --- a/mypyc/test-data/irbuild-vecs.test +++ b/mypyc/test-data/irbuild-vecs.test @@ -285,7 +285,41 @@ L0: r0 = v.len return r0 -[case testVecNestedVecI64GetItem] +[case testVecNestedGetItem] +from vecs import vec +from mypy_extensions import i64 + +def f(v: vec[vec[str]], n: i64) -> vec[str]: + return v[n] +[out] +def f(v, n): + v :: vec[vec[str]] + n :: int64 + r0 :: native_int + r1 :: bit + r2 :: bool + r3 :: object + r4 :: ptr + r5 :: int64 + r6 :: ptr + r7 :: vec[str] +L0: + r0 = v.len + r1 = n < r0 :: unsigned + if r1 goto L2 else goto L1 :: bool +L1: + r2 = raise IndexError + unreachable +L2: + r3 = v.buf + r4 = get_element_ptr r3 items :: VecbufTExtObject + r5 = n * 8 + r6 = r4 + r5 + r7 = load_mem r6 :: vec[str]* + keep_alive v + return r7 + +[case testVecNestedI64GetItem] from vecs import vec from mypy_extensions import i64 @@ -295,30 +329,118 @@ def f(v: vec[vec[i64]], n: i64) -> vec[i64]: def f(v, n): v :: vec[vec[int64]] n :: int64 - r0 :: ptr - r1 :: native_int - r2 :: bit - r3 :: bool + r0 :: native_int + r1 :: bit + r2 :: bool + r3 :: object r4 :: ptr r5 :: int64 r6 :: ptr r7 :: vec[int64] L0: - r0 = get_element_ptr v len :: VecObject - r1 = load_mem r0 :: native_int* - r2 = n < r1 :: unsigned - if r2 goto L2 else goto L1 :: bool + r0 = v.len + r1 = n < r0 :: unsigned + if r1 goto L2 else goto L1 :: bool L1: - r3 = raise IndexError + r2 = raise IndexError unreachable L2: - r4 = get_element_ptr v items :: VecTObject + r3 = v.buf + r4 = get_element_ptr r3 items :: VecbufTExtObject r5 = n * 8 r6 = r4 + r5 r7 = load_mem r6 :: vec[int64]* keep_alive v return r7 +[case testVecNestedI64GetItemWithBorrow] +from vecs import vec +from mypy_extensions import i64 + +def f(v: vec[vec[i64]], n: i64) -> i64: + return v[n][n] +[out] +def f(v, n): + v :: vec[vec[int64]] + n :: int64 + r0 :: native_int + r1 :: bit + r2 :: bool + r3 :: object + r4 :: ptr + r5 :: int64 + r6 :: ptr + r7 :: vec[int64] + r8 :: native_int + r9 :: bit + r10 :: bool + r11 :: object + r12 :: ptr + r13 :: int64 + r14 :: ptr + r15 :: int64 +L0: + r0 = v.len + r1 = n < r0 :: unsigned + if r1 goto L2 else goto L1 :: bool +L1: + r2 = raise IndexError + unreachable +L2: + r3 = v.buf + r4 = get_element_ptr r3 items :: VecbufTExtObject + r5 = n * 8 + r6 = r4 + r5 + r7 = borrow load_mem r6 :: vec[int64]* + r8 = r7.len + r9 = n < r8 :: unsigned + if r9 goto L4 else goto L3 :: bool +L3: + r10 = raise IndexError + unreachable +L4: + r11 = r7.buf + r12 = get_element_ptr r11 items :: VecbufI64Object + r13 = n * 8 + r14 = r12 + r13 + r15 = load_mem r14 :: int64* + keep_alive v, r7 + return r15 + +[case testVecDoublyNestedGetItem] +from vecs import vec +from mypy_extensions import i64 + +def f(v: vec[vec[vec[str]]], n: i64) -> vec[vec[str]]: + return v[n] +[out] +def f(v, n): + v :: vec[vec[vec[str]]] + n :: int64 + r0 :: native_int + r1 :: bit + r2 :: bool + r3 :: object + r4 :: ptr + r5 :: int64 + r6 :: ptr + r7 :: vec[vec[str]] +L0: + r0 = v.len + r1 = n < r0 :: unsigned + if r1 goto L2 else goto L1 :: bool +L1: + r2 = raise IndexError + unreachable +L2: + r3 = v.buf + r4 = get_element_ptr r3 items :: VecbufTExtObject + r5 = n * 8 + r6 = r4 + r5 + r7 = load_mem r6 :: vec[vec[str]]* + keep_alive v + return r7 + -- -- New repr -- diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index 0155fa844cb9f..5e233efaab6d9 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -1666,25 +1666,25 @@ def C.f(self, x): self :: __main__.C x :: int64 r0 :: vec[int64] - r1 :: ptr - r2 :: native_int - r3 :: bit - r4 :: bool + r1 :: native_int + r2 :: bit + r3 :: bool + r4 :: object r5 :: ptr r6 :: int64 r7 :: ptr r8 :: int64 L0: r0 = borrow self.v - r1 = get_element_ptr r0 len :: VecObject - r2 = load_mem r1 :: native_int* - r3 = x < r2 :: unsigned - if r3 goto L2 else goto L1 :: bool + r1 = r0.len + r2 = x < r1 :: unsigned + if r2 goto L2 else goto L1 :: bool L1: - r4 = raise IndexError + r3 = raise IndexError unreachable L2: - r5 = get_element_ptr r0 items :: VecI64Object + r4 = r0.buf + r5 = get_element_ptr r4 items :: VecbufI64Object r6 = x * 8 r7 = r5 + r6 r8 = load_mem r7 :: int64* @@ -1765,5 +1765,50 @@ L2: r5 = n * 8 r6 = r4 + r5 r7 = load_mem r6 :: builtins.str* - inc_ref r7 return r7 + +[case testVecNestedGetItem] +from vecs import vec +from mypy_extensions import i64 + +def f(v: vec[vec[str]], n: i64) -> None: + vv = vec[vec[str]]()[n] +[out] +def f(v, n): + v :: vec[vec[str]] + n :: int64 + r0 :: object + r1 :: ptr + r2 :: vec[vec[str]] + r3 :: native_int + r4 :: bit + r5 :: bool + r6 :: object + r7 :: ptr + r8 :: int64 + r9 :: ptr + r10, vv :: vec[str] +L0: + r0 = load_address PyUnicode_Type + inc_ref r0 + r1 = r0 + r2 = VecTExtApi.alloc(0, r1, 0, 1) + r3 = r2.len + r4 = n < r3 :: unsigned + if r4 goto L2 else goto L3 :: bool +L1: + r5 = raise IndexError + unreachable +L2: + r6 = r2.buf + r7 = get_element_ptr r6 items :: VecbufTExtObject + r8 = n * 8 + r9 = r7 + r8 + r10 = load_mem r9 :: vec[str]* + dec_ref r2 + vv = r10 + dec_ref vv + return 1 +L3: + dec_ref r2 + goto L1 From a737f7c89348213c81efedeefdb8815b7159f634 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 11 Jun 2023 15:19:13 +0100 Subject: [PATCH 024/180] Fix set item --- mypyc/analysis/dataflow.py | 8 ++++++++ mypyc/irbuild/vec.py | 4 +++- mypyc/test-data/refcount.test | 22 +++++++++++----------- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/mypyc/analysis/dataflow.py b/mypyc/analysis/dataflow.py index 779454e22fd7d..3390968f368cd 100644 --- a/mypyc/analysis/dataflow.py +++ b/mypyc/analysis/dataflow.py @@ -31,6 +31,8 @@ InitStatic, Integer, IntOp, + IncRef, + DecRef, KeepAlive, LoadAddress, LoadErrorValue, @@ -197,6 +199,12 @@ def visit_assign_multi(self, op: AssignMulti) -> GenAndKill[T]: def visit_set_mem(self, op: SetMem) -> GenAndKill[T]: raise NotImplementedError + def visit_inc_ref(self, op: IncRef) -> GenAndKill[T]: + return self.visit_register_op(op) + + def visit_dec_ref(self, op: DecRef) -> GenAndKill[T]: + return self.visit_register_op(op) + def visit_call(self, op: Call) -> GenAndKill[T]: return self.visit_register_op(op) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 6897e55101401..7b5822f99a4c0 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -12,6 +12,7 @@ Assign, BasicBlock, Branch, + DecRef, CallC, ComparisonOp, GetElement, @@ -284,7 +285,8 @@ def vec_set_item( if item_type.is_refcounted: # Read an unborrowed reference to cause a decref to be # generated for the old item. - old_item = builder.load_mem(item_addr, item_type, borrow=False) + old_item = builder.load_mem(item_addr, item_type, borrow=True) + builder.add(DecRef(old_item)) builder.set_mem(item_addr, item_type, item) builder.keep_alive([base]) diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index 5e233efaab6d9..6767651b79d61 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -1511,27 +1511,27 @@ def f(v, i, x): v :: vec[str] i :: int64 x :: str - r0 :: ptr - r1 :: native_int - r2 :: bit - r3 :: bool + r0 :: native_int + r1 :: bit + r2 :: bool + r3 :: object r4 :: ptr r5 :: int64 r6 :: ptr r7 :: str L0: - r0 = get_element_ptr v len :: VecObject - r1 = load_mem r0 :: native_int* - r2 = i < r1 :: unsigned - if r2 goto L2 else goto L1 :: bool + r0 = v.len + r1 = i < r0 :: unsigned + if r1 goto L2 else goto L1 :: bool L1: - r3 = raise IndexError + r2 = raise IndexError unreachable L2: - r4 = get_element_ptr v items :: VecTObject + r3 = v.buf + r4 = get_element_ptr r3 items :: VecbufTObject r5 = i * 8 r6 = r4 + r5 - r7 = load_mem r6 :: builtins.str* + r7 = borrow load_mem r6 :: builtins.str* dec_ref r7 inc_ref x set_mem r6, x :: builtins.str* From f866775ef76316376708173567113f76f42acbb2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 11 Jun 2023 15:42:28 +0100 Subject: [PATCH 025/180] Fix serialization --- mypyc/ir/rtypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index b9d70a6b4e704..42cd896e0fb20 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -326,7 +326,7 @@ def __hash__(self) -> int: # Similar to object_primitive, but doesn not use automatic reference # counting. Useful for temporaries. object_non_refcounted_rprimitive: Final = RPrimitive( - "builtins.object", is_unboxed=False, is_refcounted=False, is_pointer=True + "builtins.object_nrc", is_unboxed=False, is_refcounted=False, is_pointer=True ) # Arbitrary-precision integer (corresponds to Python 'int'). Small From 1f6fba9b9ea430691e61077d8e7578e45590c3a7 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 11 Jun 2023 15:46:04 +0100 Subject: [PATCH 026/180] Update tests --- mypyc/test-data/irbuild-vecs.test | 4 ++-- mypyc/test-data/refcount.test | 10 ++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/mypyc/test-data/irbuild-vecs.test b/mypyc/test-data/irbuild-vecs.test index b56c1081cb7c6..4c620790496ba 100644 --- a/mypyc/test-data/irbuild-vecs.test +++ b/mypyc/test-data/irbuild-vecs.test @@ -227,7 +227,7 @@ def f(v, vv): vv :: vec[str] r0 :: native_int r1 :: object - r2, r3 :: VecbufTExtItem{len:native_int, buf:object} + r2, r3 :: VecbufTExtItem{len:native_int, buf:object_nrc} r4 :: vec[vec[str]] L0: r0 = vv.len @@ -250,7 +250,7 @@ def f(v, vv): vv :: vec[int64] r0 :: native_int r1 :: object - r2, r3 :: VecbufTExtItem{len:native_int, buf:object} + r2, r3 :: VecbufTExtItem{len:native_int, buf:object_nrc} r4 :: vec[vec[int64]] L0: r0 = vv.len diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index 6767651b79d61..37c0708d6555b 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -1704,13 +1704,11 @@ def C.f(self, x): self :: __main__.C x :: int64 r0 :: vec[int64] - r1 :: ptr - r2 :: int64 + r1 :: native_int L0: r0 = borrow self.v - r1 = get_element_ptr r0 len :: VecObject - r2 = load_mem r1 :: int64* - return r2 + r1 = r0.len + return r1 [case testVecNestedAppend] from vecs import vec, append @@ -1723,7 +1721,7 @@ def f(vv, v): v :: vec[str] r0 :: native_int r1 :: object - r2, r3 :: VecbufTExtItem{len:native_int, buf:object} + r2, r3 :: VecbufTExtItem{len:native_int, buf:object_nrc} r4 :: vec[vec[str]] L0: r0 = v.len From 3efe0d89fea5c963db600573b6310010d433f745 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 11 Jun 2023 15:48:37 +0100 Subject: [PATCH 027/180] Update test case --- mypyc/test-data/refcount.test | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index 37c0708d6555b..711e52fc42d98 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -1589,10 +1589,9 @@ def g(s: str) -> None: pass def f(v): v :: vec[str] t :: int64 - r0 :: native_int - r1 :: ptr - r2 :: int64 - r3 :: bit + r0, r1 :: native_int + r2 :: bit + r3 :: object r4 :: ptr r5 :: native_int r6 :: ptr @@ -1603,16 +1602,15 @@ L0: t = 0 r0 = 0 L1: - r1 = get_element_ptr v len :: VecObject - r2 = load_mem r1 :: int64* - r3 = r0 < r2 :: signed - if r3 goto L2 else goto L4 :: bool + r1 = v.len + r2 = r0 < r1 :: signed + if r2 goto L2 else goto L4 :: bool L2: - r4 = get_element_ptr v items :: VecTObject + r3 = v.buf + r4 = get_element_ptr r3 items :: VecbufTObject r5 = r0 * 8 r6 = r4 + r5 r7 = load_mem r6 :: builtins.str* - inc_ref r7 s = r7 r8 = g(s) dec_ref s From c58b49ec6c38abc20f67716227d2e9f6daa73c72 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 11 Jun 2023 16:59:44 +0100 Subject: [PATCH 028/180] Add pop test case --- mypyc/test-data/irbuild-vecs.test | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/mypyc/test-data/irbuild-vecs.test b/mypyc/test-data/irbuild-vecs.test index 4c620790496ba..4f058333b36a6 100644 --- a/mypyc/test-data/irbuild-vecs.test +++ b/mypyc/test-data/irbuild-vecs.test @@ -216,6 +216,20 @@ L2: keep_alive v return r7 +[case testNewTPopLast] +from typing import Tuple +from vecs import vec, pop + +def pop_last(v: vec[str]) -> Tuple[vec[str], str]: + return pop(v) +[out] +def pop_last(v): + v :: vec[str] + r0 :: tuple[vec[str], str] +L0: + r0 = VecTApi.pop(v, -1) + return r0 + [case testVecNestedAppend] from vecs import vec, append From 2aa90b938dd4150fa6910a1f08ef6d57f2dcf973 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 11 Jun 2023 17:01:50 +0100 Subject: [PATCH 029/180] Update test case --- mypyc/test-data/refcount.test | 40 ++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index 711e52fc42d98..4ecd9c766971d 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -1548,31 +1548,37 @@ def f(n: i64) -> vec[Optional[str]]: def f(n): n :: int64 r0, r1 :: object - r2 :: vec[union[str, None]] - r3 :: ptr - r4 :: int64 - r5, r6 :: ptr - r7 :: bit - r8 :: ptr + r2, r3 :: ptr + r4 :: vec[union[str, None]] + r5 :: object + r6 :: ptr + r7 :: int64 + r8, r9 :: ptr + r10 :: bit + r11 :: ptr L0: r0 = box(None, 1) r1 = load_address PyUnicode_Type - r2 = VecTExtApi.alloc(n, r1, 1, 0) - r3 = get_element_ptr r2 items :: VecTExtObject - r4 = n * 8 - r5 = r3 + r4 - r6 = r3 + inc_ref r1 + r2 = r1 + r3 = r2 | 1 + r4 = VecTApi.alloc(n, r3) + r5 = r4.buf + r6 = get_element_ptr r5 items :: VecbufTObject + r7 = n * 8 + r8 = r6 + r7 + r9 = r6 L1: - r7 = r6 < r5 :: unsigned - if r7 goto L2 else goto L3 :: bool + r10 = r9 < r8 :: unsigned + if r10 goto L2 else goto L3 :: bool L2: inc_ref r0 - set_mem r6, r0 :: union* - r8 = r6 + 8 - r6 = r8 + set_mem r9, r0 :: union* + r11 = r9 + 8 + r9 = r11 goto L1 L3: - return r2 + return r4 [case testVecTForLoop] from vecs import vec From 4657d7175a11e27637bdd6f11e9a69d7354b4b02 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 11 Jun 2023 17:17:39 +0100 Subject: [PATCH 030/180] Fix refcounting of item type + update tests --- mypyc/test-data/refcount.test | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index 4ecd9c766971d..26c631474b3f5 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -1559,7 +1559,6 @@ def f(n): L0: r0 = box(None, 1) r1 = load_address PyUnicode_Type - inc_ref r1 r2 = r1 r3 = r2 | 1 r4 = VecTApi.alloc(n, r3) @@ -1640,21 +1639,25 @@ def f() -> vec[str]: def f(): r0, r1 :: str r2 :: object - r3 :: vec[str] - r4, r5, r6 :: ptr + r3 :: ptr + r4 :: vec[str] + r5 :: object + r6, r7, r8 :: ptr L0: r0 = 'x' r1 = 'y' r2 = load_address PyUnicode_Type - r3 = VecTApi.alloc(2, r2) - r4 = get_element_ptr r3 items :: VecTObject + r3 = r2 + r4 = VecTApi.alloc(2, r3) + r5 = r4.buf + r6 = get_element_ptr r5 items :: VecbufTObject inc_ref r0 - set_mem r4, r0 :: builtins.str* - r5 = r4 + 8 + set_mem r6, r0 :: builtins.str* + r7 = r6 + 8 inc_ref r1 - set_mem r5, r1 :: builtins.str* - r6 = r5 + 8 - return r3 + set_mem r7, r1 :: builtins.str* + r8 = r7 + 8 + return r4 [case testVecI64GetItemBorrowVec] from vecs import vec @@ -1792,7 +1795,6 @@ def f(v, n): r10, vv :: vec[str] L0: r0 = load_address PyUnicode_Type - inc_ref r0 r1 = r0 r2 = VecTExtApi.alloc(0, r1, 0, 1) r3 = r2.len From cfe392bfb898dea20144b8a31fcaa77624bcc7e0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 11 Jun 2023 17:18:43 +0100 Subject: [PATCH 031/180] Refactor vec irbuild tests --- mypyc/test-data/irbuild-vecs.test | 1242 ++++++++++++++--------------- 1 file changed, 619 insertions(+), 623 deletions(-) diff --git a/mypyc/test-data/irbuild-vecs.test b/mypyc/test-data/irbuild-vecs.test index 4f058333b36a6..f27bb9ff36ffc 100644 --- a/mypyc/test-data/irbuild-vecs.test +++ b/mypyc/test-data/irbuild-vecs.test @@ -1,162 +1,43 @@ -[case testVecTCreateEmpty] -from vecs import vec, append - -class C: pass - -def primitive() -> vec[str]: - return vec[str]() - -def native_class() -> vec[C]: - return vec[C]() -[out] -def primitive(): - r0 :: object - r1 :: ptr - r2 :: vec[str] -L0: - r0 = load_address PyUnicode_Type - r1 = r0 - r2 = VecTApi.alloc(0, r1) - return r2 -def native_class(): - r0 :: object - r1 :: ptr - r2 :: vec[__main__.C] -L0: - r0 = __main__.C :: type - r1 = r0 - r2 = VecTApi.alloc(0, r1) - return r2 - -[case testVecTAppend] -from vecs import vec, append - -def f(v: vec[str]) -> vec[str]: - return append(v, 'x') -[out] -def f(v): - v :: vec[str] - r0 :: str - r1 :: vec[str] -L0: - r0 = 'x' - r1 = VecTApi.append(v, r0) - return r1 - -[case testVecTOptionalCreateEmpty] -from vecs import vec, append -from typing import Optional - -class C: pass - -def primitive() -> vec[Optional[str]]: - return vec[Optional[str]]() - -def native_class() -> vec[Optional[C]]: - return vec[Optional[C]]() -[out] -def primitive(): - r0 :: object - r1, r2 :: ptr - r3 :: vec[union[str, None]] -L0: - r0 = load_address PyUnicode_Type - r1 = r0 - r2 = r1 | 1 - r3 = VecTApi.alloc(0, r2) - return r3 -def native_class(): - r0 :: object - r1, r2 :: ptr - r3 :: vec[union[__main__.C, None]] -L0: - r0 = __main__.C :: type - r1 = r0 - r2 = r1 | 1 - r3 = VecTApi.alloc(0, r2) - return r3 - -[case testVecTOptionalAppend] -from vecs import vec, append -from typing import Optional - -def f(v: vec[Optional[str]]) -> vec[Optional[str]]: - v = append(v, 'x') - return append(v, None) -[out] -def f(v): - v :: vec[union[str, None]] - r0 :: str - r1 :: vec[union[str, None]] - r2 :: object - r3 :: vec[union[str, None]] -L0: - r0 = 'x' - r1 = VecTApi.append(v, r0) - v = r1 - r2 = box(None, 1) - r3 = VecTApi.append(v, r2) - return r3 - -[case testVecNestedCreateEmpty] +[case testVecI64CreateEmpty] from vecs import vec, append from mypy_extensions import i64 -def f() -> vec[vec[str]]: - return vec[vec[str]]() - -def g() -> vec[vec[i64]]: - return vec[vec[i64]]() +def f() -> vec[i64]: + return vec[i64]() [out] def f(): - r0 :: object - r1 :: ptr - r2 :: vec[vec[str]] -L0: - r0 = load_address PyUnicode_Type - r1 = r0 - r2 = VecTExtApi.alloc(0, r1, 0, 1) - return r2 -def g(): - r0 :: vec[vec[int64]] + r0 :: vec[int64] L0: - r0 = VecTExtApi.alloc(0, 2, 0, 1) + r0 = VecI64Api.alloc(0) return r0 -[case testVecTLen] +[case testVecI64Len] from vecs import vec from mypy_extensions import i64 -from typing import Optional - -def f(v: vec[str]) -> i64: - return len(v) -def g(v: vec[Optional[str]]) -> i64: - return len(v) +def f(v: vec[i64]) -> i64: + l = len(v) + return l [out] def f(v): - v :: vec[str] - r0 :: native_int -L0: - r0 = v.len - return r0 -def g(v): - v :: vec[union[str, None]] + v :: vec[int64] r0 :: native_int + l :: int64 L0: r0 = v.len - return r0 + l = r0 + return l -[case testVecTGetItem] +[case testVecI64GetItem] from vecs import vec from mypy_extensions import i64 -def f(v: vec[str], n: i64) -> str: - return v[n] +def f(v: vec[i64], i: i64) -> i64: + return v[i] [out] -def f(v, n): - v :: vec[str] - n :: int64 +def f(v, i): + v :: vec[int64] + i :: int64 r0 :: native_int r1 :: bit r2 :: bool @@ -164,34 +45,48 @@ def f(v, n): r4 :: ptr r5 :: int64 r6 :: ptr - r7 :: str + r7 :: int64 L0: r0 = v.len - r1 = n < r0 :: unsigned + r1 = i < r0 :: unsigned if r1 goto L2 else goto L1 :: bool L1: r2 = raise IndexError unreachable L2: r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufTObject - r5 = n * 8 + r4 = get_element_ptr r3 items :: VecbufI64Object + r5 = i * 8 r6 = r4 + r5 - r7 = load_mem r6 :: builtins.str* + r7 = load_mem r6 :: int64* keep_alive v return r7 -[case testVecTOptionalGetItem] +[case testVecI64Append] +from vecs import vec, append +from mypy_extensions import i64 + +def f(v: vec[i64], i: i64) -> vec[i64]: + return append(v, i) +[out] +def f(v, i): + v :: vec[int64] + i :: int64 + r0 :: vec[int64] +L0: + r0 = VecI64Api.append(v, i) + return r0 + +[case testVecI64SetItem] from vecs import vec from mypy_extensions import i64 -from typing import Optional -def f(v: vec[Optional[str]], n: i64) -> Optional[str]: - return v[n] +def f(v: vec[i64], i: i64, x: i64) -> None: + v[i] = x [out] -def f(v, n): - v :: vec[union[str, None]] - n :: int64 +def f(v, i, x): + v :: vec[int64] + i, x :: int64 r0 :: native_int r1 :: bit r2 :: bool @@ -199,184 +94,252 @@ def f(v, n): r4 :: ptr r5 :: int64 r6 :: ptr - r7 :: union[str, None] L0: r0 = v.len - r1 = n < r0 :: unsigned + r1 = i < r0 :: unsigned if r1 goto L2 else goto L1 :: bool L1: r2 = raise IndexError unreachable L2: r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufTObject - r5 = n * 8 + r4 = get_element_ptr r3 items :: VecbufI64Object + r5 = i * 8 r6 = r4 + r5 - r7 = load_mem r6 :: union* + set_mem r6, x :: int64* keep_alive v - return r7 + return 1 -[case testNewTPopLast] -from typing import Tuple -from vecs import vec, pop +[case testVecI64ConstructFromListExpr] +from vecs import vec +from mypy_extensions import i64 -def pop_last(v: vec[str]) -> Tuple[vec[str], str]: - return pop(v) +def f() -> vec[i64]: + return vec[i64]([1, 5, 14]) [out] -def pop_last(v): - v :: vec[str] - r0 :: tuple[vec[str], str] +def f(): + r0 :: vec[int64] + r1 :: object + r2, r3, r4, r5 :: ptr L0: - r0 = VecTApi.pop(v, -1) + r0 = VecI64Api.alloc(3) + r1 = r0.buf + r2 = get_element_ptr r1 items :: VecbufI64Object + set_mem r2, 1 :: int64* + r3 = r2 + 8 + set_mem r3, 5 :: int64* + r4 = r3 + 8 + set_mem r4, 14 :: int64* + r5 = r4 + 8 + keep_alive r0 return r0 -[case testVecNestedAppend] -from vecs import vec, append +[case testVecI64ConstructFromListMultiply] +from vecs import vec +from mypy_extensions import i64 -def f(v: vec[vec[str]], vv: vec[str]) -> vec[vec[str]]: - return append(v, vv) +def f(n: i64) -> vec[i64]: + return vec[i64]([3] * n) [out] -def f(v, vv): - v :: vec[vec[str]] - vv :: vec[str] - r0 :: native_int +def f(n): + n :: int64 + r0 :: vec[int64] r1 :: object - r2, r3 :: VecbufTExtItem{len:native_int, buf:object_nrc} - r4 :: vec[vec[str]] + r2 :: ptr + r3 :: int64 + r4, r5 :: ptr + r6 :: bit + r7 :: ptr L0: - r0 = vv.len - r1 = vv.buf - r2 = set_element undef, len, r0 - r3 = set_element r2, buf, r1 - r4 = VecTExtApi.append(v, r3) - keep_alive vv - return r4 + r0 = VecI64Api.alloc(n) + r1 = r0.buf + r2 = get_element_ptr r1 items :: VecbufI64Object + r3 = n * 8 + r4 = r2 + r3 + r5 = r2 +L1: + r6 = r5 < r4 :: unsigned + if r6 goto L2 else goto L3 :: bool +L2: + set_mem r5, 3 :: int64* + r7 = r5 + 8 + r5 = r7 + goto L1 +L3: + keep_alive r0 + return r0 -[case testVecNestedVecI64Append] -from vecs import vec, append +[case testVecI64ConstructFromListMultiply2] +from vecs import vec from mypy_extensions import i64 -def f(v: vec[vec[i64]], vv: vec[i64]) -> vec[vec[i64]]: - return append(v, vv) +def f(n: i64, x: i64) -> vec[i64]: + return vec[i64]([x] * 3) [out] -def f(v, vv): - v :: vec[vec[int64]] - vv :: vec[int64] - r0 :: native_int +def f(n, x): + n, x :: int64 + r0 :: vec[int64] r1 :: object - r2, r3 :: VecbufTExtItem{len:native_int, buf:object_nrc} - r4 :: vec[vec[int64]] + r2 :: ptr + r3 :: native_int + r4, r5 :: ptr + r6 :: bit + r7 :: ptr L0: - r0 = vv.len - r1 = vv.buf - r2 = set_element undef, len, r0 - r3 = set_element r2, buf, r1 - r4 = VecTExtApi.append(v, r3) - keep_alive vv - return r4 + r0 = VecI64Api.alloc(3) + r1 = r0.buf + r2 = get_element_ptr r1 items :: VecbufI64Object + r3 = 3 * 8 + r4 = r2 + r3 + r5 = r2 +L1: + r6 = r5 < r4 :: unsigned + if r6 goto L2 else goto L3 :: bool +L2: + set_mem r5, x :: int64* + r7 = r5 + 8 + r5 = r7 + goto L1 +L3: + keep_alive r0 + return r0 -[case testVecNestedLen] +[case testVecI64ConstructFromListComprehension] from vecs import vec from mypy_extensions import i64 -from typing import Optional - -def f(v: vec[vec[str]]) -> i64: - return len(v) -def g(v: vec[vec[i64]]) -> i64: - return len(v) +def f(n: i64) -> vec[i64]: + return vec[i64]([x + 1 for x in range(i64(5))]) [out] -def f(v): - v :: vec[vec[str]] - r0 :: native_int -L0: - r0 = v.len - return r0 -def g(v): - v :: vec[vec[int64]] - r0 :: native_int +def f(n): + n :: int64 + r0, r1 :: vec[int64] + r2, x :: int64 + r3 :: bit + r4 :: int64 + r5 :: vec[int64] + r6 :: int64 L0: - r0 = v.len - return r0 + r0 = VecI64Api.alloc(0) + r1 = r0 + r2 = 0 + x = r2 +L1: + r3 = r2 < 5 :: signed + if r3 goto L2 else goto L4 :: bool +L2: + r4 = x + 1 + r5 = VecI64Api.append(r1, r4) + r1 = r5 +L3: + r6 = r2 + 1 + r2 = r6 + x = r6 + goto L1 +L4: + return r1 -[case testVecNestedGetItem] +[case testVecI64ForLoop] from vecs import vec from mypy_extensions import i64 -def f(v: vec[vec[str]], n: i64) -> vec[str]: - return v[n] +def f(v: vec[i64]) -> i64: + t: i64 = 0 + for x in v: + t += 1 + return t [out] -def f(v, n): - v :: vec[vec[str]] - n :: int64 - r0 :: native_int - r1 :: bit - r2 :: bool +def f(v): + v :: vec[int64] + t :: int64 + r0, r1 :: native_int + r2 :: bit r3 :: object r4 :: ptr - r5 :: int64 + r5 :: native_int r6 :: ptr - r7 :: vec[str] + r7, x, r8 :: int64 + r9 :: native_int L0: - r0 = v.len - r1 = n < r0 :: unsigned - if r1 goto L2 else goto L1 :: bool + t = 0 + r0 = 0 L1: - r2 = raise IndexError - unreachable + r1 = v.len + r2 = r0 < r1 :: signed + if r2 goto L2 else goto L4 :: bool L2: r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufTExtObject - r5 = n * 8 + r4 = get_element_ptr r3 items :: VecbufI64Object + r5 = r0 * 8 r6 = r4 + r5 - r7 = load_mem r6 :: vec[str]* + r7 = load_mem r6 :: int64* keep_alive v - return r7 + x = r7 + r8 = t + 1 + t = r8 +L3: + r9 = r0 + 1 + r0 = r9 + goto L1 +L4: + return t -[case testVecNestedI64GetItem] +[case testVecI64Contains] from vecs import vec from mypy_extensions import i64 -def f(v: vec[vec[i64]], n: i64) -> vec[i64]: - return v[n] +def contains(v: vec[i64], n: i64) -> bool: + return n in v [out] -def f(v, n): - v :: vec[vec[int64]] +def contains(v, n): + v :: vec[int64] n :: int64 r0 :: native_int - r1 :: bit - r2 :: bool - r3 :: object - r4 :: ptr - r5 :: int64 - r6 :: ptr - r7 :: vec[int64] + r1 :: object + r2 :: ptr + r3 :: native_int + r4, r5 :: ptr + r6 :: bit + r7 :: int64 + r8 :: bit + r9 :: ptr + r10 :: bool L0: r0 = v.len - r1 = n < r0 :: unsigned - if r1 goto L2 else goto L1 :: bool + r1 = v.buf + r2 = get_element_ptr r1 items :: VecbufI64Object + r3 = r0 * 8 + r4 = r2 + r3 + r5 = r2 L1: - r2 = raise IndexError - unreachable + r6 = r5 < r4 :: unsigned + if r6 goto L2 else goto L4 :: bool L2: - r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufTExtObject - r5 = n * 8 - r6 = r4 + r5 - r7 = load_mem r6 :: vec[int64]* + r7 = load_mem r5 :: int64* + r8 = r7 == n + if r8 goto L5 else goto L3 :: bool +L3: + r9 = r5 + 8 + r5 = r9 + goto L1 +L4: keep_alive v - return r7 + r10 = 0 + goto L6 +L5: + r10 = 1 +L6: + return r10 -[case testVecNestedI64GetItemWithBorrow] +[case testVecI64GetItemWithInt] from vecs import vec from mypy_extensions import i64 -def f(v: vec[vec[i64]], n: i64) -> i64: - return v[n][n] +def f(v: vec[i64]) -> i64: + return v[0] [out] -def f(v, n): - v :: vec[vec[int64]] - n :: int64 +def f(v): + v :: vec[int64] r0 :: native_int r1 :: bit r2 :: bool @@ -384,121 +347,232 @@ def f(v, n): r4 :: ptr r5 :: int64 r6 :: ptr - r7 :: vec[int64] - r8 :: native_int - r9 :: bit - r10 :: bool - r11 :: object - r12 :: ptr - r13 :: int64 - r14 :: ptr - r15 :: int64 + r7 :: int64 L0: r0 = v.len - r1 = n < r0 :: unsigned + r1 = 0 < r0 :: unsigned if r1 goto L2 else goto L1 :: bool L1: r2 = raise IndexError unreachable L2: r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufTExtObject - r5 = n * 8 + r4 = get_element_ptr r3 items :: VecbufI64Object + r5 = 0 * 8 r6 = r4 + r5 - r7 = borrow load_mem r6 :: vec[int64]* - r8 = r7.len - r9 = n < r8 :: unsigned - if r9 goto L4 else goto L3 :: bool -L3: - r10 = raise IndexError - unreachable -L4: - r11 = r7.buf - r12 = get_element_ptr r11 items :: VecbufI64Object - r13 = n * 8 - r14 = r12 + r13 - r15 = load_mem r14 :: int64* - keep_alive v, r7 - return r15 + r7 = load_mem r6 :: int64* + keep_alive v + return r7 -[case testVecDoublyNestedGetItem] +[case testVecI64Slicing] from vecs import vec from mypy_extensions import i64 -def f(v: vec[vec[vec[str]]], n: i64) -> vec[vec[str]]: - return v[n] +def f(v: vec[i64], n: i64, m: i64) -> None: + a = v[:] + b = v[n:] + c = v[n:m] + d = v[:m] + e = v[1:-2] [out] -def f(v, n): - v :: vec[vec[vec[str]]] +def f(v, n, m): + v :: vec[int64] + n, m :: int64 + r0, a, r1, b, r2, c, r3, d, r4, e :: vec[int64] +L0: + r0 = VecI64Api.slice(v, 0, 4611686018427387903) + a = r0 + r1 = VecI64Api.slice(v, n, 4611686018427387903) + b = r1 + r2 = VecI64Api.slice(v, n, m) + c = r2 + r3 = VecI64Api.slice(v, 0, m) + d = r3 + r4 = VecI64Api.slice(v, 1, -2) + e = r4 + return 1 + +[case testVecI64Remove] +from vecs import vec, remove +from mypy_extensions import i64 + +def rem(v: vec[i64], n: i64) -> None: + v = remove(v, n) +[out] +def rem(v, n): + v :: vec[int64] n :: int64 - r0 :: native_int - r1 :: bit - r2 :: bool - r3 :: object - r4 :: ptr - r5 :: int64 - r6 :: ptr - r7 :: vec[vec[str]] + r0 :: vec[int64] L0: - r0 = v.len - r1 = n < r0 :: unsigned - if r1 goto L2 else goto L1 :: bool -L1: - r2 = raise IndexError - unreachable -L2: - r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufTExtObject - r5 = n * 8 - r6 = r4 + r5 - r7 = load_mem r6 :: vec[vec[str]]* - keep_alive v - return r7 + r0 = VecI64Api.remove(v, n) + v = r0 + return 1 --- --- New repr --- +[case testVecI64PopLast] +from typing import Tuple +from vecs import vec, pop +from mypy_extensions import i64 -[case testNewVecI64CreateEmpty] -from vecs import vec, append +def pop_last(v: vec[i64]) -> Tuple[vec[i64], i64]: + return pop(v) +[out] +def pop_last(v): + v :: vec[int64] + r0 :: tuple[vec[int64], int64] +L0: + r0 = VecI64Api.pop(v, -1) + return r0 + +[case testVecI64PopNth] +from typing import Tuple +from vecs import vec, pop from mypy_extensions import i64 -def f() -> vec[i64]: - return vec[i64]() +def pop_nth(v: vec[i64], n: i64) -> Tuple[vec[i64], i64]: + return pop(v, n) [out] -def f(): - r0 :: vec[int64] +def pop_nth(v, n): + v :: vec[int64] + n :: int64 + r0 :: tuple[vec[int64], int64] L0: - r0 = VecI64Api.alloc(0) + r0 = VecI64Api.pop(v, n) return r0 -[case testNewVecI64Len] +[case testVecTCreateEmpty] +from vecs import vec, append + +class C: pass + +def primitive() -> vec[str]: + return vec[str]() + +def native_class() -> vec[C]: + return vec[C]() +[out] +def primitive(): + r0 :: object + r1 :: ptr + r2 :: vec[str] +L0: + r0 = load_address PyUnicode_Type + r1 = r0 + r2 = VecTApi.alloc(0, r1) + return r2 +def native_class(): + r0 :: object + r1 :: ptr + r2 :: vec[__main__.C] +L0: + r0 = __main__.C :: type + r1 = r0 + r2 = VecTApi.alloc(0, r1) + return r2 + +[case testVecTAppend] +from vecs import vec, append + +def f(v: vec[str]) -> vec[str]: + return append(v, 'x') +[out] +def f(v): + v :: vec[str] + r0 :: str + r1 :: vec[str] +L0: + r0 = 'x' + r1 = VecTApi.append(v, r0) + return r1 + +[case testVecTOptionalCreateEmpty] +from vecs import vec, append +from typing import Optional + +class C: pass + +def primitive() -> vec[Optional[str]]: + return vec[Optional[str]]() + +def native_class() -> vec[Optional[C]]: + return vec[Optional[C]]() +[out] +def primitive(): + r0 :: object + r1, r2 :: ptr + r3 :: vec[union[str, None]] +L0: + r0 = load_address PyUnicode_Type + r1 = r0 + r2 = r1 | 1 + r3 = VecTApi.alloc(0, r2) + return r3 +def native_class(): + r0 :: object + r1, r2 :: ptr + r3 :: vec[union[__main__.C, None]] +L0: + r0 = __main__.C :: type + r1 = r0 + r2 = r1 | 1 + r3 = VecTApi.alloc(0, r2) + return r3 + +[case testVecTOptionalAppend] +from vecs import vec, append +from typing import Optional + +def f(v: vec[Optional[str]]) -> vec[Optional[str]]: + v = append(v, 'x') + return append(v, None) +[out] +def f(v): + v :: vec[union[str, None]] + r0 :: str + r1 :: vec[union[str, None]] + r2 :: object + r3 :: vec[union[str, None]] +L0: + r0 = 'x' + r1 = VecTApi.append(v, r0) + v = r1 + r2 = box(None, 1) + r3 = VecTApi.append(v, r2) + return r3 + +[case testVecTLen] from vecs import vec from mypy_extensions import i64 +from typing import Optional -def f(v: vec[i64]) -> i64: - l = len(v) - return l +def f(v: vec[str]) -> i64: + return len(v) + +def g(v: vec[Optional[str]]) -> i64: + return len(v) [out] def f(v): - v :: vec[int64] + v :: vec[str] r0 :: native_int - l :: int64 L0: r0 = v.len - l = r0 - return l + return r0 +def g(v): + v :: vec[union[str, None]] + r0 :: native_int +L0: + r0 = v.len + return r0 -[case testNewVecI64GetItem] +[case testVecTGetItem] from vecs import vec from mypy_extensions import i64 -def f(v: vec[i64], i: i64) -> i64: - return v[i] +def f(v: vec[str], n: i64) -> str: + return v[n] [out] -def f(v, i): - v :: vec[int64] - i :: int64 +def f(v, n): + v :: vec[str] + n :: int64 r0 :: native_int r1 :: bit r2 :: bool @@ -506,48 +580,34 @@ def f(v, i): r4 :: ptr r5 :: int64 r6 :: ptr - r7 :: int64 + r7 :: str L0: r0 = v.len - r1 = i < r0 :: unsigned + r1 = n < r0 :: unsigned if r1 goto L2 else goto L1 :: bool L1: r2 = raise IndexError unreachable L2: r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufI64Object - r5 = i * 8 + r4 = get_element_ptr r3 items :: VecbufTObject + r5 = n * 8 r6 = r4 + r5 - r7 = load_mem r6 :: int64* + r7 = load_mem r6 :: builtins.str* keep_alive v return r7 -[case testNewVecI64Append] -from vecs import vec, append -from mypy_extensions import i64 - -def f(v: vec[i64], i: i64) -> vec[i64]: - return append(v, i) -[out] -def f(v, i): - v :: vec[int64] - i :: int64 - r0 :: vec[int64] -L0: - r0 = VecI64Api.append(v, i) - return r0 - -[case testNewVecI64SetItem] +[case testVecTOptionalGetItem] from vecs import vec from mypy_extensions import i64 +from typing import Optional -def f(v: vec[i64], i: i64, x: i64) -> None: - v[i] = x +def f(v: vec[Optional[str]], n: i64) -> Optional[str]: + return v[n] [out] -def f(v, i, x): - v :: vec[int64] - i, x :: int64 +def f(v, n): + v :: vec[union[str, None]] + n :: int64 r0 :: native_int r1 :: bit r2 :: bool @@ -555,252 +615,263 @@ def f(v, i, x): r4 :: ptr r5 :: int64 r6 :: ptr + r7 :: union[str, None] L0: r0 = v.len - r1 = i < r0 :: unsigned + r1 = n < r0 :: unsigned if r1 goto L2 else goto L1 :: bool L1: r2 = raise IndexError unreachable L2: r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufI64Object - r5 = i * 8 + r4 = get_element_ptr r3 items :: VecbufTObject + r5 = n * 8 r6 = r4 + r5 - set_mem r6, x :: int64* + r7 = load_mem r6 :: union* keep_alive v - return 1 + return r7 -[case testNewVecI64ConstructFromListExpr] -from vecs import vec +[case testNewTPopLast] +from typing import Tuple +from vecs import vec, pop + +def pop_last(v: vec[str]) -> Tuple[vec[str], str]: + return pop(v) +[out] +def pop_last(v): + v :: vec[str] + r0 :: tuple[vec[str], str] +L0: + r0 = VecTApi.pop(v, -1) + return r0 + +[case testVecNestedCreateEmpty] +from vecs import vec, append from mypy_extensions import i64 -def f() -> vec[i64]: - return vec[i64]([1, 5, 14]) +def f() -> vec[vec[str]]: + return vec[vec[str]]() + +def g() -> vec[vec[i64]]: + return vec[vec[i64]]() [out] def f(): - r0 :: vec[int64] - r1 :: object - r2, r3, r4, r5 :: ptr + r0 :: object + r1 :: ptr + r2 :: vec[vec[str]] L0: - r0 = VecI64Api.alloc(3) - r1 = r0.buf - r2 = get_element_ptr r1 items :: VecbufI64Object - set_mem r2, 1 :: int64* - r3 = r2 + 8 - set_mem r3, 5 :: int64* - r4 = r3 + 8 - set_mem r4, 14 :: int64* - r5 = r4 + 8 - keep_alive r0 + r0 = load_address PyUnicode_Type + r1 = r0 + r2 = VecTExtApi.alloc(0, r1, 0, 1) + return r2 +def g(): + r0 :: vec[vec[int64]] +L0: + r0 = VecTExtApi.alloc(0, 2, 0, 1) return r0 -[case testNewVecI64ConstructFromListMultiply] -from vecs import vec +[case testVecNestedAppend] +from vecs import vec, append + +def f(v: vec[vec[str]], vv: vec[str]) -> vec[vec[str]]: + return append(v, vv) +[out] +def f(v, vv): + v :: vec[vec[str]] + vv :: vec[str] + r0 :: native_int + r1 :: object + r2, r3 :: VecbufTExtItem{len:native_int, buf:object_nrc} + r4 :: vec[vec[str]] +L0: + r0 = vv.len + r1 = vv.buf + r2 = set_element undef, len, r0 + r3 = set_element r2, buf, r1 + r4 = VecTExtApi.append(v, r3) + keep_alive vv + return r4 + +[case testVecNestedVecI64Append] +from vecs import vec, append from mypy_extensions import i64 -def f(n: i64) -> vec[i64]: - return vec[i64]([3] * n) +def f(v: vec[vec[i64]], vv: vec[i64]) -> vec[vec[i64]]: + return append(v, vv) [out] -def f(n): - n :: int64 - r0 :: vec[int64] +def f(v, vv): + v :: vec[vec[int64]] + vv :: vec[int64] + r0 :: native_int r1 :: object - r2 :: ptr - r3 :: int64 - r4, r5 :: ptr - r6 :: bit - r7 :: ptr + r2, r3 :: VecbufTExtItem{len:native_int, buf:object_nrc} + r4 :: vec[vec[int64]] L0: - r0 = VecI64Api.alloc(n) - r1 = r0.buf - r2 = get_element_ptr r1 items :: VecbufI64Object - r3 = n * 8 - r4 = r2 + r3 - r5 = r2 -L1: - r6 = r5 < r4 :: unsigned - if r6 goto L2 else goto L3 :: bool -L2: - set_mem r5, 3 :: int64* - r7 = r5 + 8 - r5 = r7 - goto L1 -L3: - keep_alive r0 - return r0 + r0 = vv.len + r1 = vv.buf + r2 = set_element undef, len, r0 + r3 = set_element r2, buf, r1 + r4 = VecTExtApi.append(v, r3) + keep_alive vv + return r4 -[case testNewVecI64ConstructFromListMultiply2] +[case testVecNestedLen] from vecs import vec from mypy_extensions import i64 +from typing import Optional -def f(n: i64, x: i64) -> vec[i64]: - return vec[i64]([x] * 3) +def f(v: vec[vec[str]]) -> i64: + return len(v) + +def g(v: vec[vec[i64]]) -> i64: + return len(v) [out] -def f(n, x): - n, x :: int64 - r0 :: vec[int64] - r1 :: object - r2 :: ptr - r3 :: native_int - r4, r5 :: ptr - r6 :: bit - r7 :: ptr +def f(v): + v :: vec[vec[str]] + r0 :: native_int L0: - r0 = VecI64Api.alloc(3) - r1 = r0.buf - r2 = get_element_ptr r1 items :: VecbufI64Object - r3 = 3 * 8 - r4 = r2 + r3 - r5 = r2 -L1: - r6 = r5 < r4 :: unsigned - if r6 goto L2 else goto L3 :: bool -L2: - set_mem r5, x :: int64* - r7 = r5 + 8 - r5 = r7 - goto L1 -L3: - keep_alive r0 + r0 = v.len + return r0 +def g(v): + v :: vec[vec[int64]] + r0 :: native_int +L0: + r0 = v.len return r0 -[case testNewVecI64ConstructFromListComprehension] +[case testVecNestedGetItem] from vecs import vec from mypy_extensions import i64 -def f(n: i64) -> vec[i64]: - return vec[i64]([x + 1 for x in range(i64(5))]) +def f(v: vec[vec[str]], n: i64) -> vec[str]: + return v[n] [out] -def f(n): +def f(v, n): + v :: vec[vec[str]] n :: int64 - r0, r1 :: vec[int64] - r2, x :: int64 - r3 :: bit - r4 :: int64 - r5 :: vec[int64] - r6 :: int64 + r0 :: native_int + r1 :: bit + r2 :: bool + r3 :: object + r4 :: ptr + r5 :: int64 + r6 :: ptr + r7 :: vec[str] L0: - r0 = VecI64Api.alloc(0) - r1 = r0 - r2 = 0 - x = r2 + r0 = v.len + r1 = n < r0 :: unsigned + if r1 goto L2 else goto L1 :: bool L1: - r3 = r2 < 5 :: signed - if r3 goto L2 else goto L4 :: bool + r2 = raise IndexError + unreachable L2: - r4 = x + 1 - r5 = VecI64Api.append(r1, r4) - r1 = r5 -L3: - r6 = r2 + 1 - r2 = r6 - x = r6 - goto L1 -L4: - return r1 + r3 = v.buf + r4 = get_element_ptr r3 items :: VecbufTExtObject + r5 = n * 8 + r6 = r4 + r5 + r7 = load_mem r6 :: vec[str]* + keep_alive v + return r7 -[case testNewVecI64ForLoop] +[case testVecNestedI64GetItem] from vecs import vec from mypy_extensions import i64 -def f(v: vec[i64]) -> i64: - t: i64 = 0 - for x in v: - t += 1 - return t +def f(v: vec[vec[i64]], n: i64) -> vec[i64]: + return v[n] [out] -def f(v): - v :: vec[int64] - t :: int64 - r0, r1 :: native_int - r2 :: bit +def f(v, n): + v :: vec[vec[int64]] + n :: int64 + r0 :: native_int + r1 :: bit + r2 :: bool r3 :: object r4 :: ptr - r5 :: native_int + r5 :: int64 r6 :: ptr - r7, x, r8 :: int64 - r9 :: native_int + r7 :: vec[int64] L0: - t = 0 - r0 = 0 + r0 = v.len + r1 = n < r0 :: unsigned + if r1 goto L2 else goto L1 :: bool L1: - r1 = v.len - r2 = r0 < r1 :: signed - if r2 goto L2 else goto L4 :: bool + r2 = raise IndexError + unreachable L2: r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufI64Object - r5 = r0 * 8 + r4 = get_element_ptr r3 items :: VecbufTExtObject + r5 = n * 8 r6 = r4 + r5 - r7 = load_mem r6 :: int64* + r7 = load_mem r6 :: vec[int64]* keep_alive v - x = r7 - r8 = t + 1 - t = r8 -L3: - r9 = r0 + 1 - r0 = r9 - goto L1 -L4: - return t + return r7 -[case testNewVecI64Contains] +[case testVecNestedI64GetItemWithBorrow] from vecs import vec from mypy_extensions import i64 -def contains(v: vec[i64], n: i64) -> bool: - return n in v +def f(v: vec[vec[i64]], n: i64) -> i64: + return v[n][n] [out] -def contains(v, n): - v :: vec[int64] +def f(v, n): + v :: vec[vec[int64]] n :: int64 r0 :: native_int - r1 :: object - r2 :: ptr - r3 :: native_int - r4, r5 :: ptr - r6 :: bit - r7 :: int64 - r8 :: bit - r9 :: ptr + r1 :: bit + r2 :: bool + r3 :: object + r4 :: ptr + r5 :: int64 + r6 :: ptr + r7 :: vec[int64] + r8 :: native_int + r9 :: bit r10 :: bool + r11 :: object + r12 :: ptr + r13 :: int64 + r14 :: ptr + r15 :: int64 L0: r0 = v.len - r1 = v.buf - r2 = get_element_ptr r1 items :: VecbufI64Object - r3 = r0 * 8 - r4 = r2 + r3 - r5 = r2 + r1 = n < r0 :: unsigned + if r1 goto L2 else goto L1 :: bool L1: - r6 = r5 < r4 :: unsigned - if r6 goto L2 else goto L4 :: bool + r2 = raise IndexError + unreachable L2: - r7 = load_mem r5 :: int64* - r8 = r7 == n - if r8 goto L5 else goto L3 :: bool + r3 = v.buf + r4 = get_element_ptr r3 items :: VecbufTExtObject + r5 = n * 8 + r6 = r4 + r5 + r7 = borrow load_mem r6 :: vec[int64]* + r8 = r7.len + r9 = n < r8 :: unsigned + if r9 goto L4 else goto L3 :: bool L3: - r9 = r5 + 8 - r5 = r9 - goto L1 + r10 = raise IndexError + unreachable L4: - keep_alive v - r10 = 0 - goto L6 -L5: - r10 = 1 -L6: - return r10 + r11 = r7.buf + r12 = get_element_ptr r11 items :: VecbufI64Object + r13 = n * 8 + r14 = r12 + r13 + r15 = load_mem r14 :: int64* + keep_alive v, r7 + return r15 -[case testNewVecI64GetItemWithInt] +[case testVecDoublyNestedGetItem] from vecs import vec from mypy_extensions import i64 -def f(v: vec[i64]) -> i64: - return v[0] +def f(v: vec[vec[vec[str]]], n: i64) -> vec[vec[str]]: + return v[n] [out] -def f(v): - v :: vec[int64] +def f(v, n): + v :: vec[vec[vec[str]]] + n :: int64 r0 :: native_int r1 :: bit r2 :: bool @@ -808,94 +879,19 @@ def f(v): r4 :: ptr r5 :: int64 r6 :: ptr - r7 :: int64 + r7 :: vec[vec[str]] L0: r0 = v.len - r1 = 0 < r0 :: unsigned + r1 = n < r0 :: unsigned if r1 goto L2 else goto L1 :: bool L1: r2 = raise IndexError unreachable L2: r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufI64Object - r5 = 0 * 8 + r4 = get_element_ptr r3 items :: VecbufTExtObject + r5 = n * 8 r6 = r4 + r5 - r7 = load_mem r6 :: int64* + r7 = load_mem r6 :: vec[vec[str]]* keep_alive v return r7 - -[case testNewVecI64Slicing] -from vecs import vec -from mypy_extensions import i64 - -def f(v: vec[i64], n: i64, m: i64) -> None: - a = v[:] - b = v[n:] - c = v[n:m] - d = v[:m] - e = v[1:-2] -[out] -def f(v, n, m): - v :: vec[int64] - n, m :: int64 - r0, a, r1, b, r2, c, r3, d, r4, e :: vec[int64] -L0: - r0 = VecI64Api.slice(v, 0, 4611686018427387903) - a = r0 - r1 = VecI64Api.slice(v, n, 4611686018427387903) - b = r1 - r2 = VecI64Api.slice(v, n, m) - c = r2 - r3 = VecI64Api.slice(v, 0, m) - d = r3 - r4 = VecI64Api.slice(v, 1, -2) - e = r4 - return 1 - -[case testNewVecI64Remove] -from vecs import vec, remove -from mypy_extensions import i64 - -def rem(v: vec[i64], n: i64) -> None: - v = remove(v, n) -[out] -def rem(v, n): - v :: vec[int64] - n :: int64 - r0 :: vec[int64] -L0: - r0 = VecI64Api.remove(v, n) - v = r0 - return 1 - -[case testNewVecI64PopLast] -from typing import Tuple -from vecs import vec, pop -from mypy_extensions import i64 - -def pop_last(v: vec[i64]) -> Tuple[vec[i64], i64]: - return pop(v) -[out] -def pop_last(v): - v :: vec[int64] - r0 :: tuple[vec[int64], int64] -L0: - r0 = VecI64Api.pop(v, -1) - return r0 - -[case testNewVecI64PopNth] -from typing import Tuple -from vecs import vec, pop -from mypy_extensions import i64 - -def pop_nth(v: vec[i64], n: i64) -> Tuple[vec[i64], i64]: - return pop(v, n) -[out] -def pop_nth(v, n): - v :: vec[int64] - n :: int64 - r0 :: tuple[vec[int64], int64] -L0: - r0 = VecI64Api.pop(v, n) - return r0 From 575b130cb6c7a21c9c56773616d42d4d4a8e52c5 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 11 Jun 2023 17:31:41 +0100 Subject: [PATCH 032/180] Add vec test case --- mypyc/test-data/irbuild-vecs.test | 39 +++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/mypyc/test-data/irbuild-vecs.test b/mypyc/test-data/irbuild-vecs.test index f27bb9ff36ffc..7eb0c4218a0aa 100644 --- a/mypyc/test-data/irbuild-vecs.test +++ b/mypyc/test-data/irbuild-vecs.test @@ -646,6 +646,45 @@ L0: r0 = VecTApi.pop(v, -1) return r0 +[case testVecTConstructFromListComprehension] +from vecs import vec +from mypy_extensions import i64 + +def f(n: i64) -> vec[str]: + return vec[str](['x' for x in range(i64(5))]) +[out] +def f(n): + n :: int64 + r0 :: object + r1 :: ptr + r2, r3 :: vec[str] + r4, x :: int64 + r5 :: bit + r6 :: str + r7 :: vec[str] + r8 :: int64 +L0: + r0 = load_address PyUnicode_Type + r1 = r0 + r2 = VecTApi.alloc(0, r1) + r3 = r2 + r4 = 0 + x = r4 +L1: + r5 = r4 < 5 :: signed + if r5 goto L2 else goto L4 :: bool +L2: + r6 = 'x' + r7 = VecTApi.append(r3, r6) + r3 = r7 +L3: + r8 = r4 + 1 + r4 = r8 + x = r8 + goto L1 +L4: + return r3 + [case testVecNestedCreateEmpty] from vecs import vec, append from mypy_extensions import i64 From a54532ead8b168902fe1b5b6092d196dc7993a07 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 11 Jun 2023 17:31:49 +0100 Subject: [PATCH 033/180] Fix always defined attribute analysis --- mypyc/analysis/selfleaks.py | 8 ++++++++ mypyc/test-data/alwaysdefined.test | 10 ++++++++++ 2 files changed, 18 insertions(+) diff --git a/mypyc/analysis/selfleaks.py b/mypyc/analysis/selfleaks.py index 6b886a83b44b9..a96035814b1b4 100644 --- a/mypyc/analysis/selfleaks.py +++ b/mypyc/analysis/selfleaks.py @@ -44,6 +44,8 @@ Unborrow, Unbox, Unreachable, + IncRef, + DecRef, ) from mypyc.ir.rtypes import RInstance @@ -90,6 +92,12 @@ def visit_assign_multi(self, op: AssignMulti) -> GenAndKill: def visit_set_mem(self, op: SetMem) -> GenAndKill: return CLEAN + def visit_inc_ref(self, op: IncRef) -> GenAndKill: + return CLEAN + + def visit_dec_ref(self, op: DecRef) -> GenAndKill: + return CLEAN + def visit_call(self, op: Call) -> GenAndKill: fn = op.fn if fn.class_name and fn.name == "__init__": diff --git a/mypyc/test-data/alwaysdefined.test b/mypyc/test-data/alwaysdefined.test index ecbc8c410d6dc..49e696ef23096 100644 --- a/mypyc/test-data/alwaysdefined.test +++ b/mypyc/test-data/alwaysdefined.test @@ -729,3 +729,13 @@ class NestedFunc: -- TODO: Support nested functions. NestedFunc: [] f___init___NestedFunc_obj: [] + +[case testAlwaysDefinedWithVecSetItem] +from vecs import vec + +class C: + def __init__(self, v: vec[C]) -> None: + self.v = v + self.v[0] = self +[out] +C: [v] From 0417298c1fbe4ba6b7467a52660b4c20676842ec Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 13 Jun 2023 20:18:40 +0100 Subject: [PATCH 034/180] WIP make benchmarks compile (not working) --- mypyc/codegen/emit.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index 14822046c0559..0fc9bca7e764d 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -57,6 +57,7 @@ is_uint8_rprimitive, object_rprimitive, optional_value_type, + vec_depth, ) from mypyc.namegen import NameGenerator, exported_name from mypyc.sametype import is_same_type @@ -1063,12 +1064,20 @@ def emit_unbox( if optional: self.emit_line("}") elif isinstance(typ, RVec): - assert is_int64_rprimitive(typ.item_type) # TODO: Support more item types if declare_dest: self.emit_line(f"{self.ctype(typ)} {dest};") # TODO: Handle 'optional' # TODO: Handle 'failure' - self.emit_line(f"{dest} = VecI64Api.unbox({src});") + + # TODO: Use helper function to pick the api variant + if is_int64_rprimitive(typ.item_type): + self.emit_line(f"{dest} = VecI64Api.unbox({src});") + elif vec_depth(typ) == 0: + # TODO: Provide item type as second arg + self.emit_line(f"{dest} = VecTApi.unbox({src}, 0);") + else: + # TODO: Provide additional arguments + self.emit_line(f"{dest} = VecTExtApi.unbox({src}, 0, 0, 0);") else: assert False, "Unboxing not implemented: %s" % typ From c9be612ff1b23ae47c41a83653071dc1ccac3c51 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 17 Jun 2023 12:23:07 +0100 Subject: [PATCH 035/180] Format code --- mypyc/analysis/dataflow.py | 2 -- mypyc/analysis/selfleaks.py | 4 ++-- mypyc/irbuild/vec.py | 9 +++++---- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/mypyc/analysis/dataflow.py b/mypyc/analysis/dataflow.py index 3390968f368cd..fc14ab8b47991 100644 --- a/mypyc/analysis/dataflow.py +++ b/mypyc/analysis/dataflow.py @@ -31,8 +31,6 @@ InitStatic, Integer, IntOp, - IncRef, - DecRef, KeepAlive, LoadAddress, LoadErrorValue, diff --git a/mypyc/analysis/selfleaks.py b/mypyc/analysis/selfleaks.py index a96035814b1b4..0270cfed5640a 100644 --- a/mypyc/analysis/selfleaks.py +++ b/mypyc/analysis/selfleaks.py @@ -11,6 +11,7 @@ CallC, Cast, ComparisonOp, + DecRef, Extend, FloatComparisonOp, FloatNeg, @@ -19,6 +20,7 @@ GetElement, GetElementPtr, Goto, + IncRef, InitStatic, IntOp, KeepAlive, @@ -44,8 +46,6 @@ Unborrow, Unbox, Unreachable, - IncRef, - DecRef, ) from mypyc.ir.rtypes import RInstance diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 7b5822f99a4c0..b93d3585eeecd 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -12,9 +12,9 @@ Assign, BasicBlock, Branch, - DecRef, CallC, ComparisonOp, + DecRef, GetElement, GetElementPtr, Integer, @@ -35,11 +35,11 @@ RType, RUnion, RVec, - c_size_t_rprimitive, VecbufTExtItem, bool_rprimitive, c_int_rprimitive, c_pyssize_t_rprimitive, + c_size_t_rprimitive, int32_rprimitive, int64_rprimitive, is_c_py_ssize_t_rprimitive, @@ -241,8 +241,9 @@ def vec_check_index(builder: "LowLevelIRBuilder", lenv: Value, index: Value, lin builder.activate_block(ok) -def vec_get_item(builder: "LowLevelIRBuilder", base: Value, index: Value, line: int, *, - can_borrow: bool = False) -> Value: +def vec_get_item( + builder: "LowLevelIRBuilder", base: Value, index: Value, line: int, *, can_borrow: bool = False +) -> Value: """Generate inlined vec __getitem__ call. We inline this, since it's simple but performance-critical. From db880bf77937d9998f75890ef2375a0121db9acd Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 17 Jun 2023 12:28:27 +0100 Subject: [PATCH 036/180] Properly support vec error values --- mypyc/codegen/emit.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index 0fc9bca7e764d..be6879cee4f00 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -342,6 +342,8 @@ def c_undefined_value(self, rtype: RType) -> str: return rtype.c_undefined elif isinstance(rtype, RTuple): return self.tuple_undefined_value(rtype) + elif isinstance(rtype, RVec): + return f"({self.ctype(rtype)}) {{ -1, NULL }}" assert False, rtype def c_error_value(self, rtype: RType) -> str: @@ -503,7 +505,7 @@ def c_initializer_undefined_value(self, rtype: RType) -> str: items = ", ".join([self.c_initializer_undefined_value(t) for t in rtype.types]) return f"{{ {items} }}" elif isinstance(rtype, RVec): - res.append("{ -1, NULL }") + return "{ -1, NULL }" else: return self.c_undefined_value(rtype) From 158a609bb00fa2a0388c263fad259716eeabaa84 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 17 Jun 2023 13:16:27 +0100 Subject: [PATCH 037/180] Fix vec remove --- mypyc/irbuild/vec.py | 2 +- mypyc/test-data/exceptions.test | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index b93d3585eeecd..de315ea7be3ad 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -369,7 +369,7 @@ def vec_remove(builder: "LowLevelIRBuilder", vec: Value, item: Value, line: int) vec_type, steals=[False, False], is_borrowed=False, - error_kind=ERR_FALSE, + error_kind=ERR_MAGIC, line=line, ) return builder.add(call) diff --git a/mypyc/test-data/exceptions.test b/mypyc/test-data/exceptions.test index 9e097a18d1032..703f0ad999afc 100644 --- a/mypyc/test-data/exceptions.test +++ b/mypyc/test-data/exceptions.test @@ -719,3 +719,26 @@ L2: L3: r5 = :: float return r5 + +[case testVecNestedRemove] +from vecs import vec, remove +from mypy_extensions import i64 + +def f(v: vec[i64], i: i64, n: i64) -> None: + v = remove(v, n) +[out] +def f(v, i, n): + v :: vec[int64] + i, n :: int64 + r0 :: vec[int64] + r1 :: None +L0: + r0 = VecI64Api.remove(v, n) + if is_error(r0) goto L2 (error at f:5) else goto L1 +L1: + v = r0 + dec_ref v + return 1 +L2: + r1 = :: None + return r1 From 14f98efdfe2edf0c806dc436bcdb12956c89f03f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 18 Jun 2023 18:09:48 +0100 Subject: [PATCH 038/180] Fix nested vec get item --- mypyc/irbuild/vec.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index de315ea7be3ad..81e34d7f910eb 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -226,7 +226,13 @@ def vec_items(builder: "LowLevelIRBuilder", vecobj: Value) -> Value: def vec_item_ptr(builder: "LowLevelIRBuilder", vecobj: Value, index: Value) -> Value: items_addr = vec_items(builder, vecobj) - delta = builder.int_mul(index, 8) + assert isinstance(vecobj.type, RVec) + # TODO: Calculate item size properly and support 32-bit platforms + if isinstance(vecobj.type.item_type, RVec): + item_size = 16 + else: + item_size = 8 + delta = builder.int_mul(index, item_size) return builder.int_add(items_addr, delta) From ed24c70dfa1a05eb850ca75607944f0ed70af9ff Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 25 Jun 2023 15:00:43 +0100 Subject: [PATCH 039/180] Update some tests --- mypyc/test-data/irbuild-vecs.test | 8 ++++---- mypyc/test-data/refcount.test | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mypyc/test-data/irbuild-vecs.test b/mypyc/test-data/irbuild-vecs.test index 7eb0c4218a0aa..e5b3b7f5dd0a7 100644 --- a/mypyc/test-data/irbuild-vecs.test +++ b/mypyc/test-data/irbuild-vecs.test @@ -807,7 +807,7 @@ L1: L2: r3 = v.buf r4 = get_element_ptr r3 items :: VecbufTExtObject - r5 = n * 8 + r5 = n * 16 r6 = r4 + r5 r7 = load_mem r6 :: vec[str]* keep_alive v @@ -841,7 +841,7 @@ L1: L2: r3 = v.buf r4 = get_element_ptr r3 items :: VecbufTExtObject - r5 = n * 8 + r5 = n * 16 r6 = r4 + r5 r7 = load_mem r6 :: vec[int64]* keep_alive v @@ -883,7 +883,7 @@ L1: L2: r3 = v.buf r4 = get_element_ptr r3 items :: VecbufTExtObject - r5 = n * 8 + r5 = n * 16 r6 = r4 + r5 r7 = borrow load_mem r6 :: vec[int64]* r8 = r7.len @@ -929,7 +929,7 @@ L1: L2: r3 = v.buf r4 = get_element_ptr r3 items :: VecbufTExtObject - r5 = n * 8 + r5 = n * 16 r6 = r4 + r5 r7 = load_mem r6 :: vec[vec[str]]* keep_alive v diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index 26c631474b3f5..120ef34298994 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -1806,7 +1806,7 @@ L1: L2: r6 = r2.buf r7 = get_element_ptr r6 items :: VecbufTExtObject - r8 = n * 8 + r8 = n * 16 r9 = r7 + r8 r10 = load_mem r9 :: vec[str]* dec_ref r2 From 3028f66c18ca9479b7e6c2a0acb71ef6253032bd Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 25 Jun 2023 17:05:48 +0100 Subject: [PATCH 040/180] Implement unboxing --- mypyc/codegen/emit.py | 49 +++++++++++++++++++++++++++++++------ mypyc/codegen/emitfunc.py | 23 ++++++++--------- mypyc/ir/rtypes.py | 36 +++++++++++++++++++++++++++ mypyc/test/test_emitfunc.py | 48 ++++++++++++++++++++++++++++++++---- 4 files changed, 131 insertions(+), 25 deletions(-) diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index be6879cee4f00..d2831a0c00c00 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -21,10 +21,18 @@ REG_PREFIX, STATIC_PREFIX, TYPE_PREFIX, + TYPE_VAR_PREFIX, ) from mypyc.ir.class_ir import ClassIR, all_concrete_classes from mypyc.ir.func_ir import FUNC_STATICMETHOD, FuncDecl, FuncIR, get_text_signature -from mypyc.ir.ops import BasicBlock, Value +from mypyc.ir.ops import ( + NAMESPACE_MODULE, + NAMESPACE_STATIC, + NAMESPACE_TYPE, + NAMESPACE_TYPE_VAR, + BasicBlock, + Value, +) from mypyc.ir.rtypes import ( RInstance, RPrimitive, @@ -60,12 +68,20 @@ vec_depth, ) from mypyc.namegen import NameGenerator, exported_name +from mypyc.primitives.registry import builtin_names from mypyc.sametype import is_same_type # Whether to insert debug asserts for all error handling, to quickly # catch errors propagating without exceptions set. DEBUG_ERRORS: Final = False +PREFIX_MAP: Final = { + NAMESPACE_STATIC: STATIC_PREFIX, + NAMESPACE_TYPE: TYPE_PREFIX, + NAMESPACE_MODULE: MODULE_PREFIX, + NAMESPACE_TYPE_VAR: TYPE_VAR_PREFIX, +} + class HeaderDeclaration: """A representation of a declaration in C. @@ -1074,15 +1090,34 @@ def emit_unbox( # TODO: Use helper function to pick the api variant if is_int64_rprimitive(typ.item_type): self.emit_line(f"{dest} = VecI64Api.unbox({src});") - elif vec_depth(typ) == 0: - # TODO: Provide item type as second arg - self.emit_line(f"{dest} = VecTApi.unbox({src}, 0);") else: - # TODO: Provide additional arguments - self.emit_line(f"{dest} = VecTExtApi.unbox({src}, 0, 0, 0);") + depth = typ.depth() + optionals = typ.optional_flags() + if is_int64_rprimitive(typ.unwrap_item_type()): + type_value = "VEC_ITEM_TYPE_I64" + else: + type_value = f"(size_t)&{self.vec_item_type_c(typ)}" + if optionals & (1 << depth): + type_value = f"{type_value} | 1" + if depth == 0: + self.emit_line(f"{dest} = VecTApi.unbox({src}, {type_value});") + else: + self.emit_line( + f"{dest} = VecTExtApi.unbox({src}, {type_value}, {optionals}, {depth});") else: assert False, "Unboxing not implemented: %s" % typ + def vec_item_type_c(self, typ: RVec) -> str: + item_type = typ.unwrap_item_type() + return self.type_c_name(item_type) + + def type_c_name(self, typ: RPrimitive | RInstance) -> str | None: + if isinstance(typ, RPrimitive) and typ.is_refcounted: + return builtin_names[typ.name][1] + elif isinstance(typ, RInstance): + return self.type_struct_name(typ.class_ir) + return None + def emit_box( self, src: str, dest: str, typ: RType, declare_dest: bool = False, can_borrow: bool = False ) -> None: @@ -1140,7 +1175,7 @@ def emit_box( elif isinstance(typ, RVec): if is_int64_rprimitive(typ.item_type): api = "VecI64Api" - elif not isinstance(typ.item_type, RVec): + elif typ.depth() == 0: api = "VecTApi" else: api = "VecTExtApi" diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index 1aa7b7202f055..ffe0ecf74859f 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py @@ -5,7 +5,13 @@ from typing import Final from mypyc.analysis.blockfreq import frequently_executed_blocks -from mypyc.codegen.emit import DEBUG_ERRORS, Emitter, TracebackAndGotoHandler, c_array_initializer +from mypyc.codegen.emit import ( + DEBUG_ERRORS, + PREFIX_MAP, + Emitter, + TracebackAndGotoHandler, + c_array_initializer, +) from mypyc.common import ( GENERATOR_ATTRIBUTE_PREFIX, HAVE_IMMORTAL, @@ -20,8 +26,6 @@ from mypyc.ir.func_ir import FUNC_CLASSMETHOD, FUNC_STATICMETHOD, FuncDecl, FuncIR, all_values from mypyc.ir.ops import ( ERR_FALSE, - NAMESPACE_MODULE, - NAMESPACE_STATIC, NAMESPACE_TYPE, NAMESPACE_TYPE_VAR, Assign, @@ -537,16 +541,9 @@ def visit_set_attr(self, op: SetAttr) -> None: if op.error_kind == ERR_FALSE: self.emitter.emit_line(f"{dest} = 1;") - PREFIX_MAP: Final = { - NAMESPACE_STATIC: STATIC_PREFIX, - NAMESPACE_TYPE: TYPE_PREFIX, - NAMESPACE_MODULE: MODULE_PREFIX, - NAMESPACE_TYPE_VAR: TYPE_VAR_PREFIX, - } - def visit_load_static(self, op: LoadStatic) -> None: dest = self.reg(op) - prefix = self.PREFIX_MAP[op.namespace] + prefix = PREFIX_MAP[op.namespace] name = self.emitter.static_name(op.identifier, op.module_name, prefix) if op.namespace == NAMESPACE_TYPE: name = "(PyObject *)%s" % name @@ -554,7 +551,7 @@ def visit_load_static(self, op: LoadStatic) -> None: def visit_init_static(self, op: InitStatic) -> None: value = self.reg(op.value) - prefix = self.PREFIX_MAP[op.namespace] + prefix = PREFIX_MAP[op.namespace] name = self.emitter.static_name(op.identifier, op.module_name, prefix) if op.namespace == NAMESPACE_TYPE: value = "(PyTypeObject *)%s" % value @@ -879,7 +876,7 @@ def visit_load_address(self, op: LoadAddress) -> None: if isinstance(op.src, Register): src = self.reg(op.src) elif isinstance(op.src, LoadStatic): - prefix = self.PREFIX_MAP[op.src.namespace] + prefix = PREFIX_MAP[op.src.namespace] src = self.emitter.static_name(op.src.identifier, op.src.module_name, prefix) else: src = op.src diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 42cd896e0fb20..27ca1d19e6b79 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -1046,6 +1046,42 @@ def __init__(self, item_type: RType) -> None: self.types = [c_pyssize_t_rprimitive, VecbufTObject] self.buf_type = VecbufTObject + def unwrap_item_type(self) -> RPrimitive | RInstance: + """Return the non-optional value (non-vec) item type in a potentially nested vec.""" + item_type = self.item_type + while True: + if isinstance(item_type, RUnion): + item_type = optional_value_type(item_type) + assert item_type is not None + elif isinstance(item_type, RVec): + item_type = item_type.item_type + elif isinstance(item_type, (RPrimitive, RInstance)): + return item_type + else: + assert False, f"unexpected item type: {self.item_type}" + + def optional_flags(self) -> int: + item_type = self.item_type + if isinstance(item_type, RUnion): + item_type = optional_value_type(item_type) + assert item_type is not None + if isinstance(item_type, RVec): + return (item_type.optional_flags() << 1) | 1 + else: + return 1 + elif isinstance(item_type, RVec): + return item_type.optional_flags() << 1 + return 0 + + def depth(self) -> int: + item_type = self.item_type + if isinstance(item_type, RUnion): + item_type = optional_value_type(item_type) + assert item_type is not None + if isinstance(item_type, RVec): + return 1 + item_type.depth() + return 0 + def field_type(self, name: str) -> RType: if name == "len": return c_pyssize_t_rprimitive diff --git a/mypyc/test/test_emitfunc.py b/mypyc/test/test_emitfunc.py index 11e1ab0369b0e..378d4459da0d1 100644 --- a/mypyc/test/test_emitfunc.py +++ b/mypyc/test/test_emitfunc.py @@ -50,6 +50,8 @@ RStruct, RTuple, RType, + RUnion, + RVec, bool_rprimitive, c_int_rprimitive, cstring_rprimitive, @@ -62,6 +64,7 @@ object_rprimitive, pointer_rprimitive, short_int_rprimitive, + str_rprimitive, ) from mypyc.irbuild.vtable import compute_vtable from mypyc.namegen import NameGenerator @@ -109,6 +112,9 @@ def add_local(name: str, rtype: RType) -> Register: self.tt = add_local( "tt", RTuple([RTuple([int_rprimitive, bool_rprimitive]), bool_rprimitive]) ) + self.vi64 = add_local("vi64", RVec(int64_rprimitive)) + self.vs = add_local("vs", RVec(str_rprimitive)) + self.vvs = add_local("vvs", RVec(RVec(str_rprimitive))) ir = ClassIR("A", "mod") ir.attributes = { "x": bool_rprimitive, @@ -348,11 +354,11 @@ def test_unbox_int(self) -> None: self.assert_emit( Unbox(self.m, int_rprimitive, 55), """if (likely(PyLong_Check(cpy_r_m))) - cpy_r_r0 = CPyTagged_FromObject(cpy_r_m); - else { - CPy_TypeError("int", cpy_r_m); cpy_r_r0 = CPY_INT_TAG; - } - """, + cpy_r_r0 = CPyTagged_FromObject(cpy_r_m); + else { + CPy_TypeError("int", cpy_r_m); cpy_r_r0 = CPY_INT_TAG; + } + """, ) def test_box_i64(self) -> None: @@ -363,6 +369,38 @@ def test_unbox_i64(self) -> None: Unbox(self.o, int64_rprimitive, 55), """cpy_r_r0 = CPyLong_AsInt64(cpy_r_o);""" ) + def test_box_vec(self) -> None: + self.assert_emit(Box(self.vi64), """cpy_r_r0 = VecI64Api.box(cpy_r_vi64);""") + self.assert_emit(Box(self.vs), """cpy_r_r0 = VecTApi.box(cpy_r_vs);""") + self.assert_emit(Box(self.vvs), """cpy_r_r0 = VecTExtApi.box(cpy_r_vvs);""") + + def test_unbox_vec(self) -> None: + self.assert_emit( + Unbox(self.o, RVec(int64_rprimitive), 55), """cpy_r_r0 = VecI64Api.unbox(cpy_r_o);""" + ) + self.assert_emit( + Unbox(self.o, RVec(str_rprimitive), 55), + """cpy_r_r0 = VecTApi.unbox(cpy_r_o, (size_t)&PyUnicode_Type);""", + ) + self.assert_emit( + Unbox(self.o, RVec(RUnion([str_rprimitive, none_rprimitive])), 55), + """cpy_r_r0 = VecTApi.unbox(cpy_r_o, (size_t)&PyUnicode_Type | 1);""", + ) + self.assert_emit( + Unbox(self.o, RVec(self.r.type), 55), + """cpy_r_r0 = VecTApi.unbox(cpy_r_o, (size_t)&CPyType_A);""", + ) + + def test_unbox_vec_nested(self) -> None: + self.assert_emit(Unbox(self.o, RVec(RVec(str_rprimitive)), 55), + """cpy_r_r0 = VecTExtApi.unbox(cpy_r_o, (size_t)&PyUnicode_Type, 0, 1);""") + self.assert_emit(Unbox(self.o, RVec(RVec(RUnion([str_rprimitive, none_rprimitive]))), 55), + """cpy_r_r0 = VecTExtApi.unbox(cpy_r_o, (size_t)&PyUnicode_Type | 1, 2, 1);""") + self.assert_emit(Unbox(self.o, RVec(RUnion([RVec(str_rprimitive), none_rprimitive])), 55), + """cpy_r_r0 = VecTExtApi.unbox(cpy_r_o, (size_t)&PyUnicode_Type, 1, 1);""") + self.assert_emit(Unbox(self.o, RVec(RVec(int64_rprimitive)), 55), + """cpy_r_r0 = VecTExtApi.unbox(cpy_r_o, VEC_ITEM_TYPE_I64, 0, 1);""") + def test_list_append(self) -> None: self.assert_emit( CallC( From 24764bf1bebaa7f62c54561301225c15d511186a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 8 Jul 2023 10:48:49 +0100 Subject: [PATCH 041/180] Split vec tests into multiple files --- mypyc/test-data/irbuild-vec-i64.test | 440 +++++++++++ mypyc/test-data/irbuild-vec-nested.test | 249 +++++++ mypyc/test-data/irbuild-vec-t.test | 245 +++++++ mypyc/test-data/irbuild-vecs.test | 936 ------------------------ mypyc/test/test_irbuild.py | 4 +- 5 files changed, 937 insertions(+), 937 deletions(-) create mode 100644 mypyc/test-data/irbuild-vec-i64.test create mode 100644 mypyc/test-data/irbuild-vec-nested.test create mode 100644 mypyc/test-data/irbuild-vec-t.test delete mode 100644 mypyc/test-data/irbuild-vecs.test diff --git a/mypyc/test-data/irbuild-vec-i64.test b/mypyc/test-data/irbuild-vec-i64.test new file mode 100644 index 0000000000000..8fa2943ec68a6 --- /dev/null +++ b/mypyc/test-data/irbuild-vec-i64.test @@ -0,0 +1,440 @@ +[case testVecI64CreateEmpty] +from vecs import vec, append +from mypy_extensions import i64 + +def f() -> vec[i64]: + return vec[i64]() +[out] +def f(): + r0 :: vec[int64] +L0: + r0 = VecI64Api.alloc(0) + return r0 + +[case testVecI64Len] +from vecs import vec +from mypy_extensions import i64 + +def f(v: vec[i64]) -> i64: + l = len(v) + return l +[out] +def f(v): + v :: vec[int64] + r0 :: native_int + l :: int64 +L0: + r0 = v.len + l = r0 + return l + +[case testVecI64GetItem] +from vecs import vec +from mypy_extensions import i64 + +def f(v: vec[i64], i: i64) -> i64: + return v[i] +[out] +def f(v, i): + v :: vec[int64] + i :: int64 + r0 :: native_int + r1 :: bit + r2 :: bool + r3 :: object + r4 :: ptr + r5 :: int64 + r6 :: ptr + r7 :: int64 +L0: + r0 = v.len + r1 = i < r0 :: unsigned + if r1 goto L2 else goto L1 :: bool +L1: + r2 = raise IndexError + unreachable +L2: + r3 = v.buf + r4 = get_element_ptr r3 items :: VecbufI64Object + r5 = i * 8 + r6 = r4 + r5 + r7 = load_mem r6 :: int64* + keep_alive v + return r7 + +[case testVecI64Append] +from vecs import vec, append +from mypy_extensions import i64 + +def f(v: vec[i64], i: i64) -> vec[i64]: + return append(v, i) +[out] +def f(v, i): + v :: vec[int64] + i :: int64 + r0 :: vec[int64] +L0: + r0 = VecI64Api.append(v, i) + return r0 + +[case testVecI64SetItem] +from vecs import vec +from mypy_extensions import i64 + +def f(v: vec[i64], i: i64, x: i64) -> None: + v[i] = x +[out] +def f(v, i, x): + v :: vec[int64] + i, x :: int64 + r0 :: native_int + r1 :: bit + r2 :: bool + r3 :: object + r4 :: ptr + r5 :: int64 + r6 :: ptr +L0: + r0 = v.len + r1 = i < r0 :: unsigned + if r1 goto L2 else goto L1 :: bool +L1: + r2 = raise IndexError + unreachable +L2: + r3 = v.buf + r4 = get_element_ptr r3 items :: VecbufI64Object + r5 = i * 8 + r6 = r4 + r5 + set_mem r6, x :: int64* + keep_alive v + return 1 + +[case testVecI64ConstructFromListExpr] +from vecs import vec +from mypy_extensions import i64 + +def f() -> vec[i64]: + return vec[i64]([1, 5, 14]) +[out] +def f(): + r0 :: vec[int64] + r1 :: object + r2, r3, r4, r5 :: ptr +L0: + r0 = VecI64Api.alloc(3) + r1 = r0.buf + r2 = get_element_ptr r1 items :: VecbufI64Object + set_mem r2, 1 :: int64* + r3 = r2 + 8 + set_mem r3, 5 :: int64* + r4 = r3 + 8 + set_mem r4, 14 :: int64* + r5 = r4 + 8 + keep_alive r0 + return r0 + +[case testVecI64ConstructFromListMultiply] +from vecs import vec +from mypy_extensions import i64 + +def f(n: i64) -> vec[i64]: + return vec[i64]([3] * n) +[out] +def f(n): + n :: int64 + r0 :: vec[int64] + r1 :: object + r2 :: ptr + r3 :: int64 + r4, r5 :: ptr + r6 :: bit + r7 :: ptr +L0: + r0 = VecI64Api.alloc(n) + r1 = r0.buf + r2 = get_element_ptr r1 items :: VecbufI64Object + r3 = n * 8 + r4 = r2 + r3 + r5 = r2 +L1: + r6 = r5 < r4 :: unsigned + if r6 goto L2 else goto L3 :: bool +L2: + set_mem r5, 3 :: int64* + r7 = r5 + 8 + r5 = r7 + goto L1 +L3: + keep_alive r0 + return r0 + +[case testVecI64ConstructFromListMultiply2] +from vecs import vec +from mypy_extensions import i64 + +def f(n: i64, x: i64) -> vec[i64]: + return vec[i64]([x] * 3) +[out] +def f(n, x): + n, x :: int64 + r0 :: vec[int64] + r1 :: object + r2 :: ptr + r3 :: native_int + r4, r5 :: ptr + r6 :: bit + r7 :: ptr +L0: + r0 = VecI64Api.alloc(3) + r1 = r0.buf + r2 = get_element_ptr r1 items :: VecbufI64Object + r3 = 3 * 8 + r4 = r2 + r3 + r5 = r2 +L1: + r6 = r5 < r4 :: unsigned + if r6 goto L2 else goto L3 :: bool +L2: + set_mem r5, x :: int64* + r7 = r5 + 8 + r5 = r7 + goto L1 +L3: + keep_alive r0 + return r0 + +[case testVecI64ConstructFromListComprehension] +from vecs import vec +from mypy_extensions import i64 + +def f(n: i64) -> vec[i64]: + return vec[i64]([x + 1 for x in range(i64(5))]) +[out] +def f(n): + n :: int64 + r0, r1 :: vec[int64] + r2, x :: int64 + r3 :: bit + r4 :: int64 + r5 :: vec[int64] + r6 :: int64 +L0: + r0 = VecI64Api.alloc(0) + r1 = r0 + r2 = 0 + x = r2 +L1: + r3 = r2 < 5 :: signed + if r3 goto L2 else goto L4 :: bool +L2: + r4 = x + 1 + r5 = VecI64Api.append(r1, r4) + r1 = r5 +L3: + r6 = r2 + 1 + r2 = r6 + x = r6 + goto L1 +L4: + return r1 + +[case testVecI64ForLoop] +from vecs import vec +from mypy_extensions import i64 + +def f(v: vec[i64]) -> i64: + t: i64 = 0 + for x in v: + t += 1 + return t +[out] +def f(v): + v :: vec[int64] + t :: int64 + r0, r1 :: native_int + r2 :: bit + r3 :: object + r4 :: ptr + r5 :: native_int + r6 :: ptr + r7, x, r8 :: int64 + r9 :: native_int +L0: + t = 0 + r0 = 0 +L1: + r1 = v.len + r2 = r0 < r1 :: signed + if r2 goto L2 else goto L4 :: bool +L2: + r3 = v.buf + r4 = get_element_ptr r3 items :: VecbufI64Object + r5 = r0 * 8 + r6 = r4 + r5 + r7 = load_mem r6 :: int64* + keep_alive v + x = r7 + r8 = t + 1 + t = r8 +L3: + r9 = r0 + 1 + r0 = r9 + goto L1 +L4: + return t + +[case testVecI64Contains] +from vecs import vec +from mypy_extensions import i64 + +def contains(v: vec[i64], n: i64) -> bool: + return n in v +[out] +def contains(v, n): + v :: vec[int64] + n :: int64 + r0 :: native_int + r1 :: object + r2 :: ptr + r3 :: native_int + r4, r5 :: ptr + r6 :: bit + r7 :: int64 + r8 :: bit + r9 :: ptr + r10 :: bool +L0: + r0 = v.len + r1 = v.buf + r2 = get_element_ptr r1 items :: VecbufI64Object + r3 = r0 * 8 + r4 = r2 + r3 + r5 = r2 +L1: + r6 = r5 < r4 :: unsigned + if r6 goto L2 else goto L4 :: bool +L2: + r7 = load_mem r5 :: int64* + r8 = r7 == n + if r8 goto L5 else goto L3 :: bool +L3: + r9 = r5 + 8 + r5 = r9 + goto L1 +L4: + keep_alive v + r10 = 0 + goto L6 +L5: + r10 = 1 +L6: + return r10 + +[case testVecI64GetItemWithInt] +from vecs import vec +from mypy_extensions import i64 + +def f(v: vec[i64]) -> i64: + return v[0] +[out] +def f(v): + v :: vec[int64] + r0 :: native_int + r1 :: bit + r2 :: bool + r3 :: object + r4 :: ptr + r5 :: int64 + r6 :: ptr + r7 :: int64 +L0: + r0 = v.len + r1 = 0 < r0 :: unsigned + if r1 goto L2 else goto L1 :: bool +L1: + r2 = raise IndexError + unreachable +L2: + r3 = v.buf + r4 = get_element_ptr r3 items :: VecbufI64Object + r5 = 0 * 8 + r6 = r4 + r5 + r7 = load_mem r6 :: int64* + keep_alive v + return r7 + +[case testVecI64Slicing] +from vecs import vec +from mypy_extensions import i64 + +def f(v: vec[i64], n: i64, m: i64) -> None: + a = v[:] + b = v[n:] + c = v[n:m] + d = v[:m] + e = v[1:-2] +[out] +def f(v, n, m): + v :: vec[int64] + n, m :: int64 + r0, a, r1, b, r2, c, r3, d, r4, e :: vec[int64] +L0: + r0 = VecI64Api.slice(v, 0, 4611686018427387903) + a = r0 + r1 = VecI64Api.slice(v, n, 4611686018427387903) + b = r1 + r2 = VecI64Api.slice(v, n, m) + c = r2 + r3 = VecI64Api.slice(v, 0, m) + d = r3 + r4 = VecI64Api.slice(v, 1, -2) + e = r4 + return 1 + +[case testVecI64Remove] +from vecs import vec, remove +from mypy_extensions import i64 + +def rem(v: vec[i64], n: i64) -> None: + v = remove(v, n) +[out] +def rem(v, n): + v :: vec[int64] + n :: int64 + r0 :: vec[int64] +L0: + r0 = VecI64Api.remove(v, n) + v = r0 + return 1 + +[case testVecI64PopLast] +from typing import Tuple +from vecs import vec, pop +from mypy_extensions import i64 + +def pop_last(v: vec[i64]) -> Tuple[vec[i64], i64]: + return pop(v) +[out] +def pop_last(v): + v :: vec[int64] + r0 :: tuple[vec[int64], int64] +L0: + r0 = VecI64Api.pop(v, -1) + return r0 + +[case testVecI64PopNth] +from typing import Tuple +from vecs import vec, pop +from mypy_extensions import i64 + +def pop_nth(v: vec[i64], n: i64) -> Tuple[vec[i64], i64]: + return pop(v, n) +[out] +def pop_nth(v, n): + v :: vec[int64] + n :: int64 + r0 :: tuple[vec[int64], int64] +L0: + r0 = VecI64Api.pop(v, n) + return r0 diff --git a/mypyc/test-data/irbuild-vec-nested.test b/mypyc/test-data/irbuild-vec-nested.test new file mode 100644 index 0000000000000..a88a9690e3646 --- /dev/null +++ b/mypyc/test-data/irbuild-vec-nested.test @@ -0,0 +1,249 @@ +[case testVecNestedCreateEmpty] +from vecs import vec, append +from mypy_extensions import i64 + +def f() -> vec[vec[str]]: + return vec[vec[str]]() + +def g() -> vec[vec[i64]]: + return vec[vec[i64]]() +[out] +def f(): + r0 :: object + r1 :: ptr + r2 :: vec[vec[str]] +L0: + r0 = load_address PyUnicode_Type + r1 = r0 + r2 = VecTExtApi.alloc(0, r1, 0, 1) + return r2 +def g(): + r0 :: vec[vec[int64]] +L0: + r0 = VecTExtApi.alloc(0, 2, 0, 1) + return r0 + +[case testVecNestedAppend] +from vecs import vec, append + +def f(v: vec[vec[str]], vv: vec[str]) -> vec[vec[str]]: + return append(v, vv) +[out] +def f(v, vv): + v :: vec[vec[str]] + vv :: vec[str] + r0 :: native_int + r1 :: object + r2, r3 :: VecbufTExtItem{len:native_int, buf:object_nrc} + r4 :: vec[vec[str]] +L0: + r0 = vv.len + r1 = vv.buf + r2 = set_element undef, len, r0 + r3 = set_element r2, buf, r1 + r4 = VecTExtApi.append(v, r3) + keep_alive vv + return r4 + +[case testVecNestedVecI64Append] +from vecs import vec, append +from mypy_extensions import i64 + +def f(v: vec[vec[i64]], vv: vec[i64]) -> vec[vec[i64]]: + return append(v, vv) +[out] +def f(v, vv): + v :: vec[vec[int64]] + vv :: vec[int64] + r0 :: native_int + r1 :: object + r2, r3 :: VecbufTExtItem{len:native_int, buf:object_nrc} + r4 :: vec[vec[int64]] +L0: + r0 = vv.len + r1 = vv.buf + r2 = set_element undef, len, r0 + r3 = set_element r2, buf, r1 + r4 = VecTExtApi.append(v, r3) + keep_alive vv + return r4 + +[case testVecNestedLen] +from vecs import vec +from mypy_extensions import i64 +from typing import Optional + +def f(v: vec[vec[str]]) -> i64: + return len(v) + +def g(v: vec[vec[i64]]) -> i64: + return len(v) +[out] +def f(v): + v :: vec[vec[str]] + r0 :: native_int +L0: + r0 = v.len + return r0 +def g(v): + v :: vec[vec[int64]] + r0 :: native_int +L0: + r0 = v.len + return r0 + +[case testVecNestedGetItem] +from vecs import vec +from mypy_extensions import i64 + +def f(v: vec[vec[str]], n: i64) -> vec[str]: + return v[n] +[out] +def f(v, n): + v :: vec[vec[str]] + n :: int64 + r0 :: native_int + r1 :: bit + r2 :: bool + r3 :: object + r4 :: ptr + r5 :: int64 + r6 :: ptr + r7 :: vec[str] +L0: + r0 = v.len + r1 = n < r0 :: unsigned + if r1 goto L2 else goto L1 :: bool +L1: + r2 = raise IndexError + unreachable +L2: + r3 = v.buf + r4 = get_element_ptr r3 items :: VecbufTExtObject + r5 = n * 16 + r6 = r4 + r5 + r7 = load_mem r6 :: vec[str]* + keep_alive v + return r7 + +[case testVecNestedI64GetItem] +from vecs import vec +from mypy_extensions import i64 + +def f(v: vec[vec[i64]], n: i64) -> vec[i64]: + return v[n] +[out] +def f(v, n): + v :: vec[vec[int64]] + n :: int64 + r0 :: native_int + r1 :: bit + r2 :: bool + r3 :: object + r4 :: ptr + r5 :: int64 + r6 :: ptr + r7 :: vec[int64] +L0: + r0 = v.len + r1 = n < r0 :: unsigned + if r1 goto L2 else goto L1 :: bool +L1: + r2 = raise IndexError + unreachable +L2: + r3 = v.buf + r4 = get_element_ptr r3 items :: VecbufTExtObject + r5 = n * 16 + r6 = r4 + r5 + r7 = load_mem r6 :: vec[int64]* + keep_alive v + return r7 + +[case testVecNestedI64GetItemWithBorrow] +from vecs import vec +from mypy_extensions import i64 + +def f(v: vec[vec[i64]], n: i64) -> i64: + return v[n][n] +[out] +def f(v, n): + v :: vec[vec[int64]] + n :: int64 + r0 :: native_int + r1 :: bit + r2 :: bool + r3 :: object + r4 :: ptr + r5 :: int64 + r6 :: ptr + r7 :: vec[int64] + r8 :: native_int + r9 :: bit + r10 :: bool + r11 :: object + r12 :: ptr + r13 :: int64 + r14 :: ptr + r15 :: int64 +L0: + r0 = v.len + r1 = n < r0 :: unsigned + if r1 goto L2 else goto L1 :: bool +L1: + r2 = raise IndexError + unreachable +L2: + r3 = v.buf + r4 = get_element_ptr r3 items :: VecbufTExtObject + r5 = n * 16 + r6 = r4 + r5 + r7 = borrow load_mem r6 :: vec[int64]* + r8 = r7.len + r9 = n < r8 :: unsigned + if r9 goto L4 else goto L3 :: bool +L3: + r10 = raise IndexError + unreachable +L4: + r11 = r7.buf + r12 = get_element_ptr r11 items :: VecbufI64Object + r13 = n * 8 + r14 = r12 + r13 + r15 = load_mem r14 :: int64* + keep_alive v, r7 + return r15 + +[case testVecDoublyNestedGetItem] +from vecs import vec +from mypy_extensions import i64 + +def f(v: vec[vec[vec[str]]], n: i64) -> vec[vec[str]]: + return v[n] +[out] +def f(v, n): + v :: vec[vec[vec[str]]] + n :: int64 + r0 :: native_int + r1 :: bit + r2 :: bool + r3 :: object + r4 :: ptr + r5 :: int64 + r6 :: ptr + r7 :: vec[vec[str]] +L0: + r0 = v.len + r1 = n < r0 :: unsigned + if r1 goto L2 else goto L1 :: bool +L1: + r2 = raise IndexError + unreachable +L2: + r3 = v.buf + r4 = get_element_ptr r3 items :: VecbufTExtObject + r5 = n * 16 + r6 = r4 + r5 + r7 = load_mem r6 :: vec[vec[str]]* + keep_alive v + return r7 diff --git a/mypyc/test-data/irbuild-vec-t.test b/mypyc/test-data/irbuild-vec-t.test new file mode 100644 index 0000000000000..95b4578f87e06 --- /dev/null +++ b/mypyc/test-data/irbuild-vec-t.test @@ -0,0 +1,245 @@ +[case testVecTCreateEmpty] +from vecs import vec, append + +class C: pass + +def primitive() -> vec[str]: + return vec[str]() + +def native_class() -> vec[C]: + return vec[C]() +[out] +def primitive(): + r0 :: object + r1 :: ptr + r2 :: vec[str] +L0: + r0 = load_address PyUnicode_Type + r1 = r0 + r2 = VecTApi.alloc(0, r1) + return r2 +def native_class(): + r0 :: object + r1 :: ptr + r2 :: vec[__main__.C] +L0: + r0 = __main__.C :: type + r1 = r0 + r2 = VecTApi.alloc(0, r1) + return r2 + +[case testVecTAppend] +from vecs import vec, append + +def f(v: vec[str]) -> vec[str]: + return append(v, 'x') +[out] +def f(v): + v :: vec[str] + r0 :: str + r1 :: vec[str] +L0: + r0 = 'x' + r1 = VecTApi.append(v, r0) + return r1 + +[case testVecTOptionalCreateEmpty] +from vecs import vec, append +from typing import Optional + +class C: pass + +def primitive() -> vec[Optional[str]]: + return vec[Optional[str]]() + +def native_class() -> vec[Optional[C]]: + return vec[Optional[C]]() +[out] +def primitive(): + r0 :: object + r1, r2 :: ptr + r3 :: vec[union[str, None]] +L0: + r0 = load_address PyUnicode_Type + r1 = r0 + r2 = r1 | 1 + r3 = VecTApi.alloc(0, r2) + return r3 +def native_class(): + r0 :: object + r1, r2 :: ptr + r3 :: vec[union[__main__.C, None]] +L0: + r0 = __main__.C :: type + r1 = r0 + r2 = r1 | 1 + r3 = VecTApi.alloc(0, r2) + return r3 + +[case testVecTOptionalAppend] +from vecs import vec, append +from typing import Optional + +def f(v: vec[Optional[str]]) -> vec[Optional[str]]: + v = append(v, 'x') + return append(v, None) +[out] +def f(v): + v :: vec[union[str, None]] + r0 :: str + r1 :: vec[union[str, None]] + r2 :: object + r3 :: vec[union[str, None]] +L0: + r0 = 'x' + r1 = VecTApi.append(v, r0) + v = r1 + r2 = box(None, 1) + r3 = VecTApi.append(v, r2) + return r3 + +[case testVecTLen] +from vecs import vec +from mypy_extensions import i64 +from typing import Optional + +def f(v: vec[str]) -> i64: + return len(v) + +def g(v: vec[Optional[str]]) -> i64: + return len(v) +[out] +def f(v): + v :: vec[str] + r0 :: native_int +L0: + r0 = v.len + return r0 +def g(v): + v :: vec[union[str, None]] + r0 :: native_int +L0: + r0 = v.len + return r0 + +[case testVecTGetItem] +from vecs import vec +from mypy_extensions import i64 + +def f(v: vec[str], n: i64) -> str: + return v[n] +[out] +def f(v, n): + v :: vec[str] + n :: int64 + r0 :: native_int + r1 :: bit + r2 :: bool + r3 :: object + r4 :: ptr + r5 :: int64 + r6 :: ptr + r7 :: str +L0: + r0 = v.len + r1 = n < r0 :: unsigned + if r1 goto L2 else goto L1 :: bool +L1: + r2 = raise IndexError + unreachable +L2: + r3 = v.buf + r4 = get_element_ptr r3 items :: VecbufTObject + r5 = n * 8 + r6 = r4 + r5 + r7 = load_mem r6 :: builtins.str* + keep_alive v + return r7 + +[case testVecTOptionalGetItem] +from vecs import vec +from mypy_extensions import i64 +from typing import Optional + +def f(v: vec[Optional[str]], n: i64) -> Optional[str]: + return v[n] +[out] +def f(v, n): + v :: vec[union[str, None]] + n :: int64 + r0 :: native_int + r1 :: bit + r2 :: bool + r3 :: object + r4 :: ptr + r5 :: int64 + r6 :: ptr + r7 :: union[str, None] +L0: + r0 = v.len + r1 = n < r0 :: unsigned + if r1 goto L2 else goto L1 :: bool +L1: + r2 = raise IndexError + unreachable +L2: + r3 = v.buf + r4 = get_element_ptr r3 items :: VecbufTObject + r5 = n * 8 + r6 = r4 + r5 + r7 = load_mem r6 :: union* + keep_alive v + return r7 + +[case testNewTPopLast] +from typing import Tuple +from vecs import vec, pop + +def pop_last(v: vec[str]) -> Tuple[vec[str], str]: + return pop(v) +[out] +def pop_last(v): + v :: vec[str] + r0 :: tuple[vec[str], str] +L0: + r0 = VecTApi.pop(v, -1) + return r0 + +[case testVecTConstructFromListComprehension] +from vecs import vec +from mypy_extensions import i64 + +def f(n: i64) -> vec[str]: + return vec[str](['x' for x in range(i64(5))]) +[out] +def f(n): + n :: int64 + r0 :: object + r1 :: ptr + r2, r3 :: vec[str] + r4, x :: int64 + r5 :: bit + r6 :: str + r7 :: vec[str] + r8 :: int64 +L0: + r0 = load_address PyUnicode_Type + r1 = r0 + r2 = VecTApi.alloc(0, r1) + r3 = r2 + r4 = 0 + x = r4 +L1: + r5 = r4 < 5 :: signed + if r5 goto L2 else goto L4 :: bool +L2: + r6 = 'x' + r7 = VecTApi.append(r3, r6) + r3 = r7 +L3: + r8 = r4 + 1 + r4 = r8 + x = r8 + goto L1 +L4: + return r3 diff --git a/mypyc/test-data/irbuild-vecs.test b/mypyc/test-data/irbuild-vecs.test deleted file mode 100644 index e5b3b7f5dd0a7..0000000000000 --- a/mypyc/test-data/irbuild-vecs.test +++ /dev/null @@ -1,936 +0,0 @@ -[case testVecI64CreateEmpty] -from vecs import vec, append -from mypy_extensions import i64 - -def f() -> vec[i64]: - return vec[i64]() -[out] -def f(): - r0 :: vec[int64] -L0: - r0 = VecI64Api.alloc(0) - return r0 - -[case testVecI64Len] -from vecs import vec -from mypy_extensions import i64 - -def f(v: vec[i64]) -> i64: - l = len(v) - return l -[out] -def f(v): - v :: vec[int64] - r0 :: native_int - l :: int64 -L0: - r0 = v.len - l = r0 - return l - -[case testVecI64GetItem] -from vecs import vec -from mypy_extensions import i64 - -def f(v: vec[i64], i: i64) -> i64: - return v[i] -[out] -def f(v, i): - v :: vec[int64] - i :: int64 - r0 :: native_int - r1 :: bit - r2 :: bool - r3 :: object - r4 :: ptr - r5 :: int64 - r6 :: ptr - r7 :: int64 -L0: - r0 = v.len - r1 = i < r0 :: unsigned - if r1 goto L2 else goto L1 :: bool -L1: - r2 = raise IndexError - unreachable -L2: - r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufI64Object - r5 = i * 8 - r6 = r4 + r5 - r7 = load_mem r6 :: int64* - keep_alive v - return r7 - -[case testVecI64Append] -from vecs import vec, append -from mypy_extensions import i64 - -def f(v: vec[i64], i: i64) -> vec[i64]: - return append(v, i) -[out] -def f(v, i): - v :: vec[int64] - i :: int64 - r0 :: vec[int64] -L0: - r0 = VecI64Api.append(v, i) - return r0 - -[case testVecI64SetItem] -from vecs import vec -from mypy_extensions import i64 - -def f(v: vec[i64], i: i64, x: i64) -> None: - v[i] = x -[out] -def f(v, i, x): - v :: vec[int64] - i, x :: int64 - r0 :: native_int - r1 :: bit - r2 :: bool - r3 :: object - r4 :: ptr - r5 :: int64 - r6 :: ptr -L0: - r0 = v.len - r1 = i < r0 :: unsigned - if r1 goto L2 else goto L1 :: bool -L1: - r2 = raise IndexError - unreachable -L2: - r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufI64Object - r5 = i * 8 - r6 = r4 + r5 - set_mem r6, x :: int64* - keep_alive v - return 1 - -[case testVecI64ConstructFromListExpr] -from vecs import vec -from mypy_extensions import i64 - -def f() -> vec[i64]: - return vec[i64]([1, 5, 14]) -[out] -def f(): - r0 :: vec[int64] - r1 :: object - r2, r3, r4, r5 :: ptr -L0: - r0 = VecI64Api.alloc(3) - r1 = r0.buf - r2 = get_element_ptr r1 items :: VecbufI64Object - set_mem r2, 1 :: int64* - r3 = r2 + 8 - set_mem r3, 5 :: int64* - r4 = r3 + 8 - set_mem r4, 14 :: int64* - r5 = r4 + 8 - keep_alive r0 - return r0 - -[case testVecI64ConstructFromListMultiply] -from vecs import vec -from mypy_extensions import i64 - -def f(n: i64) -> vec[i64]: - return vec[i64]([3] * n) -[out] -def f(n): - n :: int64 - r0 :: vec[int64] - r1 :: object - r2 :: ptr - r3 :: int64 - r4, r5 :: ptr - r6 :: bit - r7 :: ptr -L0: - r0 = VecI64Api.alloc(n) - r1 = r0.buf - r2 = get_element_ptr r1 items :: VecbufI64Object - r3 = n * 8 - r4 = r2 + r3 - r5 = r2 -L1: - r6 = r5 < r4 :: unsigned - if r6 goto L2 else goto L3 :: bool -L2: - set_mem r5, 3 :: int64* - r7 = r5 + 8 - r5 = r7 - goto L1 -L3: - keep_alive r0 - return r0 - -[case testVecI64ConstructFromListMultiply2] -from vecs import vec -from mypy_extensions import i64 - -def f(n: i64, x: i64) -> vec[i64]: - return vec[i64]([x] * 3) -[out] -def f(n, x): - n, x :: int64 - r0 :: vec[int64] - r1 :: object - r2 :: ptr - r3 :: native_int - r4, r5 :: ptr - r6 :: bit - r7 :: ptr -L0: - r0 = VecI64Api.alloc(3) - r1 = r0.buf - r2 = get_element_ptr r1 items :: VecbufI64Object - r3 = 3 * 8 - r4 = r2 + r3 - r5 = r2 -L1: - r6 = r5 < r4 :: unsigned - if r6 goto L2 else goto L3 :: bool -L2: - set_mem r5, x :: int64* - r7 = r5 + 8 - r5 = r7 - goto L1 -L3: - keep_alive r0 - return r0 - -[case testVecI64ConstructFromListComprehension] -from vecs import vec -from mypy_extensions import i64 - -def f(n: i64) -> vec[i64]: - return vec[i64]([x + 1 for x in range(i64(5))]) -[out] -def f(n): - n :: int64 - r0, r1 :: vec[int64] - r2, x :: int64 - r3 :: bit - r4 :: int64 - r5 :: vec[int64] - r6 :: int64 -L0: - r0 = VecI64Api.alloc(0) - r1 = r0 - r2 = 0 - x = r2 -L1: - r3 = r2 < 5 :: signed - if r3 goto L2 else goto L4 :: bool -L2: - r4 = x + 1 - r5 = VecI64Api.append(r1, r4) - r1 = r5 -L3: - r6 = r2 + 1 - r2 = r6 - x = r6 - goto L1 -L4: - return r1 - -[case testVecI64ForLoop] -from vecs import vec -from mypy_extensions import i64 - -def f(v: vec[i64]) -> i64: - t: i64 = 0 - for x in v: - t += 1 - return t -[out] -def f(v): - v :: vec[int64] - t :: int64 - r0, r1 :: native_int - r2 :: bit - r3 :: object - r4 :: ptr - r5 :: native_int - r6 :: ptr - r7, x, r8 :: int64 - r9 :: native_int -L0: - t = 0 - r0 = 0 -L1: - r1 = v.len - r2 = r0 < r1 :: signed - if r2 goto L2 else goto L4 :: bool -L2: - r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufI64Object - r5 = r0 * 8 - r6 = r4 + r5 - r7 = load_mem r6 :: int64* - keep_alive v - x = r7 - r8 = t + 1 - t = r8 -L3: - r9 = r0 + 1 - r0 = r9 - goto L1 -L4: - return t - -[case testVecI64Contains] -from vecs import vec -from mypy_extensions import i64 - -def contains(v: vec[i64], n: i64) -> bool: - return n in v -[out] -def contains(v, n): - v :: vec[int64] - n :: int64 - r0 :: native_int - r1 :: object - r2 :: ptr - r3 :: native_int - r4, r5 :: ptr - r6 :: bit - r7 :: int64 - r8 :: bit - r9 :: ptr - r10 :: bool -L0: - r0 = v.len - r1 = v.buf - r2 = get_element_ptr r1 items :: VecbufI64Object - r3 = r0 * 8 - r4 = r2 + r3 - r5 = r2 -L1: - r6 = r5 < r4 :: unsigned - if r6 goto L2 else goto L4 :: bool -L2: - r7 = load_mem r5 :: int64* - r8 = r7 == n - if r8 goto L5 else goto L3 :: bool -L3: - r9 = r5 + 8 - r5 = r9 - goto L1 -L4: - keep_alive v - r10 = 0 - goto L6 -L5: - r10 = 1 -L6: - return r10 - -[case testVecI64GetItemWithInt] -from vecs import vec -from mypy_extensions import i64 - -def f(v: vec[i64]) -> i64: - return v[0] -[out] -def f(v): - v :: vec[int64] - r0 :: native_int - r1 :: bit - r2 :: bool - r3 :: object - r4 :: ptr - r5 :: int64 - r6 :: ptr - r7 :: int64 -L0: - r0 = v.len - r1 = 0 < r0 :: unsigned - if r1 goto L2 else goto L1 :: bool -L1: - r2 = raise IndexError - unreachable -L2: - r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufI64Object - r5 = 0 * 8 - r6 = r4 + r5 - r7 = load_mem r6 :: int64* - keep_alive v - return r7 - -[case testVecI64Slicing] -from vecs import vec -from mypy_extensions import i64 - -def f(v: vec[i64], n: i64, m: i64) -> None: - a = v[:] - b = v[n:] - c = v[n:m] - d = v[:m] - e = v[1:-2] -[out] -def f(v, n, m): - v :: vec[int64] - n, m :: int64 - r0, a, r1, b, r2, c, r3, d, r4, e :: vec[int64] -L0: - r0 = VecI64Api.slice(v, 0, 4611686018427387903) - a = r0 - r1 = VecI64Api.slice(v, n, 4611686018427387903) - b = r1 - r2 = VecI64Api.slice(v, n, m) - c = r2 - r3 = VecI64Api.slice(v, 0, m) - d = r3 - r4 = VecI64Api.slice(v, 1, -2) - e = r4 - return 1 - -[case testVecI64Remove] -from vecs import vec, remove -from mypy_extensions import i64 - -def rem(v: vec[i64], n: i64) -> None: - v = remove(v, n) -[out] -def rem(v, n): - v :: vec[int64] - n :: int64 - r0 :: vec[int64] -L0: - r0 = VecI64Api.remove(v, n) - v = r0 - return 1 - -[case testVecI64PopLast] -from typing import Tuple -from vecs import vec, pop -from mypy_extensions import i64 - -def pop_last(v: vec[i64]) -> Tuple[vec[i64], i64]: - return pop(v) -[out] -def pop_last(v): - v :: vec[int64] - r0 :: tuple[vec[int64], int64] -L0: - r0 = VecI64Api.pop(v, -1) - return r0 - -[case testVecI64PopNth] -from typing import Tuple -from vecs import vec, pop -from mypy_extensions import i64 - -def pop_nth(v: vec[i64], n: i64) -> Tuple[vec[i64], i64]: - return pop(v, n) -[out] -def pop_nth(v, n): - v :: vec[int64] - n :: int64 - r0 :: tuple[vec[int64], int64] -L0: - r0 = VecI64Api.pop(v, n) - return r0 - -[case testVecTCreateEmpty] -from vecs import vec, append - -class C: pass - -def primitive() -> vec[str]: - return vec[str]() - -def native_class() -> vec[C]: - return vec[C]() -[out] -def primitive(): - r0 :: object - r1 :: ptr - r2 :: vec[str] -L0: - r0 = load_address PyUnicode_Type - r1 = r0 - r2 = VecTApi.alloc(0, r1) - return r2 -def native_class(): - r0 :: object - r1 :: ptr - r2 :: vec[__main__.C] -L0: - r0 = __main__.C :: type - r1 = r0 - r2 = VecTApi.alloc(0, r1) - return r2 - -[case testVecTAppend] -from vecs import vec, append - -def f(v: vec[str]) -> vec[str]: - return append(v, 'x') -[out] -def f(v): - v :: vec[str] - r0 :: str - r1 :: vec[str] -L0: - r0 = 'x' - r1 = VecTApi.append(v, r0) - return r1 - -[case testVecTOptionalCreateEmpty] -from vecs import vec, append -from typing import Optional - -class C: pass - -def primitive() -> vec[Optional[str]]: - return vec[Optional[str]]() - -def native_class() -> vec[Optional[C]]: - return vec[Optional[C]]() -[out] -def primitive(): - r0 :: object - r1, r2 :: ptr - r3 :: vec[union[str, None]] -L0: - r0 = load_address PyUnicode_Type - r1 = r0 - r2 = r1 | 1 - r3 = VecTApi.alloc(0, r2) - return r3 -def native_class(): - r0 :: object - r1, r2 :: ptr - r3 :: vec[union[__main__.C, None]] -L0: - r0 = __main__.C :: type - r1 = r0 - r2 = r1 | 1 - r3 = VecTApi.alloc(0, r2) - return r3 - -[case testVecTOptionalAppend] -from vecs import vec, append -from typing import Optional - -def f(v: vec[Optional[str]]) -> vec[Optional[str]]: - v = append(v, 'x') - return append(v, None) -[out] -def f(v): - v :: vec[union[str, None]] - r0 :: str - r1 :: vec[union[str, None]] - r2 :: object - r3 :: vec[union[str, None]] -L0: - r0 = 'x' - r1 = VecTApi.append(v, r0) - v = r1 - r2 = box(None, 1) - r3 = VecTApi.append(v, r2) - return r3 - -[case testVecTLen] -from vecs import vec -from mypy_extensions import i64 -from typing import Optional - -def f(v: vec[str]) -> i64: - return len(v) - -def g(v: vec[Optional[str]]) -> i64: - return len(v) -[out] -def f(v): - v :: vec[str] - r0 :: native_int -L0: - r0 = v.len - return r0 -def g(v): - v :: vec[union[str, None]] - r0 :: native_int -L0: - r0 = v.len - return r0 - -[case testVecTGetItem] -from vecs import vec -from mypy_extensions import i64 - -def f(v: vec[str], n: i64) -> str: - return v[n] -[out] -def f(v, n): - v :: vec[str] - n :: int64 - r0 :: native_int - r1 :: bit - r2 :: bool - r3 :: object - r4 :: ptr - r5 :: int64 - r6 :: ptr - r7 :: str -L0: - r0 = v.len - r1 = n < r0 :: unsigned - if r1 goto L2 else goto L1 :: bool -L1: - r2 = raise IndexError - unreachable -L2: - r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufTObject - r5 = n * 8 - r6 = r4 + r5 - r7 = load_mem r6 :: builtins.str* - keep_alive v - return r7 - -[case testVecTOptionalGetItem] -from vecs import vec -from mypy_extensions import i64 -from typing import Optional - -def f(v: vec[Optional[str]], n: i64) -> Optional[str]: - return v[n] -[out] -def f(v, n): - v :: vec[union[str, None]] - n :: int64 - r0 :: native_int - r1 :: bit - r2 :: bool - r3 :: object - r4 :: ptr - r5 :: int64 - r6 :: ptr - r7 :: union[str, None] -L0: - r0 = v.len - r1 = n < r0 :: unsigned - if r1 goto L2 else goto L1 :: bool -L1: - r2 = raise IndexError - unreachable -L2: - r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufTObject - r5 = n * 8 - r6 = r4 + r5 - r7 = load_mem r6 :: union* - keep_alive v - return r7 - -[case testNewTPopLast] -from typing import Tuple -from vecs import vec, pop - -def pop_last(v: vec[str]) -> Tuple[vec[str], str]: - return pop(v) -[out] -def pop_last(v): - v :: vec[str] - r0 :: tuple[vec[str], str] -L0: - r0 = VecTApi.pop(v, -1) - return r0 - -[case testVecTConstructFromListComprehension] -from vecs import vec -from mypy_extensions import i64 - -def f(n: i64) -> vec[str]: - return vec[str](['x' for x in range(i64(5))]) -[out] -def f(n): - n :: int64 - r0 :: object - r1 :: ptr - r2, r3 :: vec[str] - r4, x :: int64 - r5 :: bit - r6 :: str - r7 :: vec[str] - r8 :: int64 -L0: - r0 = load_address PyUnicode_Type - r1 = r0 - r2 = VecTApi.alloc(0, r1) - r3 = r2 - r4 = 0 - x = r4 -L1: - r5 = r4 < 5 :: signed - if r5 goto L2 else goto L4 :: bool -L2: - r6 = 'x' - r7 = VecTApi.append(r3, r6) - r3 = r7 -L3: - r8 = r4 + 1 - r4 = r8 - x = r8 - goto L1 -L4: - return r3 - -[case testVecNestedCreateEmpty] -from vecs import vec, append -from mypy_extensions import i64 - -def f() -> vec[vec[str]]: - return vec[vec[str]]() - -def g() -> vec[vec[i64]]: - return vec[vec[i64]]() -[out] -def f(): - r0 :: object - r1 :: ptr - r2 :: vec[vec[str]] -L0: - r0 = load_address PyUnicode_Type - r1 = r0 - r2 = VecTExtApi.alloc(0, r1, 0, 1) - return r2 -def g(): - r0 :: vec[vec[int64]] -L0: - r0 = VecTExtApi.alloc(0, 2, 0, 1) - return r0 - -[case testVecNestedAppend] -from vecs import vec, append - -def f(v: vec[vec[str]], vv: vec[str]) -> vec[vec[str]]: - return append(v, vv) -[out] -def f(v, vv): - v :: vec[vec[str]] - vv :: vec[str] - r0 :: native_int - r1 :: object - r2, r3 :: VecbufTExtItem{len:native_int, buf:object_nrc} - r4 :: vec[vec[str]] -L0: - r0 = vv.len - r1 = vv.buf - r2 = set_element undef, len, r0 - r3 = set_element r2, buf, r1 - r4 = VecTExtApi.append(v, r3) - keep_alive vv - return r4 - -[case testVecNestedVecI64Append] -from vecs import vec, append -from mypy_extensions import i64 - -def f(v: vec[vec[i64]], vv: vec[i64]) -> vec[vec[i64]]: - return append(v, vv) -[out] -def f(v, vv): - v :: vec[vec[int64]] - vv :: vec[int64] - r0 :: native_int - r1 :: object - r2, r3 :: VecbufTExtItem{len:native_int, buf:object_nrc} - r4 :: vec[vec[int64]] -L0: - r0 = vv.len - r1 = vv.buf - r2 = set_element undef, len, r0 - r3 = set_element r2, buf, r1 - r4 = VecTExtApi.append(v, r3) - keep_alive vv - return r4 - -[case testVecNestedLen] -from vecs import vec -from mypy_extensions import i64 -from typing import Optional - -def f(v: vec[vec[str]]) -> i64: - return len(v) - -def g(v: vec[vec[i64]]) -> i64: - return len(v) -[out] -def f(v): - v :: vec[vec[str]] - r0 :: native_int -L0: - r0 = v.len - return r0 -def g(v): - v :: vec[vec[int64]] - r0 :: native_int -L0: - r0 = v.len - return r0 - -[case testVecNestedGetItem] -from vecs import vec -from mypy_extensions import i64 - -def f(v: vec[vec[str]], n: i64) -> vec[str]: - return v[n] -[out] -def f(v, n): - v :: vec[vec[str]] - n :: int64 - r0 :: native_int - r1 :: bit - r2 :: bool - r3 :: object - r4 :: ptr - r5 :: int64 - r6 :: ptr - r7 :: vec[str] -L0: - r0 = v.len - r1 = n < r0 :: unsigned - if r1 goto L2 else goto L1 :: bool -L1: - r2 = raise IndexError - unreachable -L2: - r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufTExtObject - r5 = n * 16 - r6 = r4 + r5 - r7 = load_mem r6 :: vec[str]* - keep_alive v - return r7 - -[case testVecNestedI64GetItem] -from vecs import vec -from mypy_extensions import i64 - -def f(v: vec[vec[i64]], n: i64) -> vec[i64]: - return v[n] -[out] -def f(v, n): - v :: vec[vec[int64]] - n :: int64 - r0 :: native_int - r1 :: bit - r2 :: bool - r3 :: object - r4 :: ptr - r5 :: int64 - r6 :: ptr - r7 :: vec[int64] -L0: - r0 = v.len - r1 = n < r0 :: unsigned - if r1 goto L2 else goto L1 :: bool -L1: - r2 = raise IndexError - unreachable -L2: - r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufTExtObject - r5 = n * 16 - r6 = r4 + r5 - r7 = load_mem r6 :: vec[int64]* - keep_alive v - return r7 - -[case testVecNestedI64GetItemWithBorrow] -from vecs import vec -from mypy_extensions import i64 - -def f(v: vec[vec[i64]], n: i64) -> i64: - return v[n][n] -[out] -def f(v, n): - v :: vec[vec[int64]] - n :: int64 - r0 :: native_int - r1 :: bit - r2 :: bool - r3 :: object - r4 :: ptr - r5 :: int64 - r6 :: ptr - r7 :: vec[int64] - r8 :: native_int - r9 :: bit - r10 :: bool - r11 :: object - r12 :: ptr - r13 :: int64 - r14 :: ptr - r15 :: int64 -L0: - r0 = v.len - r1 = n < r0 :: unsigned - if r1 goto L2 else goto L1 :: bool -L1: - r2 = raise IndexError - unreachable -L2: - r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufTExtObject - r5 = n * 16 - r6 = r4 + r5 - r7 = borrow load_mem r6 :: vec[int64]* - r8 = r7.len - r9 = n < r8 :: unsigned - if r9 goto L4 else goto L3 :: bool -L3: - r10 = raise IndexError - unreachable -L4: - r11 = r7.buf - r12 = get_element_ptr r11 items :: VecbufI64Object - r13 = n * 8 - r14 = r12 + r13 - r15 = load_mem r14 :: int64* - keep_alive v, r7 - return r15 - -[case testVecDoublyNestedGetItem] -from vecs import vec -from mypy_extensions import i64 - -def f(v: vec[vec[vec[str]]], n: i64) -> vec[vec[str]]: - return v[n] -[out] -def f(v, n): - v :: vec[vec[vec[str]]] - n :: int64 - r0 :: native_int - r1 :: bit - r2 :: bool - r3 :: object - r4 :: ptr - r5 :: int64 - r6 :: ptr - r7 :: vec[vec[str]] -L0: - r0 = v.len - r1 = n < r0 :: unsigned - if r1 goto L2 else goto L1 :: bool -L1: - r2 = raise IndexError - unreachable -L2: - r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufTExtObject - r5 = n * 16 - r6 = r4 + r5 - r7 = load_mem r6 :: vec[vec[str]]* - keep_alive v - return r7 diff --git a/mypyc/test/test_irbuild.py b/mypyc/test/test_irbuild.py index 9f706ec7bc426..3362c81f26f79 100644 --- a/mypyc/test/test_irbuild.py +++ b/mypyc/test/test_irbuild.py @@ -44,7 +44,9 @@ "irbuild-i32.test", "irbuild-i16.test", "irbuild-u8.test", - "irbuild-vecs.test", + "irbuild-vec-i64.test", + "irbuild-vec-t.test", + "irbuild-vec-nested.test", "irbuild-vectorcall.test", "irbuild-unreachable.test", "irbuild-isinstance.test", From d165baa560352be6b295ded40b6b15b2701444a1 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 8 Jul 2023 10:52:11 +0100 Subject: [PATCH 042/180] Add comments to tests --- mypyc/test-data/irbuild-vec-i64.test | 3 +++ mypyc/test-data/irbuild-vec-nested.test | 2 ++ mypyc/test-data/irbuild-vec-t.test | 3 +++ 3 files changed, 8 insertions(+) diff --git a/mypyc/test-data/irbuild-vec-i64.test b/mypyc/test-data/irbuild-vec-i64.test index 8fa2943ec68a6..d42472bb1130b 100644 --- a/mypyc/test-data/irbuild-vec-i64.test +++ b/mypyc/test-data/irbuild-vec-i64.test @@ -1,3 +1,6 @@ +-- Test cases for vec[i64]. These also partially cover other unboxed item types, +-- which use a similar runtime representation. + [case testVecI64CreateEmpty] from vecs import vec, append from mypy_extensions import i64 diff --git a/mypyc/test-data/irbuild-vec-nested.test b/mypyc/test-data/irbuild-vec-nested.test index a88a9690e3646..65ddc1ca3c203 100644 --- a/mypyc/test-data/irbuild-vec-nested.test +++ b/mypyc/test-data/irbuild-vec-nested.test @@ -1,3 +1,5 @@ +-- Test cases for nested vecs + [case testVecNestedCreateEmpty] from vecs import vec, append from mypy_extensions import i64 diff --git a/mypyc/test-data/irbuild-vec-t.test b/mypyc/test-data/irbuild-vec-t.test index 95b4578f87e06..64da59a9dba6c 100644 --- a/mypyc/test-data/irbuild-vec-t.test +++ b/mypyc/test-data/irbuild-vec-t.test @@ -1,3 +1,6 @@ +-- Test cases for vec[t] where t is a boxed, non-vec type (PyObject *). +-- Also tests for vec[t | None], which uses the same representation. + [case testVecTCreateEmpty] from vecs import vec, append From 58a64aef137d94db0d6d888289272c02d1301161 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 8 Jul 2023 12:58:34 +0100 Subject: [PATCH 043/180] Fix set item + add some run tests --- mypyc/irbuild/vec.py | 1 + mypyc/test-data/run-vec-i64.test | 86 ++++++++++++++++++++++++++++++++ mypyc/test/test_run.py | 1 + 3 files changed, 88 insertions(+) create mode 100644 mypyc/test-data/run-vec-i64.test diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 81e34d7f910eb..0c56866f68116 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -283,6 +283,7 @@ def vec_set_item( builder: "LowLevelIRBuilder", base: Value, index: Value, item: Value, line: int ) -> None: assert isinstance(base.type, RVec) + index = as_platform_int(builder, index, line) vtype = base.type len_val = vec_len_native(builder, base) vec_check_index(builder, len_val, index, line) diff --git a/mypyc/test-data/run-vec-i64.test b/mypyc/test-data/run-vec-i64.test new file mode 100644 index 0000000000000..4157e2ff51b90 --- /dev/null +++ b/mypyc/test-data/run-vec-i64.test @@ -0,0 +1,86 @@ +[case testVecI64BasicOps] +from typing import Final + +from mypy_extensions import i64 +from vecs import vec, append + +from testutil import assertRaises + +ERROR: Final = -113 + +def test_create_empty() -> None: + v = vec[i64]() + assert len(v) == 0 + +def test_create_from_list_and_get_item() -> None: + v = vec[i64]([]) + assert len(v) == 0 + + v = vec[i64]([ERROR, 0, 255, int() + (2**63 - 1)]) + assert len(v) == 4 + assert v[0] == -113 + assert v[1] == 0 + assert v[2] == 255 + assert v[3] == 2**63 - 1 + + v = vec[i64]([1, 2, 3, 4, 5, 6, 7, 8, 9]) + assert len(v) == 9 + for i in range(i64(9)): + assert v[i] == i + 1 + +def test_append() -> None: + v = vec[i64]() + v = append(v, int() + 3) + assert len(v) == 1 + x: i64 = ERROR + int() + v = append(v, x) + assert len(v) == 2 + assert v[0] == 3 + assert v[1] == ERROR + v = vec[i64]() + for i in range(1024): + v = append(v, i * 3) + assert len(v) == 1024 + for i in range(1024): + assert v[i] == i * 3 + +def test_get_item() -> None: + v = vec[i64]([3, 4]) + x: i64 = int() + assert v[x] == 3 + assert v[int()] == 3 + x += 1 + assert v[x] == 4 + assert v[int() + 1] == 4 + with assertRaises(IndexError): + v[x + 1] + with assertRaises(IndexError): + v[2] + with assertRaises(IndexError): + v[2**63 - 1] + +def test_set_item() -> None: + v = vec[i64]([3, 4]) + v[0] = 0 + assert v[0] == 0 + + x: i64 = int() + + v[x] = 4 + assert v[x] == 4 + + v[int()] = 5 + assert v[int()] == 5 + + x += 1 + + v[x] = -5 + assert v[x] == -5 + + v[1 + int()] = ERROR + assert v[1] == ERROR + + with assertRaises(IndexError): + v[1 + x] = 6 + with assertRaises(IndexError): + v[2 + int()] = 6 diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index 7b855841e220f..d41a5f63c38e8 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -82,6 +82,7 @@ "run-vecs-misc-interp.test", "run-vecs-t-interp.test", "run-vecs-nested-interp.test", + "run-vec-i64.test", ] if sys.version_info >= (3, 12): From 2bcf03b3bb91a8a6c683dc4fabe7670fb56fa54d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 8 Jul 2023 13:12:25 +0100 Subject: [PATCH 044/180] Add test cases --- mypyc/test-data/run-vec-i64.test | 40 +++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/mypyc/test-data/run-vec-i64.test b/mypyc/test-data/run-vec-i64.test index 4157e2ff51b90..3d4a25547e919 100644 --- a/mypyc/test-data/run-vec-i64.test +++ b/mypyc/test-data/run-vec-i64.test @@ -1,5 +1,5 @@ [case testVecI64BasicOps] -from typing import Final +from typing import Final, Any, Iterable from mypy_extensions import i64 from vecs import vec, append @@ -84,3 +84,41 @@ def test_set_item() -> None: v[1 + x] = 6 with assertRaises(IndexError): v[2 + int()] = 6 + +def test_box_and_unbox() -> None: + v = vec[i64]([3, 5]) + o: Any = v + assert len(o) == 2 + assert o[0] == 3 + assert o[1] == 5 + o[1] = 6 + v2: vec[i64] = o + assert len(v2) == 2 + assert v2[0] == 3 + assert v2[1] == 6 + o2: Any = "x" + with assertRaises(TypeError, "vec[i64] expected"): + v2 = o2 + o3: Any = vec[str]([]) + with assertRaises(TypeError, "vec[i64] expected"): + v2 = o3 + +def test_construct_from_list_multiply() -> None: + for i in range(50): + v = vec[i64]([i + 1] * i) + assert len(v) == i + for j in range(i): + assert v[j] == i + 1 + for i in range(50): + v = vec[i64](i * [i - 1]) + assert len(v) == i + for j in range(i): + assert v[j] == i - 1 + +def test_construct_from_list_comprehension() -> None: + for i in range(50): + l = [i * i for i in range(i)] + v = vec[i64]([n + 5 for n in l]) + assert len(v) == i + for j in range(i): + assert v[j] == j * j + 5 From 0af2531d597d8d941204c55923eadd03c19d1fec Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 8 Jul 2023 13:32:27 +0100 Subject: [PATCH 045/180] Add test cases --- mypyc/test-data/run-vec-i64.test | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/mypyc/test-data/run-vec-i64.test b/mypyc/test-data/run-vec-i64.test index 3d4a25547e919..caaad320df374 100644 --- a/mypyc/test-data/run-vec-i64.test +++ b/mypyc/test-data/run-vec-i64.test @@ -122,3 +122,28 @@ def test_construct_from_list_comprehension() -> None: assert len(v) == i for j in range(i): assert v[j] == j * j + 5 + +def test_str_conversion() -> None: + v = vec[i64]() + assert str(v) == "vec[i64]([])" + assert repr(v) == "vec[i64]([])" + v = vec[i64]([126]) + assert str(v) == "vec[i64]([126])" + v = append(v, -5) + assert str(v) == "vec[i64]([126, -5])" + v = append(v, ERROR) + assert str(v) == "vec[i64]([126, -5, -113])" + +def test_for_loop() -> None: + for n in vec[i64](): + assert False + a = [] + for n in vec[i64]([5]): + a.append(n) + assert a == [5] + v = vec[i64]([ERROR, 9, 8]) + a = [] + for n in v: + a.append(n) + assert a == [ERROR, 9, 8] + assert len(v) == 3 From 1f8c36a814db759ddda0cf5fd7f0b7a1ec0dbbea Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 8 Jul 2023 13:34:51 +0100 Subject: [PATCH 046/180] Test contains --- mypyc/test-data/run-vec-i64.test | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/mypyc/test-data/run-vec-i64.test b/mypyc/test-data/run-vec-i64.test index caaad320df374..eec642d22e640 100644 --- a/mypyc/test-data/run-vec-i64.test +++ b/mypyc/test-data/run-vec-i64.test @@ -147,3 +147,17 @@ def test_for_loop() -> None: a.append(n) assert a == [ERROR, 9, 8] assert len(v) == 3 + +def test_contains() -> None: + v = vec[i64]() + x: i64 = int() + assert x not in v + v = vec[i64]([0]) + assert x in v + assert x + 1 not in v + v2 = vec[i64]([ERROR, 7, 9]) + assert x + ERROR in v2 + assert x + 7 in v2 + assert int() + 9 in v2 + assert x not in v2 + assert int() not in v2 From ca6025af37532d287a9ae6c511b9fb19b4e7648f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 8 Jul 2023 14:12:08 +0100 Subject: [PATCH 047/180] Add run tests --- mypyc/test-data/run-vec-i64.test | 46 ++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/mypyc/test-data/run-vec-i64.test b/mypyc/test-data/run-vec-i64.test index eec642d22e640..e9d8dd5d7c4ea 100644 --- a/mypyc/test-data/run-vec-i64.test +++ b/mypyc/test-data/run-vec-i64.test @@ -123,6 +123,18 @@ def test_construct_from_list_comprehension() -> None: for j in range(i): assert v[j] == j * j + 5 +def test_equality() -> None: + v0 = vec[i64]() + v0b = vec[i64]() + v1 = vec[i64]([1]) + v1b = vec[i64]([3]) + assert v0 == v0 + assert v1 == v1 + assert v0 == v0b + assert v0 != v1 + assert v1 != v1b + assert v1 != v0 + def test_str_conversion() -> None: v = vec[i64]() assert str(v) == "vec[i64]([])" @@ -161,3 +173,37 @@ def test_contains() -> None: assert int() + 9 in v2 assert x not in v2 assert int() not in v2 + +def test_slicing() -> None: + v = vec[i64]() + assert v[:] == vec[i64]([]) + assert v[1:] == vec[i64]([]) + assert v[:-5] == vec[i64]([]) + v = vec[i64]([0, 1, 2, 3, 4]) + assert v[1:4] == vec[i64]([1, 2, 3]) + assert v[2:-1] == vec[i64]([2, 3]) + assert v[-2:-1] == vec[i64]([3]) + assert v[1:] == vec[i64]([1, 2, 3, 4]) + assert v[:-1] == vec[i64]([0, 1, 2, 3]) + assert v[:] == v + assert v[:] is not v + assert v[5:] == vec[i64]() + assert v[100:] == vec[i64]() + assert v[0:5] ==v + assert v[0:5] is not v + assert v[2:100] == vec[i64]([2, 3, 4]) + assert v[-100:2] == vec[i64]([0, 1]) + assert v[5:100] == vec[i64]([]) + assert v[50:100] == vec[i64]([]) + assert v[-100:-50] == vec[i64]([]) + +def test_slicing_with_step() -> None: + v = vec[i64]([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + assert v[1:5:2] == vec[i64]([1, 3]) + assert v[1:6:2] == vec[i64]([1, 3, 5]) + assert v[5:1:-2] == vec[i64]([5, 3]) + assert v[6:1:-2] == vec[i64]([6, 4, 2]) + v = vec[i64]([0, 1, 2, 3, 4]) + assert v[::-1] == vec[i64]([4, 3, 2, 1, 0]) + with assertRaises(ValueError): + v[1:3:0] From 6e13589e8624c07ea0a75d2e5a231a75f8de01e4 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 8 Jul 2023 14:15:19 +0100 Subject: [PATCH 048/180] Refactor --- mypyc/irbuild/vec.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 0c56866f68116..1617d6c3aa7a9 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -77,7 +77,7 @@ def as_platform_int(builder: LowLevelIRBuilder, v: Value, line: int) -> Value: def vec_create( - builder: "LowLevelIRBuilder", vtype: RVec, length: Union[int, Value], line: int + builder: LowLevelIRBuilder, vtype: RVec, length: Union[int, Value], line: int ) -> Value: if isinstance(length, int): length = Integer(length, c_pyssize_t_rprimitive) @@ -136,7 +136,7 @@ def vec_create( def vec_create_initialized( - builder: "LowLevelIRBuilder", vtype: RVec, length: Union[int, Value], init: Value, line: int + builder: LowLevelIRBuilder, vtype: RVec, length: Union[int, Value], init: Value, line: int ) -> Value: """Create vec with items initialized to the given value.""" if isinstance(length, int): @@ -162,7 +162,7 @@ def vec_create_initialized( def vec_create_from_values( - builder: "LowLevelIRBuilder", vtype: RVec, values: List[Value], line: int + builder: LowLevelIRBuilder, vtype: RVec, values: List[Value], line: int ) -> Value: vec = vec_create(builder, vtype, len(values), line) ptr = vec_items(builder, vec) @@ -186,7 +186,7 @@ def step_size(item_type: RType) -> int: def vec_item_type_info( - builder: "LowLevelIRBuilder", typ: RType, line: int + builder: LowLevelIRBuilder, typ: RType, line: int ) -> Tuple[Optional[Value], int, int]: if isinstance(typ, RPrimitive) and typ.is_refcounted: typ, src = builtin_names[typ.name] @@ -208,23 +208,23 @@ def vec_item_type_info( return None, 0, 0 -def vec_len(builder: "LowLevelIRBuilder", val: Value) -> Value: +def vec_len(builder: LowLevelIRBuilder, val: Value) -> Value: # TODO: what about 32-bit archs? # TODO: merge vec_len and vec_len_native return vec_len_native(builder, val) -def vec_len_native(builder: "LowLevelIRBuilder", val: Value) -> Value: +def vec_len_native(builder: LowLevelIRBuilder, val: Value) -> Value: return builder.get_element(val, "len") -def vec_items(builder: "LowLevelIRBuilder", vecobj: Value) -> Value: +def vec_items(builder: LowLevelIRBuilder, vecobj: Value) -> Value: vtype = cast(RVec, vecobj.type) buf = builder.get_element(vecobj, "buf") return builder.add(GetElementPtr(buf, vtype.buf_type, "items")) -def vec_item_ptr(builder: "LowLevelIRBuilder", vecobj: Value, index: Value) -> Value: +def vec_item_ptr(builder: LowLevelIRBuilder, vecobj: Value, index: Value) -> Value: items_addr = vec_items(builder, vecobj) assert isinstance(vecobj.type, RVec) # TODO: Calculate item size properly and support 32-bit platforms @@ -236,7 +236,7 @@ def vec_item_ptr(builder: "LowLevelIRBuilder", vecobj: Value, index: Value) -> V return builder.int_add(items_addr, delta) -def vec_check_index(builder: "LowLevelIRBuilder", lenv: Value, index: Value, line: int) -> None: +def vec_check_index(builder: LowLevelIRBuilder, lenv: Value, index: Value, line: int) -> None: ok, fail = BasicBlock(), BasicBlock() is_less = builder.comparison_op(index, lenv, ComparisonOp.ULT, line) builder.add_bool_branch(is_less, ok, fail) @@ -248,7 +248,7 @@ def vec_check_index(builder: "LowLevelIRBuilder", lenv: Value, index: Value, lin def vec_get_item( - builder: "LowLevelIRBuilder", base: Value, index: Value, line: int, *, can_borrow: bool = False + builder: LowLevelIRBuilder, base: Value, index: Value, line: int, *, can_borrow: bool = False ) -> Value: """Generate inlined vec __getitem__ call. @@ -267,7 +267,7 @@ def vec_get_item( def vec_get_item_unsafe( - builder: "LowLevelIRBuilder", base: Value, index: Value, line: int + builder: LowLevelIRBuilder, base: Value, index: Value, line: int ) -> Value: """Get vec item, assuming index is non-negative and within bounds.""" assert isinstance(base.type, RVec) @@ -280,7 +280,7 @@ def vec_get_item_unsafe( def vec_set_item( - builder: "LowLevelIRBuilder", base: Value, index: Value, item: Value, line: int + builder: LowLevelIRBuilder, base: Value, index: Value, item: Value, line: int ) -> None: assert isinstance(base.type, RVec) index = as_platform_int(builder, index, line) @@ -306,7 +306,7 @@ def convert_to_t_ext_item(builder: LowLevelIRBuilder, item: Value) -> Value: return builder.add(SetElement(temp, "buf", vec_buf)) -def vec_append(builder: "LowLevelIRBuilder", vec: Value, item: Value, line: int) -> Value: +def vec_append(builder: LowLevelIRBuilder, vec: Value, item: Value, line: int) -> Value: vec_type = vec.type assert isinstance(vec_type, RVec) item_type = vec_type.item_type @@ -334,7 +334,7 @@ def vec_append(builder: "LowLevelIRBuilder", vec: Value, item: Value, line: int) return call -def vec_pop(builder: "LowLevelIRBuilder", base: Value, index: Value, line: int) -> Value: +def vec_pop(builder: LowLevelIRBuilder, base: Value, index: Value, line: int) -> Value: assert isinstance(base.type, RVec) vec_type = base.type item_type = vec_type.item_type @@ -358,7 +358,7 @@ def vec_pop(builder: "LowLevelIRBuilder", base: Value, index: Value, line: int) return builder.add(call) -def vec_remove(builder: "LowLevelIRBuilder", vec: Value, item: Value, line: int) -> Value: +def vec_remove(builder: LowLevelIRBuilder, vec: Value, item: Value, line: int) -> Value: assert isinstance(vec.type, RVec) vec_type = vec.type item_type = vec_type.item_type @@ -382,7 +382,7 @@ def vec_remove(builder: "LowLevelIRBuilder", vec: Value, item: Value, line: int) return builder.add(call) -def vec_contains(builder: "LowLevelIRBuilder", vec: Value, target: Value, line: int) -> Value: +def vec_contains(builder: LowLevelIRBuilder, vec: Value, target: Value, line: int) -> Value: assert isinstance(vec.type, RVec) vec_type = vec.type item_type = vec_type.item_type @@ -417,7 +417,7 @@ def vec_contains(builder: "LowLevelIRBuilder", vec: Value, target: Value, line: def vec_slice( - builder: "LowLevelIRBuilder", vec: Value, begin: Value, end: Value, line: int + builder: LowLevelIRBuilder, vec: Value, begin: Value, end: Value, line: int ) -> Value: assert isinstance(vec.type, RVec) vec_type = vec.type From f9e14e8e394ad52722e798d91198c865f7819637 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 8 Jul 2023 15:20:12 +0100 Subject: [PATCH 049/180] Support construction from an arbitrary iterable --- mypyc/irbuild/expression.py | 27 +++++++++++++++++++-- mypyc/test-data/irbuild-vec-i64.test | 36 ++++++++++++++++++++++++++++ mypyc/test-data/run-vec-i64.test | 22 +++++++++++++++++ 3 files changed, 83 insertions(+), 2 deletions(-) diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index ff21a477d5131..0050ca8940d3f 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -123,6 +123,7 @@ vec_pop, vec_remove, vec_slice, + vec_append, ) from mypyc.primitives.bytes_ops import bytes_slice_op from mypyc.primitives.dict_ops import dict_get_item_op, dict_new_op, exact_dict_set_item_op @@ -582,9 +583,31 @@ def translate_vec_create_from_iterable( return vec_create_from_values(builder.builder, vec_type, items, line) if isinstance(arg, ListComprehension): return translate_vec_comprehension(builder, vec_type, arg.generator) + return vec_from_iterable(builder, vec_type, arg, line) + + +def vec_from_iterable(builder: IRBuilder, vec_type: RVec, iterable: Expression, + line: int) -> Value: + """Construct a vec from an arbitrary iterable.""" + # Translate it as a vec comprehension vec[t]([ for in + # iterable]). This way we can use various special casing supported + # by for loops and comprehensions. + vec = Register(vec_type) + builder.assign(vec, vec_create(builder.builder, vec_type, 0, line), line) + name = f"___tmp_{line}" + var = Var(name) + reg = builder.add_local(var, vec_type.item_type) + index = NameExpr(name) + index.kind = LDEF + index.node = var + loop_params: list[tuple[Expression, Expression, list[Expression], bool]] = [ + (index, iterable, [], False)] - # TODO: Construct vec from arbitrary iterable - assert False, (vec_type, arg) + def gen_inner_stmts() -> None: + builder.assign(vec, vec_append(builder.builder, vec, reg, line), line) + + comprehension_helper(builder, loop_params, gen_inner_stmts, line) + return vec def translate_cast_expr(builder: IRBuilder, expr: CastExpr) -> Value: diff --git a/mypyc/test-data/irbuild-vec-i64.test b/mypyc/test-data/irbuild-vec-i64.test index d42472bb1130b..5f846c07bbe63 100644 --- a/mypyc/test-data/irbuild-vec-i64.test +++ b/mypyc/test-data/irbuild-vec-i64.test @@ -242,6 +242,42 @@ L3: L4: return r1 +[case testVecI64ConstructFromRange] +from vecs import vec, pop +from mypy_extensions import i64 + +def f() -> vec[i64]: + return vec[i64](range(7)) +[out] +def f(): + r0, r1 :: vec[int64] + r2 :: short_int + r3, ___tmp :: int64 + r4 :: bit + r5 :: vec[int64] + r6 :: short_int + r7 :: int64 +L0: + r0 = VecI64Api.alloc(0) + r1 = r0 + r2 = 0 + r3 = r2 >> 1 + ___tmp = r3 +L1: + r4 = r2 < 14 :: signed + if r4 goto L2 else goto L4 :: bool +L2: + r5 = VecI64Api.append(r1, ___tmp) + r1 = r5 +L3: + r6 = r2 + 2 + r2 = r6 + r7 = r6 >> 1 + ___tmp = r7 + goto L1 +L4: + return r1 + [case testVecI64ForLoop] from vecs import vec from mypy_extensions import i64 diff --git a/mypyc/test-data/run-vec-i64.test b/mypyc/test-data/run-vec-i64.test index e9d8dd5d7c4ea..830055425c945 100644 --- a/mypyc/test-data/run-vec-i64.test +++ b/mypyc/test-data/run-vec-i64.test @@ -123,6 +123,28 @@ def test_construct_from_list_comprehension() -> None: for j in range(i): assert v[j] == j * j + 5 +def test_construct_from_range() -> None: + v = vec[i64](range(0)) + assert v == vec[i64]() + v = vec[i64](range(5)) + assert v == vec[i64]([0, 1, 2, 3, 4]) + v = vec[i64](range(2, 5)) + assert v == vec[i64]([2, 3, 4]) + +def test_construct_from_iterable() -> None: + for i in range(50): + it: Iterable[i64] = iter([i * i for i in range(i)]) + v = vec[i64](it) + assert len(v) == i + for j in range(i): + assert v[j] == j * j + for i in range(50): + it2: Iterable[int] = iter([i * i for i in range(i)]) + v = vec[i64](it2) + assert len(v) == i + for j in range(i): + assert v[j] == j * j + def test_equality() -> None: v0 = vec[i64]() v0b = vec[i64]() From a24fa3545d1916d679fa1a04ac30c56dccdce2fc Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 8 Jul 2023 15:22:38 +0100 Subject: [PATCH 050/180] Test remove --- mypyc/test-data/run-vec-i64.test | 34 +++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/mypyc/test-data/run-vec-i64.test b/mypyc/test-data/run-vec-i64.test index 830055425c945..4385c6ea87a6b 100644 --- a/mypyc/test-data/run-vec-i64.test +++ b/mypyc/test-data/run-vec-i64.test @@ -2,7 +2,7 @@ from typing import Final, Any, Iterable from mypy_extensions import i64 -from vecs import vec, append +from vecs import vec, append, remove, pop from testutil import assertRaises @@ -229,3 +229,35 @@ def test_slicing_with_step() -> None: assert v[::-1] == vec[i64]([4, 3, 2, 1, 0]) with assertRaises(ValueError): v[1:3:0] + +def test_remove() -> None: + a = [4, 7, ERROR] + for i in a: + v = vec[i64](a) + v = remove(v, i) + assert v == vec[i64]([j for j in a if j != i]) + v = vec[i64](a) + v = remove(v, 4) + v = remove(v, 7) + v = remove(v, ERROR) + assert v == vec[i64]() + with assertRaises(ValueError): + remove(v, 4) + v = append(v, 5) + with assertRaises(ValueError): + remove(v, 7) + v = remove(v, 5) + assert len(v) == 0 + v = vec[i64]([1, 1, 1]) + v = remove(v, 1) + assert v == vec[i64]([1, 1]) + v = remove(v, 1) + assert v == vec[i64]([1]) + v = remove(v, 1) + assert v == vec[i64]() + f: Any = 1.1 + with assertRaises(TypeError): + remove(v, f) + s: Any = 'x' + with assertRaises(TypeError): + remove(v, s) From 542ed50164bfda3d0c894cd95dff30d6747f2cc0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 8 Jul 2023 15:33:14 +0100 Subject: [PATCH 051/180] Add test case for pop --- mypyc/test-data/run-vec-i64.test | 39 ++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/mypyc/test-data/run-vec-i64.test b/mypyc/test-data/run-vec-i64.test index 4385c6ea87a6b..88c72a74f4dfb 100644 --- a/mypyc/test-data/run-vec-i64.test +++ b/mypyc/test-data/run-vec-i64.test @@ -261,3 +261,42 @@ def test_remove() -> None: s: Any = 'x' with assertRaises(TypeError): remove(v, s) + +def test_pop_last() -> None: + v = vec[i64]([4, 7, ERROR]) + v, n = pop(v) + assert n == ERROR + assert v == vec[i64]([4, 7]) + v, n = pop(v) + assert n == 7 + assert v == vec[i64]([4]) + v, n = pop(v) + assert n == 4 + assert v == vec[i64]() + with assertRaises(IndexError): + pop(v) + +def test_pop_index() -> None: + v = vec[i64]([4, 7, 9, 15, 22]) + v, n = pop(v, 0) + assert n == 4 + assert v == vec[i64]([7, 9, 15, 22]) + v, n = pop(v, -1) + assert n == 22 + assert v == vec[i64]([7, 9, 15]) + v, n = pop(v, 1) + assert n == 9 + assert v == vec[i64]([7, 15]) + + with assertRaises(IndexError): + pop(v, 2) + + with assertRaises(IndexError): + pop(v, -3) + + v, n = pop(v, -2) + assert n == 7 + assert v == vec[i64]([15]) + v, n = pop(v, 0) + assert n == 15 + assert v == vec[i64]() From 2749467141e4fa06dde435f04466be01bc1263a6 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 8 Jul 2023 15:58:48 +0100 Subject: [PATCH 052/180] Add initial vec[t] run tests --- mypyc/test-data/run-vec-t.test | 286 +++++++++++++++++++++++++++++++++ mypyc/test/test_run.py | 1 + 2 files changed, 287 insertions(+) create mode 100644 mypyc/test-data/run-vec-t.test diff --git a/mypyc/test-data/run-vec-t.test b/mypyc/test-data/run-vec-t.test new file mode 100644 index 0000000000000..22c43d97d9caf --- /dev/null +++ b/mypyc/test-data/run-vec-t.test @@ -0,0 +1,286 @@ +[case testVecTBasicOps] +from typing import Final, Any, Iterable + +from mypy_extensions import i64 +from vecs import vec, append, remove, pop + +from testutil import assertRaises + +def test_create_empty() -> None: + v = vec[str]() + assert len(v) == 0 + +def test_create_from_list_and_get_item() -> None: + v = vec[str]([]) + assert len(v) == 0 + + v = vec[str](["xyz", "0" + str(), str(), "foo"]) + assert len(v) == 4 + assert v[0] == "xyz" + assert v[1] == "0" + assert v[2] == "" + assert v[3] == "foo" + + v = vec[str](["1", "2", "3", "4", "5", "6", "7", "8", "9"]) + assert len(v) == 9 + for i in range(i64(9)): + assert v[i] == str(i + 1) + +def test_append() -> None: + v = vec[str]() + v = append(v, str() + "3") + assert len(v) == 1 + x = "xyz" + str() + v = append(v, x) + assert len(v) == 2 + assert v[0] == "3" + assert v[1] == "xyz" + v = vec[str]() + for i in range(1024): + v = append(v, str(i * 3)) + assert len(v) == 1024 + for i in range(1024): + assert v[i] == str(i * 3) + +def test_get_item() -> None: + v = vec[str](["3", "4"]) + x: i64 = int() + assert v[x] == "3" + assert v[int()] == "3" + x += 1 + assert v[x] == "4" + assert v[int() + 1] == "4" + with assertRaises(IndexError): + v[x + 1] + with assertRaises(IndexError): + v[2] + with assertRaises(IndexError): + v[2**63 - 1] + +def test_set_item() -> None: + v = vec[str](["3", "4"]) + v[0] = "0" + assert v[0] == "0" + + x: i64 = int() + + v[x] = "4" + assert v[x] == "4" + + v[int()] = "5" + assert v[int()] == "5" + + x += 1 + + v[x] = "-5" + assert v[x] == "-5" + + v[1 + int()] = "xyz" + assert v[1] == "xyz" + + with assertRaises(IndexError): + v[1 + x] = "6" + with assertRaises(IndexError): + v[2 + int()] = "6" + +def test_box_and_unbox() -> None: + v = vec[str](["3", "5"]) + o: Any = v + assert len(o) == 2 + assert o[0] == "3" + assert o[1] == "5" + o[1] = "6" + v2: vec[str] = o + assert len(v2) == 2 + assert v2[0] == "3" + assert v2[1] == "6" + o2: Any = None + with assertRaises(TypeError, "vec[t] expected"): + v2 = o2 + o3: Any = vec[bytes]([]) + with assertRaises(TypeError, "vec[t] expected"): + v2 = o3 + +def test_construct_from_list_multiply() -> None: + for i in range(50): + v = vec[str]([str(i + 1)] * i) + assert len(v) == i + for j in range(i): + assert v[j] == str(i + 1) + for i in range(50): + v = vec[str](i * [str(i - 1)]) + assert len(v) == i + for j in range(i): + assert v[j] == str(i - 1) + +def test_construct_from_list_comprehension() -> None: + for i in range(50): + l = [i * i for i in range(i)] + v = vec[str]([str(n + 5) for n in l]) + assert len(v) == i + for j in range(i): + assert v[j] == str(j * j + 5) + +def test_construct_from_iterable() -> None: + for i in range(50): + it: Iterable[str] = iter([str(i * i) for i in range(i)]) + v = vec[str](it) + assert len(v) == i + for j in range(i): + assert v[j] == str(j * j) + +def test_equality() -> None: + v0 = vec[str]() + v0b = vec[str]() + v1 = vec[str](["1"]) + v1b = vec[str](["3"]) + assert v0 == v0 + assert v1 == v1 + assert v0 == v0b + assert v0 != v1 + assert v1 != v1b + assert v1 != v0 + +def test_str_conversion() -> None: + v = vec[str]() + assert str(v) == "vec[str]([])" + assert repr(v) == "vec[str]([])" + v = vec[str](["126"]) + assert str(v) == "vec[str](['126'])" + v = append(v, "5") + assert str(v) == "vec[str](['126', '5'])" + v = append(v, "xyz") + assert str(v) == "vec[str](['126', '5', 'xyz'])" + +def test_for_loop() -> None: + for n in vec[str](): + assert False + a = [] + for n in vec[str](["5"]): + a.append(n) + assert a == ["5"] + v = vec[str](["xyz", "9", "8"]) + a = [] + for n in v: + a.append(n) + assert a == ["xyz", "9", "8"] + assert len(v) == 3 + +def test_contains() -> None: + v = vec[str]() + x = str() + assert x not in v + v = vec[str](["x"]) + assert x + "x" in v + assert x not in v + v2 = vec[str](["xyz", "7", "9"]) + assert x + "xyz" in v2 + assert x + "7" in v2 + assert x + "9" in v2 + assert x not in v2 + assert x + "x" not in v2 + +def test_slicing() -> None: + v = vec[str]() + assert v[:] == vec[str]([]) + assert v[1:] == vec[str]([]) + assert v[:-5] == vec[str]([]) + v = vec[str](["0", "1", "2", "3", "4"]) + assert v[1:4] == vec[str](["1", "2", "3"]) + assert v[2:-1] == vec[str](["2", "3"]) + assert v[-2:-1] == vec[str](["3"]) + assert v[1:] == vec[str](["1", "2", "3", "4"]) + assert v[:-1] == vec[str](["0", "1", "2", "3"]) + assert v[:] == v + assert v[:] is not v + assert v[5:] == vec[str]() + assert v[100:] == vec[str]() + assert v[0:5] ==v + assert v[0:5] is not v + assert v[2:100] == vec[str](["2", "3", "4"]) + assert v[-100:2] == vec[str](["0", "1"]) + assert v[5:100] == vec[str]([]) + assert v[50:100] == vec[str]([]) + assert v[-100:-50] == vec[str]([]) + +def test_slicing_with_step() -> None: + v = vec[str](["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]) + assert v[1:5:2] == vec[str](["1", "3"]) + assert v[1:6:2] == vec[str](["1", "3", "5"]) + assert v[5:1:-2] == vec[str](["5", "3"]) + assert v[6:1:-2] == vec[str](["6", "4", "2"]) + v = vec[str](["0", "1", "2", "3", "4"]) + assert v[::-1] == vec[str](["4", "3", "2", "1", "0"]) + with assertRaises(ValueError): + v[1:3:0] + +def test_remove() -> None: + a = ["4", "7", "xyz"] + for i in a: + v = vec[str](a) + v = remove(v, i) + assert v == vec[str]([j for j in a if j != i]) + v = vec[str](a) + v = remove(v, "4") + v = remove(v, "7") + v = remove(v, "xyz") + assert v == vec[str]() + with assertRaises(ValueError): + remove(v, "4") + v = append(v, "5") + with assertRaises(ValueError): + remove(v, "7") + v = remove(v, "5") + assert len(v) == 0 + v = vec[str](["1", "1", "1"]) + v = remove(v, "1") + assert v == vec[str](["1", "1"]) + v = remove(v, "1") + assert v == vec[str](["1"]) + v = remove(v, "1") + assert v == vec[str]() + f: Any = 1.1 + with assertRaises(TypeError): + remove(v, f) + s: Any = None + with assertRaises(TypeError): + remove(v, s) + +def test_pop_last() -> None: + v = vec[str](["4", "7", "xyz"]) + v, n = pop(v) + assert n == "xyz" + assert v == vec[str](["4", "7"]) + v, n = pop(v) + assert n == "7" + assert v == vec[str](["4"]) + v, n = pop(v) + assert n == "4" + assert v == vec[str]() + with assertRaises(IndexError): + pop(v) + +def test_pop_index() -> None: + v = vec[str](["4", "7", "9", "15", "22"]) + v, n = pop(v, 0) + assert n == "4" + assert v == vec[str](["7", "9", "15", "22"]) + v, n = pop(v, -1) + assert n == "22" + assert v == vec[str](["7", "9", "15"]) + v, n = pop(v, 1) + assert n == "9" + assert v == vec[str](["7", "15"]) + + with assertRaises(IndexError): + pop(v, 2) + + with assertRaises(IndexError): + pop(v, -3) + + v, n = pop(v, -2) + assert n == "7" + assert v == vec[str](["15"]) + v, n = pop(v, 0) + assert n == "15" + assert v == vec[str]() diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index d41a5f63c38e8..9a6b7867f5e91 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -83,6 +83,7 @@ "run-vecs-t-interp.test", "run-vecs-nested-interp.test", "run-vec-i64.test", + "run-vec-t.test", ] if sys.version_info >= (3, 12): From bb4e022d183f0bec8a34c0a650638efb8ec824d6 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 9 Jul 2023 13:02:29 +0100 Subject: [PATCH 053/180] Add test cases for optional items --- mypyc/test-data/run-vec-t.test | 64 +++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/mypyc/test-data/run-vec-t.test b/mypyc/test-data/run-vec-t.test index 22c43d97d9caf..6e54b01de7ed1 100644 --- a/mypyc/test-data/run-vec-t.test +++ b/mypyc/test-data/run-vec-t.test @@ -1,5 +1,5 @@ [case testVecTBasicOps] -from typing import Final, Any, Iterable +from typing import Final, Any, Iterable, Optional from mypy_extensions import i64 from vecs import vec, append, remove, pop @@ -9,6 +9,8 @@ from testutil import assertRaises def test_create_empty() -> None: v = vec[str]() assert len(v) == 0 + v2 = vec[Optional[str]]() + assert len(v2) == 0 def test_create_from_list_and_get_item() -> None: v = vec[str]([]) @@ -26,6 +28,12 @@ def test_create_from_list_and_get_item() -> None: for i in range(i64(9)): assert v[i] == str(i + 1) +def test_create_optional_and_get_item() -> None: + v = vec[Optional[str]](["xyz", None]) + assert len(v) == 2 + assert v[0] == "xyz" + assert v[1] is None + def test_append() -> None: v = vec[str]() v = append(v, str() + "3") @@ -83,6 +91,13 @@ def test_set_item() -> None: with assertRaises(IndexError): v[2 + int()] = "6" +def test_set_item_optional() -> None: + v = vec[Optional[str]](["3", None]) + v[1] = str() + "a" + assert v[1] == "a" + v[0] = None + assert v[0] is None + def test_box_and_unbox() -> None: v = vec[str](["3", "5"]) o: Any = v @@ -101,6 +116,27 @@ def test_box_and_unbox() -> None: with assertRaises(TypeError, "vec[t] expected"): v2 = o3 +def test_box_and_unbox_optional() -> None: + v = vec[Optional[str]](["3", "4"]) + o: Any = v + o[0] = None + v = o + assert v == vec[Optional[str]]([None, "4"]) + + v2 = vec[str](["4"]) + o2: Any = v2 + o2[0] = "5" + v2 = o2 + assert v2 == vec[str](["5"]) + + o3: Any = vec[str]() + with assertRaises(TypeError, "vec[t] expected"): + v = o3 + + o4: Any = vec[Optional[str]](["3"]) + with assertRaises(TypeError, "vec[t] expected"): + v2 = o4 + def test_construct_from_list_multiply() -> None: for i in range(50): v = vec[str]([str(i + 1)] * i) @@ -141,6 +177,17 @@ def test_equality() -> None: assert v1 != v1b assert v1 != v0 +def test_equality_optional() -> None: + v = vec[str]() + vo = vec[Optional[str]]() + assert v != vo + assert vo == vo + vo2 = vec[Optional[str]](["x", None]) + assert vo2 == vec[Optional[str]](["x", None]) + assert vo2 != vec[Optional[str]](["x", "y"]) + assert vo2 != vec[Optional[str]]([None, None]) + assert vo2 != vec[Optional[str]](["x", None, None]) + def test_str_conversion() -> None: v = vec[str]() assert str(v) == "vec[str]([])" @@ -151,6 +198,10 @@ def test_str_conversion() -> None: assert str(v) == "vec[str](['126', '5'])" v = append(v, "xyz") assert str(v) == "vec[str](['126', '5', 'xyz'])" + v2 = vec[Optional[str]]() + assert str(v2) == "vec[str | None]([])" + v2 = vec[Optional[str]](["x", None]) + assert str(v2) == "vec[str | None](['x', None])" def test_for_loop() -> None: for n in vec[str](): @@ -284,3 +335,14 @@ def test_pop_index() -> None: v, n = pop(v, 0) assert n == "15" assert v == vec[str]() + +[case testVecTBasicOps_python_3_10] +from vecs import vec, append, remove, pop + +def test_optional_item_new_syntax() -> None: + v = vec[str | None]() + assert len(v) == 0 + assert str(v) == "vec[str | None]([])" + v = append(v, 'x') + v = append(v, None) + assert str(v) == "vec[str | None](['x', None])" From 15e5610583572915345fdb15bba900bfd8c6fea4 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 9 Jul 2023 13:36:11 +0100 Subject: [PATCH 054/180] Start adding nested vec test cases --- mypyc/test-data/run-vec-i64.test | 3 + mypyc/test-data/run-vec-nested.test | 230 ++++++++++++++++++++++++++++ mypyc/test-data/run-vec-t.test | 3 + mypyc/test/test_run.py | 1 + 4 files changed, 237 insertions(+) create mode 100644 mypyc/test-data/run-vec-nested.test diff --git a/mypyc/test-data/run-vec-i64.test b/mypyc/test-data/run-vec-i64.test index 88c72a74f4dfb..a27d2e1467f5f 100644 --- a/mypyc/test-data/run-vec-i64.test +++ b/mypyc/test-data/run-vec-i64.test @@ -1,3 +1,6 @@ +-- Test cases for vec[i64]. These also partially cover other unboxed item types, +-- which use a similar runtime representation. + [case testVecI64BasicOps] from typing import Final, Any, Iterable diff --git a/mypyc/test-data/run-vec-nested.test b/mypyc/test-data/run-vec-nested.test new file mode 100644 index 0000000000000..aa268abd068dd --- /dev/null +++ b/mypyc/test-data/run-vec-nested.test @@ -0,0 +1,230 @@ +[case testVecNestedBasicOps] +from typing import Final, Any, Iterable, Optional + +from mypy_extensions import i64 +from vecs import vec, append, remove, pop + +from testutil import assertRaises + +def test_construct_empty_nested() -> None: + v = vec[vec[str]]() + assert len(v) == 0 + v2 = vec[vec[vec[vec[vec[str]]]]]() + assert len(v2) == 0 + v3 = vec[vec[i64]]() + assert len(v3) == 0 + +def test_construct_empty_nested_optional() -> None: + v = vec[vec[Optional[str]]]() + assert len(v) == 0 + +def test_construct_from_initializer_nested() -> None: + v = vec[vec[str]]([vec[str](['xyz', 'a'])]) + assert len(v) == 1 + assert len(v[0]) == 2 + assert v[0][0] == 'xyz' + assert v[0][1] == 'a' + +def test_repr() -> None: + assert str(vec[vec[str]]()) == "vec[vec[str]]([])" + assert str(vec[vec[Optional[str]]]()) == "vec[vec[str | None]]([])" + +def test_append_nested() -> None: + v = vec[vec[str]]() + vv = append(vec[str](), '1') + v = append(v, vv) + assert str(v) == "vec[vec[str]]([['1']])" + for n in range(2, 10): + vv = append(vec[str](), str(n)) + v = append(v, vv) + assert str(v) == ( + "vec[vec[str]]([['1'], ['2'], ['3'], ['4'], ['5'], ['6'], ['7'], ['8'], ['9']])" + ) + +def test_append_i64() -> None: + v = vec[vec[i64]]() + vv = append(vec[i64](), 1) + v = append(v, vv) + assert str(v) == "vec[vec[i64]]([[1]])" + for n in range(2, 10): + vv = append(vec[i64](), n) + v = append(v, vv) + assert str(v) == ( + "vec[vec[i64]]([[1], [2], [3], [4], [5], [6], [7], [8], [9]])" + ) + +def test_append_nested_optional() -> None: + v = vec[vec[Optional[str]]]() + v = append(v, vec[Optional[str]]()) + v[0] = append(v[0], None) + v[0] = append(v[0], 'x') + assert str(v) == "vec[vec[str | None]]([[None, 'x']])" + +def test_get_item() -> None: + v = append(vec[vec[str]](), append(vec[str](), 'x')) + assert repr(v[0]) == "vec[str](['x'])" + v = append(v, append(vec[str](), 'y')) + assert repr(v[0]) == "vec[str](['x'])" + assert repr(v[1]) == "vec[str](['y'])" + +def test_get_item_error() -> None: + v = append(vec[vec[str]](), vec[str](['1'])) + v = append(v, vec[str](['3'])) + with assertRaises(IndexError): + v[2] + with assertRaises(IndexError): + v[-3] + +""" +def test_len() -> None: + v = vec[Optional[vec[str]]]() + assert len(v) == 0 + v = append(v, vec[str](['1'])) + assert len(v) == 1 + v = append(v, None) + assert len(v) == 2 + for i in range(100): + v = append(v, vec[str](['i'])) + assert len(v) == i + 3 + +def test_set_item_error_nested() -> None: + v = append(vec[vec[str]](), vec[str]()) + v = append(v, vec[str]()) + v = append(v, vec[str]()) + with assertRaises(IndexError): + v[3] = vec[str]() + with assertRaises(IndexError): + v[-1] = vec[str]() + assert repr(v[0]) == 'vec[str]([])' + +def test_equality() -> None: + assert vec[vec[str]]() == vec[vec[str]]() + assert vec[vec[str]]([vec[str](['x'])]) == vec[vec[str]]([vec[str](['x'])]) + assert vec[vec[str]]([vec[str](['x'])]) != vec[vec[str]]() + assert vec[vec[str]]([vec[str](['x'])]) != vec[vec[str]]([vec[str](['y'])]) + +def test_equality_different_types() -> None: + a = vec[vec[str]]() + b = vec[str]() + assert a == a + assert b == b + assert b != a + assert a != b + + c = vec[vec[vec[str]]]() + assert c != a + assert a != c + assert c == c + + d = vec[vec[Optional[str]]]() + assert d != a + assert a != d + assert d == d + +def test_slicing() -> None: + v = vec[vec[str]](vec[str]([str(i)]) for i in range(5)) + assert v[1:4] == vec[vec[str]]([v[1], v[2], v[3]]) + assert v[2:-1] == vec[vec[str]]([v[2], v[3]]) + assert v[-2:-1] == vec[vec[str]]([v[3]]) + assert v[1:] == vec[vec[str]]([v[1], v[2], v[3], v[4]]) + assert v[:-1] == vec[vec[str]]([v[0], v[1], v[2], v[3]]) + assert v[:] == v + assert v[:] is not v + assert v[0:5] ==v + assert v[0:5] is not v + assert v[2:100] == vec[vec[str]]([v[2], v[3], v[4]]) + assert v[-100:2] == vec[vec[str]]([v[0], v[1]]) + assert v[5:100] == vec[vec[str]]([]) + assert v[50:100] == vec[vec[str]]([]) + assert v[-100:-50] == vec[vec[str]]([]) + +def test_slicing_with_step() -> None: + v = vec[vec[str]](vec[str]([str(i)]) for i in range(10)) + assert v[1:5:2] == vec[vec[str]]([v[1], v[3]]) + assert v[1:6:2] == vec[vec[str]]([v[1], v[3], v[5]]) + assert v[5:1:-2] == vec[vec[str]]([v[5], v[3]]) + assert v[6:1:-2] == vec[vec[str]]([v[6], v[4], v[2]]) + v = vec[vec[str]](vec[str]([str(i)]) for i in range(5)) + assert v[::-1] == vec[vec[str]]([v[4], v[3], v[2], v[1], v[0]]) + with assertRaises(ValueError): + v[1:3:0] + +def test_slicing_with_different_item_types() -> None: + v = vec[Optional[str]](['x', None, 'y']) + assert v[1:] == vec[Optional[str]]([None, 'y']) + vv = vec[vec[str]]([vec[str](['x']), vec[str](['y'])]) + assert vv[1:] == vec[vec[str]]([vec[str](['y'])]) + +def test_remove() -> None: + a = ['4x', '7x', '9x'] + vv = [vec[str]([s]) for s in a] + for i, s in enumerate(a): + v = vec[vec[str]](vv) + v = remove(v, vv[i]) + assert v == vec[vec[str]]([j for j in vv if j != vv[i]]) + v = vec[vec[str]](vv) + # Make sure we have a different object identity + v = remove(v, vec[str]([a[i]])) + assert v == vec[vec[str]]([j for j in vv if j != vv[i]]) + v = vec[vec[str]](vv) + v = remove(v, vv[0]) + v = remove(v, vv[1]) + v = remove(v, vv[2]) + assert v == vec[vec[str]]() + with assertRaises(ValueError): + remove(v, vec[str](['4'])) + v = append(v, vec[str](['5'])) + with assertRaises(ValueError): + remove(v, vec[str](['7'])) + v = remove(v, vec[str](['5'])) + assert len(v) == 0 + v = v0 = vec[vec[str]]([vec[str](['x']) for _ in range(3)]) + v = remove(v, vec[str](['x'])) + assert v == v0[1:] + v = remove(v, vec[str](['x'])) + assert v == v0[2:] + v = remove(v, vec[str](['x'])) + assert v == vec[vec[str]]() + vb: Any = vec[bytes]([b'x']) + with assertRaises(TypeError): + remove(v, vb) + +def test_pop_last() -> None: + v = vec[Optional[vec[str]]]([vec[str](['4']), None, vec[str](['9'])]) + v, item = pop(v) + assert item == vec[str](['9']) + assert v == vec[Optional[vec[str]]]([vec[str](['4']), None]) + v, item = pop(v) + assert item is None + assert v == vec[Optional[vec[str]]]([vec[str](['4'])]) + v, item = pop(v) + assert item == vec[str](['4']) + assert v == vec[Optional[vec[str]]]() + with assertRaises(IndexError): + pop(v) + +def test_pop_index() -> None: + v = vec[vec[str]](vec[str]([s]) for s in ['4', '7', '9', '15', '22']) + v, item = pop(v, 0) + assert item == vec[str](['4']) + assert v == vec[vec[str]](vec[str]([s]) for s in ['7', '9', '15', '22']) + v, item = pop(v, -1) + assert item == vec[str](['22']) + assert v == vec[vec[str]](vec[str]([s]) for s in ['7', '9', '15']) + v, item = pop(v, 1) + assert item == vec[str](['9']) + assert v == vec[vec[str]](vec[str]([s]) for s in ['7', '15']) + + with assertRaises(IndexError): + pop(v, 2) + + with assertRaises(IndexError): + pop(v, -3) + + v, item = pop(v, -2) + assert item == vec[str](['7']) + assert v == vec[vec[str]]([vec[str](['15'])]) + v, item = pop(v, 0) + assert item == vec[str](['15']) + assert v == vec[vec[str]]() +""" diff --git a/mypyc/test-data/run-vec-t.test b/mypyc/test-data/run-vec-t.test index 6e54b01de7ed1..9898e9dda7d45 100644 --- a/mypyc/test-data/run-vec-t.test +++ b/mypyc/test-data/run-vec-t.test @@ -1,3 +1,6 @@ +-- Test cases for vec[t] where t is a boxed, non-vec type (PyObject *). +-- Also tests for vec[t | None], which uses the same representation. + [case testVecTBasicOps] from typing import Final, Any, Iterable, Optional diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index 9a6b7867f5e91..14db3b1e64805 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -84,6 +84,7 @@ "run-vecs-nested-interp.test", "run-vec-i64.test", "run-vec-t.test", + "run-vec-nested.test", ] if sys.version_info >= (3, 12): From 0d614371e3e5de8d80c3227742faba06e1d59c0d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 19 Aug 2023 13:16:44 +0100 Subject: [PATCH 055/180] Don't support vec[...] | None as item type, to match runtime --- mypyc/codegen/emit.py | 6 +++--- mypyc/ir/rtypes.py | 13 ++++--------- mypyc/irbuild/vec.py | 18 ++++++++---------- mypyc/test-data/irbuild-vec-i64.test | 8 ++++---- mypyc/test-data/irbuild-vec-nested.test | 4 ++-- mypyc/test-data/refcount.test | 2 +- mypyc/test/test_emitfunc.py | 8 +++----- 7 files changed, 25 insertions(+), 34 deletions(-) diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index d2831a0c00c00..ee1194e9d6c3a 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -1092,18 +1092,18 @@ def emit_unbox( self.emit_line(f"{dest} = VecI64Api.unbox({src});") else: depth = typ.depth() - optionals = typ.optional_flags() + optional = typ.is_optional() if is_int64_rprimitive(typ.unwrap_item_type()): type_value = "VEC_ITEM_TYPE_I64" else: type_value = f"(size_t)&{self.vec_item_type_c(typ)}" - if optionals & (1 << depth): + if optional: type_value = f"{type_value} | 1" if depth == 0: self.emit_line(f"{dest} = VecTApi.unbox({src}, {type_value});") else: self.emit_line( - f"{dest} = VecTExtApi.unbox({src}, {type_value}, {optionals}, {depth});") + f"{dest} = VecTExtApi.unbox({src}, {type_value}, {depth});") else: assert False, "Unboxing not implemented: %s" % typ diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 27ca1d19e6b79..31688932ff3e9 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -1060,18 +1060,13 @@ def unwrap_item_type(self) -> RPrimitive | RInstance: else: assert False, f"unexpected item type: {self.item_type}" - def optional_flags(self) -> int: + def is_optional(self) -> bool: item_type = self.item_type if isinstance(item_type, RUnion): - item_type = optional_value_type(item_type) - assert item_type is not None - if isinstance(item_type, RVec): - return (item_type.optional_flags() << 1) | 1 - else: - return 1 + return True elif isinstance(item_type, RVec): - return item_type.optional_flags() << 1 - return 0 + return item_type.is_optional() + return False def depth(self) -> int: item_type = self.item_type diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 1617d6c3aa7a9..2a48d2295c106 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -90,7 +90,7 @@ def vec_create( ) return builder.add(call) - typeobj, optionals, depth = vec_item_type_info(builder, item_type, line) + typeobj, optional, depth = vec_item_type_info(builder, item_type, line) if typeobj is not None: typeval: Value if isinstance(typeobj, Integer): @@ -100,7 +100,7 @@ def vec_create( # Assign implicitly coerces between pointer/integer types. typeval = Register(pointer_rprimitive) builder.add(Assign(typeval, typeobj)) - if optionals & 1: + if optional: typeval = builder.add( IntOp(pointer_rprimitive, typeval, Integer(1, pointer_rprimitive), IntOp.OR) ) @@ -121,7 +121,6 @@ def vec_create( [ length, typeval, - Integer(optionals, int32_rprimitive), Integer(depth, int32_rprimitive), ], vtype, @@ -187,7 +186,7 @@ def step_size(item_type: RType) -> int: def vec_item_type_info( builder: LowLevelIRBuilder, typ: RType, line: int -) -> Tuple[Optional[Value], int, int]: +) -> Tuple[Optional[Value], bool, int]: if isinstance(typ, RPrimitive) and typ.is_refcounted: typ, src = builtin_names[typ.name] return builder.load_address(src, typ), 0, 0 @@ -197,14 +196,13 @@ def vec_item_type_info( return Integer(VEC_TYPE_INFO_I64, c_size_t_rprimitive), 0, 0 elif isinstance(typ, RUnion): non_opt = optional_value_type(typ) - if non_opt is not None: - typeval, optionals, depth = vec_item_type_info(builder, non_opt, line) - if typeval is not None: - return typeval, optionals | 1, depth + typeval, _, _ = vec_item_type_info(builder, non_opt, line) + if typeval is not None: + return typeval, True, 0 elif isinstance(typ, RVec): - typeval, optionals, depth = vec_item_type_info(builder, typ.item_type, line) + typeval, optional, depth = vec_item_type_info(builder, typ.item_type, line) if typeval is not None: - return typeval, optionals << 1, depth + 1 + return typeval, optional, depth + 1 return None, 0, 0 diff --git a/mypyc/test-data/irbuild-vec-i64.test b/mypyc/test-data/irbuild-vec-i64.test index 5f846c07bbe63..db90ff752a581 100644 --- a/mypyc/test-data/irbuild-vec-i64.test +++ b/mypyc/test-data/irbuild-vec-i64.test @@ -252,7 +252,7 @@ def f() -> vec[i64]: def f(): r0, r1 :: vec[int64] r2 :: short_int - r3, ___tmp :: int64 + r3, ___tmp_5 :: int64 r4 :: bit r5 :: vec[int64] r6 :: short_int @@ -262,18 +262,18 @@ L0: r1 = r0 r2 = 0 r3 = r2 >> 1 - ___tmp = r3 + ___tmp_5 = r3 L1: r4 = r2 < 14 :: signed if r4 goto L2 else goto L4 :: bool L2: - r5 = VecI64Api.append(r1, ___tmp) + r5 = VecI64Api.append(r1, ___tmp_5) r1 = r5 L3: r6 = r2 + 2 r2 = r6 r7 = r6 >> 1 - ___tmp = r7 + ___tmp_5 = r7 goto L1 L4: return r1 diff --git a/mypyc/test-data/irbuild-vec-nested.test b/mypyc/test-data/irbuild-vec-nested.test index 65ddc1ca3c203..58f11016b3dea 100644 --- a/mypyc/test-data/irbuild-vec-nested.test +++ b/mypyc/test-data/irbuild-vec-nested.test @@ -17,12 +17,12 @@ def f(): L0: r0 = load_address PyUnicode_Type r1 = r0 - r2 = VecTExtApi.alloc(0, r1, 0, 1) + r2 = VecTExtApi.alloc(0, r1, 1) return r2 def g(): r0 :: vec[vec[int64]] L0: - r0 = VecTExtApi.alloc(0, 2, 0, 1) + r0 = VecTExtApi.alloc(0, 2, 1) return r0 [case testVecNestedAppend] diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index 120ef34298994..6d09d0341255b 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -1796,7 +1796,7 @@ def f(v, n): L0: r0 = load_address PyUnicode_Type r1 = r0 - r2 = VecTExtApi.alloc(0, r1, 0, 1) + r2 = VecTExtApi.alloc(0, r1, 1) r3 = r2.len r4 = n < r3 :: unsigned if r4 goto L2 else goto L3 :: bool diff --git a/mypyc/test/test_emitfunc.py b/mypyc/test/test_emitfunc.py index 378d4459da0d1..06ef147afd297 100644 --- a/mypyc/test/test_emitfunc.py +++ b/mypyc/test/test_emitfunc.py @@ -393,13 +393,11 @@ def test_unbox_vec(self) -> None: def test_unbox_vec_nested(self) -> None: self.assert_emit(Unbox(self.o, RVec(RVec(str_rprimitive)), 55), - """cpy_r_r0 = VecTExtApi.unbox(cpy_r_o, (size_t)&PyUnicode_Type, 0, 1);""") + """cpy_r_r0 = VecTExtApi.unbox(cpy_r_o, (size_t)&PyUnicode_Type, 1);""") self.assert_emit(Unbox(self.o, RVec(RVec(RUnion([str_rprimitive, none_rprimitive]))), 55), - """cpy_r_r0 = VecTExtApi.unbox(cpy_r_o, (size_t)&PyUnicode_Type | 1, 2, 1);""") - self.assert_emit(Unbox(self.o, RVec(RUnion([RVec(str_rprimitive), none_rprimitive])), 55), - """cpy_r_r0 = VecTExtApi.unbox(cpy_r_o, (size_t)&PyUnicode_Type, 1, 1);""") + """cpy_r_r0 = VecTExtApi.unbox(cpy_r_o, (size_t)&PyUnicode_Type | 1, 1);""") self.assert_emit(Unbox(self.o, RVec(RVec(int64_rprimitive)), 55), - """cpy_r_r0 = VecTExtApi.unbox(cpy_r_o, VEC_ITEM_TYPE_I64, 0, 1);""") + """cpy_r_r0 = VecTExtApi.unbox(cpy_r_o, VEC_ITEM_TYPE_I64, 1);""") def test_list_append(self) -> None: self.assert_emit( From e60aca1099ca487592199056b458b378f3a17570 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 19 Aug 2023 14:32:19 +0100 Subject: [PATCH 056/180] Allow empty vec[t] and vec[t | None] to be represented with a NULL buf This avoids some allocations. --- mypyc/codegen/emit.py | 21 +++++++----- mypyc/irbuild/vec.py | 22 +++++++++++-- mypyc/test-data/irbuild-vec-t.test | 53 ++++++++++++++++++++---------- mypyc/test/test_emitfunc.py | 6 +++- 4 files changed, 73 insertions(+), 29 deletions(-) diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index ee1194e9d6c3a..0203a2eda4fb1 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -1092,13 +1092,10 @@ def emit_unbox( self.emit_line(f"{dest} = VecI64Api.unbox({src});") else: depth = typ.depth() - optional = typ.is_optional() if is_int64_rprimitive(typ.unwrap_item_type()): type_value = "VEC_ITEM_TYPE_I64" else: - type_value = f"(size_t)&{self.vec_item_type_c(typ)}" - if optional: - type_value = f"{type_value} | 1" + type_value = self.vec_item_type_c(typ) if depth == 0: self.emit_line(f"{dest} = VecTApi.unbox({src}, {type_value});") else: @@ -1109,7 +1106,10 @@ def emit_unbox( def vec_item_type_c(self, typ: RVec) -> str: item_type = typ.unwrap_item_type() - return self.type_c_name(item_type) + type_value = f"(size_t)&{self.type_c_name(item_type)}" + if typ.is_optional(): + type_value = f"{type_value} | 1" + return type_value def type_c_name(self, typ: RPrimitive | RInstance) -> str | None: if isinstance(typ, RPrimitive) and typ.is_refcounted: @@ -1175,10 +1175,15 @@ def emit_box( elif isinstance(typ, RVec): if is_int64_rprimitive(typ.item_type): api = "VecI64Api" - elif typ.depth() == 0: - api = "VecTApi" - else: + elif typ.depth() > 0: api = "VecTExtApi" + else: + api = "VecTApi" + # Empty vecs of this sort don't describe item type, so it needs to be + # passed explicitly. + item_type = self.vec_item_type_c(typ) + self.emit_line(f"{declaration}{dest} = {api}.box({src}, {item_type});") + return self.emit_line(f"{declaration}{dest} = {api}.box({src});") else: assert not typ.is_unboxed diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 2a48d2295c106..f55c5e16e0c8f 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -304,24 +304,42 @@ def convert_to_t_ext_item(builder: LowLevelIRBuilder, item: Value) -> Value: return builder.add(SetElement(temp, "buf", vec_buf)) +def vec_item_type(builder: LowLevelIRBuilder, item_type: RType, line: int) -> Value: + typeobj, optional, depth = vec_item_type_info(builder, item_type, line) + if isinstance(typeobj, Integer): + return typeobj + else: + # Create an integer which will hold the type object * as an integral value. + # Assign implicitly coerces between pointer/integer types. + typeval = Register(pointer_rprimitive) + builder.add(Assign(typeval, typeobj)) + if optional: + typeval = builder.add( + IntOp(pointer_rprimitive, typeval, Integer(1, pointer_rprimitive), IntOp.OR) + ) + return typeval + + def vec_append(builder: LowLevelIRBuilder, vec: Value, item: Value, line: int) -> Value: vec_type = vec.type assert isinstance(vec_type, RVec) item_type = vec_type.item_type coerced_item = builder.coerce(item, item_type, line) + item_type_arg = [] if is_int64_rprimitive(item_type): name = "VecI64Api.append" elif vec_depth(vec_type) == 0: name = "VecTApi.append" + item_type_arg = [vec_item_type(builder, item_type, line)] else: coerced_item = convert_to_t_ext_item(builder, coerced_item) name = "VecTExtApi.append" call = builder.add( CallC( name, - [vec, coerced_item], + [vec, coerced_item] + item_type_arg, vec_type, - steals=[True, False], + steals=[True, False] + ([False] if item_type_arg else []), is_borrowed=False, error_kind=ERR_MAGIC, line=line, diff --git a/mypyc/test-data/irbuild-vec-t.test b/mypyc/test-data/irbuild-vec-t.test index 64da59a9dba6c..01e6cf1b56ad4 100644 --- a/mypyc/test-data/irbuild-vec-t.test +++ b/mypyc/test-data/irbuild-vec-t.test @@ -40,11 +40,15 @@ def f(v: vec[str]) -> vec[str]: def f(v): v :: vec[str] r0 :: str - r1 :: vec[str] + r1 :: object + r2 :: ptr + r3 :: vec[str] L0: r0 = 'x' - r1 = VecTApi.append(v, r0) - return r1 + r1 = load_address PyUnicode_Type + r2 = r1 + r3 = VecTApi.append(v, r0, r2) + return r3 [case testVecTOptionalCreateEmpty] from vecs import vec, append @@ -90,16 +94,25 @@ def f(v: vec[Optional[str]]) -> vec[Optional[str]]: def f(v): v :: vec[union[str, None]] r0 :: str - r1 :: vec[union[str, None]] - r2 :: object - r3 :: vec[union[str, None]] + r1 :: object + r2, r3 :: ptr + r4 :: vec[union[str, None]] + r5, r6 :: object + r7, r8 :: ptr + r9 :: vec[union[str, None]] L0: r0 = 'x' - r1 = VecTApi.append(v, r0) - v = r1 - r2 = box(None, 1) - r3 = VecTApi.append(v, r2) - return r3 + r1 = load_address PyUnicode_Type + r2 = r1 + r3 = r2 | 1 + r4 = VecTApi.append(v, r0, r3) + v = r4 + r5 = box(None, 1) + r6 = load_address PyUnicode_Type + r7 = r6 + r8 = r7 | 1 + r9 = VecTApi.append(v, r5, r8) + return r9 [case testVecTLen] from vecs import vec @@ -223,8 +236,10 @@ def f(n): r4, x :: int64 r5 :: bit r6 :: str - r7 :: vec[str] - r8 :: int64 + r7 :: object + r8 :: ptr + r9 :: vec[str] + r10 :: int64 L0: r0 = load_address PyUnicode_Type r1 = r0 @@ -237,12 +252,14 @@ L1: if r5 goto L2 else goto L4 :: bool L2: r6 = 'x' - r7 = VecTApi.append(r3, r6) - r3 = r7 + r7 = load_address PyUnicode_Type + r8 = r7 + r9 = VecTApi.append(r3, r6, r8) + r3 = r9 L3: - r8 = r4 + 1 - r4 = r8 - x = r8 + r10 = r4 + 1 + r4 = r10 + x = r10 goto L1 L4: return r3 diff --git a/mypyc/test/test_emitfunc.py b/mypyc/test/test_emitfunc.py index 06ef147afd297..ba75250d01151 100644 --- a/mypyc/test/test_emitfunc.py +++ b/mypyc/test/test_emitfunc.py @@ -114,6 +114,7 @@ def add_local(name: str, rtype: RType) -> Register: ) self.vi64 = add_local("vi64", RVec(int64_rprimitive)) self.vs = add_local("vs", RVec(str_rprimitive)) + self.vs_opt = add_local("vs", RVec(RUnion([str_rprimitive, none_rprimitive]))) self.vvs = add_local("vvs", RVec(RVec(str_rprimitive))) ir = ClassIR("A", "mod") ir.attributes = { @@ -371,7 +372,10 @@ def test_unbox_i64(self) -> None: def test_box_vec(self) -> None: self.assert_emit(Box(self.vi64), """cpy_r_r0 = VecI64Api.box(cpy_r_vi64);""") - self.assert_emit(Box(self.vs), """cpy_r_r0 = VecTApi.box(cpy_r_vs);""") + self.assert_emit(Box(self.vs), + """cpy_r_r0 = VecTApi.box(cpy_r_vs, (size_t)&PyUnicode_Type);""") + self.assert_emit(Box(self.vs_opt), + """cpy_r_r0 = VecTApi.box(cpy_r_vs, (size_t)&PyUnicode_Type | 1);""") self.assert_emit(Box(self.vvs), """cpy_r_r0 = VecTExtApi.box(cpy_r_vvs);""") def test_unbox_vec(self) -> None: From 1c727f268ee1049c42ba8946419fb31b229d1e63 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 19 Aug 2023 15:38:51 +0100 Subject: [PATCH 057/180] Use the latest alloc API that accepts capacity as well --- mypyc/irbuild/vec.py | 6 ++++-- mypyc/test-data/irbuild-vec-i64.test | 12 ++++++------ mypyc/test-data/irbuild-vec-nested.test | 4 ++-- mypyc/test-data/irbuild-vec-t.test | 10 +++++----- mypyc/test-data/refcount.test | 6 +++--- 5 files changed, 20 insertions(+), 18 deletions(-) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index f55c5e16e0c8f..1a3c42b51705c 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -86,7 +86,8 @@ def vec_create( item_type = vtype.item_type if is_int64_rprimitive(item_type): call = CallC( - "VecI64Api.alloc", [length], vtype, False, False, error_kind=ERR_MAGIC, line=line + "VecI64Api.alloc", [length, length], vtype, False, False, error_kind=ERR_MAGIC, + line=line ) return builder.add(call) @@ -107,7 +108,7 @@ def vec_create( if depth == 0: call = CallC( "VecTApi.alloc", - [length, typeval], + [length, length, typeval], vtype, False, False, @@ -119,6 +120,7 @@ def vec_create( call = CallC( "VecTExtApi.alloc", [ + length, length, typeval, Integer(depth, int32_rprimitive), diff --git a/mypyc/test-data/irbuild-vec-i64.test b/mypyc/test-data/irbuild-vec-i64.test index db90ff752a581..fdcbd2b19c97e 100644 --- a/mypyc/test-data/irbuild-vec-i64.test +++ b/mypyc/test-data/irbuild-vec-i64.test @@ -11,7 +11,7 @@ def f() -> vec[i64]: def f(): r0 :: vec[int64] L0: - r0 = VecI64Api.alloc(0) + r0 = VecI64Api.alloc(0, 0) return r0 [case testVecI64Len] @@ -125,7 +125,7 @@ def f(): r1 :: object r2, r3, r4, r5 :: ptr L0: - r0 = VecI64Api.alloc(3) + r0 = VecI64Api.alloc(3, 3) r1 = r0.buf r2 = get_element_ptr r1 items :: VecbufI64Object set_mem r2, 1 :: int64* @@ -154,7 +154,7 @@ def f(n): r6 :: bit r7 :: ptr L0: - r0 = VecI64Api.alloc(n) + r0 = VecI64Api.alloc(n, n) r1 = r0.buf r2 = get_element_ptr r1 items :: VecbufI64Object r3 = n * 8 @@ -189,7 +189,7 @@ def f(n, x): r6 :: bit r7 :: ptr L0: - r0 = VecI64Api.alloc(3) + r0 = VecI64Api.alloc(3, 3) r1 = r0.buf r2 = get_element_ptr r1 items :: VecbufI64Object r3 = 3 * 8 @@ -223,7 +223,7 @@ def f(n): r5 :: vec[int64] r6 :: int64 L0: - r0 = VecI64Api.alloc(0) + r0 = VecI64Api.alloc(0, 0) r1 = r0 r2 = 0 x = r2 @@ -258,7 +258,7 @@ def f(): r6 :: short_int r7 :: int64 L0: - r0 = VecI64Api.alloc(0) + r0 = VecI64Api.alloc(0, 0) r1 = r0 r2 = 0 r3 = r2 >> 1 diff --git a/mypyc/test-data/irbuild-vec-nested.test b/mypyc/test-data/irbuild-vec-nested.test index 58f11016b3dea..c833f3c1aff7a 100644 --- a/mypyc/test-data/irbuild-vec-nested.test +++ b/mypyc/test-data/irbuild-vec-nested.test @@ -17,12 +17,12 @@ def f(): L0: r0 = load_address PyUnicode_Type r1 = r0 - r2 = VecTExtApi.alloc(0, r1, 1) + r2 = VecTExtApi.alloc(0, 0, r1, 1) return r2 def g(): r0 :: vec[vec[int64]] L0: - r0 = VecTExtApi.alloc(0, 2, 1) + r0 = VecTExtApi.alloc(0, 0, 2, 1) return r0 [case testVecNestedAppend] diff --git a/mypyc/test-data/irbuild-vec-t.test b/mypyc/test-data/irbuild-vec-t.test index 01e6cf1b56ad4..b59c7eafb0958 100644 --- a/mypyc/test-data/irbuild-vec-t.test +++ b/mypyc/test-data/irbuild-vec-t.test @@ -19,7 +19,7 @@ def primitive(): L0: r0 = load_address PyUnicode_Type r1 = r0 - r2 = VecTApi.alloc(0, r1) + r2 = VecTApi.alloc(0, 0, r1) return r2 def native_class(): r0 :: object @@ -28,7 +28,7 @@ def native_class(): L0: r0 = __main__.C :: type r1 = r0 - r2 = VecTApi.alloc(0, r1) + r2 = VecTApi.alloc(0, 0, r1) return r2 [case testVecTAppend] @@ -70,7 +70,7 @@ L0: r0 = load_address PyUnicode_Type r1 = r0 r2 = r1 | 1 - r3 = VecTApi.alloc(0, r2) + r3 = VecTApi.alloc(0, 0, r2) return r3 def native_class(): r0 :: object @@ -80,7 +80,7 @@ L0: r0 = __main__.C :: type r1 = r0 r2 = r1 | 1 - r3 = VecTApi.alloc(0, r2) + r3 = VecTApi.alloc(0, 0, r2) return r3 [case testVecTOptionalAppend] @@ -243,7 +243,7 @@ def f(n): L0: r0 = load_address PyUnicode_Type r1 = r0 - r2 = VecTApi.alloc(0, r1) + r2 = VecTApi.alloc(0, 0, r1) r3 = r2 r4 = 0 x = r4 diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index 6d09d0341255b..542d77a549333 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -1561,7 +1561,7 @@ L0: r1 = load_address PyUnicode_Type r2 = r1 r3 = r2 | 1 - r4 = VecTApi.alloc(n, r3) + r4 = VecTApi.alloc(n, n, r3) r5 = r4.buf r6 = get_element_ptr r5 items :: VecbufTObject r7 = n * 8 @@ -1648,7 +1648,7 @@ L0: r1 = 'y' r2 = load_address PyUnicode_Type r3 = r2 - r4 = VecTApi.alloc(2, r3) + r4 = VecTApi.alloc(2, 2, r3) r5 = r4.buf r6 = get_element_ptr r5 items :: VecbufTObject inc_ref r0 @@ -1796,7 +1796,7 @@ def f(v, n): L0: r0 = load_address PyUnicode_Type r1 = r0 - r2 = VecTExtApi.alloc(0, r1, 1) + r2 = VecTExtApi.alloc(0, 0, r1, 1) r3 = r2.len r4 = n < r3 :: unsigned if r4 goto L2 else goto L3 :: bool From 93124891172ae84ca7f158dcadfd000fc54c5204 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 19 Aug 2023 17:59:24 +0100 Subject: [PATCH 058/180] Make operator assignment faster --- mypyc/irbuild/ll_builder.py | 22 +++++++++++ mypyc/test-data/irbuild-vec-i64.test | 55 ++++++++++++++++++++++++++++ mypyc/test-data/run-vec-i64.test | 13 +++++++ 3 files changed, 90 insertions(+) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 81bc4699e7406..bdfa9a38801fe 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -2839,12 +2839,34 @@ def translate_special_method_call( Return None if no translation found; otherwise return the target register. """ + low_level_op = self._translate_special_low_level_method_call( + base_reg, name, args, result_type, line, can_borrow) + if low_level_op is not None: + return low_level_op primitive_ops_candidates = method_call_ops.get(name, []) primitive_op = self.matching_primitive_op( primitive_ops_candidates, [base_reg] + args, line, result_type, can_borrow=can_borrow ) return primitive_op + def _translate_special_low_level_method_call( + self, + base_reg: Value, + name: str, + args: list[Value], + result_type: RType | None, + line: int, + can_borrow: bool = False, + ) -> Value | None: + if name == "__getitem__" and len(args) == 1: + arg = args[0] + if isinstance(base_reg.type, RVec) and (is_int64_rprimitive(arg.type) or + is_tagged(arg.type)): + if is_tagged(arg.type): + arg = self.coerce(arg, int64_rprimitive, line) + return vec_get_item(self, base_reg, arg, line) + return None + def translate_eq_cmp(self, lreg: Value, rreg: Value, expr_op: str, line: int) -> Value | None: """Add an equality comparison operation. diff --git a/mypyc/test-data/irbuild-vec-i64.test b/mypyc/test-data/irbuild-vec-i64.test index fdcbd2b19c97e..ca71ae3aa0af1 100644 --- a/mypyc/test-data/irbuild-vec-i64.test +++ b/mypyc/test-data/irbuild-vec-i64.test @@ -477,3 +477,58 @@ def pop_nth(v, n): L0: r0 = VecI64Api.pop(v, n) return r0 + +[case testVecI64InPlaceOp] +from vecs import vec, remove +from mypy_extensions import i64 + +def inplace(v: vec[i64], n: i64, m: i64) -> None: + v[n] += m +[out] +def inplace(v, n, m): + v :: vec[int64] + n, m :: int64 + r0 :: native_int + r1 :: bit + r2 :: bool + r3 :: object + r4 :: ptr + r5 :: int64 + r6 :: ptr + r7, r8 :: int64 + r9 :: native_int + r10 :: bit + r11 :: bool + r12 :: object + r13 :: ptr + r14 :: int64 + r15 :: ptr +L0: + r0 = v.len + r1 = n < r0 :: unsigned + if r1 goto L2 else goto L1 :: bool +L1: + r2 = raise IndexError + unreachable +L2: + r3 = v.buf + r4 = get_element_ptr r3 items :: VecbufI64Object + r5 = n * 8 + r6 = r4 + r5 + r7 = load_mem r6 :: int64* + keep_alive v + r8 = r7 + m + r9 = v.len + r10 = n < r9 :: unsigned + if r10 goto L4 else goto L3 :: bool +L3: + r11 = raise IndexError + unreachable +L4: + r12 = v.buf + r13 = get_element_ptr r12 items :: VecbufI64Object + r14 = n * 8 + r15 = r13 + r14 + set_mem r15, r8 :: int64* + keep_alive v + return 1 diff --git a/mypyc/test-data/run-vec-i64.test b/mypyc/test-data/run-vec-i64.test index a27d2e1467f5f..30e86fed664b8 100644 --- a/mypyc/test-data/run-vec-i64.test +++ b/mypyc/test-data/run-vec-i64.test @@ -88,6 +88,19 @@ def test_set_item() -> None: with assertRaises(IndexError): v[2 + int()] = 6 +def test_operator_assignment() -> None: + v = vec[i64]([3, 4]) + x: i64 = int() + v[x] += 2 + assert v[0] == 5 + v[x + 1] -= 10 + assert v[1] == -6 + y = int() + v[y] += 6 + assert v[y] == 11 + v[y + 1] *= 2 + assert v[1 + y] == -12 + def test_box_and_unbox() -> None: v = vec[i64]([3, 5]) o: Any = v From d23195dac518637d79072033d04e6bad10bce12e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 26 Aug 2023 12:14:09 +0100 Subject: [PATCH 059/180] Add some checks for vec item types (incomplete) --- mypy/typeanal.py | 1 + mypyc/test-data/irbuild-vec-t.test | 46 ++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/mypy/typeanal.py b/mypy/typeanal.py index 5190eeb2df2c5..cc4aa5a9ec9df 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -883,6 +883,7 @@ def analyze_type_with_type_info( ): return AnyType(TypeOfAny.from_error) + # Check type argument count. instance.args = tuple(flatten_nested_tuples(instance.args)) if not (self.defining_alias and self.nesting_level == 0) and not validate_instance( diff --git a/mypyc/test-data/irbuild-vec-t.test b/mypyc/test-data/irbuild-vec-t.test index b59c7eafb0958..c0618c92a2269 100644 --- a/mypyc/test-data/irbuild-vec-t.test +++ b/mypyc/test-data/irbuild-vec-t.test @@ -263,3 +263,49 @@ L3: goto L1 L4: return r3 + +[case testVecTCheckItemType] +from vecs import vec +from typing import Tuple + +def bad1(v: vec[Tuple[str, str]]) -> None: pass +def bad2(v: vec[int]) -> None: pass +def bad3(v: vec) -> None: pass + +def ok1(v: vec[str], v2: vec[bytes]) -> None: pass +def ok2(v: vec[Tuple[str, ...]]) -> None: pass +def ok3(v: vec[object]) -> None: pass +[out] +main:4: error: Invalid item type for "vec" +main:5: error: Invalid item type for "vec" +main:6: error: Invalid item type for "vec" + +[case testVecTCheckItemTypeUnion] +from vecs import vec +from typing import Tuple, Union, Optional +from mypy_extensions import i64 + +def bad1(v: vec[Union[str, bytes]]) -> None: pass +def bad2(v: vec[Union[str, bytes, None]]) -> None: pass +def bad3(v: vec[Union[int, None]]) -> None: pass +def bad4(v: vec[Union[None, int]]) -> None: pass +def bad5(v: vec[Union[i64, None]]) -> None: pass +def bad6(v: vec[Union[None, i64]]) -> None: pass +def bad7(v: vec[Union[bool, None]]) -> None: pass +def bad9(v: vec[Union[float, None]]) -> None: pass +def bad10(v: vec[Union[vec[str], None]]) -> None: pass +def bad11(v: vec[Optional[vec[str]]]) -> None: pass + +def ok1(v: vec[Union[str, None]]) -> None: pass +def ok2(v: vec[Union[None, str]]) -> None: pass +[out] +main:5: error: Invalid item type for "vec" +main:6: error: Invalid item type for "vec" +main:7: error: Invalid item type for "vec" +main:8: error: Invalid item type for "vec" +main:9: error: Invalid item type for "vec" +main:10: error: Invalid item type for "vec" +main:11: error: Invalid item type for "vec" +main:12: error: Invalid item type for "vec" +main:13: error: Invalid item type for "vec" +main:14: error: Invalid item type for "vec" From 517975b7be72cad852054ceeafc288b667437f97 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 26 Aug 2023 12:41:23 +0100 Subject: [PATCH 060/180] Check vec item types during construction --- mypy/semanal.py | 4 ++++ mypyc/test-data/irbuild-vec-t.test | 14 ++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/mypy/semanal.py b/mypy/semanal.py index f38a71cb16e30..60e82ed8707cf 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -251,6 +251,7 @@ fix_instance, has_any_from_unimported_type, type_constructors, + check_vec_type_args, validate_instance, ) from mypy.typeops import function_type, get_type_vars, try_getting_str_literals_from_type @@ -6178,6 +6179,9 @@ def analyze_type_application(self, expr: IndexExpr) -> None: expr.analyzed = TypeApplication(base, types) expr.analyzed.line = expr.line expr.analyzed.column = expr.column + n = self.lookup_type_node(base) + if n and n.fullname == "vecs.vec": + check_vec_type_args(types, expr, self) if isinstance(base, RefExpr) and base.fullname == "librt.vecs.vec": # Apply restrictions specific to vec diff --git a/mypyc/test-data/irbuild-vec-t.test b/mypyc/test-data/irbuild-vec-t.test index c0618c92a2269..ae60d626b5564 100644 --- a/mypyc/test-data/irbuild-vec-t.test +++ b/mypyc/test-data/irbuild-vec-t.test @@ -309,3 +309,17 @@ main:11: error: Invalid item type for "vec" main:12: error: Invalid item type for "vec" main:13: error: Invalid item type for "vec" main:14: error: Invalid item type for "vec" + +[case testVecTCheckItemTypeDuringConstruction] +from vecs import vec +from typing import Optional, Union + +def f() -> None: + vec[Optional[int]]() + vec[Union[str, bytes]]() + vec[Optional[vec[str]]]() + +[out] +main:5: error: Invalid item type for "vec" +main:6: error: Invalid item type for "vec" +main:7: error: Invalid item type for "vec" From f7b079159aaec964a51e7556473b7980db985c2f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 26 Aug 2023 12:57:06 +0100 Subject: [PATCH 061/180] Enable nested vec test cases --- mypyc/test-data/run-vec-nested.test | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/mypyc/test-data/run-vec-nested.test b/mypyc/test-data/run-vec-nested.test index aa268abd068dd..87876a922fc39 100644 --- a/mypyc/test-data/run-vec-nested.test +++ b/mypyc/test-data/run-vec-nested.test @@ -75,13 +75,12 @@ def test_get_item_error() -> None: with assertRaises(IndexError): v[-3] -""" def test_len() -> None: - v = vec[Optional[vec[str]]]() + v = vec[vec[str]]() assert len(v) == 0 v = append(v, vec[str](['1'])) assert len(v) == 1 - v = append(v, None) + v = append(v, vec[str]()) assert len(v) == 2 for i in range(100): v = append(v, vec[str](['i'])) @@ -121,6 +120,10 @@ def test_equality_different_types() -> None: assert a != d assert d == d + assert vec[vec[str]]() != vec[vec[bytes]]() + assert vec[vec[str]]() != vec[vec[i64]]() + +""" def test_slicing() -> None: v = vec[vec[str]](vec[str]([str(i)]) for i in range(5)) assert v[1:4] == vec[vec[str]]([v[1], v[2], v[3]]) From ec4512c85ff66bd4c596c55ae34a81c4e5e992e5 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 26 Aug 2023 14:22:50 +0100 Subject: [PATCH 062/180] Fix step size calculation for nested vecs --- mypyc/irbuild/vec.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 1a3c42b51705c..54e7cdb802a7f 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -179,6 +179,8 @@ def vec_create_from_values( def step_size(item_type: RType) -> int: if isinstance(item_type, RPrimitive): return item_type.size + elif isinstance(item_type, RVec): + return PLATFORM_SIZE * 2 else: return PLATFORM_SIZE From d65bf6d0395a93a7beb135897142e179f055bca2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 26 Aug 2023 14:24:42 +0100 Subject: [PATCH 063/180] Enable test cases --- mypyc/test-data/run-vec-nested.test | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/mypyc/test-data/run-vec-nested.test b/mypyc/test-data/run-vec-nested.test index 87876a922fc39..f1ce0f4784955 100644 --- a/mypyc/test-data/run-vec-nested.test +++ b/mypyc/test-data/run-vec-nested.test @@ -123,7 +123,6 @@ def test_equality_different_types() -> None: assert vec[vec[str]]() != vec[vec[bytes]]() assert vec[vec[str]]() != vec[vec[i64]]() -""" def test_slicing() -> None: v = vec[vec[str]](vec[str]([str(i)]) for i in range(5)) assert v[1:4] == vec[vec[str]]([v[1], v[2], v[3]]) @@ -153,11 +152,10 @@ def test_slicing_with_step() -> None: v[1:3:0] def test_slicing_with_different_item_types() -> None: - v = vec[Optional[str]](['x', None, 'y']) - assert v[1:] == vec[Optional[str]]([None, 'y']) - vv = vec[vec[str]]([vec[str](['x']), vec[str](['y'])]) - assert vv[1:] == vec[vec[str]]([vec[str](['y'])]) + v = vec[vec[i64]]([vec[i64]([11]), vec[i64]([22])]) + assert v[1:] == vec[vec[i64]]([vec[i64]([22])]) +""" def test_remove() -> None: a = ['4x', '7x', '9x'] vv = [vec[str]([s]) for s in a] @@ -193,16 +191,16 @@ def test_remove() -> None: remove(v, vb) def test_pop_last() -> None: - v = vec[Optional[vec[str]]]([vec[str](['4']), None, vec[str](['9'])]) + v = vec[vec[str]]([vec[str](['4']), vec[str](['6']), vec[str](['9'])]) v, item = pop(v) assert item == vec[str](['9']) - assert v == vec[Optional[vec[str]]]([vec[str](['4']), None]) + assert v == vec[vec[str]]([vec[str](['4']), vec[str](['6'])]) v, item = pop(v) - assert item is None - assert v == vec[Optional[vec[str]]]([vec[str](['4'])]) + assert item == vec[str](['6']) + assert v == vec[vec[str]]([vec[str](['4'])]) v, item = pop(v) assert item == vec[str](['4']) - assert v == vec[Optional[vec[str]]]() + assert v == vec[vec[str]]() with assertRaises(IndexError): pop(v) From afffa93b07c90d2809c843b55d07f2440f2dff83 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 2 Sep 2023 17:56:37 +0100 Subject: [PATCH 064/180] Fix tests after rebase --- mypyc/test-data/exceptions.test | 6 +- mypyc/test-data/irbuild-dict.test | 10 +- mypyc/test-data/irbuild-vec-i64.test | 132 ++++++++++++------------ mypyc/test-data/irbuild-vec-nested.test | 44 ++++---- mypyc/test-data/irbuild-vec-t.test | 14 +-- mypyc/test-data/refcount.test | 32 +++--- 6 files changed, 119 insertions(+), 119 deletions(-) diff --git a/mypyc/test-data/exceptions.test b/mypyc/test-data/exceptions.test index 703f0ad999afc..8089549c194be 100644 --- a/mypyc/test-data/exceptions.test +++ b/mypyc/test-data/exceptions.test @@ -728,9 +728,9 @@ def f(v: vec[i64], i: i64, n: i64) -> None: v = remove(v, n) [out] def f(v, i, n): - v :: vec[int64] - i, n :: int64 - r0 :: vec[int64] + v :: vec[i64] + i, n :: i64 + r0 :: vec[i64] r1 :: None L0: r0 = VecI64Api.remove(v, n) diff --git a/mypyc/test-data/irbuild-dict.test b/mypyc/test-data/irbuild-dict.test index dd39c7f3977de..e209cb73632b2 100644 --- a/mypyc/test-data/irbuild-dict.test +++ b/mypyc/test-data/irbuild-dict.test @@ -159,7 +159,7 @@ def increment(d): r6 :: object r7, k :: str r8, r9, r10 :: object - r11 :: int32 + r11 :: i32 r12, r13, r14 :: bit L0: r0 = 0 @@ -249,7 +249,7 @@ def print_dict_methods(d1, d2): r6 :: object r7, v :: int r8 :: object - r9 :: int32 + r9 :: i32 r10 :: bit r11 :: bool r12, r13 :: bit @@ -262,7 +262,7 @@ def print_dict_methods(d1, d2): r20, r21 :: object r22, r23, k :: int r24, r25, r26, r27, r28 :: object - r29 :: int32 + r29 :: i32 r30, r31, r32 :: bit L0: r0 = 0 @@ -281,7 +281,7 @@ L2: r8 = box(int, v) r9 = PyDict_Contains(d2, r8) r10 = r9 >= 0 :: signed - r11 = truncate r9: int32 to builtins.bool + r11 = truncate r9: i32 to builtins.bool if r11 goto L3 else goto L4 :: bool L3: return 1 @@ -384,7 +384,7 @@ def typeddict(d): r8, k :: str v :: object r9 :: str - r10 :: int32 + r10 :: i32 r11 :: bit r12 :: object r13, r14, r15 :: bit diff --git a/mypyc/test-data/irbuild-vec-i64.test b/mypyc/test-data/irbuild-vec-i64.test index ca71ae3aa0af1..fb8ebc5f80e7c 100644 --- a/mypyc/test-data/irbuild-vec-i64.test +++ b/mypyc/test-data/irbuild-vec-i64.test @@ -9,7 +9,7 @@ def f() -> vec[i64]: return vec[i64]() [out] def f(): - r0 :: vec[int64] + r0 :: vec[i64] L0: r0 = VecI64Api.alloc(0, 0) return r0 @@ -23,9 +23,9 @@ def f(v: vec[i64]) -> i64: return l [out] def f(v): - v :: vec[int64] + v :: vec[i64] r0 :: native_int - l :: int64 + l :: i64 L0: r0 = v.len l = r0 @@ -39,16 +39,16 @@ def f(v: vec[i64], i: i64) -> i64: return v[i] [out] def f(v, i): - v :: vec[int64] - i :: int64 + v :: vec[i64] + i :: i64 r0 :: native_int r1 :: bit r2 :: bool r3 :: object r4 :: ptr - r5 :: int64 + r5 :: i64 r6 :: ptr - r7 :: int64 + r7 :: i64 L0: r0 = v.len r1 = i < r0 :: unsigned @@ -61,7 +61,7 @@ L2: r4 = get_element_ptr r3 items :: VecbufI64Object r5 = i * 8 r6 = r4 + r5 - r7 = load_mem r6 :: int64* + r7 = load_mem r6 :: i64* keep_alive v return r7 @@ -73,9 +73,9 @@ def f(v: vec[i64], i: i64) -> vec[i64]: return append(v, i) [out] def f(v, i): - v :: vec[int64] - i :: int64 - r0 :: vec[int64] + v :: vec[i64] + i :: i64 + r0 :: vec[i64] L0: r0 = VecI64Api.append(v, i) return r0 @@ -88,14 +88,14 @@ def f(v: vec[i64], i: i64, x: i64) -> None: v[i] = x [out] def f(v, i, x): - v :: vec[int64] - i, x :: int64 + v :: vec[i64] + i, x :: i64 r0 :: native_int r1 :: bit r2 :: bool r3 :: object r4 :: ptr - r5 :: int64 + r5 :: i64 r6 :: ptr L0: r0 = v.len @@ -109,7 +109,7 @@ L2: r4 = get_element_ptr r3 items :: VecbufI64Object r5 = i * 8 r6 = r4 + r5 - set_mem r6, x :: int64* + set_mem r6, x :: i64* keep_alive v return 1 @@ -121,18 +121,18 @@ def f() -> vec[i64]: return vec[i64]([1, 5, 14]) [out] def f(): - r0 :: vec[int64] + r0 :: vec[i64] r1 :: object r2, r3, r4, r5 :: ptr L0: r0 = VecI64Api.alloc(3, 3) r1 = r0.buf r2 = get_element_ptr r1 items :: VecbufI64Object - set_mem r2, 1 :: int64* + set_mem r2, 1 :: i64* r3 = r2 + 8 - set_mem r3, 5 :: int64* + set_mem r3, 5 :: i64* r4 = r3 + 8 - set_mem r4, 14 :: int64* + set_mem r4, 14 :: i64* r5 = r4 + 8 keep_alive r0 return r0 @@ -145,11 +145,11 @@ def f(n: i64) -> vec[i64]: return vec[i64]([3] * n) [out] def f(n): - n :: int64 - r0 :: vec[int64] + n :: i64 + r0 :: vec[i64] r1 :: object r2 :: ptr - r3 :: int64 + r3 :: i64 r4, r5 :: ptr r6 :: bit r7 :: ptr @@ -164,7 +164,7 @@ L1: r6 = r5 < r4 :: unsigned if r6 goto L2 else goto L3 :: bool L2: - set_mem r5, 3 :: int64* + set_mem r5, 3 :: i64* r7 = r5 + 8 r5 = r7 goto L1 @@ -180,8 +180,8 @@ def f(n: i64, x: i64) -> vec[i64]: return vec[i64]([x] * 3) [out] def f(n, x): - n, x :: int64 - r0 :: vec[int64] + n, x :: i64 + r0 :: vec[i64] r1 :: object r2 :: ptr r3 :: native_int @@ -199,7 +199,7 @@ L1: r6 = r5 < r4 :: unsigned if r6 goto L2 else goto L3 :: bool L2: - set_mem r5, x :: int64* + set_mem r5, x :: i64* r7 = r5 + 8 r5 = r7 goto L1 @@ -215,13 +215,13 @@ def f(n: i64) -> vec[i64]: return vec[i64]([x + 1 for x in range(i64(5))]) [out] def f(n): - n :: int64 - r0, r1 :: vec[int64] - r2, x :: int64 + n :: i64 + r0, r1 :: vec[i64] + r2, x :: i64 r3 :: bit - r4 :: int64 - r5 :: vec[int64] - r6 :: int64 + r4 :: i64 + r5 :: vec[i64] + r6 :: i64 L0: r0 = VecI64Api.alloc(0, 0) r1 = r0 @@ -250,13 +250,13 @@ def f() -> vec[i64]: return vec[i64](range(7)) [out] def f(): - r0, r1 :: vec[int64] + r0, r1 :: vec[i64] r2 :: short_int - r3, ___tmp_5 :: int64 + r3, ___tmp_5 :: i64 r4 :: bit - r5 :: vec[int64] + r5 :: vec[i64] r6 :: short_int - r7 :: int64 + r7 :: i64 L0: r0 = VecI64Api.alloc(0, 0) r1 = r0 @@ -289,15 +289,15 @@ def f(v: vec[i64]) -> i64: return t [out] def f(v): - v :: vec[int64] - t :: int64 + v :: vec[i64] + t :: i64 r0, r1 :: native_int r2 :: bit r3 :: object r4 :: ptr r5 :: native_int r6 :: ptr - r7, x, r8 :: int64 + r7, x, r8 :: i64 r9 :: native_int L0: t = 0 @@ -311,7 +311,7 @@ L2: r4 = get_element_ptr r3 items :: VecbufI64Object r5 = r0 * 8 r6 = r4 + r5 - r7 = load_mem r6 :: int64* + r7 = load_mem r6 :: i64* keep_alive v x = r7 r8 = t + 1 @@ -331,15 +331,15 @@ def contains(v: vec[i64], n: i64) -> bool: return n in v [out] def contains(v, n): - v :: vec[int64] - n :: int64 + v :: vec[i64] + n :: i64 r0 :: native_int r1 :: object r2 :: ptr r3 :: native_int r4, r5 :: ptr r6 :: bit - r7 :: int64 + r7 :: i64 r8 :: bit r9 :: ptr r10 :: bool @@ -354,7 +354,7 @@ L1: r6 = r5 < r4 :: unsigned if r6 goto L2 else goto L4 :: bool L2: - r7 = load_mem r5 :: int64* + r7 = load_mem r5 :: i64* r8 = r7 == n if r8 goto L5 else goto L3 :: bool L3: @@ -378,15 +378,15 @@ def f(v: vec[i64]) -> i64: return v[0] [out] def f(v): - v :: vec[int64] + v :: vec[i64] r0 :: native_int r1 :: bit r2 :: bool r3 :: object r4 :: ptr - r5 :: int64 + r5 :: i64 r6 :: ptr - r7 :: int64 + r7 :: i64 L0: r0 = v.len r1 = 0 < r0 :: unsigned @@ -399,7 +399,7 @@ L2: r4 = get_element_ptr r3 items :: VecbufI64Object r5 = 0 * 8 r6 = r4 + r5 - r7 = load_mem r6 :: int64* + r7 = load_mem r6 :: i64* keep_alive v return r7 @@ -415,9 +415,9 @@ def f(v: vec[i64], n: i64, m: i64) -> None: e = v[1:-2] [out] def f(v, n, m): - v :: vec[int64] - n, m :: int64 - r0, a, r1, b, r2, c, r3, d, r4, e :: vec[int64] + v :: vec[i64] + n, m :: i64 + r0, a, r1, b, r2, c, r3, d, r4, e :: vec[i64] L0: r0 = VecI64Api.slice(v, 0, 4611686018427387903) a = r0 @@ -439,9 +439,9 @@ def rem(v: vec[i64], n: i64) -> None: v = remove(v, n) [out] def rem(v, n): - v :: vec[int64] - n :: int64 - r0 :: vec[int64] + v :: vec[i64] + n :: i64 + r0 :: vec[i64] L0: r0 = VecI64Api.remove(v, n) v = r0 @@ -456,8 +456,8 @@ def pop_last(v: vec[i64]) -> Tuple[vec[i64], i64]: return pop(v) [out] def pop_last(v): - v :: vec[int64] - r0 :: tuple[vec[int64], int64] + v :: vec[i64] + r0 :: tuple[vec[i64], i64] L0: r0 = VecI64Api.pop(v, -1) return r0 @@ -471,9 +471,9 @@ def pop_nth(v: vec[i64], n: i64) -> Tuple[vec[i64], i64]: return pop(v, n) [out] def pop_nth(v, n): - v :: vec[int64] - n :: int64 - r0 :: tuple[vec[int64], int64] + v :: vec[i64] + n :: i64 + r0 :: tuple[vec[i64], i64] L0: r0 = VecI64Api.pop(v, n) return r0 @@ -486,22 +486,22 @@ def inplace(v: vec[i64], n: i64, m: i64) -> None: v[n] += m [out] def inplace(v, n, m): - v :: vec[int64] - n, m :: int64 + v :: vec[i64] + n, m :: i64 r0 :: native_int r1 :: bit r2 :: bool r3 :: object r4 :: ptr - r5 :: int64 + r5 :: i64 r6 :: ptr - r7, r8 :: int64 + r7, r8 :: i64 r9 :: native_int r10 :: bit r11 :: bool r12 :: object r13 :: ptr - r14 :: int64 + r14 :: i64 r15 :: ptr L0: r0 = v.len @@ -515,7 +515,7 @@ L2: r4 = get_element_ptr r3 items :: VecbufI64Object r5 = n * 8 r6 = r4 + r5 - r7 = load_mem r6 :: int64* + r7 = load_mem r6 :: i64* keep_alive v r8 = r7 + m r9 = v.len @@ -529,6 +529,6 @@ L4: r13 = get_element_ptr r12 items :: VecbufI64Object r14 = n * 8 r15 = r13 + r14 - set_mem r15, r8 :: int64* + set_mem r15, r8 :: i64* keep_alive v return 1 diff --git a/mypyc/test-data/irbuild-vec-nested.test b/mypyc/test-data/irbuild-vec-nested.test index c833f3c1aff7a..d1fede5ad127e 100644 --- a/mypyc/test-data/irbuild-vec-nested.test +++ b/mypyc/test-data/irbuild-vec-nested.test @@ -20,7 +20,7 @@ L0: r2 = VecTExtApi.alloc(0, 0, r1, 1) return r2 def g(): - r0 :: vec[vec[int64]] + r0 :: vec[vec[i64]] L0: r0 = VecTExtApi.alloc(0, 0, 2, 1) return r0 @@ -55,12 +55,12 @@ def f(v: vec[vec[i64]], vv: vec[i64]) -> vec[vec[i64]]: return append(v, vv) [out] def f(v, vv): - v :: vec[vec[int64]] - vv :: vec[int64] + v :: vec[vec[i64]] + vv :: vec[i64] r0 :: native_int r1 :: object r2, r3 :: VecbufTExtItem{len:native_int, buf:object_nrc} - r4 :: vec[vec[int64]] + r4 :: vec[vec[i64]] L0: r0 = vv.len r1 = vv.buf @@ -88,7 +88,7 @@ L0: r0 = v.len return r0 def g(v): - v :: vec[vec[int64]] + v :: vec[vec[i64]] r0 :: native_int L0: r0 = v.len @@ -103,13 +103,13 @@ def f(v: vec[vec[str]], n: i64) -> vec[str]: [out] def f(v, n): v :: vec[vec[str]] - n :: int64 + n :: i64 r0 :: native_int r1 :: bit r2 :: bool r3 :: object r4 :: ptr - r5 :: int64 + r5 :: i64 r6 :: ptr r7 :: vec[str] L0: @@ -136,16 +136,16 @@ def f(v: vec[vec[i64]], n: i64) -> vec[i64]: return v[n] [out] def f(v, n): - v :: vec[vec[int64]] - n :: int64 + v :: vec[vec[i64]] + n :: i64 r0 :: native_int r1 :: bit r2 :: bool r3 :: object r4 :: ptr - r5 :: int64 + r5 :: i64 r6 :: ptr - r7 :: vec[int64] + r7 :: vec[i64] L0: r0 = v.len r1 = n < r0 :: unsigned @@ -158,7 +158,7 @@ L2: r4 = get_element_ptr r3 items :: VecbufTExtObject r5 = n * 16 r6 = r4 + r5 - r7 = load_mem r6 :: vec[int64]* + r7 = load_mem r6 :: vec[i64]* keep_alive v return r7 @@ -170,24 +170,24 @@ def f(v: vec[vec[i64]], n: i64) -> i64: return v[n][n] [out] def f(v, n): - v :: vec[vec[int64]] - n :: int64 + v :: vec[vec[i64]] + n :: i64 r0 :: native_int r1 :: bit r2 :: bool r3 :: object r4 :: ptr - r5 :: int64 + r5 :: i64 r6 :: ptr - r7 :: vec[int64] + r7 :: vec[i64] r8 :: native_int r9 :: bit r10 :: bool r11 :: object r12 :: ptr - r13 :: int64 + r13 :: i64 r14 :: ptr - r15 :: int64 + r15 :: i64 L0: r0 = v.len r1 = n < r0 :: unsigned @@ -200,7 +200,7 @@ L2: r4 = get_element_ptr r3 items :: VecbufTExtObject r5 = n * 16 r6 = r4 + r5 - r7 = borrow load_mem r6 :: vec[int64]* + r7 = borrow load_mem r6 :: vec[i64]* r8 = r7.len r9 = n < r8 :: unsigned if r9 goto L4 else goto L3 :: bool @@ -212,7 +212,7 @@ L4: r12 = get_element_ptr r11 items :: VecbufI64Object r13 = n * 8 r14 = r12 + r13 - r15 = load_mem r14 :: int64* + r15 = load_mem r14 :: i64* keep_alive v, r7 return r15 @@ -225,13 +225,13 @@ def f(v: vec[vec[vec[str]]], n: i64) -> vec[vec[str]]: [out] def f(v, n): v :: vec[vec[vec[str]]] - n :: int64 + n :: i64 r0 :: native_int r1 :: bit r2 :: bool r3 :: object r4 :: ptr - r5 :: int64 + r5 :: i64 r6 :: ptr r7 :: vec[vec[str]] L0: diff --git a/mypyc/test-data/irbuild-vec-t.test b/mypyc/test-data/irbuild-vec-t.test index ae60d626b5564..36528a89be58b 100644 --- a/mypyc/test-data/irbuild-vec-t.test +++ b/mypyc/test-data/irbuild-vec-t.test @@ -147,13 +147,13 @@ def f(v: vec[str], n: i64) -> str: [out] def f(v, n): v :: vec[str] - n :: int64 + n :: i64 r0 :: native_int r1 :: bit r2 :: bool r3 :: object r4 :: ptr - r5 :: int64 + r5 :: i64 r6 :: ptr r7 :: str L0: @@ -182,13 +182,13 @@ def f(v: vec[Optional[str]], n: i64) -> Optional[str]: [out] def f(v, n): v :: vec[union[str, None]] - n :: int64 + n :: i64 r0 :: native_int r1 :: bit r2 :: bool r3 :: object r4 :: ptr - r5 :: int64 + r5 :: i64 r6 :: ptr r7 :: union[str, None] L0: @@ -229,17 +229,17 @@ def f(n: i64) -> vec[str]: return vec[str](['x' for x in range(i64(5))]) [out] def f(n): - n :: int64 + n :: i64 r0 :: object r1 :: ptr r2, r3 :: vec[str] - r4, x :: int64 + r4, x :: i64 r5 :: bit r6 :: str r7 :: object r8 :: ptr r9 :: vec[str] - r10 :: int64 + r10 :: i64 L0: r0 = load_address PyUnicode_Type r1 = r0 diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index 542d77a549333..8a84cb82f0c10 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -1509,14 +1509,14 @@ def f(v: vec[str], i: i64, x: str) -> None: [out] def f(v, i, x): v :: vec[str] - i :: int64 + i :: i64 x :: str r0 :: native_int r1 :: bit r2 :: bool r3 :: object r4 :: ptr - r5 :: int64 + r5 :: i64 r6 :: ptr r7 :: str L0: @@ -1546,13 +1546,13 @@ def f(n: i64) -> vec[Optional[str]]: return vec[Optional[str]]([None] * n) [out] def f(n): - n :: int64 + n :: i64 r0, r1 :: object r2, r3 :: ptr r4 :: vec[union[str, None]] r5 :: object r6 :: ptr - r7 :: int64 + r7 :: i64 r8, r9 :: ptr r10 :: bit r11 :: ptr @@ -1593,7 +1593,7 @@ def g(s: str) -> None: pass [out] def f(v): v :: vec[str] - t :: int64 + t :: i64 r0, r1 :: native_int r2 :: bit r3 :: object @@ -1671,16 +1671,16 @@ class C: [out] def C.f(self, x): self :: __main__.C - x :: int64 - r0 :: vec[int64] + x :: i64 + r0 :: vec[i64] r1 :: native_int r2 :: bit r3 :: bool r4 :: object r5 :: ptr - r6 :: int64 + r6 :: i64 r7 :: ptr - r8 :: int64 + r8 :: i64 L0: r0 = borrow self.v r1 = r0.len @@ -1694,7 +1694,7 @@ L2: r5 = get_element_ptr r4 items :: VecbufI64Object r6 = x * 8 r7 = r5 + r6 - r8 = load_mem r7 :: int64* + r8 = load_mem r7 :: i64* return r8 [case testVecI64LenBorrowVec] @@ -1709,8 +1709,8 @@ class C: [out] def C.f(self, x): self :: __main__.C - x :: int64 - r0 :: vec[int64] + x :: i64 + r0 :: vec[i64] r1 :: native_int L0: r0 = borrow self.v @@ -1748,13 +1748,13 @@ def f(v: vec[str], n: i64) -> str: [out] def f(v, n): v :: vec[str] - n :: int64 + n :: i64 r0 :: native_int r1 :: bit r2 :: bool r3 :: object r4 :: ptr - r5 :: int64 + r5 :: i64 r6 :: ptr r7 :: str L0: @@ -1781,7 +1781,7 @@ def f(v: vec[vec[str]], n: i64) -> None: [out] def f(v, n): v :: vec[vec[str]] - n :: int64 + n :: i64 r0 :: object r1 :: ptr r2 :: vec[vec[str]] @@ -1790,7 +1790,7 @@ def f(v, n): r5 :: bool r6 :: object r7 :: ptr - r8 :: int64 + r8 :: i64 r9 :: ptr r10, vv :: vec[str] L0: From 08b1826c7b89eb50f4e09c9bdaf280b60a302cc6 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 2 Sep 2023 18:41:43 +0100 Subject: [PATCH 065/180] Support nested vec pop --- mypyc/ir/rtypes.py | 23 ++++++++- mypyc/irbuild/vec.py | 75 +++++++++++++++++++++-------- mypyc/test-data/refcount.test | 33 +++++++++++++ mypyc/test-data/run-vec-nested.test | 5 +- 4 files changed, 112 insertions(+), 24 deletions(-) diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 31688932ff3e9..e72a75bf305b9 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -745,6 +745,8 @@ def visit_rtuple(self, t: RTuple) -> str: return "T{}{}".format(len(parts), "".join(parts)) def visit_rstruct(self, t: RStruct) -> str: + if t.name == "VecbufTExtItem": + return "Vi" assert False, "RStruct not supported in tuple" def visit_rarray(self, t: RArray) -> str: @@ -1036,14 +1038,17 @@ def __init__(self, item_type: RType) -> None: if is_int64_rprimitive(item_type): self._ctype = "VecI64" self.types = [c_pyssize_t_rprimitive, VecbufI64Object] + self.struct_type = VecI64 self.buf_type = VecbufI64Object elif isinstance(non_opt, RVec): self._ctype = "VecTExt" self.types = [c_pyssize_t_rprimitive, VecbufTObject] + self.struct_type = VecTExt self.buf_type = VecbufTExtObject else: self._ctype = "VecT" self.types = [c_pyssize_t_rprimitive, VecbufTObject] + self.struct_type = VecT self.buf_type = VecbufTObject def unwrap_item_type(self) -> RPrimitive | RInstance: @@ -1331,7 +1336,7 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: # "VecbufI64Object", is_unboxed=False, is_refcounted=True, ctype="VecbufI64Object *" # ) -# vec[i64] +# Struct type for vec[i64] (in most cases use RVec instead). VecI64 = RStruct( name="VecI64", names=["len", "buf"], types=[c_pyssize_t_rprimitive, object_rprimitive] ) @@ -1344,6 +1349,11 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: types=[PyVarObject, c_pyssize_t_rprimitive, object_rprimitive], ) +# Struct type for vec[t] (in most cases use RVec instead). +VecT = RStruct( + name="VecT", names=["len", "buf"], types=[c_pyssize_t_rprimitive, object_rprimitive] +) + VecbufTExtItem = RStruct( name="VecbufTExtItem", names=["len", "buf"], @@ -1363,6 +1373,11 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: ], ) +# Struct type for vec[vec[...]] (in most cases use RVec instead). +VecTExt = RStruct( + name="VecTExt", names=["len", "buf"], types=[c_pyssize_t_rprimitive, object_rprimitive] +) + VecbufTExtObject_rprimitive = RPrimitive( "VecbufTExtObject_ptr", is_unboxed=False, @@ -1370,3 +1385,9 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: is_refcounted=True, ctype="VecbufTExtObject *", ) + +VecTExtPopResult = RStruct( + name="VecTExtPopResult", + names=["vec", "item"], + types=[VecTExt, VecbufTExtItem], +) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 54e7cdb802a7f..fa3da814ee47b 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -9,6 +9,7 @@ from mypyc.ir.ops import ( ERR_FALSE, ERR_MAGIC, + ERR_NEVER, Assign, BasicBlock, Branch, @@ -27,6 +28,9 @@ Undef, Unreachable, Value, + TupleGet, + TupleSet, + Unborrow, ) from mypyc.ir.rtypes import ( RInstance, @@ -308,6 +312,19 @@ def convert_to_t_ext_item(builder: LowLevelIRBuilder, item: Value) -> Value: return builder.add(SetElement(temp, "buf", vec_buf)) +def convert_from_t_ext_item(builder: LowLevelIRBuilder, item: Value, vec_type: RVec) -> Value: + """Convert a value of type VecbufTExtItem to the corresponding RVec value.""" + if is_int64_rprimitive(vec_type.item_type): + name = "VecI64Api.convert_from_nested" + elif isinstance(vec_type.item_type, RVec): + name = "VecTExtApi.convert_from_nested" + else: + name = "VecTApi.convert_from_nested" + + return builder.add(CallC( + name, [item], vec_type, steals=[True], is_borrowed=False, error_kind=ERR_NEVER, line=-1)) + + def vec_item_type(builder: LowLevelIRBuilder, item_type: RType, line: int) -> Value: typeobj, optional, depth = vec_item_type_info(builder, item_type, line) if isinstance(typeobj, Integer): @@ -362,44 +379,62 @@ def vec_pop(builder: LowLevelIRBuilder, base: Value, index: Value, line: int) -> if is_int64_rprimitive(item_type): name = "VecI64Api.pop" - elif vec_depth(vec_type) == 0 and not isinstance(item_type, RUnion): + elif vec_depth(vec_type) == 0 and not isinstance(item_type, RUnion): # TODO fix union name = "VecTApi.pop" else: name = "VecTExtApi.pop" - call = CallC( - name, - [base, index], - RTuple([vec_type, item_type]), - steals=[False, False], - is_borrowed=False, - error_kind=ERR_MAGIC, - line=line, + # Nested vecs return a generic vec struct. + item_type = VecbufTExtItem + result = builder.add( + CallC( + name, + [base, index], + RTuple([vec_type, item_type]), + steals=[False, False], + is_borrowed=False, + error_kind=ERR_MAGIC, + line=line, + ) ) - return builder.add(call) + if vec_depth(vec_type) > 0: + orig = result + x = builder.add(TupleGet(result, 0, borrow=True)) + x = builder.add(Unborrow(x)) + y = builder.add(TupleGet(result, 1, borrow=True)) + y = builder.add(Unborrow(y)) + z = convert_from_t_ext_item(builder, y, vec_type.item_type) + result = builder.add(TupleSet([x, z], line)) + builder.keep_alive([orig], steal=True) + return result def vec_remove(builder: LowLevelIRBuilder, vec: Value, item: Value, line: int) -> Value: assert isinstance(vec.type, RVec) vec_type = vec.type item_type = vec_type.item_type - item = builder.coerce(item, item_type, line) + coerced_item = builder.coerce(item, item_type, line) if is_int64_rprimitive(item_type): name = "VecI64Api.remove" elif vec_depth(vec_type) == 0 and not isinstance(item_type, RUnion): name = "VecTApi.remove" else: + coerced_item = convert_to_t_ext_item(builder, coerced_item) name = "VecTExtApi.remove" - call = CallC( - name, - [vec, item], - vec_type, - steals=[False, False], - is_borrowed=False, - error_kind=ERR_MAGIC, - line=line, + call = builder.add( + CallC( + name, + [vec, coerced_item], + vec_type, + steals=[False, False], + is_borrowed=False, + error_kind=ERR_MAGIC, + line=line, + ) ) - return builder.add(call) + if vec_depth(vec_type) > 0: + builder.keep_alive([item]) + return call def vec_contains(builder: LowLevelIRBuilder, vec: Value, target: Value, line: int) -> Value: diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index 8a84cb82f0c10..687fef66761e6 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -1816,3 +1816,36 @@ L2: L3: dec_ref r2 goto L1 + +[case testVecNestedPop] +from vecs import vec, pop + +def f(v: vec[vec[str]]) -> vec[str]: + vv, x = pop(v) + return x +[out] +def f(v): + v :: vec[vec[str]] + r0 :: tuple[vec[vec[str]], VecbufTExtItem{len:native_int, buf:object_nrc}] + r1, r2 :: vec[vec[str]] + r3, r4 :: VecbufTExtItem{len:native_int, buf:object_nrc} + r5 :: vec[str] + r6 :: tuple[vec[vec[str]], vec[str]] + r7, r8, vv :: vec[vec[str]] + r9, r10, x :: vec[str] +L0: + r0 = VecTExtApi.pop(v, -1) + r1 = borrow r0[0] + r2 = unborrow r1 + r3 = borrow r0[1] + r4 = unborrow r3 + r5 = VecTApi.convert_from_nested(r4) + r6 = (r2, r5) + r7 = borrow r6[0] + r8 = unborrow r7 + vv = r8 + dec_ref vv + r9 = borrow r6[1] + r10 = unborrow r9 + x = r10 + return x diff --git a/mypyc/test-data/run-vec-nested.test b/mypyc/test-data/run-vec-nested.test index f1ce0f4784955..b3a244683fe01 100644 --- a/mypyc/test-data/run-vec-nested.test +++ b/mypyc/test-data/run-vec-nested.test @@ -62,7 +62,8 @@ def test_append_nested_optional() -> None: def test_get_item() -> None: v = append(vec[vec[str]](), append(vec[str](), 'x')) - assert repr(v[0]) == "vec[str](['x'])" + vv = v[0] + assert vv == vec[str](['x']) v = append(v, append(vec[str](), 'y')) assert repr(v[0]) == "vec[str](['x'])" assert repr(v[1]) == "vec[str](['y'])" @@ -155,7 +156,6 @@ def test_slicing_with_different_item_types() -> None: v = vec[vec[i64]]([vec[i64]([11]), vec[i64]([22])]) assert v[1:] == vec[vec[i64]]([vec[i64]([22])]) -""" def test_remove() -> None: a = ['4x', '7x', '9x'] vv = [vec[str]([s]) for s in a] @@ -228,4 +228,3 @@ def test_pop_index() -> None: v, item = pop(v, 0) assert item == vec[str](['15']) assert v == vec[vec[str]]() -""" From 8f5745bae0b85f367c3434041f72151afb54fdde Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 3 Sep 2023 14:17:17 +0100 Subject: [PATCH 066/180] Add tests for double nested vecs --- mypyc/test-data/run-vec-nested.test | 60 +++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/mypyc/test-data/run-vec-nested.test b/mypyc/test-data/run-vec-nested.test index b3a244683fe01..22e9d98a73d80 100644 --- a/mypyc/test-data/run-vec-nested.test +++ b/mypyc/test-data/run-vec-nested.test @@ -228,3 +228,63 @@ def test_pop_index() -> None: v, item = pop(v, 0) assert item == vec[str](['15']) assert v == vec[vec[str]]() + +def test_doubly_nested_operations() -> None: + a = vec[vec[str]]([vec[str](["x"])]) + b = vec[vec[str]]([vec[str]()]) + v = vec[vec[vec[str]]]() + + # append + v = append(v, a) + assert repr(v) == "vec[vec[vec[str]]]([[['x']]])" + v = append(v, b) + assert len(v) == 2 + + # get item + assert v[0] == a + assert v[1] == b + + # set item + v[0] = b + assert v[0] == b + v[0] = a + with assertRaises(IndexError): + v[2 + int()] + with assertRaises(IndexError): + v[2 + int()] = a + + # slicing + vv = v[:1] + assert len(vv) == 1 + assert vv[0] == a + vv = v[1:] + assert len(vv) == 1 + assert vv[0] == b + + # remove + v = remove(v, a) + assert len(v) == 1 + assert v[0] == b + v = remove(v, b) + assert len(v) == 0 + + # pop last + v = append(v, a) + v = append(v, b) + v, x = pop(v) + assert len(v) == 1 + assert v[0] == a + assert x == b + v, x = pop(v) + assert len(v) == 0 + assert x == a + with assertRaises(IndexError): + pop(v) + + # pop index + v = append(v, a) + v = append(v, b) + v, x = pop(v, 0) + assert len(v) == 1 + assert v[0] == b + assert x == a From 56b86a4662e78056c21891a88dc26b78e7632671 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 3 Sep 2023 14:43:22 +0100 Subject: [PATCH 067/180] Test nested vec[i64] --- mypyc/test-data/run-vec-nested.test | 62 +++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/mypyc/test-data/run-vec-nested.test b/mypyc/test-data/run-vec-nested.test index 22e9d98a73d80..ceae8a9f7bbc0 100644 --- a/mypyc/test-data/run-vec-nested.test +++ b/mypyc/test-data/run-vec-nested.test @@ -288,3 +288,65 @@ def test_doubly_nested_operations() -> None: assert len(v) == 1 assert v[0] == b assert x == a + + # TODO: box/unbox + +def test_vec_i64_nested_operations() -> None: + a = vec[i64]([11]) + b = vec[i64]([-33]) + v = vec[vec[i64]]() + + # append + v = append(v, a) + assert repr(v) == "vec[vec[i64]]([[11]])" + v = append(v, b) + assert len(v) == 2 + + # get item + assert v[0] == a + assert v[1] == b + + # set item + v[0] = b + assert v[0] == b + v[0] = a + with assertRaises(IndexError): + v[2 + int()] + with assertRaises(IndexError): + v[2 + int()] = a + + # slicing + vv = v[:1] + assert len(vv) == 1 + assert vv[0] == a + vv = v[1:] + assert len(vv) == 1 + assert vv[0] == b + + # remove + v = remove(v, a) + assert len(v) == 1 + assert v[0] == b + v = remove(v, b) + assert len(v) == 0 + + # pop last + v = append(v, a) + v = append(v, b) + v, x = pop(v) + assert len(v) == 1 + assert v[0] == a + assert x == b + v, x = pop(v) + assert len(v) == 0 + assert x == a + with assertRaises(IndexError): + pop(v) + + # pop index + v = append(v, a) + v = append(v, b) + v, x = pop(v, 0) + assert len(v) == 1 + assert v[0] == b + assert x == a From d00fb068be5de04c1496317a5995a4ccfb286964 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 3 Sep 2023 15:44:01 +0100 Subject: [PATCH 068/180] Add tests --- mypyc/test-data/run-vec-nested.test | 69 ++++++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/mypyc/test-data/run-vec-nested.test b/mypyc/test-data/run-vec-nested.test index ceae8a9f7bbc0..4c8898faedd9b 100644 --- a/mypyc/test-data/run-vec-nested.test +++ b/mypyc/test-data/run-vec-nested.test @@ -1,5 +1,6 @@ [case testVecNestedBasicOps] from typing import Final, Any, Iterable, Optional +import sys from mypy_extensions import i64 from vecs import vec, append, remove, pop @@ -28,6 +29,8 @@ def test_construct_from_initializer_nested() -> None: def test_repr() -> None: assert str(vec[vec[str]]()) == "vec[vec[str]]([])" assert str(vec[vec[Optional[str]]]()) == "vec[vec[str | None]]([])" + if sys.version_info[1] >= 10: + assert str(vec[vec[str | None]]()) == "vec[vec[str | None]]([])" def test_append_nested() -> None: v = vec[vec[str]]() @@ -293,7 +296,7 @@ def test_doubly_nested_operations() -> None: def test_vec_i64_nested_operations() -> None: a = vec[i64]([11]) - b = vec[i64]([-33]) + b = vec[i64]() v = vec[vec[i64]]() # append @@ -350,3 +353,67 @@ def test_vec_i64_nested_operations() -> None: assert len(v) == 1 assert v[0] == b assert x == a + + # TODO: box/unbox + +def test_doubly_nested_vec_i64_operations() -> None: + a = vec[vec[i64]]([vec[i64]([11])]) + b = vec[vec[i64]]([vec[i64]()]) + v = vec[vec[vec[i64]]]() + + # append + v = append(v, a) + assert repr(v) == "vec[vec[vec[i64]]]([[[11]]])" + v = append(v, b) + assert len(v) == 2 + + # get item + assert v[0] == a + assert v[1] == b + + # set item + v[0] = b + assert v[0] == b + v[0] = a + with assertRaises(IndexError): + v[2 + int()] + with assertRaises(IndexError): + v[2 + int()] = a + + # slicing + vv = v[:1] + assert len(vv) == 1 + assert vv[0] == a + vv = v[1:] + assert len(vv) == 1 + assert vv[0] == b + + # remove + v = remove(v, a) + assert len(v) == 1 + assert v[0] == b + v = remove(v, b) + assert len(v) == 0 + + # pop last + v = append(v, a) + v = append(v, b) + v, x = pop(v) + assert len(v) == 1 + assert v[0] == a + assert x == b + v, x = pop(v) + assert len(v) == 0 + assert x == a + with assertRaises(IndexError): + pop(v) + + # pop index + v = append(v, a) + v = append(v, b) + v, x = pop(v, 0) + assert len(v) == 1 + assert v[0] == b + assert x == a + + # TODO: box/unbox From e897f29f64e37d1ac8a3f4ad777541b32de9dda8 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 3 Sep 2023 17:28:26 +0100 Subject: [PATCH 069/180] Support negative indexes --- mypyc/irbuild/vec.py | 29 +++++-- mypyc/test-data/irbuild-vec-i64.test | 116 ++++++++++++++++++--------- mypyc/test-data/refcount.test | 40 +++++---- mypyc/test-data/run-vec-i64.test | 43 ++++++++++ mypyc/test-data/run-vec-nested.test | 2 +- 5 files changed, 170 insertions(+), 60 deletions(-) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index fa3da814ee47b..6e5f1a95ffb0c 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -242,15 +242,34 @@ def vec_item_ptr(builder: LowLevelIRBuilder, vecobj: Value, index: Value) -> Val return builder.int_add(items_addr, delta) -def vec_check_index(builder: LowLevelIRBuilder, lenv: Value, index: Value, line: int) -> None: - ok, fail = BasicBlock(), BasicBlock() +def vec_check_and_adjust_index( + builder: LowLevelIRBuilder, lenv: Value, index: Value, line: int) -> Value: + r = Register(int64_rprimitive) + ok, ok2, ok3 = BasicBlock(), BasicBlock(), BasicBlock() + fail, fail2 = BasicBlock(), BasicBlock() is_less = builder.comparison_op(index, lenv, ComparisonOp.ULT, line) - builder.add_bool_branch(is_less, ok, fail) + builder.add_bool_branch(is_less, ok2, fail) builder.activate_block(fail) + + x = builder.int_add(index, lenv) + is_less2 = builder.comparison_op(x, lenv, ComparisonOp.ULT, line) + builder.add_bool_branch(is_less2, ok, fail2) + + builder.activate_block(fail2) # TODO: Include index in exception builder.add(RaiseStandardError(RaiseStandardError.INDEX_ERROR, None, line)) builder.add(Unreachable()) + builder.activate_block(ok) + builder.assign(r, x) + builder.goto(ok3) + + builder.activate_block(ok2) + builder.assign(r, index) + builder.goto(ok3) + + builder.activate_block(ok3) + return r def vec_get_item( @@ -265,7 +284,7 @@ def vec_get_item( # TODO: Support more item types # TODO: Support more index types len_val = vec_len_native(builder, base) - vec_check_index(builder, len_val, index, line) + index = vec_check_and_adjust_index(builder, len_val, index, line) item_addr = vec_item_ptr(builder, base, index) result = builder.load_mem(item_addr, vtype.item_type, borrow=can_borrow) builder.keep_alives.append(base) @@ -292,7 +311,7 @@ def vec_set_item( index = as_platform_int(builder, index, line) vtype = base.type len_val = vec_len_native(builder, base) - vec_check_index(builder, len_val, index, line) + index = vec_check_and_adjust_index(builder, len_val, index, line) item_addr = vec_item_ptr(builder, base, index) item_type = vtype.item_type item = builder.coerce(item, item_type, line) diff --git a/mypyc/test-data/irbuild-vec-i64.test b/mypyc/test-data/irbuild-vec-i64.test index fb8ebc5f80e7c..e1cca7e77f893 100644 --- a/mypyc/test-data/irbuild-vec-i64.test +++ b/mypyc/test-data/irbuild-vec-i64.test @@ -43,27 +43,39 @@ def f(v, i): i :: i64 r0 :: native_int r1 :: bit - r2 :: bool - r3 :: object - r4 :: ptr + r2 :: i64 + r3 :: bit + r4 :: bool r5 :: i64 - r6 :: ptr - r7 :: i64 + r6 :: object + r7 :: ptr + r8 :: i64 + r9 :: ptr + r10 :: i64 L0: r0 = v.len r1 = i < r0 :: unsigned - if r1 goto L2 else goto L1 :: bool + if r1 goto L4 else goto L1 :: bool L1: - r2 = raise IndexError - unreachable + r2 = i + r0 + r3 = r2 < r0 :: unsigned + if r3 goto L3 else goto L2 :: bool L2: - r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufI64Object - r5 = i * 8 - r6 = r4 + r5 - r7 = load_mem r6 :: i64* + r4 = raise IndexError + unreachable +L3: + r5 = r2 + goto L5 +L4: + r5 = i +L5: + r6 = v.buf + r7 = get_element_ptr r6 items :: VecbufI64Object + r8 = r5 * 8 + r9 = r7 + r8 + r10 = load_mem r9 :: i64* keep_alive v - return r7 + return r10 [case testVecI64Append] from vecs import vec, append @@ -92,24 +104,36 @@ def f(v, i, x): i, x :: i64 r0 :: native_int r1 :: bit - r2 :: bool - r3 :: object - r4 :: ptr + r2 :: i64 + r3 :: bit + r4 :: bool r5 :: i64 - r6 :: ptr + r6 :: object + r7 :: ptr + r8 :: i64 + r9 :: ptr L0: r0 = v.len r1 = i < r0 :: unsigned - if r1 goto L2 else goto L1 :: bool + if r1 goto L4 else goto L1 :: bool L1: - r2 = raise IndexError - unreachable + r2 = i + r0 + r3 = r2 < r0 :: unsigned + if r3 goto L3 else goto L2 :: bool L2: - r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufI64Object - r5 = i * 8 - r6 = r4 + r5 - set_mem r6, x :: i64* + r4 = raise IndexError + unreachable +L3: + r5 = r2 + goto L5 +L4: + r5 = i +L5: + r6 = v.buf + r7 = get_element_ptr r6 items :: VecbufI64Object + r8 = r5 * 8 + r9 = r7 + r8 + set_mem r9, x :: i64* keep_alive v return 1 @@ -381,27 +405,39 @@ def f(v): v :: vec[i64] r0 :: native_int r1 :: bit - r2 :: bool - r3 :: object - r4 :: ptr + r2 :: i64 + r3 :: bit + r4 :: bool r5 :: i64 - r6 :: ptr - r7 :: i64 + r6 :: object + r7 :: ptr + r8 :: i64 + r9 :: ptr + r10 :: i64 L0: r0 = v.len r1 = 0 < r0 :: unsigned - if r1 goto L2 else goto L1 :: bool + if r1 goto L4 else goto L1 :: bool L1: - r2 = raise IndexError - unreachable + r2 = 0 + r0 + r3 = r2 < r0 :: unsigned + if r3 goto L3 else goto L2 :: bool L2: - r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufI64Object - r5 = 0 * 8 - r6 = r4 + r5 - r7 = load_mem r6 :: i64* + r4 = raise IndexError + unreachable +L3: + r5 = r2 + goto L5 +L4: + r5 = 0 +L5: + r6 = v.buf + r7 = get_element_ptr r6 items :: VecbufI64Object + r8 = r5 * 8 + r9 = r7 + r8 + r10 = load_mem r9 :: i64* keep_alive v - return r7 + return r10 [case testVecI64Slicing] from vecs import vec diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index 687fef66761e6..e9e05753727aa 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -1675,27 +1675,39 @@ def C.f(self, x): r0 :: vec[i64] r1 :: native_int r2 :: bit - r3 :: bool - r4 :: object - r5 :: ptr + r3 :: i64 + r4 :: bit + r5 :: bool r6 :: i64 - r7 :: ptr - r8 :: i64 + r7 :: object + r8 :: ptr + r9 :: i64 + r10 :: ptr + r11 :: i64 L0: r0 = borrow self.v r1 = r0.len r2 = x < r1 :: unsigned - if r2 goto L2 else goto L1 :: bool + if r2 goto L4 else goto L1 :: bool L1: - r3 = raise IndexError - unreachable + r3 = x + r1 + r4 = r3 < r1 :: unsigned + if r4 goto L3 else goto L2 :: bool L2: - r4 = r0.buf - r5 = get_element_ptr r4 items :: VecbufI64Object - r6 = x * 8 - r7 = r5 + r6 - r8 = load_mem r7 :: i64* - return r8 + r5 = raise IndexError + unreachable +L3: + r6 = r3 + goto L5 +L4: + r6 = x +L5: + r7 = r0.buf + r8 = get_element_ptr r7 items :: VecbufI64Object + r9 = r6 * 8 + r10 = r8 + r9 + r11 = load_mem r10 :: i64* + return r11 [case testVecI64LenBorrowVec] from vecs import vec diff --git a/mypyc/test-data/run-vec-i64.test b/mypyc/test-data/run-vec-i64.test index 30e86fed664b8..945dc0db879ad 100644 --- a/mypyc/test-data/run-vec-i64.test +++ b/mypyc/test-data/run-vec-i64.test @@ -62,6 +62,23 @@ def test_get_item() -> None: with assertRaises(IndexError): v[2**63 - 1] +def test_get_item_negative() -> None: + v = vec[i64]([3, 4]) + x: i64 = int() - 1 + assert v[x] == 4 + assert v[int() - 2] == 3 + x -= 1 + assert v[x] == 3 + assert v[int() - 1] == 4 + with assertRaises(IndexError): + v[x - 1] + with assertRaises(IndexError): + v[-3] + with assertRaises(IndexError): + v[-2**63] + with assertRaises(IndexError): + v[int() - 3] + def test_set_item() -> None: v = vec[i64]([3, 4]) v[0] = 0 @@ -88,6 +105,32 @@ def test_set_item() -> None: with assertRaises(IndexError): v[2 + int()] = 6 +def test_set_item_negative() -> None: + v = vec[i64]([3, 4]) + v[-1] = 0 + assert v[-1] == 0 + + x: i64 = int() - 1 + + v[x] = 4 + assert v[x] == 4 + + v[int() - 2] = 5 + assert v[int() - 2] == 5 + + x -= 1 + + v[x] = -5 + assert v[x] == -5 + + v[-1 + int()] = ERROR + assert v[-1] == ERROR + + with assertRaises(IndexError): + v[x - 1] = 6 + with assertRaises(IndexError): + v[int() - 3] = 6 + def test_operator_assignment() -> None: v = vec[i64]([3, 4]) x: i64 = int() diff --git a/mypyc/test-data/run-vec-nested.test b/mypyc/test-data/run-vec-nested.test index 4c8898faedd9b..08d54ca5dbb3e 100644 --- a/mypyc/test-data/run-vec-nested.test +++ b/mypyc/test-data/run-vec-nested.test @@ -97,7 +97,7 @@ def test_set_item_error_nested() -> None: with assertRaises(IndexError): v[3] = vec[str]() with assertRaises(IndexError): - v[-1] = vec[str]() + v[-4] = vec[str]() assert repr(v[0]) == 'vec[str]([])' def test_equality() -> None: From 0099332b52c00cf3126d0174236b6755d37b3f3a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 26 Dec 2023 15:43:13 +0000 Subject: [PATCH 070/180] Update irbuild tests for negative indexing --- mypyc/test-data/irbuild-vec-i64.test | 87 ++++++---- mypyc/test-data/irbuild-vec-nested.test | 209 +++++++++++++++--------- mypyc/test-data/irbuild-vec-t.test | 80 +++++---- mypyc/test-data/refcount.test | 126 +++++++++----- 4 files changed, 324 insertions(+), 178 deletions(-) diff --git a/mypyc/test-data/irbuild-vec-i64.test b/mypyc/test-data/irbuild-vec-i64.test index e1cca7e77f893..9f0df375f85a8 100644 --- a/mypyc/test-data/irbuild-vec-i64.test +++ b/mypyc/test-data/irbuild-vec-i64.test @@ -526,45 +526,70 @@ def inplace(v, n, m): n, m :: i64 r0 :: native_int r1 :: bit - r2 :: bool - r3 :: object - r4 :: ptr + r2 :: i64 + r3 :: bit + r4 :: bool r5 :: i64 - r6 :: ptr - r7, r8 :: i64 - r9 :: native_int - r10 :: bit - r11 :: bool - r12 :: object - r13 :: ptr + r6 :: object + r7 :: ptr + r8 :: i64 + r9 :: ptr + r10, r11 :: i64 + r12 :: native_int + r13 :: bit r14 :: i64 - r15 :: ptr + r15 :: bit + r16 :: bool + r17 :: i64 + r18 :: object + r19 :: ptr + r20 :: i64 + r21 :: ptr L0: r0 = v.len r1 = n < r0 :: unsigned - if r1 goto L2 else goto L1 :: bool + if r1 goto L4 else goto L1 :: bool L1: - r2 = raise IndexError - unreachable + r2 = n + r0 + r3 = r2 < r0 :: unsigned + if r3 goto L3 else goto L2 :: bool L2: - r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufI64Object - r5 = n * 8 - r6 = r4 + r5 - r7 = load_mem r6 :: i64* - keep_alive v - r8 = r7 + m - r9 = v.len - r10 = n < r9 :: unsigned - if r10 goto L4 else goto L3 :: bool -L3: - r11 = raise IndexError + r4 = raise IndexError unreachable +L3: + r5 = r2 + goto L5 L4: - r12 = v.buf - r13 = get_element_ptr r12 items :: VecbufI64Object - r14 = n * 8 - r15 = r13 + r14 - set_mem r15, r8 :: i64* + r5 = n +L5: + r6 = v.buf + r7 = get_element_ptr r6 items :: VecbufI64Object + r8 = r5 * 8 + r9 = r7 + r8 + r10 = load_mem r9 :: i64* + keep_alive v + r11 = r10 + m + r12 = v.len + r13 = n < r12 :: unsigned + if r13 goto L9 else goto L6 :: bool +L6: + r14 = n + r12 + r15 = r14 < r12 :: unsigned + if r15 goto L8 else goto L7 :: bool +L7: + r16 = raise IndexError + unreachable +L8: + r17 = r14 + goto L10 +L9: + r17 = n +L10: + r18 = v.buf + r19 = get_element_ptr r18 items :: VecbufI64Object + r20 = r17 * 8 + r21 = r19 + r20 + set_mem r21, r11 :: i64* keep_alive v return 1 + diff --git a/mypyc/test-data/irbuild-vec-nested.test b/mypyc/test-data/irbuild-vec-nested.test index d1fede5ad127e..8daad6103a630 100644 --- a/mypyc/test-data/irbuild-vec-nested.test +++ b/mypyc/test-data/irbuild-vec-nested.test @@ -106,27 +106,39 @@ def f(v, n): n :: i64 r0 :: native_int r1 :: bit - r2 :: bool - r3 :: object - r4 :: ptr + r2 :: i64 + r3 :: bit + r4 :: bool r5 :: i64 - r6 :: ptr - r7 :: vec[str] + r6 :: object + r7 :: ptr + r8 :: i64 + r9 :: ptr + r10 :: vec[str] L0: r0 = v.len r1 = n < r0 :: unsigned - if r1 goto L2 else goto L1 :: bool + if r1 goto L4 else goto L1 :: bool L1: - r2 = raise IndexError - unreachable + r2 = n + r0 + r3 = r2 < r0 :: unsigned + if r3 goto L3 else goto L2 :: bool L2: - r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufTExtObject - r5 = n * 16 - r6 = r4 + r5 - r7 = load_mem r6 :: vec[str]* + r4 = raise IndexError + unreachable +L3: + r5 = r2 + goto L5 +L4: + r5 = n +L5: + r6 = v.buf + r7 = get_element_ptr r6 items :: VecbufTExtObject + r8 = r5 * 16 + r9 = r7 + r8 + r10 = load_mem r9 :: vec[str]* keep_alive v - return r7 + return r10 [case testVecNestedI64GetItem] from vecs import vec @@ -140,27 +152,39 @@ def f(v, n): n :: i64 r0 :: native_int r1 :: bit - r2 :: bool - r3 :: object - r4 :: ptr + r2 :: i64 + r3 :: bit + r4 :: bool r5 :: i64 - r6 :: ptr - r7 :: vec[i64] + r6 :: object + r7 :: ptr + r8 :: i64 + r9 :: ptr + r10 :: vec[i64] L0: r0 = v.len r1 = n < r0 :: unsigned - if r1 goto L2 else goto L1 :: bool + if r1 goto L4 else goto L1 :: bool L1: - r2 = raise IndexError - unreachable + r2 = n + r0 + r3 = r2 < r0 :: unsigned + if r3 goto L3 else goto L2 :: bool L2: - r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufTExtObject - r5 = n * 16 - r6 = r4 + r5 - r7 = load_mem r6 :: vec[i64]* + r4 = raise IndexError + unreachable +L3: + r5 = r2 + goto L5 +L4: + r5 = n +L5: + r6 = v.buf + r7 = get_element_ptr r6 items :: VecbufTExtObject + r8 = r5 * 16 + r9 = r7 + r8 + r10 = load_mem r9 :: vec[i64]* keep_alive v - return r7 + return r10 [case testVecNestedI64GetItemWithBorrow] from vecs import vec @@ -174,47 +198,71 @@ def f(v, n): n :: i64 r0 :: native_int r1 :: bit - r2 :: bool - r3 :: object - r4 :: ptr + r2 :: i64 + r3 :: bit + r4 :: bool r5 :: i64 - r6 :: ptr - r7 :: vec[i64] - r8 :: native_int - r9 :: bit - r10 :: bool - r11 :: object - r12 :: ptr + r6 :: object + r7 :: ptr + r8 :: i64 + r9 :: ptr + r10 :: vec[i64] + r11 :: native_int + r12 :: bit r13 :: i64 - r14 :: ptr - r15 :: i64 + r14 :: bit + r15 :: bool + r16 :: i64 + r17 :: object + r18 :: ptr + r19 :: i64 + r20 :: ptr + r21 :: i64 L0: r0 = v.len r1 = n < r0 :: unsigned - if r1 goto L2 else goto L1 :: bool + if r1 goto L4 else goto L1 :: bool L1: - r2 = raise IndexError - unreachable + r2 = n + r0 + r3 = r2 < r0 :: unsigned + if r3 goto L3 else goto L2 :: bool L2: - r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufTExtObject - r5 = n * 16 - r6 = r4 + r5 - r7 = borrow load_mem r6 :: vec[i64]* - r8 = r7.len - r9 = n < r8 :: unsigned - if r9 goto L4 else goto L3 :: bool -L3: - r10 = raise IndexError + r4 = raise IndexError unreachable +L3: + r5 = r2 + goto L5 L4: - r11 = r7.buf - r12 = get_element_ptr r11 items :: VecbufI64Object - r13 = n * 8 - r14 = r12 + r13 - r15 = load_mem r14 :: i64* - keep_alive v, r7 - return r15 + r5 = n +L5: + r6 = v.buf + r7 = get_element_ptr r6 items :: VecbufTExtObject + r8 = r5 * 16 + r9 = r7 + r8 + r10 = borrow load_mem r9 :: vec[i64]* + r11 = r10.len + r12 = n < r11 :: unsigned + if r12 goto L9 else goto L6 :: bool +L6: + r13 = n + r11 + r14 = r13 < r11 :: unsigned + if r14 goto L8 else goto L7 :: bool +L7: + r15 = raise IndexError + unreachable +L8: + r16 = r13 + goto L10 +L9: + r16 = n +L10: + r17 = r10.buf + r18 = get_element_ptr r17 items :: VecbufI64Object + r19 = r16 * 8 + r20 = r18 + r19 + r21 = load_mem r20 :: i64* + keep_alive v, r10 + return r21 [case testVecDoublyNestedGetItem] from vecs import vec @@ -228,24 +276,37 @@ def f(v, n): n :: i64 r0 :: native_int r1 :: bit - r2 :: bool - r3 :: object - r4 :: ptr + r2 :: i64 + r3 :: bit + r4 :: bool r5 :: i64 - r6 :: ptr - r7 :: vec[vec[str]] + r6 :: object + r7 :: ptr + r8 :: i64 + r9 :: ptr + r10 :: vec[vec[str]] L0: r0 = v.len r1 = n < r0 :: unsigned - if r1 goto L2 else goto L1 :: bool + if r1 goto L4 else goto L1 :: bool L1: - r2 = raise IndexError - unreachable + r2 = n + r0 + r3 = r2 < r0 :: unsigned + if r3 goto L3 else goto L2 :: bool L2: - r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufTExtObject - r5 = n * 16 - r6 = r4 + r5 - r7 = load_mem r6 :: vec[vec[str]]* + r4 = raise IndexError + unreachable +L3: + r5 = r2 + goto L5 +L4: + r5 = n +L5: + r6 = v.buf + r7 = get_element_ptr r6 items :: VecbufTExtObject + r8 = r5 * 16 + r9 = r7 + r8 + r10 = load_mem r9 :: vec[vec[str]]* keep_alive v - return r7 + return r10 + diff --git a/mypyc/test-data/irbuild-vec-t.test b/mypyc/test-data/irbuild-vec-t.test index 36528a89be58b..2b311bfb04c83 100644 --- a/mypyc/test-data/irbuild-vec-t.test +++ b/mypyc/test-data/irbuild-vec-t.test @@ -150,27 +150,39 @@ def f(v, n): n :: i64 r0 :: native_int r1 :: bit - r2 :: bool - r3 :: object - r4 :: ptr + r2 :: i64 + r3 :: bit + r4 :: bool r5 :: i64 - r6 :: ptr - r7 :: str + r6 :: object + r7 :: ptr + r8 :: i64 + r9 :: ptr + r10 :: str L0: r0 = v.len r1 = n < r0 :: unsigned - if r1 goto L2 else goto L1 :: bool + if r1 goto L4 else goto L1 :: bool L1: - r2 = raise IndexError - unreachable + r2 = n + r0 + r3 = r2 < r0 :: unsigned + if r3 goto L3 else goto L2 :: bool L2: - r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufTObject - r5 = n * 8 - r6 = r4 + r5 - r7 = load_mem r6 :: builtins.str* + r4 = raise IndexError + unreachable +L3: + r5 = r2 + goto L5 +L4: + r5 = n +L5: + r6 = v.buf + r7 = get_element_ptr r6 items :: VecbufTObject + r8 = r5 * 8 + r9 = r7 + r8 + r10 = load_mem r9 :: builtins.str* keep_alive v - return r7 + return r10 [case testVecTOptionalGetItem] from vecs import vec @@ -185,27 +197,39 @@ def f(v, n): n :: i64 r0 :: native_int r1 :: bit - r2 :: bool - r3 :: object - r4 :: ptr + r2 :: i64 + r3 :: bit + r4 :: bool r5 :: i64 - r6 :: ptr - r7 :: union[str, None] + r6 :: object + r7 :: ptr + r8 :: i64 + r9 :: ptr + r10 :: union[str, None] L0: r0 = v.len r1 = n < r0 :: unsigned - if r1 goto L2 else goto L1 :: bool + if r1 goto L4 else goto L1 :: bool L1: - r2 = raise IndexError - unreachable + r2 = n + r0 + r3 = r2 < r0 :: unsigned + if r3 goto L3 else goto L2 :: bool L2: - r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufTObject - r5 = n * 8 - r6 = r4 + r5 - r7 = load_mem r6 :: union* + r4 = raise IndexError + unreachable +L3: + r5 = r2 + goto L5 +L4: + r5 = n +L5: + r6 = v.buf + r7 = get_element_ptr r6 items :: VecbufTObject + r8 = r5 * 8 + r9 = r7 + r8 + r10 = load_mem r9 :: union* keep_alive v - return r7 + return r10 [case testNewTPopLast] from typing import Tuple diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index e9e05753727aa..5e002d5089249 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -1513,28 +1513,40 @@ def f(v, i, x): x :: str r0 :: native_int r1 :: bit - r2 :: bool - r3 :: object - r4 :: ptr + r2 :: i64 + r3 :: bit + r4 :: bool r5 :: i64 - r6 :: ptr - r7 :: str + r6 :: object + r7 :: ptr + r8 :: i64 + r9 :: ptr + r10 :: str L0: r0 = v.len r1 = i < r0 :: unsigned - if r1 goto L2 else goto L1 :: bool + if r1 goto L4 else goto L1 :: bool L1: - r2 = raise IndexError - unreachable + r2 = i + r0 + r3 = r2 < r0 :: unsigned + if r3 goto L3 else goto L2 :: bool L2: - r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufTObject - r5 = i * 8 - r6 = r4 + r5 - r7 = borrow load_mem r6 :: builtins.str* - dec_ref r7 + r4 = raise IndexError + unreachable +L3: + r5 = r2 + goto L5 +L4: + r5 = i +L5: + r6 = v.buf + r7 = get_element_ptr r6 items :: VecbufTObject + r8 = r5 * 8 + r9 = r7 + r8 + r10 = borrow load_mem r9 :: builtins.str* + dec_ref r10 inc_ref x - set_mem r6, x :: builtins.str* + set_mem r9, x :: builtins.str* return 1 [case testVecTConstructFromListMultiply] @@ -1763,26 +1775,38 @@ def f(v, n): n :: i64 r0 :: native_int r1 :: bit - r2 :: bool - r3 :: object - r4 :: ptr + r2 :: i64 + r3 :: bit + r4 :: bool r5 :: i64 - r6 :: ptr - r7 :: str + r6 :: object + r7 :: ptr + r8 :: i64 + r9 :: ptr + r10 :: str L0: r0 = v.len r1 = n < r0 :: unsigned - if r1 goto L2 else goto L1 :: bool + if r1 goto L4 else goto L1 :: bool L1: - r2 = raise IndexError - unreachable + r2 = n + r0 + r3 = r2 < r0 :: unsigned + if r3 goto L3 else goto L2 :: bool L2: - r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufTObject - r5 = n * 8 - r6 = r4 + r5 - r7 = load_mem r6 :: builtins.str* - return r7 + r4 = raise IndexError + unreachable +L3: + r5 = r2 + goto L5 +L4: + r5 = n +L5: + r6 = v.buf + r7 = get_element_ptr r6 items :: VecbufTObject + r8 = r5 * 8 + r9 = r7 + r8 + r10 = load_mem r9 :: builtins.str* + return r10 [case testVecNestedGetItem] from vecs import vec @@ -1799,35 +1823,47 @@ def f(v, n): r2 :: vec[vec[str]] r3 :: native_int r4 :: bit - r5 :: bool - r6 :: object - r7 :: ptr + r5 :: i64 + r6 :: bit + r7 :: bool r8 :: i64 - r9 :: ptr - r10, vv :: vec[str] + r9 :: object + r10 :: ptr + r11 :: i64 + r12 :: ptr + r13, vv :: vec[str] L0: r0 = load_address PyUnicode_Type r1 = r0 r2 = VecTExtApi.alloc(0, 0, r1, 1) r3 = r2.len r4 = n < r3 :: unsigned - if r4 goto L2 else goto L3 :: bool + if r4 goto L4 else goto L1 :: bool L1: - r5 = raise IndexError - unreachable + r5 = n + r3 + r6 = r5 < r3 :: unsigned + if r6 goto L3 else goto L6 :: bool L2: - r6 = r2.buf - r7 = get_element_ptr r6 items :: VecbufTExtObject - r8 = n * 16 - r9 = r7 + r8 - r10 = load_mem r9 :: vec[str]* + r7 = raise IndexError + unreachable +L3: + r8 = r5 + goto L5 +L4: + r8 = n +L5: + r9 = r2.buf + r10 = get_element_ptr r9 items :: VecbufTExtObject + r11 = r8 * 16 + r12 = r10 + r11 + r13 = load_mem r12 :: vec[str]* dec_ref r2 - vv = r10 + vv = r13 dec_ref vv return 1 -L3: +L6: dec_ref r2 - goto L1 + goto L2 [case testVecNestedPop] from vecs import vec, pop From 812ecc313e2cc3288ab6fa49d4e80720a1b20cbc Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 26 Dec 2023 15:58:45 +0000 Subject: [PATCH 071/180] Detect additional invalid vec item types --- mypyc/test-data/irbuild-vec-t.test | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/mypyc/test-data/irbuild-vec-t.test b/mypyc/test-data/irbuild-vec-t.test index 2b311bfb04c83..4b72a76465c00 100644 --- a/mypyc/test-data/irbuild-vec-t.test +++ b/mypyc/test-data/irbuild-vec-t.test @@ -290,23 +290,30 @@ L4: [case testVecTCheckItemType] from vecs import vec -from typing import Tuple +from typing import Tuple, Any, List, TypeVar def bad1(v: vec[Tuple[str, str]]) -> None: pass def bad2(v: vec[int]) -> None: pass def bad3(v: vec) -> None: pass +def bad4(v: vec[Any]) -> None: pass +T = TypeVar("T") +def bad5(v: vec[T]) -> None: pass def ok1(v: vec[str], v2: vec[bytes]) -> None: pass def ok2(v: vec[Tuple[str, ...]]) -> None: pass def ok3(v: vec[object]) -> None: pass +def ok4(v: vec[List[Any]]) -> None: pass +def ok5(v: vec[vec[str]]) -> None: pass [out] main:4: error: Invalid item type for "vec" main:5: error: Invalid item type for "vec" main:6: error: Invalid item type for "vec" +main:7: error: Invalid item type for "vec" +main:9: error: Invalid item type for "vec" [case testVecTCheckItemTypeUnion] from vecs import vec -from typing import Tuple, Union, Optional +from typing import Tuple, Union, Optional, Any, TypeVar from mypy_extensions import i64 def bad1(v: vec[Union[str, bytes]]) -> None: pass @@ -319,6 +326,9 @@ def bad7(v: vec[Union[bool, None]]) -> None: pass def bad9(v: vec[Union[float, None]]) -> None: pass def bad10(v: vec[Union[vec[str], None]]) -> None: pass def bad11(v: vec[Optional[vec[str]]]) -> None: pass +def bad12(v: vec[Union[str, Any]]) -> None: pass +T = TypeVar("T") +def bad13(v: vec[Union[str, T]]) -> None: pass def ok1(v: vec[Union[str, None]]) -> None: pass def ok2(v: vec[Union[None, str]]) -> None: pass @@ -333,6 +343,8 @@ main:11: error: Invalid item type for "vec" main:12: error: Invalid item type for "vec" main:13: error: Invalid item type for "vec" main:14: error: Invalid item type for "vec" +main:15: error: Invalid item type for "vec" +main:17: error: Invalid item type for "vec" [case testVecTCheckItemTypeDuringConstruction] from vecs import vec From 87d8f7d4344378f362de3997f5982f68f4c47616 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 26 Dec 2023 16:13:40 +0000 Subject: [PATCH 072/180] Support final vec values --- mypyc/ir/rtypes.py | 8 +++++++- mypyc/test-data/run-vec-i64.test | 5 +++++ mypyc/test-data/run-vec-nested.test | 5 +++++ mypyc/test-data/run-vec-t.test | 5 +++++ 4 files changed, 22 insertions(+), 1 deletion(-) diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index e72a75bf305b9..01b33dafb65f0 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -136,6 +136,8 @@ def deserialize_type(data: JsonDict | str, ctx: DeserMaps) -> RType: return RTuple.deserialize(data, ctx) elif data[".class"] == "RUnion": return RUnion.deserialize(data, ctx) + elif data[".class"] == "RVec": + return RVec.deserialize(data, ctx) raise NotImplementedError("unexpected .class {}".format(data[".class"])) @@ -1102,7 +1104,11 @@ def __hash__(self) -> int: return hash(self.item_type) ^ 1 def serialize(self) -> str: - return self.name + return {".class": "RVec", "item_type": self.item_type.serialize()} + + @classmethod + def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> RTuple: + return RVec(deserialize_type(data["item_type"], ctx)) def vec_depth(t: RVec) -> int: diff --git a/mypyc/test-data/run-vec-i64.test b/mypyc/test-data/run-vec-i64.test index 945dc0db879ad..4669721db00b9 100644 --- a/mypyc/test-data/run-vec-i64.test +++ b/mypyc/test-data/run-vec-i64.test @@ -359,3 +359,8 @@ def test_pop_index() -> None: v, n = pop(v, 0) assert n == 15 assert v == vec[i64]() + +v: Final = vec[i64]([5, 6]) + +def test_final() -> None: + assert v == vec[i64]([5, 6]) diff --git a/mypyc/test-data/run-vec-nested.test b/mypyc/test-data/run-vec-nested.test index 08d54ca5dbb3e..0888e5c063446 100644 --- a/mypyc/test-data/run-vec-nested.test +++ b/mypyc/test-data/run-vec-nested.test @@ -417,3 +417,8 @@ def test_doubly_nested_vec_i64_operations() -> None: assert x == a # TODO: box/unbox + +v: Final = vec[vec[str]]([vec[str](["x"])]) + +def test_final() -> None: + assert v == vec[vec[str]]([vec[str](["x"])]) diff --git a/mypyc/test-data/run-vec-t.test b/mypyc/test-data/run-vec-t.test index 9898e9dda7d45..3dc2695ec6c36 100644 --- a/mypyc/test-data/run-vec-t.test +++ b/mypyc/test-data/run-vec-t.test @@ -339,6 +339,11 @@ def test_pop_index() -> None: assert n == "15" assert v == vec[str]() +v: Final = vec[str](["x", "y"]) + +def test_final() -> None: + assert v == vec[str](["x", "y"]) + [case testVecTBasicOps_python_3_10] from vecs import vec, append, remove, pop From 588c5da9fcf09fdc35ca64d0f58ae98561379b44 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 26 Dec 2023 17:02:37 +0000 Subject: [PATCH 073/180] Test vec as default argument value --- mypyc/test-data/run-vec-i64.test | 7 +++++++ mypyc/test-data/run-vec-nested.test | 8 ++++++++ mypyc/test-data/run-vec-t.test | 7 +++++++ 3 files changed, 22 insertions(+) diff --git a/mypyc/test-data/run-vec-i64.test b/mypyc/test-data/run-vec-i64.test index 4669721db00b9..00f9ca47aed16 100644 --- a/mypyc/test-data/run-vec-i64.test +++ b/mypyc/test-data/run-vec-i64.test @@ -364,3 +364,10 @@ v: Final = vec[i64]([5, 6]) def test_final() -> None: assert v == vec[i64]([5, 6]) + +def fdefault(v: vec[i64] = vec[i64]([5])) -> vec[i64]: + return v + +def test_default_arg() -> None: + assert fdefault() == vec[i64]([5]) + assert fdefault(vec[i64]()) == vec[i64]() diff --git a/mypyc/test-data/run-vec-nested.test b/mypyc/test-data/run-vec-nested.test index 0888e5c063446..680f56edf09a1 100644 --- a/mypyc/test-data/run-vec-nested.test +++ b/mypyc/test-data/run-vec-nested.test @@ -422,3 +422,11 @@ v: Final = vec[vec[str]]([vec[str](["x"])]) def test_final() -> None: assert v == vec[vec[str]]([vec[str](["x"])]) + +def fdefault(v: vec[vec[str]] = vec[vec[str]]()) -> vec[vec[str]]: + return v + +def test_default_arg() -> None: + assert fdefault() == vec[vec[str]]() + v = vec[vec[str]]([vec[str](["x"])]) + assert fdefault(v) == v diff --git a/mypyc/test-data/run-vec-t.test b/mypyc/test-data/run-vec-t.test index 3dc2695ec6c36..780abbf37f177 100644 --- a/mypyc/test-data/run-vec-t.test +++ b/mypyc/test-data/run-vec-t.test @@ -344,6 +344,13 @@ v: Final = vec[str](["x", "y"]) def test_final() -> None: assert v == vec[str](["x", "y"]) +def fdefault(v: vec[str] = vec[str](["x"])) -> vec[str]: + return v + +def test_default_arg() -> None: + assert fdefault() == vec[str](["x"]) + assert fdefault(vec[str](["y"])) == vec[str](["y"]) + [case testVecTBasicOps_python_3_10] from vecs import vec, append, remove, pop From b37ee5c5d75bacf66b99bc1ac7f41d23dbd46618 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 26 Dec 2023 17:16:33 +0000 Subject: [PATCH 074/180] Test vec as tuple item --- mypyc/test-data/run-vec-i64.test | 18 +++++++++++++++++- mypyc/test-data/run-vec-nested.test | 18 +++++++++++++++++- mypyc/test-data/run-vec-t.test | 21 ++++++++++++++++++++- 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/mypyc/test-data/run-vec-i64.test b/mypyc/test-data/run-vec-i64.test index 00f9ca47aed16..21b695d481886 100644 --- a/mypyc/test-data/run-vec-i64.test +++ b/mypyc/test-data/run-vec-i64.test @@ -2,7 +2,7 @@ -- which use a similar runtime representation. [case testVecI64BasicOps] -from typing import Final, Any, Iterable +from typing import Final, Any, Iterable, Tuple from mypy_extensions import i64 from vecs import vec, append, remove, pop @@ -371,3 +371,19 @@ def fdefault(v: vec[i64] = vec[i64]([5])) -> vec[i64]: def test_default_arg() -> None: assert fdefault() == vec[i64]([5]) assert fdefault(vec[i64]()) == vec[i64]() + +def ftuple(b: bool) -> Tuple[vec[i64], vec[i64]]: + if b: + raise RuntimeError() + return vec[i64]([5]), vec[i64]() + +def test_tuple() -> None: + t = ftuple(False) + assert len(t) == 2 + assert t[0] == vec[i64]([5]) + assert t[1] == vec[i64]() + a, b = t + assert a == t[0] + assert b == t[1] + with assertRaises(RuntimeError): + ftuple(True) diff --git a/mypyc/test-data/run-vec-nested.test b/mypyc/test-data/run-vec-nested.test index 680f56edf09a1..350ef9cffe981 100644 --- a/mypyc/test-data/run-vec-nested.test +++ b/mypyc/test-data/run-vec-nested.test @@ -1,5 +1,5 @@ [case testVecNestedBasicOps] -from typing import Final, Any, Iterable, Optional +from typing import Final, Any, Iterable, Optional, Tuple import sys from mypy_extensions import i64 @@ -430,3 +430,19 @@ def test_default_arg() -> None: assert fdefault() == vec[vec[str]]() v = vec[vec[str]]([vec[str](["x"])]) assert fdefault(v) == v + +def ftuple(b: bool) -> Tuple[vec[vec[str]], vec[vec[str]]]: + if b: + raise RuntimeError() + return vec[vec[str]]([vec[str]()]), vec[vec[str]]() + +def test_tuple() -> None: + t = ftuple(False) + assert len(t) == 2 + assert t[0] == vec[vec[str]]([vec[str]()]) + assert t[1] == vec[vec[str]]() + a, b = t + assert a == t[0] + assert b == t[1] + with assertRaises(RuntimeError): + ftuple(True) diff --git a/mypyc/test-data/run-vec-t.test b/mypyc/test-data/run-vec-t.test index 780abbf37f177..3cde38a38ab07 100644 --- a/mypyc/test-data/run-vec-t.test +++ b/mypyc/test-data/run-vec-t.test @@ -2,7 +2,7 @@ -- Also tests for vec[t | None], which uses the same representation. [case testVecTBasicOps] -from typing import Final, Any, Iterable, Optional +from typing import Final, Any, Iterable, Optional, Tuple from mypy_extensions import i64 from vecs import vec, append, remove, pop @@ -351,6 +351,25 @@ def test_default_arg() -> None: assert fdefault() == vec[str](["x"]) assert fdefault(vec[str](["y"])) == vec[str](["y"]) +def ftuple(b: bool) -> Tuple[vec[str], vec[str]]: + if b: + raise RuntimeError() + return vec[str](["x"]), vec[str]() + +def test_tuple() -> None: + t = ftuple(False) + assert len(t) == 2 + assert t[0] == vec[str](["x"]) + assert t[1] == vec[str]() + a, b = t + assert a == t[0] + assert b == t[1] + try: + ftuple(True) + assert False + except RuntimeError: + pass + [case testVecTBasicOps_python_3_10] from vecs import vec, append, remove, pop From ac8a09a38b37cd949976d8400675464ea8d79d70 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 26 Dec 2023 18:07:36 +0000 Subject: [PATCH 075/180] Fix wrapper functions --- mypyc/codegen/emit.py | 10 +++++++++- mypyc/test-data/run-vec-i64.test | 3 +++ mypyc/test-data/run-vec-nested.test | 8 ++++++++ mypyc/test-data/run-vec-t.test | 13 +++++++++---- 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index 0203a2eda4fb1..4cbc8d41aa68c 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -950,6 +950,7 @@ def emit_unbox( declare_dest: If True, also declare the variable 'dest' error: What happens on error raise_exception: If True, also raise TypeError on failure + optional: If True, NULL src value is allowed and will map to error value borrow: If True, create a borrowed reference """ @@ -1084,9 +1085,13 @@ def emit_unbox( elif isinstance(typ, RVec): if declare_dest: self.emit_line(f"{self.ctype(typ)} {dest};") - # TODO: Handle 'optional' # TODO: Handle 'failure' + if optional: + self.emit_line(f"if ({src} == NULL) {{") + self.emit_line(f"{dest} = {self.c_error_value(typ)};") + self.emit_line("} else {") + # TODO: Use helper function to pick the api variant if is_int64_rprimitive(typ.item_type): self.emit_line(f"{dest} = VecI64Api.unbox({src});") @@ -1101,6 +1106,9 @@ def emit_unbox( else: self.emit_line( f"{dest} = VecTExtApi.unbox({src}, {type_value}, {depth});") + + if optional: + self.emit_line("}") else: assert False, "Unboxing not implemented: %s" % typ diff --git a/mypyc/test-data/run-vec-i64.test b/mypyc/test-data/run-vec-i64.test index 21b695d481886..ab4a9075335a5 100644 --- a/mypyc/test-data/run-vec-i64.test +++ b/mypyc/test-data/run-vec-i64.test @@ -371,6 +371,9 @@ def fdefault(v: vec[i64] = vec[i64]([5])) -> vec[i64]: def test_default_arg() -> None: assert fdefault() == vec[i64]([5]) assert fdefault(vec[i64]()) == vec[i64]() + f_any: Any = fdefault + assert f_any() == vec[i64]([5]) + assert f_any(vec[i64]([6])) == vec[i64]([6]) def ftuple(b: bool) -> Tuple[vec[i64], vec[i64]]: if b: diff --git a/mypyc/test-data/run-vec-nested.test b/mypyc/test-data/run-vec-nested.test index 350ef9cffe981..e7678b2a38a64 100644 --- a/mypyc/test-data/run-vec-nested.test +++ b/mypyc/test-data/run-vec-nested.test @@ -430,6 +430,9 @@ def test_default_arg() -> None: assert fdefault() == vec[vec[str]]() v = vec[vec[str]]([vec[str](["x"])]) assert fdefault(v) == v + f_any: Any = fdefault + assert f_any() == vec[vec[str]]() + assert f_any(v) == v def ftuple(b: bool) -> Tuple[vec[vec[str]], vec[vec[str]]]: if b: @@ -446,3 +449,8 @@ def test_tuple() -> None: assert b == t[1] with assertRaises(RuntimeError): ftuple(True) + f_any: Any = ftuple + t2 = f_any(False) + assert t == t2 + with assertRaises(RuntimeError): + f_any(True) diff --git a/mypyc/test-data/run-vec-t.test b/mypyc/test-data/run-vec-t.test index 3cde38a38ab07..512483d1525c5 100644 --- a/mypyc/test-data/run-vec-t.test +++ b/mypyc/test-data/run-vec-t.test @@ -350,6 +350,9 @@ def fdefault(v: vec[str] = vec[str](["x"])) -> vec[str]: def test_default_arg() -> None: assert fdefault() == vec[str](["x"]) assert fdefault(vec[str](["y"])) == vec[str](["y"]) + f_any: Any = fdefault + assert f_any() == vec[str](["x"]) + assert f_any(vec[str](["y"])) == vec[str](["y"]) def ftuple(b: bool) -> Tuple[vec[str], vec[str]]: if b: @@ -364,11 +367,13 @@ def test_tuple() -> None: a, b = t assert a == t[0] assert b == t[1] - try: + with assertRaises(RuntimeError): ftuple(True) - assert False - except RuntimeError: - pass + f_any: Any = ftuple + t2 = f_any(False) + assert t == t2 + with assertRaises(RuntimeError): + f_any(True) [case testVecTBasicOps_python_3_10] from vecs import vec, append, remove, pop From cddaea6b8704f8939e99016cd5890336a35a31f9 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 26 Dec 2023 19:14:27 +0000 Subject: [PATCH 076/180] Fix memory corruption bug --- mypyc/codegen/emit.py | 6 +++--- mypyc/test/test_emitfunc.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index 4cbc8d41aa68c..ab8e82b2dae1c 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -1114,14 +1114,14 @@ def emit_unbox( def vec_item_type_c(self, typ: RVec) -> str: item_type = typ.unwrap_item_type() - type_value = f"(size_t)&{self.type_c_name(item_type)}" + type_value = f"(size_t){self.type_c_ptr(item_type)}" if typ.is_optional(): type_value = f"{type_value} | 1" return type_value - def type_c_name(self, typ: RPrimitive | RInstance) -> str | None: + def type_c_ptr(self, typ: RPrimitive | RInstance) -> str | None: if isinstance(typ, RPrimitive) and typ.is_refcounted: - return builtin_names[typ.name][1] + return "&" + builtin_names[typ.name][1] elif isinstance(typ, RInstance): return self.type_struct_name(typ.class_ir) return None diff --git a/mypyc/test/test_emitfunc.py b/mypyc/test/test_emitfunc.py index ba75250d01151..2f993ea2f3bee 100644 --- a/mypyc/test/test_emitfunc.py +++ b/mypyc/test/test_emitfunc.py @@ -392,7 +392,7 @@ def test_unbox_vec(self) -> None: ) self.assert_emit( Unbox(self.o, RVec(self.r.type), 55), - """cpy_r_r0 = VecTApi.unbox(cpy_r_o, (size_t)&CPyType_A);""", + """cpy_r_r0 = VecTApi.unbox(cpy_r_o, (size_t)CPyType_A);""", ) def test_unbox_vec_nested(self) -> None: From d09b2d2ffbe91e2f9fc8bee154f8f5d7a4c018ab Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 26 Dec 2023 20:06:11 +0000 Subject: [PATCH 077/180] Fix vec argument type checking in wrapper functions --- mypyc/codegen/emit.py | 5 ++- mypyc/ir/rtypes.py | 9 ++++- mypyc/test-data/irbuild-vec-t.test | 14 ++++---- mypyc/test-data/refcount.test | 2 +- mypyc/test-data/run-vec-i64.test | 13 ++++++-- mypyc/test-data/run-vec-nested.test | 12 +++++++ mypyc/test-data/run-vec-t.test | 17 +++++++--- mypyc/test/test_emitfunc.py | 52 +++++++++++++++++++++++------ 8 files changed, 98 insertions(+), 26 deletions(-) diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index ab8e82b2dae1c..e4212eb762c6f 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -1085,7 +1085,6 @@ def emit_unbox( elif isinstance(typ, RVec): if declare_dest: self.emit_line(f"{self.ctype(typ)} {dest};") - # TODO: Handle 'failure' if optional: self.emit_line(f"if ({src} == NULL) {{") @@ -1107,6 +1106,10 @@ def emit_unbox( self.emit_line( f"{dest} = VecTExtApi.unbox({src}, {type_value}, {depth});") + self.emit_line(f"if (VEC_IS_ERROR({dest})) {{") + self.emit_line(failure) + self.emit_line("}") + if optional: self.emit_line("}") else: diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 01b33dafb65f0..087ce86018065 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -1094,8 +1094,15 @@ def field_type(self, name: str) -> RType: def accept(self, visitor: "RTypeVisitor[T]") -> T: return visitor.visit_rvec(self) + def __str__(self) -> str: + if self.is_optional() and self.depth() == 0: + type_str = f"{self.unwrap_item_type()} | None" + else: + type_str = str(self.item_type) + return f"vec[{type_str}]" + def __repr__(self) -> str: - return "" % self.item_type + return "" % self.item_type_str def __eq__(self, other: object) -> bool: return isinstance(other, RVec) and other.item_type == self.item_type diff --git a/mypyc/test-data/irbuild-vec-t.test b/mypyc/test-data/irbuild-vec-t.test index 4b72a76465c00..13d73e9d5c22a 100644 --- a/mypyc/test-data/irbuild-vec-t.test +++ b/mypyc/test-data/irbuild-vec-t.test @@ -65,7 +65,7 @@ def native_class() -> vec[Optional[C]]: def primitive(): r0 :: object r1, r2 :: ptr - r3 :: vec[union[str, None]] + r3 :: vec[str | None] L0: r0 = load_address PyUnicode_Type r1 = r0 @@ -75,7 +75,7 @@ L0: def native_class(): r0 :: object r1, r2 :: ptr - r3 :: vec[union[__main__.C, None]] + r3 :: vec[__main__.C | None] L0: r0 = __main__.C :: type r1 = r0 @@ -92,14 +92,14 @@ def f(v: vec[Optional[str]]) -> vec[Optional[str]]: return append(v, None) [out] def f(v): - v :: vec[union[str, None]] + v :: vec[str | None] r0 :: str r1 :: object r2, r3 :: ptr - r4 :: vec[union[str, None]] + r4 :: vec[str | None] r5, r6 :: object r7, r8 :: ptr - r9 :: vec[union[str, None]] + r9 :: vec[str | None] L0: r0 = 'x' r1 = load_address PyUnicode_Type @@ -132,7 +132,7 @@ L0: r0 = v.len return r0 def g(v): - v :: vec[union[str, None]] + v :: vec[str | None] r0 :: native_int L0: r0 = v.len @@ -193,7 +193,7 @@ def f(v: vec[Optional[str]], n: i64) -> Optional[str]: return v[n] [out] def f(v, n): - v :: vec[union[str, None]] + v :: vec[str | None] n :: i64 r0 :: native_int r1 :: bit diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index 5e002d5089249..901f6f728b70e 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -1561,7 +1561,7 @@ def f(n): n :: i64 r0, r1 :: object r2, r3 :: ptr - r4 :: vec[union[str, None]] + r4 :: vec[str | None] r5 :: object r6 :: ptr r7 :: i64 diff --git a/mypyc/test-data/run-vec-i64.test b/mypyc/test-data/run-vec-i64.test index ab4a9075335a5..da165ffdb634d 100644 --- a/mypyc/test-data/run-vec-i64.test +++ b/mypyc/test-data/run-vec-i64.test @@ -156,10 +156,11 @@ def test_box_and_unbox() -> None: assert v2[0] == 3 assert v2[1] == 6 o2: Any = "x" - with assertRaises(TypeError, "vec[i64] expected"): + with assertRaises(TypeError, "vec[i64] object expected; got str"): v2 = o2 o3: Any = vec[str]([]) - with assertRaises(TypeError, "vec[i64] expected"): + # TODO: Better message + with assertRaises(TypeError, "vec[i64] object expected; got vec"): v2 = o3 def test_construct_from_list_multiply() -> None: @@ -360,6 +361,14 @@ def test_pop_index() -> None: assert n == 15 assert v == vec[i64]() +def f(x: vec[i64]) -> None: + pass + +def test_wrapper_arg_check() -> None: + f_any: Any = f + with assertRaises(TypeError): + f_any([]) + v: Final = vec[i64]([5, 6]) def test_final() -> None: diff --git a/mypyc/test-data/run-vec-nested.test b/mypyc/test-data/run-vec-nested.test index e7678b2a38a64..89b9556914965 100644 --- a/mypyc/test-data/run-vec-nested.test +++ b/mypyc/test-data/run-vec-nested.test @@ -418,6 +418,18 @@ def test_doubly_nested_vec_i64_operations() -> None: # TODO: box/unbox +def f(x: vec[vec[str]]) -> None: + pass + +def test_wrapper_arg_check() -> None: + f_any: Any = f + with assertRaises(TypeError): + f_any(vec[str]()) + with assertRaises(TypeError): + f_any(None) + with assertRaises(TypeError): + f_any([]) + v: Final = vec[vec[str]]([vec[str](["x"])]) def test_final() -> None: diff --git a/mypyc/test-data/run-vec-t.test b/mypyc/test-data/run-vec-t.test index 512483d1525c5..f7003f1855452 100644 --- a/mypyc/test-data/run-vec-t.test +++ b/mypyc/test-data/run-vec-t.test @@ -113,10 +113,10 @@ def test_box_and_unbox() -> None: assert v2[0] == "3" assert v2[1] == "6" o2: Any = None - with assertRaises(TypeError, "vec[t] expected"): + with assertRaises(TypeError, "vec[str] object expected; got None"): v2 = o2 o3: Any = vec[bytes]([]) - with assertRaises(TypeError, "vec[t] expected"): + with assertRaises(TypeError, "vec[str] object expected; got vec"): v2 = o3 def test_box_and_unbox_optional() -> None: @@ -133,11 +133,12 @@ def test_box_and_unbox_optional() -> None: assert v2 == vec[str](["5"]) o3: Any = vec[str]() - with assertRaises(TypeError, "vec[t] expected"): + # TODO: Use str | None instead union + with assertRaises(TypeError, "vec[str | None] object expected; got vec"): v = o3 o4: Any = vec[Optional[str]](["3"]) - with assertRaises(TypeError, "vec[t] expected"): + with assertRaises(TypeError, "vec[str] object expected; got vec"): v2 = o4 def test_construct_from_list_multiply() -> None: @@ -339,6 +340,14 @@ def test_pop_index() -> None: assert n == "15" assert v == vec[str]() +def f(x: vec[str]) -> None: + pass + +def test_wrapper_arg_check() -> None: + f_any: Any = f + with assertRaises(TypeError): + f_any([]) + v: Final = vec[str](["x", "y"]) def test_final() -> None: diff --git a/mypyc/test/test_emitfunc.py b/mypyc/test/test_emitfunc.py index 2f993ea2f3bee..84781e01d3901 100644 --- a/mypyc/test/test_emitfunc.py +++ b/mypyc/test/test_emitfunc.py @@ -380,28 +380,60 @@ def test_box_vec(self) -> None: def test_unbox_vec(self) -> None: self.assert_emit( - Unbox(self.o, RVec(int64_rprimitive), 55), """cpy_r_r0 = VecI64Api.unbox(cpy_r_o);""" + Unbox(self.o, RVec(int64_rprimitive), 55), + """cpy_r_r0 = VecI64Api.unbox(cpy_r_o); + if (VEC_IS_ERROR(cpy_r_r0)) { + CPy_TypeError("vec[i64]", cpy_r_o); cpy_r_r0 = (VecI64) { -1, NULL }; + } + """ ) self.assert_emit( Unbox(self.o, RVec(str_rprimitive), 55), - """cpy_r_r0 = VecTApi.unbox(cpy_r_o, (size_t)&PyUnicode_Type);""", + """cpy_r_r0 = VecTApi.unbox(cpy_r_o, (size_t)&PyUnicode_Type); + if (VEC_IS_ERROR(cpy_r_r0)) { + CPy_TypeError("vec[str]", cpy_r_o); cpy_r_r0 = (VecT) { -1, NULL }; + } + """, ) self.assert_emit( Unbox(self.o, RVec(RUnion([str_rprimitive, none_rprimitive])), 55), - """cpy_r_r0 = VecTApi.unbox(cpy_r_o, (size_t)&PyUnicode_Type | 1);""", + """cpy_r_r0 = VecTApi.unbox(cpy_r_o, (size_t)&PyUnicode_Type | 1); + if (VEC_IS_ERROR(cpy_r_r0)) { + CPy_TypeError("vec[str | None]", cpy_r_o); cpy_r_r0 = (VecT) { -1, NULL }; + } + """, ) self.assert_emit( Unbox(self.o, RVec(self.r.type), 55), - """cpy_r_r0 = VecTApi.unbox(cpy_r_o, (size_t)CPyType_A);""", + """cpy_r_r0 = VecTApi.unbox(cpy_r_o, (size_t)CPyType_A); + if (VEC_IS_ERROR(cpy_r_r0)) { + CPy_TypeError("vec[mod.A]", cpy_r_o); cpy_r_r0 = (VecT) { -1, NULL }; + } + """, ) def test_unbox_vec_nested(self) -> None: - self.assert_emit(Unbox(self.o, RVec(RVec(str_rprimitive)), 55), - """cpy_r_r0 = VecTExtApi.unbox(cpy_r_o, (size_t)&PyUnicode_Type, 1);""") - self.assert_emit(Unbox(self.o, RVec(RVec(RUnion([str_rprimitive, none_rprimitive]))), 55), - """cpy_r_r0 = VecTExtApi.unbox(cpy_r_o, (size_t)&PyUnicode_Type | 1, 1);""") - self.assert_emit(Unbox(self.o, RVec(RVec(int64_rprimitive)), 55), - """cpy_r_r0 = VecTExtApi.unbox(cpy_r_o, VEC_ITEM_TYPE_I64, 1);""") + self.assert_emit( + Unbox(self.o, RVec(RVec(str_rprimitive)), 55), + """cpy_r_r0 = VecTExtApi.unbox(cpy_r_o, (size_t)&PyUnicode_Type, 1); + if (VEC_IS_ERROR(cpy_r_r0)) { + CPy_TypeError("vec[vec[str]]", cpy_r_o); cpy_r_r0 = (VecTExt) { -1, NULL }; + } + """) + self.assert_emit( + Unbox(self.o, RVec(RVec(RUnion([str_rprimitive, none_rprimitive]))), 55), + """cpy_r_r0 = VecTExtApi.unbox(cpy_r_o, (size_t)&PyUnicode_Type | 1, 1); + if (VEC_IS_ERROR(cpy_r_r0)) { + CPy_TypeError("vec[vec[str | None]]", cpy_r_o); cpy_r_r0 = (VecTExt) { -1, NULL }; + } + """) + self.assert_emit( + Unbox(self.o, RVec(RVec(int64_rprimitive)), 55), + """cpy_r_r0 = VecTExtApi.unbox(cpy_r_o, VEC_ITEM_TYPE_I64, 1); + if (VEC_IS_ERROR(cpy_r_r0)) { + CPy_TypeError("vec[vec[i64]]", cpy_r_o); cpy_r_r0 = (VecTExt) { -1, NULL }; + } + """) def test_list_append(self) -> None: self.assert_emit( From fda4c6bae43eeb90275b0b840b3e50bd2135f1a5 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 27 Dec 2023 10:06:38 +0000 Subject: [PATCH 078/180] Update to work with the latest vec implementation --- mypyc/codegen/emit.py | 4 +- mypyc/codegen/emitmodule.py | 4 +- mypyc/ir/rtypes.py | 54 ++++++++++++------------- mypyc/irbuild/vec.py | 20 ++++----- mypyc/test-data/irbuild-vec-i64.test | 20 ++++----- mypyc/test-data/irbuild-vec-nested.test | 22 +++++----- mypyc/test-data/irbuild-vec-t.test | 4 +- mypyc/test-data/refcount.test | 27 +++++++------ mypyc/test/test_emitfunc.py | 14 +++---- 9 files changed, 85 insertions(+), 84 deletions(-) diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index e4212eb762c6f..b5a0b4f4d5256 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -1104,7 +1104,7 @@ def emit_unbox( self.emit_line(f"{dest} = VecTApi.unbox({src}, {type_value});") else: self.emit_line( - f"{dest} = VecTExtApi.unbox({src}, {type_value}, {depth});") + f"{dest} = VecNestedApi.unbox({src}, {type_value}, {depth});") self.emit_line(f"if (VEC_IS_ERROR({dest})) {{") self.emit_line(failure) @@ -1187,7 +1187,7 @@ def emit_box( if is_int64_rprimitive(typ.item_type): api = "VecI64Api" elif typ.depth() > 0: - api = "VecTExtApi" + api = "VecNestedApi" else: api = "VecTApi" # Empty vecs of this sort don't describe item type, so it needs to be diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 1bd5246906861..c68f8186890b0 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -586,7 +586,7 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]: self.declare_global("VecCapsule *", "VecApi") self.declare_global("VecI64Features ", "VecI64Api") self.declare_global("VecTFeatures ", "VecTApi") - self.declare_global("VecTExtFeatures ", "VecTExtApi") + self.declare_global("VecNestedFeatures ", "VecNestedApi") for module_name, module in self.modules.items(): if multi_file: @@ -974,7 +974,7 @@ def generate_globals_init(self, emitter: Emitter) -> None: "if (!VecApi) return -1;", "VecI64Api = *VecApi->i64;", "VecTApi = *VecApi->t;", - "VecTExtApi = *VecApi->t_ext;", + "VecNestedApi = *VecApi->nested;", ) emitter.emit_lines("is_initialized = 1;", "return 0;", "}") diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 087ce86018065..180cebf757be8 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -747,7 +747,7 @@ def visit_rtuple(self, t: RTuple) -> str: return "T{}{}".format(len(parts), "".join(parts)) def visit_rstruct(self, t: RStruct) -> str: - if t.name == "VecbufTExtItem": + if t.name == "VecNestedBufItem": return "Vi" assert False, "RStruct not supported in tuple" @@ -1039,19 +1039,19 @@ def __init__(self, item_type: RType) -> None: non_opt = item_type if is_int64_rprimitive(item_type): self._ctype = "VecI64" - self.types = [c_pyssize_t_rprimitive, VecbufI64Object] + self.types = [c_pyssize_t_rprimitive, VecI64BufObject] self.struct_type = VecI64 - self.buf_type = VecbufI64Object + self.buf_type = VecI64BufObject elif isinstance(non_opt, RVec): - self._ctype = "VecTExt" - self.types = [c_pyssize_t_rprimitive, VecbufTObject] - self.struct_type = VecTExt - self.buf_type = VecbufTExtObject + self._ctype = "VecNested" + self.types = [c_pyssize_t_rprimitive, VecTBufObject] + self.struct_type = VecNested + self.buf_type = VecNestedBufObject else: self._ctype = "VecT" - self.types = [c_pyssize_t_rprimitive, VecbufTObject] + self.types = [c_pyssize_t_rprimitive, VecTBufObject] self.struct_type = VecT - self.buf_type = VecbufTObject + self.buf_type = VecTBufObject def unwrap_item_type(self) -> RPrimitive | RInstance: """Return the non-optional value (non-vec) item type in a potentially nested vec.""" @@ -1339,14 +1339,14 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: # Buffer for vec[i64] -VecbufI64Object = RStruct( - name="VecbufI64Object", +VecI64BufObject = RStruct( + name="VecI64BufObject", names=["ob_base", "len", "items"], types=[PyVarObject, int64_rprimitive], ) # vecbuf_i64_rprimitive: Final = RPrimitive( -# "VecbufI64Object", is_unboxed=False, is_refcounted=True, ctype="VecbufI64Object *" +# "VecI64BufObject", is_unboxed=False, is_refcounted=True, ctype="VecI64BufObject *" # ) # Struct type for vec[i64] (in most cases use RVec instead). @@ -1356,8 +1356,8 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: # Buffer for vec[t] -VecbufTObject = RStruct( - name="VecbufTObject", +VecTBufObject = RStruct( + name="VecTBufObject", names=["ob_base", "item_type", "items"], types=[PyVarObject, c_pyssize_t_rprimitive, object_rprimitive], ) @@ -1367,40 +1367,40 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: name="VecT", names=["len", "buf"], types=[c_pyssize_t_rprimitive, object_rprimitive] ) -VecbufTExtItem = RStruct( - name="VecbufTExtItem", +VecNestedBufItem = RStruct( + name="VecNestedBufItem", names=["len", "buf"], types=[c_pyssize_t_rprimitive, object_non_refcounted_rprimitive], ) # Buffer for vec[vec[t]] -VecbufTExtObject = RStruct( - name="VecbufTExtObject", +VecNestedBufObject = RStruct( + name="VecNestedBufObject", names=["ob_base", "item_type", "depth", "optionals", "items"], types=[ PyVarObject, c_pyssize_t_rprimitive, int32_rprimitive, int32_rprimitive, - VecbufTExtItem, + VecNestedBufItem, ], ) # Struct type for vec[vec[...]] (in most cases use RVec instead). -VecTExt = RStruct( - name="VecTExt", names=["len", "buf"], types=[c_pyssize_t_rprimitive, object_rprimitive] +VecNested = RStruct( + name="VecNested", names=["len", "buf"], types=[c_pyssize_t_rprimitive, object_rprimitive] ) -VecbufTExtObject_rprimitive = RPrimitive( - "VecbufTExtObject_ptr", +VecNestedBufObject_rprimitive = RPrimitive( + "VecNestedBufObject_ptr", is_unboxed=False, is_pointer=True, is_refcounted=True, - ctype="VecbufTExtObject *", + ctype="VecNestedBufObject *", ) -VecTExtPopResult = RStruct( - name="VecTExtPopResult", +VecNestedPopResult = RStruct( + name="VecNestedPopResult", names=["vec", "item"], - types=[VecTExt, VecbufTExtItem], + types=[VecNested, VecNestedBufItem], ) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 6e5f1a95ffb0c..f1b02208282d4 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -39,7 +39,7 @@ RType, RUnion, RVec, - VecbufTExtItem, + VecNestedBufItem, bool_rprimitive, c_int_rprimitive, c_pyssize_t_rprimitive, @@ -122,7 +122,7 @@ def vec_create( return builder.add(call) else: call = CallC( - "VecTExtApi.alloc", + "VecNestedApi.alloc", [ length, length, @@ -327,16 +327,16 @@ def vec_set_item( def convert_to_t_ext_item(builder: LowLevelIRBuilder, item: Value) -> Value: vec_len = builder.add(GetElement(item, "len")) vec_buf = builder.add(GetElement(item, "buf")) - temp = builder.add(SetElement(Undef(VecbufTExtItem), "len", vec_len)) + temp = builder.add(SetElement(Undef(VecNestedBufItem), "len", vec_len)) return builder.add(SetElement(temp, "buf", vec_buf)) def convert_from_t_ext_item(builder: LowLevelIRBuilder, item: Value, vec_type: RVec) -> Value: - """Convert a value of type VecbufTExtItem to the corresponding RVec value.""" + """Convert a value of type VecNestedBufItem to the corresponding RVec value.""" if is_int64_rprimitive(vec_type.item_type): name = "VecI64Api.convert_from_nested" elif isinstance(vec_type.item_type, RVec): - name = "VecTExtApi.convert_from_nested" + name = "VecNestedApi.convert_from_nested" else: name = "VecTApi.convert_from_nested" @@ -373,7 +373,7 @@ def vec_append(builder: LowLevelIRBuilder, vec: Value, item: Value, line: int) - item_type_arg = [vec_item_type(builder, item_type, line)] else: coerced_item = convert_to_t_ext_item(builder, coerced_item) - name = "VecTExtApi.append" + name = "VecNestedApi.append" call = builder.add( CallC( name, @@ -401,9 +401,9 @@ def vec_pop(builder: LowLevelIRBuilder, base: Value, index: Value, line: int) -> elif vec_depth(vec_type) == 0 and not isinstance(item_type, RUnion): # TODO fix union name = "VecTApi.pop" else: - name = "VecTExtApi.pop" + name = "VecNestedApi.pop" # Nested vecs return a generic vec struct. - item_type = VecbufTExtItem + item_type = VecNestedBufItem result = builder.add( CallC( name, @@ -439,7 +439,7 @@ def vec_remove(builder: LowLevelIRBuilder, vec: Value, item: Value, line: int) - name = "VecTApi.remove" else: coerced_item = convert_to_t_ext_item(builder, coerced_item) - name = "VecTExtApi.remove" + name = "VecNestedApi.remove" call = builder.add( CallC( name, @@ -503,7 +503,7 @@ def vec_slice( elif vec_depth(vec_type) == 0 and not isinstance(item_type, RUnion): name = "VecTApi.slice" else: - name = "VecTExtApi.slice" + name = "VecNestedApi.slice" call = CallC( name, [vec, begin, end], diff --git a/mypyc/test-data/irbuild-vec-i64.test b/mypyc/test-data/irbuild-vec-i64.test index 9f0df375f85a8..982a09b84f287 100644 --- a/mypyc/test-data/irbuild-vec-i64.test +++ b/mypyc/test-data/irbuild-vec-i64.test @@ -70,7 +70,7 @@ L4: r5 = i L5: r6 = v.buf - r7 = get_element_ptr r6 items :: VecbufI64Object + r7 = get_element_ptr r6 items :: VecI64BufObject r8 = r5 * 8 r9 = r7 + r8 r10 = load_mem r9 :: i64* @@ -130,7 +130,7 @@ L4: r5 = i L5: r6 = v.buf - r7 = get_element_ptr r6 items :: VecbufI64Object + r7 = get_element_ptr r6 items :: VecI64BufObject r8 = r5 * 8 r9 = r7 + r8 set_mem r9, x :: i64* @@ -151,7 +151,7 @@ def f(): L0: r0 = VecI64Api.alloc(3, 3) r1 = r0.buf - r2 = get_element_ptr r1 items :: VecbufI64Object + r2 = get_element_ptr r1 items :: VecI64BufObject set_mem r2, 1 :: i64* r3 = r2 + 8 set_mem r3, 5 :: i64* @@ -180,7 +180,7 @@ def f(n): L0: r0 = VecI64Api.alloc(n, n) r1 = r0.buf - r2 = get_element_ptr r1 items :: VecbufI64Object + r2 = get_element_ptr r1 items :: VecI64BufObject r3 = n * 8 r4 = r2 + r3 r5 = r2 @@ -215,7 +215,7 @@ def f(n, x): L0: r0 = VecI64Api.alloc(3, 3) r1 = r0.buf - r2 = get_element_ptr r1 items :: VecbufI64Object + r2 = get_element_ptr r1 items :: VecI64BufObject r3 = 3 * 8 r4 = r2 + r3 r5 = r2 @@ -332,7 +332,7 @@ L1: if r2 goto L2 else goto L4 :: bool L2: r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufI64Object + r4 = get_element_ptr r3 items :: VecI64BufObject r5 = r0 * 8 r6 = r4 + r5 r7 = load_mem r6 :: i64* @@ -370,7 +370,7 @@ def contains(v, n): L0: r0 = v.len r1 = v.buf - r2 = get_element_ptr r1 items :: VecbufI64Object + r2 = get_element_ptr r1 items :: VecI64BufObject r3 = r0 * 8 r4 = r2 + r3 r5 = r2 @@ -432,7 +432,7 @@ L4: r5 = 0 L5: r6 = v.buf - r7 = get_element_ptr r6 items :: VecbufI64Object + r7 = get_element_ptr r6 items :: VecI64BufObject r8 = r5 * 8 r9 = r7 + r8 r10 = load_mem r9 :: i64* @@ -563,7 +563,7 @@ L4: r5 = n L5: r6 = v.buf - r7 = get_element_ptr r6 items :: VecbufI64Object + r7 = get_element_ptr r6 items :: VecI64BufObject r8 = r5 * 8 r9 = r7 + r8 r10 = load_mem r9 :: i64* @@ -586,7 +586,7 @@ L9: r17 = n L10: r18 = v.buf - r19 = get_element_ptr r18 items :: VecbufI64Object + r19 = get_element_ptr r18 items :: VecI64BufObject r20 = r17 * 8 r21 = r19 + r20 set_mem r21, r11 :: i64* diff --git a/mypyc/test-data/irbuild-vec-nested.test b/mypyc/test-data/irbuild-vec-nested.test index 8daad6103a630..c88a99eddfebf 100644 --- a/mypyc/test-data/irbuild-vec-nested.test +++ b/mypyc/test-data/irbuild-vec-nested.test @@ -17,12 +17,12 @@ def f(): L0: r0 = load_address PyUnicode_Type r1 = r0 - r2 = VecTExtApi.alloc(0, 0, r1, 1) + r2 = VecNestedApi.alloc(0, 0, r1, 1) return r2 def g(): r0 :: vec[vec[i64]] L0: - r0 = VecTExtApi.alloc(0, 0, 2, 1) + r0 = VecNestedApi.alloc(0, 0, 2, 1) return r0 [case testVecNestedAppend] @@ -36,14 +36,14 @@ def f(v, vv): vv :: vec[str] r0 :: native_int r1 :: object - r2, r3 :: VecbufTExtItem{len:native_int, buf:object_nrc} + r2, r3 :: VecNestedBufItem{len:native_int, buf:object_nrc} r4 :: vec[vec[str]] L0: r0 = vv.len r1 = vv.buf r2 = set_element undef, len, r0 r3 = set_element r2, buf, r1 - r4 = VecTExtApi.append(v, r3) + r4 = VecNestedApi.append(v, r3) keep_alive vv return r4 @@ -59,14 +59,14 @@ def f(v, vv): vv :: vec[i64] r0 :: native_int r1 :: object - r2, r3 :: VecbufTExtItem{len:native_int, buf:object_nrc} + r2, r3 :: VecNestedBufItem{len:native_int, buf:object_nrc} r4 :: vec[vec[i64]] L0: r0 = vv.len r1 = vv.buf r2 = set_element undef, len, r0 r3 = set_element r2, buf, r1 - r4 = VecTExtApi.append(v, r3) + r4 = VecNestedApi.append(v, r3) keep_alive vv return r4 @@ -133,7 +133,7 @@ L4: r5 = n L5: r6 = v.buf - r7 = get_element_ptr r6 items :: VecbufTExtObject + r7 = get_element_ptr r6 items :: VecNestedBufObject r8 = r5 * 16 r9 = r7 + r8 r10 = load_mem r9 :: vec[str]* @@ -179,7 +179,7 @@ L4: r5 = n L5: r6 = v.buf - r7 = get_element_ptr r6 items :: VecbufTExtObject + r7 = get_element_ptr r6 items :: VecNestedBufObject r8 = r5 * 16 r9 = r7 + r8 r10 = load_mem r9 :: vec[i64]* @@ -236,7 +236,7 @@ L4: r5 = n L5: r6 = v.buf - r7 = get_element_ptr r6 items :: VecbufTExtObject + r7 = get_element_ptr r6 items :: VecNestedBufObject r8 = r5 * 16 r9 = r7 + r8 r10 = borrow load_mem r9 :: vec[i64]* @@ -257,7 +257,7 @@ L9: r16 = n L10: r17 = r10.buf - r18 = get_element_ptr r17 items :: VecbufI64Object + r18 = get_element_ptr r17 items :: VecI64BufObject r19 = r16 * 8 r20 = r18 + r19 r21 = load_mem r20 :: i64* @@ -303,7 +303,7 @@ L4: r5 = n L5: r6 = v.buf - r7 = get_element_ptr r6 items :: VecbufTExtObject + r7 = get_element_ptr r6 items :: VecNestedBufObject r8 = r5 * 16 r9 = r7 + r8 r10 = load_mem r9 :: vec[vec[str]]* diff --git a/mypyc/test-data/irbuild-vec-t.test b/mypyc/test-data/irbuild-vec-t.test index 13d73e9d5c22a..aefa7ed5a6c8f 100644 --- a/mypyc/test-data/irbuild-vec-t.test +++ b/mypyc/test-data/irbuild-vec-t.test @@ -177,7 +177,7 @@ L4: r5 = n L5: r6 = v.buf - r7 = get_element_ptr r6 items :: VecbufTObject + r7 = get_element_ptr r6 items :: VecTBufObject r8 = r5 * 8 r9 = r7 + r8 r10 = load_mem r9 :: builtins.str* @@ -224,7 +224,7 @@ L4: r5 = n L5: r6 = v.buf - r7 = get_element_ptr r6 items :: VecbufTObject + r7 = get_element_ptr r6 items :: VecTBufObject r8 = r5 * 8 r9 = r7 + r8 r10 = load_mem r9 :: union* diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index 901f6f728b70e..cc6d75b1f0309 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -1540,7 +1540,7 @@ L4: r5 = i L5: r6 = v.buf - r7 = get_element_ptr r6 items :: VecbufTObject + r7 = get_element_ptr r6 items :: VecTBufObject r8 = r5 * 8 r9 = r7 + r8 r10 = borrow load_mem r9 :: builtins.str* @@ -1575,7 +1575,7 @@ L0: r3 = r2 | 1 r4 = VecTApi.alloc(n, n, r3) r5 = r4.buf - r6 = get_element_ptr r5 items :: VecbufTObject + r6 = get_element_ptr r5 items :: VecTBufObject r7 = n * 8 r8 = r6 + r7 r9 = r6 @@ -1624,7 +1624,7 @@ L1: if r2 goto L2 else goto L4 :: bool L2: r3 = v.buf - r4 = get_element_ptr r3 items :: VecbufTObject + r4 = get_element_ptr r3 items :: VecTBufObject r5 = r0 * 8 r6 = r4 + r5 r7 = load_mem r6 :: builtins.str* @@ -1662,7 +1662,7 @@ L0: r3 = r2 r4 = VecTApi.alloc(2, 2, r3) r5 = r4.buf - r6 = get_element_ptr r5 items :: VecbufTObject + r6 = get_element_ptr r5 items :: VecTBufObject inc_ref r0 set_mem r6, r0 :: builtins.str* r7 = r6 + 8 @@ -1715,7 +1715,7 @@ L4: r6 = x L5: r7 = r0.buf - r8 = get_element_ptr r7 items :: VecbufI64Object + r8 = get_element_ptr r7 items :: VecI64BufObject r9 = r6 * 8 r10 = r8 + r9 r11 = load_mem r10 :: i64* @@ -1752,7 +1752,7 @@ def f(vv, v): v :: vec[str] r0 :: native_int r1 :: object - r2, r3 :: VecbufTExtItem{len:native_int, buf:object_nrc} + r2, r3 :: VecNestedBufItem{len:native_int, buf:object_nrc} r4 :: vec[vec[str]] L0: r0 = v.len @@ -1760,7 +1760,7 @@ L0: r2 = set_element undef, len, r0 r3 = set_element r2, buf, r1 inc_ref vv - r4 = VecTExtApi.append(vv, r3) + r4 = VecNestedApi.append(vv, r3) return r4 [case testVecTGetItem] @@ -1802,7 +1802,7 @@ L4: r5 = n L5: r6 = v.buf - r7 = get_element_ptr r6 items :: VecbufTObject + r7 = get_element_ptr r6 items :: VecTBufObject r8 = r5 * 8 r9 = r7 + r8 r10 = load_mem r9 :: builtins.str* @@ -1835,7 +1835,7 @@ def f(v, n): L0: r0 = load_address PyUnicode_Type r1 = r0 - r2 = VecTExtApi.alloc(0, 0, r1, 1) + r2 = VecNestedApi.alloc(0, 0, r1, 1) r3 = r2.len r4 = n < r3 :: unsigned if r4 goto L4 else goto L1 :: bool @@ -1853,7 +1853,7 @@ L4: r8 = n L5: r9 = r2.buf - r10 = get_element_ptr r9 items :: VecbufTExtObject + r10 = get_element_ptr r9 items :: VecNestedBufObject r11 = r8 * 16 r12 = r10 + r11 r13 = load_mem r12 :: vec[str]* @@ -1874,15 +1874,15 @@ def f(v: vec[vec[str]]) -> vec[str]: [out] def f(v): v :: vec[vec[str]] - r0 :: tuple[vec[vec[str]], VecbufTExtItem{len:native_int, buf:object_nrc}] + r0 :: tuple[vec[vec[str]], VecNestedBufItem{len:native_int, buf:object_nrc}] r1, r2 :: vec[vec[str]] - r3, r4 :: VecbufTExtItem{len:native_int, buf:object_nrc} + r3, r4 :: VecNestedBufItem{len:native_int, buf:object_nrc} r5 :: vec[str] r6 :: tuple[vec[vec[str]], vec[str]] r7, r8, vv :: vec[vec[str]] r9, r10, x :: vec[str] L0: - r0 = VecTExtApi.pop(v, -1) + r0 = VecNestedApi.pop(v, -1) r1 = borrow r0[0] r2 = unborrow r1 r3 = borrow r0[1] @@ -1897,3 +1897,4 @@ L0: r10 = unborrow r9 x = r10 return x + diff --git a/mypyc/test/test_emitfunc.py b/mypyc/test/test_emitfunc.py index 84781e01d3901..3446ea6a2541e 100644 --- a/mypyc/test/test_emitfunc.py +++ b/mypyc/test/test_emitfunc.py @@ -376,7 +376,7 @@ def test_box_vec(self) -> None: """cpy_r_r0 = VecTApi.box(cpy_r_vs, (size_t)&PyUnicode_Type);""") self.assert_emit(Box(self.vs_opt), """cpy_r_r0 = VecTApi.box(cpy_r_vs, (size_t)&PyUnicode_Type | 1);""") - self.assert_emit(Box(self.vvs), """cpy_r_r0 = VecTExtApi.box(cpy_r_vvs);""") + self.assert_emit(Box(self.vvs), """cpy_r_r0 = VecNestedApi.box(cpy_r_vvs);""") def test_unbox_vec(self) -> None: self.assert_emit( @@ -415,23 +415,23 @@ def test_unbox_vec(self) -> None: def test_unbox_vec_nested(self) -> None: self.assert_emit( Unbox(self.o, RVec(RVec(str_rprimitive)), 55), - """cpy_r_r0 = VecTExtApi.unbox(cpy_r_o, (size_t)&PyUnicode_Type, 1); + """cpy_r_r0 = VecNestedApi.unbox(cpy_r_o, (size_t)&PyUnicode_Type, 1); if (VEC_IS_ERROR(cpy_r_r0)) { - CPy_TypeError("vec[vec[str]]", cpy_r_o); cpy_r_r0 = (VecTExt) { -1, NULL }; + CPy_TypeError("vec[vec[str]]", cpy_r_o); cpy_r_r0 = (VecNested) { -1, NULL }; } """) self.assert_emit( Unbox(self.o, RVec(RVec(RUnion([str_rprimitive, none_rprimitive]))), 55), - """cpy_r_r0 = VecTExtApi.unbox(cpy_r_o, (size_t)&PyUnicode_Type | 1, 1); + """cpy_r_r0 = VecNestedApi.unbox(cpy_r_o, (size_t)&PyUnicode_Type | 1, 1); if (VEC_IS_ERROR(cpy_r_r0)) { - CPy_TypeError("vec[vec[str | None]]", cpy_r_o); cpy_r_r0 = (VecTExt) { -1, NULL }; + CPy_TypeError("vec[vec[str | None]]", cpy_r_o); cpy_r_r0 = (VecNested) { -1, NULL }; } """) self.assert_emit( Unbox(self.o, RVec(RVec(int64_rprimitive)), 55), - """cpy_r_r0 = VecTExtApi.unbox(cpy_r_o, VEC_ITEM_TYPE_I64, 1); + """cpy_r_r0 = VecNestedApi.unbox(cpy_r_o, VEC_ITEM_TYPE_I64, 1); if (VEC_IS_ERROR(cpy_r_r0)) { - CPy_TypeError("vec[vec[i64]]", cpy_r_o); cpy_r_r0 = (VecTExt) { -1, NULL }; + CPy_TypeError("vec[vec[i64]]", cpy_r_o); cpy_r_r0 = (VecNested) { -1, NULL }; } """) From 39189acf61e95a54779329f23af21ca73647c0a7 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 27 Dec 2023 12:05:25 +0000 Subject: [PATCH 079/180] Only import vec capsule if needed --- mypyc/analysis/vecusage.py | 43 ++++++++++++++++++++++ mypyc/codegen/emitmodule.py | 28 +++++++++------ mypyc/lib-rt/CPy.h | 1 - mypyc/test-data/vecusage.test | 68 +++++++++++++++++++++++++++++++++++ mypyc/test/test_vecusage.py | 40 +++++++++++++++++++++ 5 files changed, 168 insertions(+), 12 deletions(-) create mode 100644 mypyc/analysis/vecusage.py create mode 100644 mypyc/test-data/vecusage.test create mode 100644 mypyc/test/test_vecusage.py diff --git a/mypyc/analysis/vecusage.py b/mypyc/analysis/vecusage.py new file mode 100644 index 0000000000000..fba39eefeee7f --- /dev/null +++ b/mypyc/analysis/vecusage.py @@ -0,0 +1,43 @@ +"""Analysis to decide whether a module needs the vec capsule.""" + +from mypyc.ir.module_ir import ModuleIR +from mypyc.ir.func_ir import FuncIR +from mypyc.ir.rtypes import RType, RVec, RUnion, RTuple, RStruct + + +def needs_vec_capsule(module: ModuleIR) -> bool: + for f in module.functions: + if func_needs_vec(f): + return True + for cl in module.classes: + for base in cl.mro: + for f in base.methods.values(): + if func_needs_vec(f): + return True + for t in base.attributes.values(): + if uses_vec_type(t): + return True + return False + + +def func_needs_vec(func: FuncIR) -> bool: + for arg in func.arg_regs: + if uses_vec_type(arg.type): + return True + if uses_vec_type(func.decl.sig.ret_type): + return True + for block in func.blocks: + for op in block.ops: + if uses_vec_type(op.type) or any(uses_vec_type(s.type) for s in op.sources()): + return True + return False + + +def uses_vec_type(typ: RType) -> None: + if isinstance(typ, RVec): + return True + if isinstance(typ, RUnion) and any(uses_vec_type(t) for t in typ.items): + return True + if isinstance(typ, (RTuple, RStruct)) and any(uses_vec_type(t) for t in typ.types): + return True + return False diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index c68f8186890b0..25f1cfde2beef 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -74,6 +74,7 @@ from mypyc.transform.refcount import insert_ref_count_opcodes from mypyc.transform.spill import insert_spills from mypyc.transform.uninit import insert_uninit_checks +from mypyc.analysis.vecusage import needs_vec_capsule # All the modules being compiled are divided into "groups". A group # is a set of modules that are placed into the same shared library. @@ -549,6 +550,7 @@ def __init__( # Multi-phase init is needed to enable free-threading. In the future we'll # probably want to enable it always, but we'll wait until it's stable. self.multi_phase_init = IS_FREE_THREADED + self.use_vec_capsule = any(needs_vec_capsule(m) for m in self.modules.values()) @property def group_suffix(self) -> str: @@ -583,10 +585,11 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]: self.generate_literal_tables() - self.declare_global("VecCapsule *", "VecApi") - self.declare_global("VecI64Features ", "VecI64Api") - self.declare_global("VecTFeatures ", "VecTApi") - self.declare_global("VecNestedFeatures ", "VecNestedApi") + if self.use_vec_capsule: + self.declare_global("VecCapsule *", "VecApi") + self.declare_global("VecI64Features ", "VecI64Api") + self.declare_global("VecTFeatures ", "VecTApi") + self.declare_global("VecNestedFeatures ", "VecNestedApi") for module_name, module in self.modules.items(): if multi_file: @@ -641,6 +644,8 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]: source_deps = collect_source_dependencies(self.modules) for source_dep in sorted(source_deps, key=lambda d: d.path): ext_declarations.emit_line(f'#include "{source_dep.get_header()}"') + if self.use_vec_capsule: + ext_declarations.emit_line('#include "vecs.h"') declarations = Emitter(self.context) declarations.emit_line(f"#ifndef MYPYC_LIBRT_INTERNAL{self.group_suffix}_H") @@ -969,13 +974,14 @@ def generate_globals_init(self, emitter: Emitter) -> None: f"if (CPyStatics_Initialize(CPyStatics, {values}) < 0) {{", "return -1;", "}" ) - emitter.emit_lines( - 'VecApi = PyCapsule_Import("vecs._C_API", 0);', - "if (!VecApi) return -1;", - "VecI64Api = *VecApi->i64;", - "VecTApi = *VecApi->t;", - "VecNestedApi = *VecApi->nested;", - ) + if self.use_vec_capsule: + emitter.emit_lines( + 'VecApi = PyCapsule_Import("vecs._C_API", 0);', + "if (!VecApi) return -1;", + "VecI64Api = *VecApi->i64;", + "VecTApi = *VecApi->t;", + "VecNestedApi = *VecApi->nested;", + ) emitter.emit_lines("is_initialized = 1;", "return 0;", "}") diff --git a/mypyc/lib-rt/CPy.h b/mypyc/lib-rt/CPy.h index f77c8e663ef48..cc8b8bafaf994 100644 --- a/mypyc/lib-rt/CPy.h +++ b/mypyc/lib-rt/CPy.h @@ -9,7 +9,6 @@ #include #include #include -#include "vecs.h" #include "pythonsupport.h" #include "mypyc_util.h" diff --git a/mypyc/test-data/vecusage.test b/mypyc/test-data/vecusage.test new file mode 100644 index 0000000000000..afb357258b046 --- /dev/null +++ b/mypyc/test-data/vecusage.test @@ -0,0 +1,68 @@ +[case testNoVecNeeded] +class vec: pass +class RVec: pass +def append(v: vec) -> None: + pass + +def f() -> None: + append(vec()) +[out] +False + +[case testVecUsedInFunction] +from vecs import vec + +def f() -> None: + vec[str]() +[out] +True + +[case testVecUsedInMethod] +from vecs import vec + +class C: + def m(self) -> None: + vec[str]() +[out] +True + +[case testVecUsedAtTopLevel] +from vecs import vec + +vec[str]() +[out] +True + +[case testVecUsedInArgTypeOnly] +from vecs import vec + +def f(v: vec[str]) -> None: + pass +[out] +True + +[case testVecUsedInReturnTypeOnly] +from vecs import vec + +def f() -> vec[str]: + assert False +[out] +True + +[case testVecUsedInAttributeTypeOnly] +from vecs import vec + +class C: + v: vec[str] +[out] +True + +[case testVecUsedInPropertyTypeOnly] +from vecs import vec + +class C: + @property + def f(self) -> vec[str]: + assert False +[out] +True diff --git a/mypyc/test/test_vecusage.py b/mypyc/test/test_vecusage.py new file mode 100644 index 0000000000000..e10bdd2b2886b --- /dev/null +++ b/mypyc/test/test_vecusage.py @@ -0,0 +1,40 @@ +"""Test cases for analysing if vec capsule is needed.""" + +from __future__ import annotations + +import os.path + +from mypy.errors import CompileError +from mypy.test.config import test_temp_dir +from mypy.test.data import DataDrivenTestCase +from mypyc.test.testutil import ( + ICODE_GEN_BUILTINS, + MypycDataSuite, + assert_test_output, + build_ir_for_single_file2, + infer_ir_build_options_from_test_name, + use_custom_builtins, +) +from mypyc.analysis.vecusage import needs_vec_capsule + +files = ["vecusage.test"] + + +class TestVecUsage(MypycDataSuite): + files = files + base_path = test_temp_dir + + def run_case(self, testcase: DataDrivenTestCase) -> None: + options = infer_ir_build_options_from_test_name(testcase.name) + if options is None: + # Skipped test case + return + with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase): + try: + ir = build_ir_for_single_file2(testcase.input, options) + except CompileError as e: + actual = e.messages + else: + actual = [str(needs_vec_capsule(ir))] + + assert_test_output(testcase, actual, "Invalid test output", testcase.output) From 85ec9ae5ca6dfd7831297dbad43e3d38bb1fcd35 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 27 Dec 2023 14:05:29 +0000 Subject: [PATCH 080/180] Fix self check --- mypyc/analysis/vecusage.py | 2 +- mypyc/ir/rtypes.py | 16 +++++++++------- mypyc/irbuild/vec.py | 12 ++++++++---- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/mypyc/analysis/vecusage.py b/mypyc/analysis/vecusage.py index fba39eefeee7f..6950149aab32a 100644 --- a/mypyc/analysis/vecusage.py +++ b/mypyc/analysis/vecusage.py @@ -33,7 +33,7 @@ def func_needs_vec(func: FuncIR) -> bool: return False -def uses_vec_type(typ: RType) -> None: +def uses_vec_type(typ: RType) -> bool: if isinstance(typ, RVec): return True if isinstance(typ, RUnion) and any(uses_vec_type(t) for t in typ.items): diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 180cebf757be8..d09fd9344f377 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -1058,8 +1058,9 @@ def unwrap_item_type(self) -> RPrimitive | RInstance: item_type = self.item_type while True: if isinstance(item_type, RUnion): - item_type = optional_value_type(item_type) - assert item_type is not None + value_type = optional_value_type(item_type) + assert value_type is not None + item_type = value_type elif isinstance(item_type, RVec): item_type = item_type.item_type elif isinstance(item_type, (RPrimitive, RInstance)): @@ -1078,8 +1079,9 @@ def is_optional(self) -> bool: def depth(self) -> int: item_type = self.item_type if isinstance(item_type, RUnion): - item_type = optional_value_type(item_type) - assert item_type is not None + value_type = optional_value_type(item_type) + assert value_type is not None + item_type = value_type if isinstance(item_type, RVec): return 1 + item_type.depth() return 0 @@ -1102,7 +1104,7 @@ def __str__(self) -> str: return f"vec[{type_str}]" def __repr__(self) -> str: - return "" % self.item_type_str + return "" % self.item_type def __eq__(self, other: object) -> bool: return isinstance(other, RVec) and other.item_type == self.item_type @@ -1110,11 +1112,11 @@ def __eq__(self, other: object) -> bool: def __hash__(self) -> int: return hash(self.item_type) ^ 1 - def serialize(self) -> str: + def serialize(self) -> JsonDict: return {".class": "RVec", "item_type": self.item_type.serialize()} @classmethod - def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> RTuple: + def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> RVec: return RVec(deserialize_type(data["item_type"], ctx)) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index f1b02208282d4..82b567d4de43d 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -197,13 +197,14 @@ def vec_item_type_info( ) -> Tuple[Optional[Value], bool, int]: if isinstance(typ, RPrimitive) and typ.is_refcounted: typ, src = builtin_names[typ.name] - return builder.load_address(src, typ), 0, 0 + return builder.load_address(src, typ), False, 0 elif isinstance(typ, RInstance): - return builder.load_native_type_object(typ.name), 0, 0 + return builder.load_native_type_object(typ.name), False, 0 elif is_int64_rprimitive(typ): - return Integer(VEC_TYPE_INFO_I64, c_size_t_rprimitive), 0, 0 + return Integer(VEC_TYPE_INFO_I64, c_size_t_rprimitive), False, 0 elif isinstance(typ, RUnion): non_opt = optional_value_type(typ) + assert non_opt is not None typeval, _, _ = vec_item_type_info(builder, non_opt, line) if typeval is not None: return typeval, True, 0 @@ -211,7 +212,7 @@ def vec_item_type_info( typeval, optional, depth = vec_item_type_info(builder, typ.item_type, line) if typeval is not None: return typeval, optional, depth + 1 - return None, 0, 0 + return None, False, 0 def vec_len(builder: LowLevelIRBuilder, val: Value) -> Value: @@ -346,11 +347,13 @@ def convert_from_t_ext_item(builder: LowLevelIRBuilder, item: Value, vec_type: R def vec_item_type(builder: LowLevelIRBuilder, item_type: RType, line: int) -> Value: typeobj, optional, depth = vec_item_type_info(builder, item_type, line) + assert typeobj is not None if isinstance(typeobj, Integer): return typeobj else: # Create an integer which will hold the type object * as an integral value. # Assign implicitly coerces between pointer/integer types. + typeval: Value typeval = Register(pointer_rprimitive) builder.add(Assign(typeval, typeobj)) if optional: @@ -421,6 +424,7 @@ def vec_pop(builder: LowLevelIRBuilder, base: Value, index: Value, line: int) -> x = builder.add(Unborrow(x)) y = builder.add(TupleGet(result, 1, borrow=True)) y = builder.add(Unborrow(y)) + assert isinstance(vec_type.item_type, RVec) z = convert_from_t_ext_item(builder, y, vec_type.item_type) result = builder.add(TupleSet([x, z], line)) builder.keep_alive([orig], steal=True) From 58a0b6f7204b0662e653cb0bf7bc8beba5103392 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 27 Dec 2023 14:04:00 +0000 Subject: [PATCH 081/180] WIP add failing test --- mypyc/test-data/irbuild-vec-i64.test | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/mypyc/test-data/irbuild-vec-i64.test b/mypyc/test-data/irbuild-vec-i64.test index 982a09b84f287..97433dad9a404 100644 --- a/mypyc/test-data/irbuild-vec-i64.test +++ b/mypyc/test-data/irbuild-vec-i64.test @@ -266,6 +266,16 @@ L3: L4: return r1 +[case testVecI64XXX] +from vecs import vec +from mypy_extensions import i64 +from typing import List + +def f(n: i64, l: List[i64]) -> vec[i64]: + return vec[i64]([x + 1 for x in l]) +[out] + + [case testVecI64ConstructFromRange] from vecs import vec, pop from mypy_extensions import i64 @@ -592,4 +602,3 @@ L10: set_mem r21, r11 :: i64* keep_alive v return 1 - From 4287e8ba2e34b8b20046b37a11a20e95092127ef Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 27 Dec 2023 15:24:10 +0000 Subject: [PATCH 082/180] Support fast list vec comprehensions (from list/tuple) --- mypyc/irbuild/for_helpers.py | 23 +++++++++++-- mypyc/irbuild/specialize.py | 10 ++++-- mypyc/irbuild/vec.py | 13 ++++++++ mypyc/test-data/irbuild-vec-i64.test | 49 ++++++++++++++++++++++++++-- 4 files changed, 88 insertions(+), 7 deletions(-) diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index 2666345818158..3afa7d86f13c6 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -70,7 +70,7 @@ from mypyc.irbuild.constant_fold import constant_fold_expr from mypyc.irbuild.prepare import GENERATOR_HELPER_NAME from mypyc.irbuild.targets import AssignmentTarget, AssignmentTargetTuple -from mypyc.irbuild.vec import vec_append, vec_create, vec_get_item_unsafe +from mypyc.irbuild.vec import vec_append, vec_create, vec_get_item_unsafe, vec_init_item_unsafe from mypyc.primitives.dict_ops import ( dict_check_size_op, dict_item_iter_op, @@ -213,7 +213,7 @@ def sequence_from_generator_preallocate_helper( builder: IRBuilder, gen: GeneratorExpr, empty_op_llbuilder: Callable[[Value, int], Value], - set_item_op: CFunctionDescription, + set_item_op: Callable[[Value, Value, Value, int], None], # CFunctionDescription, ) -> Value | None: """Generate a new tuple or list from a simple generator expression. @@ -300,12 +300,15 @@ def translate_list_comprehension(builder: IRBuilder, gen: GeneratorExpr) -> Valu if raise_error_if_contains_unreachable_names(builder, gen): return builder.none() + def set_item(x: Value, y: Value, z: Value, line: int) -> None: + builder.call_c(new_list_set_item_op, [x, y, z], line) + # Try simplest list comprehension, otherwise fall back to general one val = sequence_from_generator_preallocate_helper( builder, gen, empty_op_llbuilder=builder.builder.new_list_op_with_length, - set_item_op=new_list_set_item_op, + set_item_op=set_item, ) if val is not None: return val @@ -358,6 +361,20 @@ def gen_inner_stmts() -> None: def translate_vec_comprehension(builder: IRBuilder, vec_type: RVec, gen: GeneratorExpr) -> Value: + def set_item(x: Value, y: Value, z: Value, line: int) -> None: + vec_init_item_unsafe(builder.builder, x, y, z, line) + + # Try simplest comprehension, otherwise fall back to general one + val = sequence_from_generator_preallocate_helper( + builder, + gen, + empty_op_llbuilder=lambda length, line: vec_create(builder.builder, + vec_type, length, line), + set_item_op=set_item, + ) + if val is not None: + return val + vec = Register(vec_type) builder.assign(vec, vec_create(builder.builder, vec_type, 0, gen.line), gen.line) loop_params = list(zip(gen.indices, gen.sequences, gen.condlists, gen.is_async)) diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index 6ff6700c55144..9ec7aaba4b11f 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -380,11 +380,14 @@ def translate_list_from_generator_call( and expr.arg_kinds[0] == ARG_POS and isinstance(expr.args[0], GeneratorExpr) ): + def set_item(x: Value, y: Value, z: Value, line: int) -> None: + builder.call_c(new_list_set_item_op, [x, y, z], line) + return sequence_from_generator_preallocate_helper( builder, expr.args[0], empty_op_llbuilder=builder.builder.new_list_op_with_length, - set_item_op=new_list_set_item_op, + set_item_op=set_item, ) return None @@ -405,11 +408,14 @@ def translate_tuple_from_generator_call( and expr.arg_kinds[0] == ARG_POS and isinstance(expr.args[0], GeneratorExpr) ): + def set_item(x: Value, y: Value, z: Value, line: int) -> None: + builder.call_c(new_tuple_set_item_op, [x, y, z], line) + return sequence_from_generator_preallocate_helper( builder, expr.args[0], empty_op_llbuilder=builder.builder.new_tuple_with_length, - set_item_op=new_tuple_set_item_op, + set_item_op=set_item, ) return None diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 82b567d4de43d..fdc3a3fb0a6ff 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -325,6 +325,19 @@ def vec_set_item( builder.keep_alive([base]) +def vec_init_item_unsafe( + builder: LowLevelIRBuilder, base: Value, index: Value, item: Value, line: int +) -> None: + assert isinstance(base.type, RVec) + index = as_platform_int(builder, index, line) + vtype = base.type + item_addr = vec_item_ptr(builder, base, index) + item_type = vtype.item_type + item = builder.coerce(item, item_type, line) + builder.set_mem(item_addr, item_type, item) + builder.keep_alive([base]) + + def convert_to_t_ext_item(builder: LowLevelIRBuilder, item: Value) -> Value: vec_len = builder.add(GetElement(item, "len")) vec_buf = builder.add(GetElement(item, "buf")) diff --git a/mypyc/test-data/irbuild-vec-i64.test b/mypyc/test-data/irbuild-vec-i64.test index 97433dad9a404..66c69cf8a51ab 100644 --- a/mypyc/test-data/irbuild-vec-i64.test +++ b/mypyc/test-data/irbuild-vec-i64.test @@ -266,7 +266,7 @@ L3: L4: return r1 -[case testVecI64XXX] +[case testVecI64FastListComprehension] from vecs import vec from mypy_extensions import i64 from typing import List @@ -274,7 +274,52 @@ from typing import List def f(n: i64, l: List[i64]) -> vec[i64]: return vec[i64]([x + 1 for x in l]) [out] - +def f(n, l): + n :: i64 + l :: list + r0 :: ptr + r1 :: native_int + r2 :: vec[i64] + r3 :: native_int + r4 :: ptr + r5 :: native_int + r6 :: bit + r7 :: object + r8, x, r9 :: i64 + r10 :: object + r11 :: ptr + r12 :: native_int + r13 :: ptr + r14 :: native_int +L0: + r0 = get_element_ptr l ob_size :: PyVarObject + r1 = load_mem r0 :: native_int* + keep_alive l + r2 = VecI64Api.alloc(r1, r1) + r3 = 0 +L1: + r4 = get_element_ptr l ob_size :: PyVarObject + r5 = load_mem r4 :: native_int* + keep_alive l + r6 = r3 < r5 :: signed + if r6 goto L2 else goto L4 :: bool +L2: + r7 = CPyList_GetItemUnsafe(l, r3) + r8 = unbox(i64, r7) + x = r8 + r9 = x + 1 + r10 = r2.buf + r11 = get_element_ptr r10 items :: VecI64BufObject + r12 = r3 * 8 + r13 = r11 + r12 + set_mem r13, r9 :: i64* + keep_alive r2 +L3: + r14 = r3 + 1 + r3 = r14 + goto L1 +L4: + return r2 [case testVecI64ConstructFromRange] from vecs import vec, pop From ea8490f33c5989e846e46e5587fcca1de86caf97 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 27 Dec 2023 17:08:16 +0000 Subject: [PATCH 083/180] Support fast comprehensions iterating over vecs --- mypyc/irbuild/for_helpers.py | 4 +- mypyc/test-data/irbuild-vec-i64.test | 57 +++++++++++++++++++++++++++- 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index 3afa7d86f13c6..8f756cdd4ccfd 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -174,7 +174,7 @@ def for_loop_helper_with_index( body_insts: a function that generates the body of the loop. It needs a index as parameter. """ - assert is_sequence_rprimitive(expr_reg.type), (expr_reg, expr_reg.type) + assert is_sequence_rprimitive(expr_reg.type) or isinstance(expr_reg.type, RVec), (expr_reg, expr_reg.type) target_type = builder.get_sequence_type(expr) body_block = BasicBlock() @@ -241,7 +241,7 @@ def sequence_from_generator_preallocate_helper( line = gen.line sequence_expr = gen.sequences[0] rtype = builder.node_type(sequence_expr) - if not (is_sequence_rprimitive(rtype) or isinstance(rtype, RTuple)): + if not (is_sequence_rprimitive(rtype) or isinstance(rtype, RTuple) or isinstance(rtype, RVec)): return None if isinstance(rtype, RTuple): diff --git a/mypyc/test-data/irbuild-vec-i64.test b/mypyc/test-data/irbuild-vec-i64.test index 66c69cf8a51ab..c5951583656e6 100644 --- a/mypyc/test-data/irbuild-vec-i64.test +++ b/mypyc/test-data/irbuild-vec-i64.test @@ -266,7 +266,7 @@ L3: L4: return r1 -[case testVecI64FastListComprehension] +[case testVecI64FastComprehensionFromList] from vecs import vec from mypy_extensions import i64 from typing import List @@ -321,6 +321,61 @@ L3: L4: return r2 +[case testVecI64FastComprehensionFromVec] +from vecs import vec +from mypy_extensions import i64 +from typing import List + +def f(n: i64, v: vec[i64]) -> vec[i64]: + return vec[i64]([x + 1 for x in v]) +[out] +def f(n, v): + n :: i64 + v :: vec[i64] + r0 :: native_int + r1 :: vec[i64] + r2, r3 :: native_int + r4 :: bit + r5 :: object + r6 :: ptr + r7 :: native_int + r8 :: ptr + r9, x, r10 :: i64 + r11 :: object + r12 :: ptr + r13 :: native_int + r14 :: ptr + r15 :: native_int +L0: + r0 = v.len + r1 = VecI64Api.alloc(r0, r0) + r2 = 0 +L1: + r3 = v.len + r4 = r2 < r3 :: signed + if r4 goto L2 else goto L4 :: bool +L2: + r5 = v.buf + r6 = get_element_ptr r5 items :: VecI64BufObject + r7 = r2 * 8 + r8 = r6 + r7 + r9 = load_mem r8 :: i64* + keep_alive v + x = r9 + r10 = x + 1 + r11 = r1.buf + r12 = get_element_ptr r11 items :: VecI64BufObject + r13 = r2 * 8 + r14 = r12 + r13 + set_mem r14, r10 :: i64* + keep_alive r1 +L3: + r15 = r2 + 1 + r2 = r15 + goto L1 +L4: + return r1 + [case testVecI64ConstructFromRange] from vecs import vec, pop from mypy_extensions import i64 From 269325b4c7f0bc61fbcdae788795714672e866f8 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 28 Jan 2024 14:32:14 +0000 Subject: [PATCH 084/180] Support additional item types in vec creation --- mypyc/irbuild/vec.py | 20 +++++++++-- mypyc/test-data/irbuild-vec-misc.test | 49 +++++++++++++++++++++++++++ mypyc/test/test_irbuild.py | 1 + 3 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 mypyc/test-data/irbuild-vec-misc.test diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index fdc3a3fb0a6ff..42d7b8ca15017 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -44,8 +44,11 @@ c_int_rprimitive, c_pyssize_t_rprimitive, c_size_t_rprimitive, - int32_rprimitive, int64_rprimitive, + int32_rprimitive, + int16_rprimitive, + uint8_rprimitive, + float_rprimitive, is_c_py_ssize_t_rprimitive, is_int32_rprimitive, is_int64_rprimitive, @@ -64,6 +67,16 @@ from mypyc.irbuild.ll_builder import LowLevelIRBuilder +vec_api_by_item_type = { + int64_rprimitive: "VecI64Api", + int32_rprimitive: "VecI32Api", + int16_rprimitive: "VecI16Api", + uint8_rprimitive: "VecU8Api", + float_rprimitive: "VecFloatApi", + bool_rprimitive: "VecBoolApi", +} + + def as_platform_int(builder: LowLevelIRBuilder, v: Value, line: int) -> Value: rtype = v.type if is_c_py_ssize_t_rprimitive(rtype): @@ -88,9 +101,10 @@ def vec_create( length = as_platform_int(builder, length, line) item_type = vtype.item_type - if is_int64_rprimitive(item_type): + api_name = vec_api_by_item_type.get(item_type) + if api_name is not None: call = CallC( - "VecI64Api.alloc", [length, length], vtype, False, False, error_kind=ERR_MAGIC, + f"{api_name}.alloc", [length, length], vtype, False, False, error_kind=ERR_MAGIC, line=line ) return builder.add(call) diff --git a/mypyc/test-data/irbuild-vec-misc.test b/mypyc/test-data/irbuild-vec-misc.test new file mode 100644 index 0000000000000..3a1f6e68e5a24 --- /dev/null +++ b/mypyc/test-data/irbuild-vec-misc.test @@ -0,0 +1,49 @@ +-- Test cases for packed/specialized vec item types other than vec[i64], such as +-- vec[i32] and vec[float]. Since many of the code paths are the same as for +-- vec[i64], only test a subset of functionality. + +[case testVecMiscCreateEmpty] +from vecs import vec, append + +from mypy_extensions import i32, i16, u8 + +def create_float() -> vec[float]: + return vec[float]() + +def create_i32() -> vec[i32]: + return vec[i32]() + +def create_i16() -> vec[i16]: + return vec[i16]() + +def create_u8() -> vec[u8]: + return vec[u8]() + +def create_bool() -> vec[bool]: + return vec[bool]() +[out] +def create_float(): + r0 :: vec[float] +L0: + r0 = VecFloatApi.alloc(0, 0) + return r0 +def create_i32(): + r0 :: vec[i32] +L0: + r0 = VecI32Api.alloc(0, 0) + return r0 +def create_i16(): + r0 :: vec[i16] +L0: + r0 = VecI16Api.alloc(0, 0) + return r0 +def create_u8(): + r0 :: vec[u8] +L0: + r0 = VecU8Api.alloc(0, 0) + return r0 +def create_bool(): + r0 :: vec[bool] +L0: + r0 = VecBoolApi.alloc(0, 0) + return r0 diff --git a/mypyc/test/test_irbuild.py b/mypyc/test/test_irbuild.py index 3362c81f26f79..92048277565d3 100644 --- a/mypyc/test/test_irbuild.py +++ b/mypyc/test/test_irbuild.py @@ -45,6 +45,7 @@ "irbuild-i16.test", "irbuild-u8.test", "irbuild-vec-i64.test", + "irbuild-vec-misc.test", "irbuild-vec-t.test", "irbuild-vec-nested.test", "irbuild-vectorcall.test", From db6b555a1b0f767497fbbe31e856ce0a085720cf Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 28 Jan 2024 14:38:31 +0000 Subject: [PATCH 085/180] Support append with additional item types Also test get item with float item type (code path is generic, so need for further tests). --- mypyc/irbuild/vec.py | 5 +- mypyc/test-data/irbuild-vec-misc.test | 71 ++++++++++++++++++++++++++- 2 files changed, 73 insertions(+), 3 deletions(-) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 42d7b8ca15017..bd08d9d1d3cc4 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -396,8 +396,9 @@ def vec_append(builder: LowLevelIRBuilder, vec: Value, item: Value, line: int) - item_type = vec_type.item_type coerced_item = builder.coerce(item, item_type, line) item_type_arg = [] - if is_int64_rprimitive(item_type): - name = "VecI64Api.append" + api_name = vec_api_by_item_type.get(item_type) + if api_name is not None: + name = f"{api_name}.append" elif vec_depth(vec_type) == 0: name = "VecTApi.append" item_type_arg = [vec_item_type(builder, item_type, line)] diff --git a/mypyc/test-data/irbuild-vec-misc.test b/mypyc/test-data/irbuild-vec-misc.test index 3a1f6e68e5a24..6a233254d4816 100644 --- a/mypyc/test-data/irbuild-vec-misc.test +++ b/mypyc/test-data/irbuild-vec-misc.test @@ -3,7 +3,7 @@ -- vec[i64], only test a subset of functionality. [case testVecMiscCreateEmpty] -from vecs import vec, append +from vecs import vec from mypy_extensions import i32, i16, u8 @@ -47,3 +47,72 @@ def create_bool(): L0: r0 = VecBoolApi.alloc(0, 0) return r0 + +[case testVecMiscAppend] +from vecs import vec, append + +from mypy_extensions import i32 + +def append_float(v: vec[float]) -> vec[float]: + return append(v, 1.5) + +def append_i32(v: vec[i32]) -> vec[i32]: + return append(v, 123) +[out] +def append_float(v): + v, r0 :: vec[float] +L0: + r0 = VecFloatApi.append(v, 1.5) + return r0 +def append_i32(v): + v, r0 :: vec[i32] +L0: + r0 = VecI32Api.append(v, 123) + return r0 + +[case testVecMiscGetItem] +from vecs import vec + +from mypy_extensions import i64 + +def get_item_float(v: vec[float], i: i64) -> float: + return v[i] +[out] +def get_item_float(v, i): + v :: vec[float] + i :: i64 + r0 :: native_int + r1 :: bit + r2 :: i64 + r3 :: bit + r4 :: bool + r5 :: i64 + r6 :: object + r7 :: ptr + r8 :: i64 + r9 :: ptr + r10 :: float +L0: + r0 = v.len + r1 = i < r0 :: unsigned + if r1 goto L4 else goto L1 :: bool +L1: + r2 = i + r0 + r3 = r2 < r0 :: unsigned + if r3 goto L3 else goto L2 :: bool +L2: + r4 = raise IndexError + unreachable +L3: + r5 = r2 + goto L5 +L4: + r5 = i +L5: + r6 = v.buf + r7 = get_element_ptr r6 items :: VecTBufObject + r8 = r5 * 8 + r9 = r7 + r8 + r10 = load_mem r9 :: builtins.float* + keep_alive v + return r10 From e514bd260e7c7a67d6e28df2c23369dca7883536 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 28 Jan 2024 14:43:26 +0000 Subject: [PATCH 086/180] Support pop with additional item types --- mypyc/irbuild/vec.py | 5 +++-- mypyc/test-data/irbuild-vec-misc.test | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index bd08d9d1d3cc4..c2dbdadce2c65 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -427,8 +427,9 @@ def vec_pop(builder: LowLevelIRBuilder, base: Value, index: Value, line: int) -> item_type = vec_type.item_type index = as_platform_int(builder, index, line) - if is_int64_rprimitive(item_type): - name = "VecI64Api.pop" + api_name = vec_api_by_item_type.get(item_type) + if api_name is not None: + name = f"{api_name}.pop" elif vec_depth(vec_type) == 0 and not isinstance(item_type, RUnion): # TODO fix union name = "VecTApi.pop" else: diff --git a/mypyc/test-data/irbuild-vec-misc.test b/mypyc/test-data/irbuild-vec-misc.test index 6a233254d4816..d7a83550a6465 100644 --- a/mypyc/test-data/irbuild-vec-misc.test +++ b/mypyc/test-data/irbuild-vec-misc.test @@ -116,3 +116,29 @@ L5: r10 = load_mem r9 :: builtins.float* keep_alive v return r10 + +[case testVecMiscPop] +from vecs import vec, pop + +from mypy_extensions import i64 + +def pop_float(v: vec[float], i: i64) -> float: + v, x = pop(v) + return x +[out] +def pop_float(v, i): + v :: vec[float] + i :: i64 + r0 :: tuple[vec[float], float] + r1, r2 :: vec[float] + r3, r4, x :: float +L0: + r0 = VecFloatApi.pop(v, -1) + r1 = borrow r0[0] + r2 = unborrow r1 + v = r2 + r3 = borrow r0[1] + r4 = unborrow r3 + x = r4 + keep_alive steal r0 + return x From c300cd4632f8b994d06784f94b63603318e25d21 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 28 Jan 2024 14:50:55 +0000 Subject: [PATCH 087/180] Support additional item types with remove --- mypyc/test-data/irbuild-vec-misc.test | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/mypyc/test-data/irbuild-vec-misc.test b/mypyc/test-data/irbuild-vec-misc.test index d7a83550a6465..4c874da99cc08 100644 --- a/mypyc/test-data/irbuild-vec-misc.test +++ b/mypyc/test-data/irbuild-vec-misc.test @@ -142,3 +142,17 @@ L0: x = r4 keep_alive steal r0 return x + +[case testVecMiscRemove] +from vecs import vec, remove + +from mypy_extensions import i64 + +def remove_float(v: vec[float]) -> vec[float]: + return remove(v, 1.5) +[out] +def remove_float(v): + v, r0 :: vec[float] +L0: + r0 = VecTApi.remove(v, 1.5) + return r0 From eb8af92e95b40c32ed8a4e10471afbb34c0e5fba Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 28 Jan 2024 14:52:40 +0000 Subject: [PATCH 088/180] Support slicing with additional item types --- mypyc/irbuild/vec.py | 5 +++-- mypyc/test-data/irbuild-vec-misc.test | 18 ++++++++++++++++-- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index c2dbdadce2c65..9a35dccc9e316 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -531,8 +531,9 @@ def vec_slice( item_type = vec_type.item_type begin = builder.coerce(begin, int64_rprimitive, line) end = builder.coerce(end, int64_rprimitive, line) - if is_int64_rprimitive(item_type): - name = "VecI64Api.slice" + api_name = vec_api_by_item_type.get(item_type) + if api_name is not None: + name = f"{api_name}.slice" elif vec_depth(vec_type) == 0 and not isinstance(item_type, RUnion): name = "VecTApi.slice" else: diff --git a/mypyc/test-data/irbuild-vec-misc.test b/mypyc/test-data/irbuild-vec-misc.test index 4c874da99cc08..b7b399a3be9c5 100644 --- a/mypyc/test-data/irbuild-vec-misc.test +++ b/mypyc/test-data/irbuild-vec-misc.test @@ -146,8 +146,6 @@ L0: [case testVecMiscRemove] from vecs import vec, remove -from mypy_extensions import i64 - def remove_float(v: vec[float]) -> vec[float]: return remove(v, 1.5) [out] @@ -156,3 +154,19 @@ def remove_float(v): L0: r0 = VecTApi.remove(v, 1.5) return r0 + +[case testVecMiscSlice] +from vecs import vec, remove + +from mypy_extensions import i64 + +def remove_float(v: vec[float], x: i64, y: i64) -> vec[float]: + return v[x:y] +[out] +def remove_float(v, x, y): + v :: vec[float] + x, y :: i64 + r0 :: vec[float] +L0: + r0 = VecFloatApi.slice(v, x, y) + return r0 From 5a3ab6c85b55dda6689faf39788ce2656c69e443 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 28 Jan 2024 15:03:54 +0000 Subject: [PATCH 089/180] Support construction of vec with misc item types from a list expression --- mypyc/ir/rtypes.py | 53 ++++++++++++++++++++++++--- mypyc/irbuild/vec.py | 2 +- mypyc/test-data/irbuild-vec-misc.test | 25 ++++++++++++- 3 files changed, 72 insertions(+), 8 deletions(-) diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index d09fd9344f377..46ff391740f53 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -1037,11 +1037,11 @@ def __init__(self, item_type: RType) -> None: non_opt = optional_value_type(item_type) else: non_opt = item_type - if is_int64_rprimitive(item_type): - self._ctype = "VecI64" - self.types = [c_pyssize_t_rprimitive, VecI64BufObject] - self.struct_type = VecI64 - self.buf_type = VecI64BufObject + if item_type in vec_buf_types: + self._ctype = "VecI64" # TODO + self.buf_type = vec_buf_types[item_type] + self.types = [c_pyssize_t_rprimitive, self.buf_type] + self.struct_type = VecI64 # TODO elif isinstance(non_opt, RVec): self._ctype = "VecNested" self.types = [c_pyssize_t_rprimitive, VecTBufObject] @@ -1340,13 +1340,54 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: """ -# Buffer for vec[i64] +# Buffers for vec item types that have a packed representation + VecI64BufObject = RStruct( name="VecI64BufObject", names=["ob_base", "len", "items"], types=[PyVarObject, int64_rprimitive], ) +VecI32BufObject = RStruct( + name="VecI32BufObject", + names=["ob_base", "len", "items"], + types=[PyVarObject, int64_rprimitive], +) + +VecI16BufObject = RStruct( + name="VecI16BufObject", + names=["ob_base", "len", "items"], + types=[PyVarObject, int64_rprimitive], +) + +VecU8BufObject = RStruct( + name="VecU8BufObject", + names=["ob_base", "len", "items"], + types=[PyVarObject, int64_rprimitive], +) + +VecFloatBufObject = RStruct( + name="VecFloatBufObject", + names=["ob_base", "len", "items"], + types=[PyVarObject, int64_rprimitive], +) + +VecBoolBufObject = RStruct( + name="VecBoolBufObject", + names=["ob_base", "len", "items"], + types=[PyVarObject, int64_rprimitive], +) + +vec_buf_types: Final = { + int64_rprimitive: VecI64BufObject, + int32_rprimitive: VecI32BufObject, + int16_rprimitive: VecI16BufObject, + uint8_rprimitive: VecU8BufObject, + float_rprimitive: VecFloatBufObject, + bool_rprimitive: VecBoolBufObject, +} + + # vecbuf_i64_rprimitive: Final = RPrimitive( # "VecI64BufObject", is_unboxed=False, is_refcounted=True, ctype="VecI64BufObject *" # ) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 9a35dccc9e316..011b876e16b2e 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -67,7 +67,7 @@ from mypyc.irbuild.ll_builder import LowLevelIRBuilder -vec_api_by_item_type = { +vec_api_by_item_type: Final = { int64_rprimitive: "VecI64Api", int32_rprimitive: "VecI32Api", int16_rprimitive: "VecI16Api", diff --git a/mypyc/test-data/irbuild-vec-misc.test b/mypyc/test-data/irbuild-vec-misc.test index b7b399a3be9c5..7af48f2559687 100644 --- a/mypyc/test-data/irbuild-vec-misc.test +++ b/mypyc/test-data/irbuild-vec-misc.test @@ -48,6 +48,29 @@ L0: r0 = VecBoolApi.alloc(0, 0) return r0 +[case testVecMiscCreateFromList] +from vecs import vec, remove + +from mypy_extensions import i64 + +def create_float() -> vec[float]: + return vec[float]([1.5, 2.5]) +[out] +def create_float(): + r0 :: vec[float] + r1 :: object + r2, r3, r4 :: ptr +L0: + r0 = VecFloatApi.alloc(2, 2) + r1 = r0.buf + r2 = get_element_ptr r1 items :: VecFloatBufObject + set_mem r2, 1.5 :: builtins.float* + r3 = r2 + 8 + set_mem r3, 2.5 :: builtins.float* + r4 = r3 + 8 + keep_alive r0 + return r0 + [case testVecMiscAppend] from vecs import vec, append @@ -110,7 +133,7 @@ L4: r5 = i L5: r6 = v.buf - r7 = get_element_ptr r6 items :: VecTBufObject + r7 = get_element_ptr r6 items :: VecFloatBufObject r8 = r5 * 8 r9 = r7 + r8 r10 = load_mem r9 :: builtins.float* From e388ecaa592603a8230731e344cc1c91e6caaf53 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 28 Jan 2024 15:08:39 +0000 Subject: [PATCH 090/180] Test for loop over vec[float] --- mypyc/test-data/irbuild-vec-misc.test | 46 +++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/mypyc/test-data/irbuild-vec-misc.test b/mypyc/test-data/irbuild-vec-misc.test index 7af48f2559687..3a95d4c069266 100644 --- a/mypyc/test-data/irbuild-vec-misc.test +++ b/mypyc/test-data/irbuild-vec-misc.test @@ -193,3 +193,49 @@ def remove_float(v, x, y): L0: r0 = VecFloatApi.slice(v, x, y) return r0 + +[case testVecMiscForLoop] +from vecs import vec, remove + +from mypy_extensions import i64 + +def for_float(v: vec[float]) -> float: + s = 0.0 + for x in v: + s += x + return s +[out] +def for_float(v): + v :: vec[float] + s :: float + r0, r1 :: native_int + r2 :: bit + r3 :: object + r4 :: ptr + r5 :: native_int + r6 :: ptr + r7, x, r8 :: float + r9 :: native_int +L0: + s = 0.0 + r0 = 0 +L1: + r1 = v.len + r2 = r0 < r1 :: signed + if r2 goto L2 else goto L4 :: bool +L2: + r3 = v.buf + r4 = get_element_ptr r3 items :: VecFloatBufObject + r5 = r0 * 8 + r6 = r4 + r5 + r7 = load_mem r6 :: builtins.float* + keep_alive v + x = r7 + r8 = s + x + s = r8 +L3: + r9 = r0 + 1 + r0 = r9 + goto L1 +L4: + return s From de27f6d1236ebcacd4fa1be9928100aac63730d6 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 28 Jan 2024 15:27:08 +0000 Subject: [PATCH 091/180] Fix vec item size calculation and update tests --- mypyc/irbuild/vec.py | 7 +--- mypyc/test-data/irbuild-vec-misc.test | 56 +++++++++++++-------------- 2 files changed, 30 insertions(+), 33 deletions(-) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 011b876e16b2e..121de5a1739a3 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -248,11 +248,8 @@ def vec_items(builder: LowLevelIRBuilder, vecobj: Value) -> Value: def vec_item_ptr(builder: LowLevelIRBuilder, vecobj: Value, index: Value) -> Value: items_addr = vec_items(builder, vecobj) assert isinstance(vecobj.type, RVec) - # TODO: Calculate item size properly and support 32-bit platforms - if isinstance(vecobj.type.item_type, RVec): - item_size = 16 - else: - item_size = 8 + # TODO: Do we need to care about alignment? + item_size = vecobj.type.item_type.size delta = builder.int_mul(index, item_size) return builder.int_add(items_addr, delta) diff --git a/mypyc/test-data/irbuild-vec-misc.test b/mypyc/test-data/irbuild-vec-misc.test index 3a95d4c069266..f360a9b531d5e 100644 --- a/mypyc/test-data/irbuild-vec-misc.test +++ b/mypyc/test-data/irbuild-vec-misc.test @@ -51,23 +51,23 @@ L0: [case testVecMiscCreateFromList] from vecs import vec, remove -from mypy_extensions import i64 +from mypy_extensions import i64, i32 -def create_float() -> vec[float]: - return vec[float]([1.5, 2.5]) +def create_i32() -> vec[i32]: + return vec[i32]([1, -5]) [out] -def create_float(): - r0 :: vec[float] +def create_i32(): + r0 :: vec[i32] r1 :: object r2, r3, r4 :: ptr L0: - r0 = VecFloatApi.alloc(2, 2) + r0 = VecI32Api.alloc(2, 2) r1 = r0.buf - r2 = get_element_ptr r1 items :: VecFloatBufObject - set_mem r2, 1.5 :: builtins.float* - r3 = r2 + 8 - set_mem r3, 2.5 :: builtins.float* - r4 = r3 + 8 + r2 = get_element_ptr r1 items :: VecI32BufObject + set_mem r2, 1 :: i32* + r3 = r2 + 4 + set_mem r3, -5 :: i32* + r4 = r3 + 4 keep_alive r0 return r0 @@ -98,11 +98,11 @@ from vecs import vec from mypy_extensions import i64 -def get_item_float(v: vec[float], i: i64) -> float: +def get_item_bool(v: vec[bool], i: i64) -> bool: return v[i] [out] -def get_item_float(v, i): - v :: vec[float] +def get_item_bool(v, i): + v :: vec[bool] i :: i64 r0 :: native_int r1 :: bit @@ -114,7 +114,7 @@ def get_item_float(v, i): r7 :: ptr r8 :: i64 r9 :: ptr - r10 :: float + r10 :: bool L0: r0 = v.len r1 = i < r0 :: unsigned @@ -133,10 +133,10 @@ L4: r5 = i L5: r6 = v.buf - r7 = get_element_ptr r6 items :: VecFloatBufObject + r7 = get_element_ptr r6 items :: VecBoolBufObject r8 = r5 * 8 r9 = r7 + r8 - r10 = load_mem r9 :: builtins.float* + r10 = load_mem r9 :: builtins.bool* keep_alive v return r10 @@ -197,27 +197,27 @@ L0: [case testVecMiscForLoop] from vecs import vec, remove -from mypy_extensions import i64 +from mypy_extensions import i64, i16 -def for_float(v: vec[float]) -> float: - s = 0.0 +def for_bool(v: vec[i16]) -> i16: + s: i16 = 0 for x in v: s += x return s [out] -def for_float(v): - v :: vec[float] - s :: float +def for_bool(v): + v :: vec[i16] + s :: i16 r0, r1 :: native_int r2 :: bit r3 :: object r4 :: ptr r5 :: native_int r6 :: ptr - r7, x, r8 :: float + r7, x, r8 :: i16 r9 :: native_int L0: - s = 0.0 + s = 0 r0 = 0 L1: r1 = v.len @@ -225,10 +225,10 @@ L1: if r2 goto L2 else goto L4 :: bool L2: r3 = v.buf - r4 = get_element_ptr r3 items :: VecFloatBufObject - r5 = r0 * 8 + r4 = get_element_ptr r3 items :: VecI16BufObject + r5 = r0 * 2 r6 = r4 + r5 - r7 = load_mem r6 :: builtins.float* + r7 = load_mem r6 :: i16* keep_alive v x = r7 r8 = s + x From 603ab44461380c3f7e91714f0db39c29ecffb41c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 28 Jan 2024 15:29:19 +0000 Subject: [PATCH 092/180] Update tests --- mypyc/test-data/irbuild-vec-misc.test | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/mypyc/test-data/irbuild-vec-misc.test b/mypyc/test-data/irbuild-vec-misc.test index f360a9b531d5e..fcfacccdb067f 100644 --- a/mypyc/test-data/irbuild-vec-misc.test +++ b/mypyc/test-data/irbuild-vec-misc.test @@ -49,7 +49,7 @@ L0: return r0 [case testVecMiscCreateFromList] -from vecs import vec, remove +from vecs import vec from mypy_extensions import i64, i32 @@ -71,6 +71,21 @@ L0: keep_alive r0 return r0 +[case testVecMiscLen] +from vecs import vec + +from mypy_extensions import i64, i32 + +def len_i32(v: vec[i32]) -> i64: + return len(v) +[out] +def len_i32(v): + v :: vec[i32] + r0 :: native_int +L0: + r0 = v.len + return r0 + [case testVecMiscAppend] from vecs import vec, append @@ -134,7 +149,7 @@ L4: L5: r6 = v.buf r7 = get_element_ptr r6 items :: VecBoolBufObject - r8 = r5 * 8 + r8 = r5 * 1 r9 = r7 + r8 r10 = load_mem r9 :: builtins.bool* keep_alive v From 504f1ac528beddd38dd7ee1bb7e659c279e917cd Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 28 Jan 2024 15:48:12 +0000 Subject: [PATCH 093/180] Fix nested vecs and boxing vecs --- mypyc/codegen/emit.py | 6 +- mypyc/ir/rtypes.py | 10 ++++ mypyc/irbuild/vec.py | 25 ++++---- mypyc/test-data/irbuild-vec-misc.test | 84 +++++++++++++++++++++++++++ mypyc/test/test_emitfunc.py | 2 + 5 files changed, 112 insertions(+), 15 deletions(-) diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index b5a0b4f4d5256..958a68fb14ad2 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -66,6 +66,7 @@ object_rprimitive, optional_value_type, vec_depth, + vec_api_by_item_type, ) from mypyc.namegen import NameGenerator, exported_name from mypyc.primitives.registry import builtin_names @@ -1184,8 +1185,9 @@ def emit_box( self.emit_box(f"{src}.f{i}", inner_name, typ.types[i], declare_dest=True) self.emit_line(f"PyTuple_SET_ITEM({dest}, {i}, {inner_name});") elif isinstance(typ, RVec): - if is_int64_rprimitive(typ.item_type): - api = "VecI64Api" + specialized_api_name = vec_api_by_item_type.get(typ.item_type) + if specialized_api_name is not None: + api = specialized_api_name elif typ.depth() > 0: api = "VecNestedApi" else: diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 46ff391740f53..5359f6ea5d9ed 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -1447,3 +1447,13 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: names=["vec", "item"], types=[VecNested, VecNestedBufItem], ) + + +vec_api_by_item_type: Final = { + int64_rprimitive: "VecI64Api", + int32_rprimitive: "VecI32Api", + int16_rprimitive: "VecI16Api", + uint8_rprimitive: "VecU8Api", + float_rprimitive: "VecFloatApi", + bool_rprimitive: "VecBoolApi", +} diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 121de5a1739a3..073c7b855aff7 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -60,6 +60,7 @@ optional_value_type, pointer_rprimitive, vec_depth, + vec_api_by_item_type, ) from mypyc.primitives.registry import builtin_names @@ -67,16 +68,6 @@ from mypyc.irbuild.ll_builder import LowLevelIRBuilder -vec_api_by_item_type: Final = { - int64_rprimitive: "VecI64Api", - int32_rprimitive: "VecI32Api", - int16_rprimitive: "VecI16Api", - uint8_rprimitive: "VecU8Api", - float_rprimitive: "VecFloatApi", - bool_rprimitive: "VecBoolApi", -} - - def as_platform_int(builder: LowLevelIRBuilder, v: Value, line: int) -> Value: rtype = v.type if is_c_py_ssize_t_rprimitive(rtype): @@ -249,7 +240,14 @@ def vec_item_ptr(builder: LowLevelIRBuilder, vecobj: Value, index: Value) -> Val items_addr = vec_items(builder, vecobj) assert isinstance(vecobj.type, RVec) # TODO: Do we need to care about alignment? - item_size = vecobj.type.item_type.size + item_type = vecobj.type.item_type + if isinstance(item_type, RPrimitive): + item_size = vecobj.type.item_type.size + elif isinstance(item_type, RVec): + # TODO: Support 32-bit platforms + item_size = 16 + else: + item_size = object_rprimitive.size delta = builder.int_mul(index, item_size) return builder.int_add(items_addr, delta) @@ -358,8 +356,9 @@ def convert_to_t_ext_item(builder: LowLevelIRBuilder, item: Value) -> Value: def convert_from_t_ext_item(builder: LowLevelIRBuilder, item: Value, vec_type: RVec) -> Value: """Convert a value of type VecNestedBufItem to the corresponding RVec value.""" - if is_int64_rprimitive(vec_type.item_type): - name = "VecI64Api.convert_from_nested" + api_name = vec_api_by_item_type.get(vec_type.item_type) + if api_name is not None: + name = f"{api_name}.convert_from_nested" elif isinstance(vec_type.item_type, RVec): name = "VecNestedApi.convert_from_nested" else: diff --git a/mypyc/test-data/irbuild-vec-misc.test b/mypyc/test-data/irbuild-vec-misc.test index fcfacccdb067f..65d63b8da3d08 100644 --- a/mypyc/test-data/irbuild-vec-misc.test +++ b/mypyc/test-data/irbuild-vec-misc.test @@ -254,3 +254,87 @@ L3: goto L1 L4: return s + +[case testVecMiscNestedGetItem] +from vecs import vec + +from mypy_extensions import i64, i32 + +def get_item_nested(v: vec[vec[i32]], i: i64) -> vec[i32]: + return v[i] +[out] +def get_item_nested(v, i): + v :: vec[vec[i32]] + i :: i64 + r0 :: native_int + r1 :: bit + r2 :: i64 + r3 :: bit + r4 :: bool + r5 :: i64 + r6 :: object + r7 :: ptr + r8 :: i64 + r9 :: ptr + r10 :: vec[i32] +L0: + r0 = v.len + r1 = i < r0 :: unsigned + if r1 goto L4 else goto L1 :: bool +L1: + r2 = i + r0 + r3 = r2 < r0 :: unsigned + if r3 goto L3 else goto L2 :: bool +L2: + r4 = raise IndexError + unreachable +L3: + r5 = r2 + goto L5 +L4: + r5 = i +L5: + r6 = v.buf + r7 = get_element_ptr r6 items :: VecNestedBufObject + r8 = r5 * 16 + r9 = r7 + r8 + r10 = load_mem r9 :: vec[i32]* + keep_alive v + return r10 + +[case testVecMiscNestedPop] +from vecs import vec, pop + +from mypy_extensions import i64, i32 + +def get_item_nested(v: vec[vec[i32]], i: i64) -> vec[i32]: + v, x = pop(v, i) + return x +[out] +def get_item_nested(v, i): + v :: vec[vec[i32]] + i :: i64 + r0 :: tuple[vec[vec[i32]], VecNestedBufItem{len:native_int, buf:object_nrc}] + r1, r2 :: vec[vec[i32]] + r3, r4 :: VecNestedBufItem{len:native_int, buf:object_nrc} + r5 :: vec[i32] + r6 :: tuple[vec[vec[i32]], vec[i32]] + r7, r8 :: vec[vec[i32]] + r9, r10, x :: vec[i32] +L0: + r0 = VecNestedApi.pop(v, i) + r1 = borrow r0[0] + r2 = unborrow r1 + r3 = borrow r0[1] + r4 = unborrow r3 + r5 = VecI32Api.convert_from_nested(r4) + r6 = (r2, r5) + keep_alive steal r0 + r7 = borrow r6[0] + r8 = unborrow r7 + v = r8 + r9 = borrow r6[1] + r10 = unborrow r9 + x = r10 + keep_alive steal r6 + return x diff --git a/mypyc/test/test_emitfunc.py b/mypyc/test/test_emitfunc.py index 3446ea6a2541e..5274c1b6222a1 100644 --- a/mypyc/test/test_emitfunc.py +++ b/mypyc/test/test_emitfunc.py @@ -113,6 +113,7 @@ def add_local(name: str, rtype: RType) -> Register: "tt", RTuple([RTuple([int_rprimitive, bool_rprimitive]), bool_rprimitive]) ) self.vi64 = add_local("vi64", RVec(int64_rprimitive)) + self.vi32 = add_local("vi32", RVec(int32_rprimitive)) self.vs = add_local("vs", RVec(str_rprimitive)) self.vs_opt = add_local("vs", RVec(RUnion([str_rprimitive, none_rprimitive]))) self.vvs = add_local("vvs", RVec(RVec(str_rprimitive))) @@ -372,6 +373,7 @@ def test_unbox_i64(self) -> None: def test_box_vec(self) -> None: self.assert_emit(Box(self.vi64), """cpy_r_r0 = VecI64Api.box(cpy_r_vi64);""") + self.assert_emit(Box(self.vi32), """cpy_r_r0 = VecI32Api.box(cpy_r_vi32);""") self.assert_emit(Box(self.vs), """cpy_r_r0 = VecTApi.box(cpy_r_vs, (size_t)&PyUnicode_Type);""") self.assert_emit(Box(self.vs_opt), From 61b69319de840acb4904461b5ac3eac2bf46826e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 28 Jan 2024 16:12:27 +0000 Subject: [PATCH 094/180] Fix vec unboxing with additional item types --- mypyc/codegen/emit.py | 6 +++--- mypyc/ir/rtypes.py | 29 +++++++++++++++++++---------- mypyc/test/test_emitfunc.py | 8 ++++++++ 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index 958a68fb14ad2..18447bfcd680e 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -1092,9 +1092,9 @@ def emit_unbox( self.emit_line(f"{dest} = {self.c_error_value(typ)};") self.emit_line("} else {") - # TODO: Use helper function to pick the api variant - if is_int64_rprimitive(typ.item_type): - self.emit_line(f"{dest} = VecI64Api.unbox({src});") + specialized_api_name = vec_api_by_item_type.get(typ.item_type) + if specialized_api_name is not None: + self.emit_line(f"{dest} = {specialized_api_name}.unbox({src});") else: depth = typ.depth() if is_int64_rprimitive(typ.unwrap_item_type()): diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 5359f6ea5d9ed..a40275ed1e220 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -1038,7 +1038,7 @@ def __init__(self, item_type: RType) -> None: else: non_opt = item_type if item_type in vec_buf_types: - self._ctype = "VecI64" # TODO + self._ctype = vec_c_types[item_type] self.buf_type = vec_buf_types[item_type] self.types = [c_pyssize_t_rprimitive, self.buf_type] self.struct_type = VecI64 # TODO @@ -1378,15 +1378,6 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: types=[PyVarObject, int64_rprimitive], ) -vec_buf_types: Final = { - int64_rprimitive: VecI64BufObject, - int32_rprimitive: VecI32BufObject, - int16_rprimitive: VecI16BufObject, - uint8_rprimitive: VecU8BufObject, - float_rprimitive: VecFloatBufObject, - bool_rprimitive: VecBoolBufObject, -} - # vecbuf_i64_rprimitive: Final = RPrimitive( # "VecI64BufObject", is_unboxed=False, is_refcounted=True, ctype="VecI64BufObject *" @@ -1449,6 +1440,24 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: ) +vec_buf_types: Final = { + int64_rprimitive: VecI64BufObject, + int32_rprimitive: VecI32BufObject, + int16_rprimitive: VecI16BufObject, + uint8_rprimitive: VecU8BufObject, + float_rprimitive: VecFloatBufObject, + bool_rprimitive: VecBoolBufObject, +} + +vec_c_types: Final = { + int64_rprimitive: "VecI64", + int32_rprimitive: "VecI32", + int16_rprimitive: "VecI16", + uint8_rprimitive: "VecU8", + float_rprimitive: "VecFloat", + bool_rprimitive: "VecBool", +} + vec_api_by_item_type: Final = { int64_rprimitive: "VecI64Api", int32_rprimitive: "VecI32Api", diff --git a/mypyc/test/test_emitfunc.py b/mypyc/test/test_emitfunc.py index 5274c1b6222a1..348bb984a4979 100644 --- a/mypyc/test/test_emitfunc.py +++ b/mypyc/test/test_emitfunc.py @@ -389,6 +389,14 @@ def test_unbox_vec(self) -> None: } """ ) + self.assert_emit( + Unbox(self.o, RVec(int32_rprimitive), 55), + """cpy_r_r0 = VecI32Api.unbox(cpy_r_o); + if (VEC_IS_ERROR(cpy_r_r0)) { + CPy_TypeError("vec[i32]", cpy_r_o); cpy_r_r0 = (VecI32) { -1, NULL }; + } + """ + ) self.assert_emit( Unbox(self.o, RVec(str_rprimitive), 55), """cpy_r_r0 = VecTApi.unbox(cpy_r_o, (size_t)&PyUnicode_Type); From 6b2fd35dd21826e7ae719e4caf0706fda7853d1a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 28 Jan 2024 16:14:22 +0000 Subject: [PATCH 095/180] Fix vec remove --- mypyc/irbuild/vec.py | 4 ++-- mypyc/test-data/irbuild-vec-misc.test | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 073c7b855aff7..808457d7aee17 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -462,8 +462,8 @@ def vec_remove(builder: LowLevelIRBuilder, vec: Value, item: Value, line: int) - item_type = vec_type.item_type coerced_item = builder.coerce(item, item_type, line) - if is_int64_rprimitive(item_type): - name = "VecI64Api.remove" + if item_type in vec_api_by_item_type: + name = f"{vec_api_by_item_type[item_type]}.remove" elif vec_depth(vec_type) == 0 and not isinstance(item_type, RUnion): name = "VecTApi.remove" else: diff --git a/mypyc/test-data/irbuild-vec-misc.test b/mypyc/test-data/irbuild-vec-misc.test index 65d63b8da3d08..83c92418d4d66 100644 --- a/mypyc/test-data/irbuild-vec-misc.test +++ b/mypyc/test-data/irbuild-vec-misc.test @@ -190,7 +190,7 @@ def remove_float(v: vec[float]) -> vec[float]: def remove_float(v): v, r0 :: vec[float] L0: - r0 = VecTApi.remove(v, 1.5) + r0 = VecFloatApi.remove(v, 1.5) return r0 [case testVecMiscSlice] From e35763be2264f7bd008c2dce6d3fe711e97b07bf Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 28 Jan 2024 16:22:41 +0000 Subject: [PATCH 096/180] Minimal runtime support for new vec item types (no tests) --- mypyc/codegen/emitmodule.py | 9 ++++++--- mypyc/ir/rtypes.py | 9 +++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 25f1cfde2beef..9c68667b636ce 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -60,7 +60,7 @@ from mypyc.ir.func_ir import FuncIR from mypyc.ir.module_ir import ModuleIR, ModuleIRs, deserialize_modules from mypyc.ir.ops import DeserMaps, LoadLiteral -from mypyc.ir.rtypes import RType +from mypyc.ir.rtypes import RType, vec_c_types, vec_api_fields, vec_api_by_item_type from mypyc.irbuild.main import build_ir from mypyc.irbuild.mapper import Mapper from mypyc.irbuild.prepare import load_type_map @@ -587,7 +587,8 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]: if self.use_vec_capsule: self.declare_global("VecCapsule *", "VecApi") - self.declare_global("VecI64Features ", "VecI64Api") + for vec_type in sorted(vec_c_types.values()): + self.declare_global(f"{vec_type}Features ", f"{vec_type}Api") self.declare_global("VecTFeatures ", "VecTApi") self.declare_global("VecNestedFeatures ", "VecNestedApi") @@ -978,10 +979,12 @@ def generate_globals_init(self, emitter: Emitter) -> None: emitter.emit_lines( 'VecApi = PyCapsule_Import("vecs._C_API", 0);', "if (!VecApi) return -1;", - "VecI64Api = *VecApi->i64;", "VecTApi = *VecApi->t;", "VecNestedApi = *VecApi->nested;", ) + for item_type, api_name in vec_api_by_item_type.items(): + field_name = vec_api_fields[item_type] + emitter.emit_line(f"{api_name} = *VecApi->{field_name};") emitter.emit_lines("is_initialized = 1;", "return 0;", "}") diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index a40275ed1e220..95d764cc3c47b 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -1458,6 +1458,15 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: bool_rprimitive: "VecBool", } +vec_api_fields: Final = { + int64_rprimitive: "i64", + int32_rprimitive: "i32", + int16_rprimitive: "i16", + uint8_rprimitive: "u8", + float_rprimitive: "float_", + bool_rprimitive: "bool_", +} + vec_api_by_item_type: Final = { int64_rprimitive: "VecI64Api", int32_rprimitive: "VecI32Api", From ca07aab40678720f1c4050ca6d640fab593c731b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 28 Jan 2024 16:41:19 +0000 Subject: [PATCH 097/180] Add a run test for additional item types --- mypyc/test-data/irbuild-vec-misc.test | 4 +++- mypyc/test-data/run-vec-misc.test | 16 ++++++++++++++++ mypyc/test/test_run.py | 1 + 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 mypyc/test-data/run-vec-misc.test diff --git a/mypyc/test-data/irbuild-vec-misc.test b/mypyc/test-data/irbuild-vec-misc.test index 83c92418d4d66..0b44214e9ce1f 100644 --- a/mypyc/test-data/irbuild-vec-misc.test +++ b/mypyc/test-data/irbuild-vec-misc.test @@ -1,6 +1,8 @@ --- Test cases for packed/specialized vec item types other than vec[i64], such as +-- Test cases for packed/specialized vec item types other than i64, such as -- vec[i32] and vec[float]. Since many of the code paths are the same as for -- vec[i64], only test a subset of functionality. +-- +-- vec[i64] test cases are in irbuild-vec-i64.test. [case testVecMiscCreateEmpty] from vecs import vec diff --git a/mypyc/test-data/run-vec-misc.test b/mypyc/test-data/run-vec-misc.test new file mode 100644 index 0000000000000..e8c43fdd75397 --- /dev/null +++ b/mypyc/test-data/run-vec-misc.test @@ -0,0 +1,16 @@ +-- Test cases for packed/specialized vec item types other than i64, such as +-- vec[i32] and vec[float]. Since many of the code paths are the same as for +-- vec[i64], only test a subset of functionality. +-- +-- vec[i64] test cases are in run-vec-i64.test. + +[case testVecMiscBasicOps] +from mypy_extensions import i64, i32, i16, u8 +from vecs import vec, append, remove, pop + +def test_create_empty() -> None: + assert str(vec[i32]()) == "vec[i32]([])" + assert str(vec[i16]()) == "vec[i16]([])" + assert str(vec[u8]()) == "vec[u8]([])" + assert str(vec[float]()) == "vec[float]([])" + assert str(vec[bool]()) == "vec[bool]([])" diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index 14db3b1e64805..ed68c52878f25 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -83,6 +83,7 @@ "run-vecs-t-interp.test", "run-vecs-nested-interp.test", "run-vec-i64.test", + "run-vec-misc.test", "run-vec-t.test", "run-vec-nested.test", ] From b4f084b9c3f04b5415f312ecd92f0732313570b2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 28 Jan 2024 16:42:54 +0000 Subject: [PATCH 098/180] Add run test case --- mypyc/test-data/run-vec-misc.test | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mypyc/test-data/run-vec-misc.test b/mypyc/test-data/run-vec-misc.test index e8c43fdd75397..6f0421fb3e1f5 100644 --- a/mypyc/test-data/run-vec-misc.test +++ b/mypyc/test-data/run-vec-misc.test @@ -14,3 +14,10 @@ def test_create_empty() -> None: assert str(vec[u8]()) == "vec[u8]([])" assert str(vec[float]()) == "vec[float]([])" assert str(vec[bool]()) == "vec[bool]([])" + +def test_create_from_values() -> None: + assert str(vec[i32]([5, -12])) == "vec[i32]([5, -12])" + assert str(vec[i16]([6, -8])) == "vec[i16]([6, -8])" + assert str(vec[u8]([0, 123, 255])) == "vec[u8]([0, 123, 255])" + assert str(vec[float]([1.5, -3.5])) == "vec[float]([1.5, -3.5])" + assert str(vec[bool]([False, True])) == "vec[bool]([False, True])" From a26b5ba3d082c974809f7d5a9ad78d4ecef97848 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 28 Jan 2024 16:46:48 +0000 Subject: [PATCH 099/180] Add run tests --- mypyc/test-data/run-vec-misc.test | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/mypyc/test-data/run-vec-misc.test b/mypyc/test-data/run-vec-misc.test index 6f0421fb3e1f5..d4b0ff161336d 100644 --- a/mypyc/test-data/run-vec-misc.test +++ b/mypyc/test-data/run-vec-misc.test @@ -21,3 +21,24 @@ def test_create_from_values() -> None: assert str(vec[u8]([0, 123, 255])) == "vec[u8]([0, 123, 255])" assert str(vec[float]([1.5, -3.5])) == "vec[float]([1.5, -3.5])" assert str(vec[bool]([False, True])) == "vec[bool]([False, True])" + +def test_get_item() -> None: + assert vec[i32]([5, -12])[1] == -12 + assert vec[i16]([6, -8])[1] == -8 + assert vec[u8]([0, 123, 255])[2] == 255 + assert vec[float]([1.5, -3.5])[1] == -3.5 + assert vec[bool]([False, True])[1] == True + +def test_get_item_negative() -> None: + assert vec[i32]([5, -12])[-2] == 5 + assert vec[i16]([6, -8])[-2] == 6 + assert vec[u8]([0, 123, 255])[-2] == 123 + assert vec[float]([1.5, -3.5])[-2] == 1.5 + assert vec[bool]([False, True])[-2] == False + +def test_append() -> None: + assert append(vec[i32](), 123) == vec[i32]([123]) + assert append(vec[i16](), 123) == vec[i16]([123]) + assert append(vec[u8](), 123) == vec[u8]([123]) + assert append(vec[float](), 123.5) == vec[float]([123.5]) + assert append(vec[bool](), True) == vec[bool]([True]) From 02d083af086478c0db52dae98b6a3fbb378c913e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 10 Feb 2024 17:45:47 +0000 Subject: [PATCH 100/180] Fixes to nested vecs with specialized item types --- mypyc/codegen/emit.py | 6 +- mypyc/ir/rtypes.py | 12 +++ mypyc/irbuild/vec.py | 5 +- mypyc/test-data/run-vec-misc.test | 122 ++++++++++++++++++++++++++++++ mypyc/test/test_emitfunc.py | 2 +- 5 files changed, 142 insertions(+), 5 deletions(-) diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index 18447bfcd680e..6c067904dc13b 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -67,6 +67,7 @@ optional_value_type, vec_depth, vec_api_by_item_type, + vec_item_type_tags, ) from mypyc.namegen import NameGenerator, exported_name from mypyc.primitives.registry import builtin_names @@ -1097,8 +1098,9 @@ def emit_unbox( self.emit_line(f"{dest} = {specialized_api_name}.unbox({src});") else: depth = typ.depth() - if is_int64_rprimitive(typ.unwrap_item_type()): - type_value = "VEC_ITEM_TYPE_I64" + unwrapped = typ.unwrap_item_type() + if unwrapped in vec_item_type_tags: + type_value = str(vec_item_type_tags[unwrapped]) else: type_value = self.vec_item_type_c(typ) if depth == 0: diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 95d764cc3c47b..b6a4ebb19d00b 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -1475,3 +1475,15 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: float_rprimitive: "VecFloatApi", bool_rprimitive: "VecBoolApi", } + +# These are special type item type contants used in nested vecs to represent +# item types with specialized representations. These must match definitions +# in the vecs module (see VEC_ITEM_TYPE_I64 etc.). +vec_item_type_tags: Final = { + int64_rprimitive: 2, + int32_rprimitive: 6, + int16_rprimitive: 10, + uint8_rprimitive: 14, + float_rprimitive: 18, + bool_rprimitive: 22, +} diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 808457d7aee17..5eee4a0ad0b7e 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -61,6 +61,7 @@ pointer_rprimitive, vec_depth, vec_api_by_item_type, + vec_item_type_tags, ) from mypyc.primitives.registry import builtin_names @@ -205,8 +206,8 @@ def vec_item_type_info( return builder.load_address(src, typ), False, 0 elif isinstance(typ, RInstance): return builder.load_native_type_object(typ.name), False, 0 - elif is_int64_rprimitive(typ): - return Integer(VEC_TYPE_INFO_I64, c_size_t_rprimitive), False, 0 + elif typ in vec_item_type_tags: + return Integer(vec_item_type_tags[typ], c_size_t_rprimitive), False, 0 elif isinstance(typ, RUnion): non_opt = optional_value_type(typ) assert non_opt is not None diff --git a/mypyc/test-data/run-vec-misc.test b/mypyc/test-data/run-vec-misc.test index d4b0ff161336d..369a8d8490c3e 100644 --- a/mypyc/test-data/run-vec-misc.test +++ b/mypyc/test-data/run-vec-misc.test @@ -5,8 +5,12 @@ -- vec[i64] test cases are in run-vec-i64.test. [case testVecMiscBasicOps] +# mypy: allow-redefinition + +from typing import Any, cast from mypy_extensions import i64, i32, i16, u8 from vecs import vec, append, remove, pop +from testutil import assertRaises def test_create_empty() -> None: assert str(vec[i32]()) == "vec[i32]([])" @@ -42,3 +46,121 @@ def test_append() -> None: assert append(vec[u8](), 123) == vec[u8]([123]) assert append(vec[float](), 123.5) == vec[float]([123.5]) assert append(vec[bool](), True) == vec[bool]([True]) + +def test_set_item() -> None: + v = vec[i32]([5, -12]) + v[1] = 55 + assert v[0] == 5 and v[1] == 55 + + v = vec[i16]([5, -12]) + v[1] = 55 + assert v[0] == 5 and v[1] == 55 + + v = vec[u8]([5, 237]) + v[1] = 55 + assert v[0] == 5 and v[1] == 55 + + v = vec[float]([1.5, -3.5]) + v[1] = 5.5 + assert v[0] == 1.5 and v[1] == 5.5 + + v = vec[bool]([True, False]) + v[1] = True + assert v[0] and v[1] + +def test_set_item_negative() -> None: + v = vec[i32]([5, -12]) + v[-1] = 55 + assert v[0] == 5 and v[1] == 55 + + v = vec[i16]([5, -12]) + v[-1] = 55 + assert v[0] == 5 and v[1] == 55 + + v = vec[u8]([5, 237]) + v[-1] = 55 + assert v[0] == 5 and v[1] == 55 + + v = vec[float]([1.5, -3.5]) + v[-1] = 5.5 + assert v[0] == 1.5 and v[1] == 5.5 + + v = vec[bool]([True, False]) + v[-1] = True + assert v[0] and v[1] + +def test_nested_basics() -> None: + v = vec[vec[i32]]([vec[i32]([5, 6]), + vec[i32]([-12])]) + assert str(v) == "vec[vec[i32]]([[5, 6], [-12]])" + assert v[0][1] == 6 + assert v[1][0] == -12 + + v = vec[vec[i16]]([vec[i16]([5, 6]), + vec[i16]([-12])]) + assert str(v) == "vec[vec[i16]]([[5, 6], [-12]])" + assert v[0][1] == 6 + assert v[1][0] == -12 + + v = vec[vec[u8]]([vec[u8]([5, 6]), + vec[u8]([237])]) + assert str(v) == "vec[vec[u8]]([[5, 6], [237]])" + assert v[0][1] == 6 + assert v[1][0] == 237 + + v = vec[vec[float]]([vec[float]([1.5, -3.5]), + vec[float]([3.0])]) + assert str(v) == "vec[vec[float]]([[1.5, -3.5], [3.0]])" + assert v[0][1] == -3.5 + assert v[1][0] == 3.0 + + v = vec[vec[bool]]([vec[bool]([False, True]), + vec[bool]([False])]) + assert str(v) == "vec[vec[bool]]([[False, True], [False]])" + assert v[0][1] == True + assert v[1][0] == False + +def test_nested_unbox() -> None: + vv: vec[vec[i64]] + + a = cast(Any, vec[vec[i32]]([vec[i32]([5])])) + v_i32: vec[vec[i32]] = a + assert str(v_i32) == "vec[vec[i32]]([[5]])" + with assertRaises(TypeError): + vv = a + + a = cast(Any, vec[vec[i16]]([vec[i16]([5])])) + v_i16: vec[vec[i16]] = a + with assertRaises(TypeError): + vv = a + + a = cast(Any, vec[vec[u8]]([vec[u8]([5])])) + v_u8: vec[vec[u8]] = a + with assertRaises(TypeError): + vv = a + + a = cast(Any, vec[vec[float]]([vec[float]([5])])) + v_float: vec[vec[float]] = a + with assertRaises(TypeError): + vv = a + + a = cast(Any, vec[vec[bool]]([vec[bool]([True])])) + v_bool: vec[vec[bool]] = a + with assertRaises(TypeError): + vv = a + +def test_nested_misc() -> None: + v = vec[vec[float]]() + assert len(v) == 0 + v = append(v, vec[float]([1.5])) + assert len(v) == 1 + for x in v: + assert x == vec[float]([1.5]) + vv, x = pop(v) + assert vv == vec[vec[float]]() + assert x == vec[float]([1.5]) + v[0] = vec[float]([-2.5]) + assert v[0] == vec[float]([-2.5]) + v[0][0] = 3.5 + assert v[0] == vec[float]([3.5]) + assert v[:5] == vec[vec[float]]([vec[float]([3.5])]) diff --git a/mypyc/test/test_emitfunc.py b/mypyc/test/test_emitfunc.py index 348bb984a4979..00e206bb73492 100644 --- a/mypyc/test/test_emitfunc.py +++ b/mypyc/test/test_emitfunc.py @@ -439,7 +439,7 @@ def test_unbox_vec_nested(self) -> None: """) self.assert_emit( Unbox(self.o, RVec(RVec(int64_rprimitive)), 55), - """cpy_r_r0 = VecNestedApi.unbox(cpy_r_o, VEC_ITEM_TYPE_I64, 1); + """cpy_r_r0 = VecNestedApi.unbox(cpy_r_o, 2, 1); if (VEC_IS_ERROR(cpy_r_r0)) { CPy_TypeError("vec[vec[i64]]", cpy_r_o); cpy_r_r0 = (VecNested) { -1, NULL }; } From 262148d14731d2fe7766089f4effa307b0e89ef5 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 10 Feb 2024 19:36:03 +0000 Subject: [PATCH 101/180] Test unboxing --- mypyc/test-data/run-vec-misc.test | 33 +++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/mypyc/test-data/run-vec-misc.test b/mypyc/test-data/run-vec-misc.test index 369a8d8490c3e..be125306d740e 100644 --- a/mypyc/test-data/run-vec-misc.test +++ b/mypyc/test-data/run-vec-misc.test @@ -47,6 +47,39 @@ def test_append() -> None: assert append(vec[float](), 123.5) == vec[float]([123.5]) assert append(vec[bool](), True) == vec[bool]([True]) +def test_unbox() -> None: + v: vec[i64] + + a = cast(Any, vec[i32]([1, 2])) + v_i32: vec[i32] = a + assert v_i32 == vec[i32]([1, 2]) + with assertRaises(TypeError): + v = a + + a = cast(Any, vec[i16]([1, 2])) + v_i16: vec[i16] = a + assert v_i16 == vec[i16]([1, 2]) + with assertRaises(TypeError): + v = a + + a = cast(Any, vec[u8]([1, 2])) + v_u8: vec[u8] = a + assert v_u8 == vec[u8]([1, 2]) + with assertRaises(TypeError): + v = a + + a = cast(Any, vec[float]([1, 2])) + v_float: vec[float] = a + assert v_float == vec[float]([1, 2]) + with assertRaises(TypeError): + v = a + + a = cast(Any, vec[bool]([True, False])) + v_bool: vec[bool] = a + assert v_bool == vec[bool]([True, False]) + with assertRaises(TypeError): + v = a + def test_set_item() -> None: v = vec[i32]([5, -12]) v[1] = 55 From 45c42414c84a41ca448a538354697b1a681a9838 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 10 Feb 2024 19:38:14 +0000 Subject: [PATCH 102/180] Test for loop --- mypyc/test-data/run-vec-misc.test | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/mypyc/test-data/run-vec-misc.test b/mypyc/test-data/run-vec-misc.test index be125306d740e..4f4c33ad3cc32 100644 --- a/mypyc/test-data/run-vec-misc.test +++ b/mypyc/test-data/run-vec-misc.test @@ -122,6 +122,22 @@ def test_set_item_negative() -> None: v[-1] = True assert v[0] and v[1] +def test_for_loop() -> None: + a: Any = [] + for x in vec[float]([1.5, -3.5]): + a.append(x) + assert a == [1.5, -3.5] + + a = [] + for x in vec[bool]([True, False, False, True]): + a.append(x) + assert a == [True, False, False, True] + + a = [] + for x in vec[u8]([0, 5, 237, 255]): + a.append(x) + assert a == [0, 5, 237, 255] + def test_nested_basics() -> None: v = vec[vec[i32]]([vec[i32]([5, 6]), vec[i32]([-12])]) From 8c074eb059ed9d57933a43f878a6a102c706ba69 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 10 Feb 2024 19:40:52 +0000 Subject: [PATCH 103/180] Add comprehension tests --- mypyc/test-data/run-vec-misc.test | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mypyc/test-data/run-vec-misc.test b/mypyc/test-data/run-vec-misc.test index 4f4c33ad3cc32..f9b5435191bdf 100644 --- a/mypyc/test-data/run-vec-misc.test +++ b/mypyc/test-data/run-vec-misc.test @@ -138,6 +138,13 @@ def test_for_loop() -> None: a.append(x) assert a == [0, 5, 237, 255] +def test_comprehension() -> None: + v = vec[i32]([x + 1 for x in [1, 4, -5]]) + assert list(v) == [2, 5, -4] + + v = vec[float]([x + 1.5 for x in range(4)]) + assert list(v) == [1.5, 2.5, 3.5, 4.5] + def test_nested_basics() -> None: v = vec[vec[i32]]([vec[i32]([5, 6]), vec[i32]([-12])]) From a6503d1ef2372ee26ef9ef427b679009a0d75f3c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 19 Jan 2026 10:53:02 +0000 Subject: [PATCH 104/180] Fix duplicate visit_set_element methods after rebase --- mypyc/analysis/dataflow.py | 3 --- mypyc/analysis/ircheck.py | 3 --- mypyc/analysis/selfleaks.py | 3 --- mypyc/codegen/emitfunc.py | 20 -------------------- mypyc/ir/ops.py | 4 ---- mypyc/ir/pprint.py | 3 --- 6 files changed, 36 deletions(-) diff --git a/mypyc/analysis/dataflow.py b/mypyc/analysis/dataflow.py index fc14ab8b47991..2a916f426f41e 100644 --- a/mypyc/analysis/dataflow.py +++ b/mypyc/analysis/dataflow.py @@ -281,9 +281,6 @@ def visit_load_mem(self, op: LoadMem) -> GenAndKill[T]: def visit_get_element(self, op: GetElement) -> GenAndKill[T]: return self.visit_register_op(op) - def visit_set_element(self, op: SetElement) -> GenAndKill[T]: - return self.visit_register_op(op) - def visit_get_element_ptr(self, op: GetElementPtr) -> GenAndKill[T]: return self.visit_register_op(op) diff --git a/mypyc/analysis/ircheck.py b/mypyc/analysis/ircheck.py index 920ba40f8be1f..238d416729241 100644 --- a/mypyc/analysis/ircheck.py +++ b/mypyc/analysis/ircheck.py @@ -453,9 +453,6 @@ def visit_set_mem(self, op: SetMem) -> None: def visit_get_element(self, op: GetElement) -> None: pass - def visit_set_element(self, op: SetElement) -> None: - pass - def visit_get_element_ptr(self, op: GetElementPtr) -> None: pass diff --git a/mypyc/analysis/selfleaks.py b/mypyc/analysis/selfleaks.py index 0270cfed5640a..a3940f76d5b9e 100644 --- a/mypyc/analysis/selfleaks.py +++ b/mypyc/analysis/selfleaks.py @@ -191,9 +191,6 @@ def visit_load_mem(self, op: LoadMem) -> GenAndKill: def visit_get_element(self, op: GetElement) -> GenAndKill: return CLEAN - def visit_set_element(self, op: SetElement) -> GenAndKill: - return CLEAN - def visit_get_element_ptr(self, op: GetElementPtr) -> GenAndKill: return CLEAN diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index ffe0ecf74859f..5939782b81288 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py @@ -813,26 +813,6 @@ def visit_get_element(self, op: GetElement) -> None: dest_type = self.ctype(op.type) self.emit_line(f"{dest} = ({dest_type}){src}.{op.field};") - def visit_set_element(self, op: SetElement) -> None: - # TODO: do properly - dest = self.reg(op) - item = self.reg(op.item) - field = op.field - if isinstance(op.src, Undef): - self.emit_line(f"{dest}.{field} = {item};") - else: - src = self.reg(op.src) - # TODO: Support tuples (or use RStruct for tuples) - src_type = op.src.type - assert isinstance(src_type, RStruct), src_type - init_items = [] - for n in src_type.names: - if n != field: - init_items.append(f"{src}.{n}") - else: - init_items.append(item) - self.emit_line(f"{dest} = ({self.ctype(src_type)}) {{ {', '.join(init_items)} }};") - def visit_get_element_ptr(self, op: GetElementPtr) -> None: dest = self.reg(op) src = self.reg(op.src) diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py index 17a703c13ed14..d64d9add5905b 100644 --- a/mypyc/ir/ops.py +++ b/mypyc/ir/ops.py @@ -2044,10 +2044,6 @@ def visit_set_mem(self, op: SetMem) -> T: def visit_get_element(self, op: GetElement) -> T: raise NotImplementedError - @abstractmethod - def visit_set_element(self, op: SetElement) -> T: - raise NotImplementedError - @abstractmethod def visit_get_element_ptr(self, op: GetElementPtr) -> T: raise NotImplementedError diff --git a/mypyc/ir/pprint.py b/mypyc/ir/pprint.py index ee4ffcaad6f86..d0db9f2460a1d 100644 --- a/mypyc/ir/pprint.py +++ b/mypyc/ir/pprint.py @@ -284,9 +284,6 @@ def visit_set_mem(self, op: SetMem) -> str: def visit_get_element(self, op: GetElement) -> str: return self.format("%r = %r.%s", op, op.src, op.field) - def visit_set_element(self, op: SetElement) -> str: - return self.format("%r = set_element %r, %s, %r", op, op.src, op.field, op.item) - def visit_get_element_ptr(self, op: GetElementPtr) -> str: return self.format("%r = get_element_ptr %r %s :: %t", op, op.src, op.field, op.src_type) From 9d4adb17f6cec10f3de7b9366995704e9157c460 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 19 Jan 2026 10:56:54 +0000 Subject: [PATCH 105/180] Fix issues due to rebase --- mypyc/ir/rtypes.py | 4 ++++ mypyc/irbuild/ll_builder.py | 2 +- mypyc/irbuild/specialize.py | 6 +++--- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index b6a4ebb19d00b..5bd41536a84b5 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -1053,6 +1053,10 @@ def __init__(self, item_type: RType) -> None: self.struct_type = VecT self.buf_type = VecTBufObject + @property + def may_be_immortal(self) -> bool: + return False + def unwrap_item_type(self) -> RPrimitive | RInstance: """Return the non-optional value (non-vec) item type in a potentially nested vec.""" item_type = self.item_type diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index bdfa9a38801fe..2b43a8e92a524 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -1991,7 +1991,7 @@ def get_item( self, base: Value, item: Value, - result_type: Optional[RType], + result_type: RType | None, line: int, *, can_borrow: bool = False, diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index 9ec7aaba4b11f..c6988941b8789 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -1509,7 +1509,7 @@ def translate_bytes_get_item( @specialize_function("vecs.append") -def translate_vec_append(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Optional[Value]: +def translate_vec_append(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: if len(expr.args) == 2 and expr.arg_kinds == [ARG_POS, ARG_POS]: vec_arg = expr.args[0] item_arg = expr.args[1] @@ -1523,7 +1523,7 @@ def translate_vec_append(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> @specialize_function("vecs.remove") -def translate_vec_remove(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Optional[Value]: +def translate_vec_remove(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: if len(expr.args) == 2 and expr.arg_kinds == [ARG_POS, ARG_POS]: vec_arg = expr.args[0] item_arg = expr.args[1] @@ -1537,7 +1537,7 @@ def translate_vec_remove(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> @specialize_function("vecs.pop") -def translate_vec_pop(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Optional[Value]: +def translate_vec_pop(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: if 1 <= len(expr.args) <= 2 and all(kind == ARG_POS for kind in expr.arg_kinds): vec_arg = expr.args[0] vec_type = builder.node_type(vec_arg) From 260393ed9e175c24dd18a764b7b1073af4629589 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 19 Jan 2026 11:06:11 +0000 Subject: [PATCH 106/180] Fix self check errors --- mypyc/ir/rtypes.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 5bd41536a84b5..8f849762a8ee6 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -1444,7 +1444,7 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: ) -vec_buf_types: Final = { +vec_buf_types: Final[dict[RType, RStruct]] = { int64_rprimitive: VecI64BufObject, int32_rprimitive: VecI32BufObject, int16_rprimitive: VecI16BufObject, @@ -1453,7 +1453,7 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: bool_rprimitive: VecBoolBufObject, } -vec_c_types: Final = { +vec_c_types: Final[dict[RType, str]] = { int64_rprimitive: "VecI64", int32_rprimitive: "VecI32", int16_rprimitive: "VecI16", @@ -1471,7 +1471,7 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: bool_rprimitive: "bool_", } -vec_api_by_item_type: Final = { +vec_api_by_item_type: Final[dict[RType, str]] = { int64_rprimitive: "VecI64Api", int32_rprimitive: "VecI32Api", int16_rprimitive: "VecI16Api", @@ -1483,7 +1483,7 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: # These are special type item type contants used in nested vecs to represent # item types with specialized representations. These must match definitions # in the vecs module (see VEC_ITEM_TYPE_I64 etc.). -vec_item_type_tags: Final = { +vec_item_type_tags: Final[dict[RType, int]] = { int64_rprimitive: 2, int32_rprimitive: 6, int16_rprimitive: 10, From fda233c638b3a9c3ca0cc046f5b205dbd5ee278c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 19 Jan 2026 11:26:07 +0000 Subject: [PATCH 107/180] Fix self check --- mypyc/ir/rtypes.py | 2 +- mypyc/irbuild/for_helpers.py | 2 +- mypyc/irbuild/vec.py | 2 +- mypyc/test/test_vecusage.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 8f849762a8ee6..f4cc4aa0730fb 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -1462,7 +1462,7 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: bool_rprimitive: "VecBool", } -vec_api_fields: Final = { +vec_api_fields: Final[dict[RType, str]] = { int64_rprimitive: "i64", int32_rprimitive: "i32", int16_rprimitive: "i16", diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index 8f756cdd4ccfd..ff771b4031ee2 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -286,7 +286,7 @@ def sequence_from_generator_preallocate_helper( def set_item(item_index: Value) -> None: e = builder.accept(gen.left_expr) - builder.call_c(set_item_op, [target_op, item_index, e], line) + set_item_op(target_op, item_index, e, line) for_loop_helper_with_index( builder, gen.indices[0], sequence_expr, sequence, set_item, line, length diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 5eee4a0ad0b7e..3d8281608718b 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -243,7 +243,7 @@ def vec_item_ptr(builder: LowLevelIRBuilder, vecobj: Value, index: Value) -> Val # TODO: Do we need to care about alignment? item_type = vecobj.type.item_type if isinstance(item_type, RPrimitive): - item_size = vecobj.type.item_type.size + item_size = item_type.size elif isinstance(item_type, RVec): # TODO: Support 32-bit platforms item_size = 16 diff --git a/mypyc/test/test_vecusage.py b/mypyc/test/test_vecusage.py index e10bdd2b2886b..676dab5919b06 100644 --- a/mypyc/test/test_vecusage.py +++ b/mypyc/test/test_vecusage.py @@ -31,7 +31,7 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: return with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase): try: - ir = build_ir_for_single_file2(testcase.input, options) + ir, _, _, _ = build_ir_for_single_file2(testcase.input, options) except CompileError as e: actual = e.messages else: From f8f7cd24d7473613a0e0a148172d92fd5b3bdd64 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 19 Jan 2026 12:36:54 +0000 Subject: [PATCH 108/180] Update irbuild tests to pass --- mypyc/test-data/irbuild-dict.test | 83 ++++--- mypyc/test-data/irbuild-lists.test | 350 +++++++++++++---------------- 2 files changed, 190 insertions(+), 243 deletions(-) diff --git a/mypyc/test-data/irbuild-dict.test b/mypyc/test-data/irbuild-dict.test index e209cb73632b2..89e19710ac2e6 100644 --- a/mypyc/test-data/irbuild-dict.test +++ b/mypyc/test-data/irbuild-dict.test @@ -184,7 +184,7 @@ L3: r13 = CPyDict_CheckSize(d, r1) goto L1 L4: - r14 = CPy_NoErrOccured() + r14 = CPy_NoErrOccurred() L5: return d @@ -290,7 +290,7 @@ L5: r12 = CPyDict_CheckSize(d1, r1) goto L1 L6: - r13 = CPy_NoErrOccured() + r13 = CPy_NoErrOccurred() L7: r14 = 0 r15 = PyDict_Size(d2) @@ -319,7 +319,7 @@ L10: r31 = CPyDict_CheckSize(d2, r15) goto L8 L11: - r32 = CPy_NoErrOccured() + r32 = CPy_NoErrOccurred() L12: return 1 def union_of_dicts(d): @@ -335,11 +335,14 @@ def union_of_dicts(d): r10 :: union[int, str] k :: str v :: union[int, str] - r11, r12 :: object - r13 :: int + r11 :: object + r12 :: object[1] + r13 :: object_ptr r14 :: object - r15 :: i32 - r16, r17, r18 :: bit + r15 :: int + r16 :: object + r17 :: i32 + r18, r19, r20 :: bit L0: r0 = PyDict_New() new = r0 @@ -360,16 +363,19 @@ L2: k = r9 v = r10 r11 = load_address PyLong_Type - r12 = PyObject_CallFunctionObjArgs(r11, v, 0) - r13 = unbox(int, r12) - r14 = box(int, r13) - r15 = CPyDict_SetItem(new, k, r14) - r16 = r15 >= 0 :: signed + r12 = [v] + r13 = load_address r12 + r14 = PyObject_Vectorcall(r11, r13, 1, 0) + keep_alive v + r15 = unbox(int, r14) + r16 = box(int, r15) + r17 = CPyDict_SetItem(new, k, r16) + r18 = r17 >= 0 :: signed L3: - r17 = CPyDict_CheckSize(d, r2) + r19 = CPyDict_CheckSize(d, r2) goto L1 L4: - r18 = CPy_NoErrOccured() + r20 = CPy_NoErrOccurred() L5: return 1 def typeddict(d): @@ -384,12 +390,9 @@ def typeddict(d): r8, k :: str v :: object r9 :: str - r10 :: i32 - r11 :: bit - r12 :: object - r13, r14, r15 :: bit + r10 :: bool name :: object - r16, r17 :: bit + r11, r12 :: bit L0: r0 = 0 r1 = PyDict_Size(d) @@ -399,7 +402,7 @@ L1: r4 = r3[1] r0 = r4 r5 = r3[0] - if r5 goto L2 else goto L9 :: bool + if r5 goto L2 else goto L6 :: bool L2: r6 = r3[2] r7 = r3[3] @@ -407,27 +410,17 @@ L2: k = r8 v = r7 r9 = 'name' - r10 = PyUnicode_Compare(k, r9) - r11 = r10 == -1 - if r11 goto L3 else goto L5 :: bool + r10 = CPyStr_EqualLiteral(k, r9, 4) + if r10 goto L3 else goto L4 :: bool L3: - r12 = PyErr_Occurred() - r13 = r12 != 0 - if r13 goto L4 else goto L5 :: bool + name = v L4: - r14 = CPy_KeepPropagating() L5: - r15 = r10 == 0 - if r15 goto L6 else goto L7 :: bool + r11 = CPyDict_CheckSize(d, r1) + goto L1 L6: - name = v + r12 = CPy_NoErrOccurred() L7: -L8: - r16 = CPyDict_CheckSize(d, r1) - goto L1 -L9: - r17 = CPy_NoErrOccured() -L10: return 1 [case testDictLoadAddress] @@ -527,8 +520,8 @@ def f3(d, flag): r2 :: str r3 :: list r4 :: object - r5, r6 :: ptr - r7, r8 :: object + r5 :: ptr + r6, r7 :: object L0: if flag goto L1 else goto L2 :: bool L1: @@ -539,15 +532,14 @@ L2: r2 = 'a' r3 = PyList_New(1) r4 = object 1 - r5 = get_element_ptr r3 ob_item :: PyListObject - r6 = load_mem r5 :: ptr* - set_mem r6, r4 :: builtins.object* + r5 = list_items r3 + buf_init_item r5, 0, r4 keep_alive r3 - r7 = CPyDict_SetDefault(d, r2, r3) - return r7 + r6 = CPyDict_SetDefault(d, r2, r3) + return r6 L3: - r8 = box(None, 1) - return r8 + r7 = box(None, 1) + return r7 def f4(d, flag): d :: dict flag :: bool @@ -573,3 +565,4 @@ L2: L3: r7 = box(None, 1) return r7 + diff --git a/mypyc/test-data/irbuild-lists.test b/mypyc/test-data/irbuild-lists.test index 2e5f8553fe8c8..9a07749f780d1 100644 --- a/mypyc/test-data/irbuild-lists.test +++ b/mypyc/test-data/irbuild-lists.test @@ -108,17 +108,15 @@ def f() -> None: def f(): r0 :: list r1, r2 :: object - r3, r4, r5 :: ptr + r3 :: ptr x :: list L0: r0 = PyList_New(2) r1 = object 1 r2 = object 2 - r3 = get_element_ptr r0 ob_item :: PyListObject - r4 = load_mem r3 :: ptr* - set_mem r4, r1 :: builtins.object* - r5 = r4 + WORD_SIZE*1 - set_mem r5, r2 :: builtins.object* + r3 = list_items r0 + buf_init_item r3, 0, r1 + buf_init_item r3, 1, r2 keep_alive r0 x = r0 return 1 @@ -156,19 +154,18 @@ def f(a: List[int]) -> None: def f(a): a, r0, b, r1 :: list r2 :: object - r3, r4 :: ptr - r5 :: list + r3 :: ptr + r4 :: list L0: r0 = CPySequence_Multiply(a, 4) b = r0 r1 = PyList_New(1) r2 = object 4 - r3 = get_element_ptr r1 ob_item :: PyListObject - r4 = load_mem r3 :: ptr* - set_mem r4, r2 :: builtins.object* + r3 = list_items r1 + buf_init_item r3, 0, r2 keep_alive r1 - r5 = CPySequence_RMultiply(6, r1) - b = r5 + r4 = CPySequence_RMultiply(6, r1) + b = r4 return 1 [case testListLen] @@ -178,15 +175,12 @@ def f(a: List[int]) -> int: [out] def f(a): a :: list - r0 :: ptr - r1 :: native_int - r2 :: short_int + r0 :: native_int + r1 :: short_int L0: - r0 = get_element_ptr a ob_size :: PyVarObject - r1 = load_mem r0 :: native_int* - keep_alive a - r2 = r1 << 1 - return r2 + r0 = var_object_size a + r1 = r0 << 1 + return r1 [case testListAppend] from typing import List @@ -214,33 +208,30 @@ def increment(l: List[int]) -> List[int]: [out] def increment(l): l :: list - r0 :: ptr - r1 :: native_int - r2, r3 :: short_int + r0 :: native_int + r1, r2 :: short_int i :: int - r4 :: bit - r5, r6, r7 :: object - r8 :: bit - r9 :: short_int + r3 :: bit + r4, r5, r6 :: object + r7 :: bit + r8 :: short_int L0: - r0 = get_element_ptr l ob_size :: PyVarObject - r1 = load_mem r0 :: native_int* - keep_alive l - r2 = r1 << 1 - r3 = 0 - i = r3 + r0 = var_object_size l + r1 = r0 << 1 + r2 = 0 + i = r2 L1: - r4 = r3 < r2 :: signed - if r4 goto L2 else goto L4 :: bool + r3 = int_lt r2, r1 + if r3 goto L2 else goto L4 :: bool L2: - r5 = CPyList_GetItem(l, i) - r6 = object 1 - r7 = PyNumber_InPlaceAdd(r5, r6) - r8 = CPyList_SetItem(l, i, r7) + r4 = CPyList_GetItem(l, i) + r5 = object 1 + r6 = PyNumber_InPlaceAdd(r4, r5) + r7 = CPyList_SetItem(l, i, r6) L3: - r9 = r3 + 2 - r3 = r9 - i = r9 + r8 = r2 + 2 + r2 = r8 + i = r8 goto L1 L4: return l @@ -253,25 +244,23 @@ def f(x: List[int], y: List[int]) -> List[int]: def f(x, y): x, y, r0 :: list r1, r2 :: object - r3, r4, r5 :: ptr - r6, r7, r8 :: object - r9 :: i32 - r10 :: bit + r3 :: ptr + r4, r5, r6 :: object + r7 :: i32 + r8 :: bit L0: r0 = PyList_New(2) r1 = object 1 r2 = object 2 - r3 = get_element_ptr r0 ob_item :: PyListObject - r4 = load_mem r3 :: ptr* - set_mem r4, r1 :: builtins.object* - r5 = r4 + WORD_SIZE*1 - set_mem r5, r2 :: builtins.object* + r3 = list_items r0 + buf_init_item r3, 0, r1 + buf_init_item r3, 1, r2 keep_alive r0 - r6 = CPyList_Extend(r0, x) - r7 = CPyList_Extend(r0, y) - r8 = object 3 - r9 = PyList_Append(r0, r8) - r10 = r9 >= 0 :: signed + r4 = CPyList_Extend(r0, x) + r5 = CPyList_Extend(r0, y) + r6 = object 3 + r7 = PyList_Append(r0, r6) + r8 = r7 >= 0 :: signed return r0 [case testListIn] @@ -318,79 +307,65 @@ def f(source: List[int]) -> None: [out] def f(source): source :: list - r0 :: ptr - r1 :: native_int - r2 :: list - r3 :: native_int - r4 :: ptr - r5 :: native_int - r6 :: bit - r7 :: object - r8, x, r9 :: int - r10 :: object - r11 :: native_int + r0 :: native_int + r1 :: list + r2, r3 :: native_int + r4 :: bit + r5 :: object + r6, x, r7 :: int + r8 :: object + r9 :: native_int a :: list - r12 :: ptr - r13 :: native_int - r14 :: list - r15 :: native_int - r16 :: ptr - r17 :: native_int - r18 :: bit - r19 :: object - r20, x_2, r21 :: int - r22 :: object - r23 :: native_int + r10 :: native_int + r11 :: list + r12, r13 :: native_int + r14 :: bit + r15 :: object + r16, x_2, r17 :: int + r18 :: object + r19 :: native_int b :: list L0: - r0 = get_element_ptr source ob_size :: PyVarObject - r1 = load_mem r0 :: native_int* - keep_alive source - r2 = PyList_New(r1) - r3 = 0 + r0 = var_object_size source + r1 = PyList_New(r0) + r2 = 0 L1: - r4 = get_element_ptr source ob_size :: PyVarObject - r5 = load_mem r4 :: native_int* - keep_alive source - r6 = r3 < r5 :: signed - if r6 goto L2 else goto L4 :: bool + r3 = var_object_size source + r4 = r2 < r3 :: signed + if r4 goto L2 else goto L4 :: bool L2: - r7 = CPyList_GetItemUnsafe(source, r3) - r8 = unbox(int, r7) - x = r8 - r9 = CPyTagged_Add(x, 2) - r10 = box(int, r9) - CPyList_SetItemUnsafe(r2, r3, r10) + r5 = list_get_item_unsafe source, r2 + r6 = unbox(int, r5) + x = r6 + r7 = CPyTagged_Add(x, 2) + r8 = box(int, r7) + CPyList_SetItemUnsafe(r1, r2, r8) L3: - r11 = r3 + 1 - r3 = r11 + r9 = r2 + 1 + r2 = r9 goto L1 L4: - a = r2 - r12 = get_element_ptr source ob_size :: PyVarObject - r13 = load_mem r12 :: native_int* - keep_alive source - r14 = PyList_New(r13) - r15 = 0 + a = r1 + r10 = var_object_size source + r11 = PyList_New(r10) + r12 = 0 L5: - r16 = get_element_ptr source ob_size :: PyVarObject - r17 = load_mem r16 :: native_int* - keep_alive source - r18 = r15 < r17 :: signed - if r18 goto L6 else goto L8 :: bool + r13 = var_object_size source + r14 = r12 < r13 :: signed + if r14 goto L6 else goto L8 :: bool L6: - r19 = CPyList_GetItemUnsafe(source, r15) - r20 = unbox(int, r19) - x_2 = r20 - r21 = CPyTagged_Add(x_2, 2) - r22 = box(int, r21) - CPyList_SetItemUnsafe(r14, r15, r22) + r15 = list_get_item_unsafe source, r12 + r16 = unbox(int, r15) + x_2 = r16 + r17 = CPyTagged_Add(x_2, 2) + r18 = box(int, r17) + CPyList_SetItemUnsafe(r11, r12, r18) L7: - r23 = r15 + 1 - r15 = r23 + r19 = r12 + 1 + r12 = r19 goto L5 L8: - b = r14 + b = r11 return 1 [case testGeneratorNext] @@ -401,41 +376,37 @@ def test(x: List[int]) -> None: [out] def test(x): x :: list - r0 :: native_int - r1 :: ptr - r2 :: native_int - r3 :: bit - r4 :: object - r5, i :: int - r6 :: object - r7 :: union[int, None] - r8 :: native_int - r9 :: object + r0, r1 :: native_int + r2 :: bit + r3 :: object + r4, i :: int + r5 :: object + r6 :: union[int, None] + r7 :: native_int + r8 :: object res :: union[int, None] L0: r0 = 0 L1: - r1 = get_element_ptr x ob_size :: PyVarObject - r2 = load_mem r1 :: native_int* - keep_alive x - r3 = r0 < r2 :: signed - if r3 goto L2 else goto L4 :: bool + r1 = var_object_size x + r2 = r0 < r1 :: signed + if r2 goto L2 else goto L4 :: bool L2: - r4 = CPyList_GetItemUnsafe(x, r0) - r5 = unbox(int, r4) - i = r5 - r6 = box(int, i) - r7 = r6 + r3 = list_get_item_unsafe x, r0 + r4 = unbox(int, r3) + i = r4 + r5 = box(int, i) + r6 = r5 goto L5 L3: - r8 = r0 + 1 - r0 = r8 + r7 = r0 + 1 + r0 = r7 goto L1 L4: - r9 = box(None, 1) - r7 = r9 + r8 = box(None, 1) + r6 = r8 L5: - res = r7 + res = r6 return 1 [case testSimplifyListUnion] @@ -454,83 +425,66 @@ def nested_union(a: Union[List[str], List[Optional[str]]]) -> None: [out] def narrow(a): a :: union[list, int] - r0 :: object - r1 :: i32 - r2 :: bit - r3 :: bool - r4 :: list - r5 :: ptr - r6 :: native_int - r7 :: short_int - r8 :: int + r0 :: bit + r1 :: list + r2 :: native_int + r3 :: short_int + r4 :: int L0: - r0 = load_address PyList_Type - r1 = PyObject_IsInstance(a, r0) - r2 = r1 >= 0 :: signed - r3 = truncate r1: i32 to builtins.bool - if r3 goto L1 else goto L2 :: bool + r0 = PyList_Check(a) + if r0 goto L1 else goto L2 :: bool L1: - r4 = borrow cast(list, a) - r5 = get_element_ptr r4 ob_size :: PyVarObject - r6 = load_mem r5 :: native_int* - keep_alive r4 - r7 = r6 << 1 + r1 = borrow cast(list, a) + r2 = var_object_size r1 + r3 = r2 << 1 keep_alive a - return r7 + return r3 L2: - r8 = unbox(int, a) - return r8 + r4 = unbox(int, a) + return r4 def loop(a): a :: list - r0 :: native_int - r1 :: ptr - r2 :: native_int - r3 :: bit - r4 :: object - r5, x :: union[str, bytes] - r6 :: native_int + r0, r1 :: native_int + r2 :: bit + r3 :: object + r4, x :: union[str, bytes] + r5 :: native_int L0: r0 = 0 L1: - r1 = get_element_ptr a ob_size :: PyVarObject - r2 = load_mem r1 :: native_int* - keep_alive a - r3 = r0 < r2 :: signed - if r3 goto L2 else goto L4 :: bool + r1 = var_object_size a + r2 = r0 < r1 :: signed + if r2 goto L2 else goto L4 :: bool L2: - r4 = CPyList_GetItemUnsafe(a, r0) - r5 = cast(union[str, bytes], r4) - x = r5 + r3 = list_get_item_unsafe a, r0 + r4 = cast(union[str, bytes], r3) + x = r4 L3: - r6 = r0 + 1 - r0 = r6 + r5 = r0 + 1 + r0 = r5 goto L1 L4: return 1 def nested_union(a): a :: list - r0 :: native_int - r1 :: ptr - r2 :: native_int - r3 :: bit - r4 :: object - r5, x :: union[str, None] - r6 :: native_int + r0, r1 :: native_int + r2 :: bit + r3 :: object + r4, x :: union[str, None] + r5 :: native_int L0: r0 = 0 L1: - r1 = get_element_ptr a ob_size :: PyVarObject - r2 = load_mem r1 :: native_int* - keep_alive a - r3 = r0 < r2 :: signed - if r3 goto L2 else goto L4 :: bool + r1 = var_object_size a + r2 = r0 < r1 :: signed + if r2 goto L2 else goto L4 :: bool L2: - r4 = CPyList_GetItemUnsafe(a, r0) - r5 = cast(union[str, None], r4) - x = r5 + r3 = list_get_item_unsafe a, r0 + r4 = cast(union[str, None], r3) + x = r4 L3: - r6 = r0 + 1 - r0 = r6 + r5 = r0 + 1 + r0 = r5 goto L1 L4: return 1 From a9edd69bb4fe0ae644b5302615c7f7a4b14f827a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 19 Jan 2026 12:40:34 +0000 Subject: [PATCH 109/180] Update some vec irbuild test outputs --- mypyc/test-data/irbuild-vec-i64.test | 65 +++++++++++-------------- mypyc/test-data/irbuild-vec-nested.test | 4 +- 2 files changed, 31 insertions(+), 38 deletions(-) diff --git a/mypyc/test-data/irbuild-vec-i64.test b/mypyc/test-data/irbuild-vec-i64.test index c5951583656e6..583af33ba0407 100644 --- a/mypyc/test-data/irbuild-vec-i64.test +++ b/mypyc/test-data/irbuild-vec-i64.test @@ -277,49 +277,42 @@ def f(n: i64, l: List[i64]) -> vec[i64]: def f(n, l): n :: i64 l :: list - r0 :: ptr - r1 :: native_int - r2 :: vec[i64] - r3 :: native_int - r4 :: ptr - r5 :: native_int - r6 :: bit - r7 :: object - r8, x, r9 :: i64 - r10 :: object + r0 :: native_int + r1 :: vec[i64] + r2, r3 :: native_int + r4 :: bit + r5 :: object + r6, x, r7 :: i64 + r8 :: object + r9 :: ptr + r10 :: native_int r11 :: ptr r12 :: native_int - r13 :: ptr - r14 :: native_int L0: - r0 = get_element_ptr l ob_size :: PyVarObject - r1 = load_mem r0 :: native_int* - keep_alive l - r2 = VecI64Api.alloc(r1, r1) - r3 = 0 + r0 = var_object_size l + r1 = VecI64Api.alloc(r0, r0) + r2 = 0 L1: - r4 = get_element_ptr l ob_size :: PyVarObject - r5 = load_mem r4 :: native_int* - keep_alive l - r6 = r3 < r5 :: signed - if r6 goto L2 else goto L4 :: bool + r3 = var_object_size l + r4 = r2 < r3 :: signed + if r4 goto L2 else goto L4 :: bool L2: - r7 = CPyList_GetItemUnsafe(l, r3) - r8 = unbox(i64, r7) - x = r8 - r9 = x + 1 - r10 = r2.buf - r11 = get_element_ptr r10 items :: VecI64BufObject - r12 = r3 * 8 - r13 = r11 + r12 - set_mem r13, r9 :: i64* - keep_alive r2 + r5 = list_get_item_unsafe l, r2 + r6 = unbox(i64, r5) + x = r6 + r7 = x + 1 + r8 = r1.buf + r9 = get_element_ptr r8 items :: VecI64BufObject + r10 = r2 * 8 + r11 = r9 + r10 + set_mem r11, r7 :: i64* + keep_alive r1 L3: - r14 = r3 + 1 - r3 = r14 + r12 = r2 + 1 + r2 = r12 goto L1 L4: - return r2 + return r1 [case testVecI64FastComprehensionFromVec] from vecs import vec @@ -398,7 +391,7 @@ L0: r3 = r2 >> 1 ___tmp_5 = r3 L1: - r4 = r2 < 14 :: signed + r4 = int_lt r2, 14 if r4 goto L2 else goto L4 :: bool L2: r5 = VecI64Api.append(r1, ___tmp_5) diff --git a/mypyc/test-data/irbuild-vec-nested.test b/mypyc/test-data/irbuild-vec-nested.test index c88a99eddfebf..975ef614f8066 100644 --- a/mypyc/test-data/irbuild-vec-nested.test +++ b/mypyc/test-data/irbuild-vec-nested.test @@ -41,7 +41,7 @@ def f(v, vv): L0: r0 = vv.len r1 = vv.buf - r2 = set_element undef, len, r0 + r2 = set_element undef VecNestedBufItem, len, r0 r3 = set_element r2, buf, r1 r4 = VecNestedApi.append(v, r3) keep_alive vv @@ -64,7 +64,7 @@ def f(v, vv): L0: r0 = vv.len r1 = vv.buf - r2 = set_element undef, len, r0 + r2 = set_element undef VecNestedBufItem, len, r0 r3 = set_element r2, buf, r1 r4 = VecNestedApi.append(v, r3) keep_alive vv From 77f584a10c7c032bc1fa8b3541b821d383c9fed2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 19 Jan 2026 13:07:49 +0000 Subject: [PATCH 110/180] Fix a few vec test failures --- mypyc/irbuild/for_helpers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index ff771b4031ee2..84c34a7ebc8ae 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -880,7 +880,8 @@ class ForSequence(ForGenerator): def init( self, expr_reg: Value, target_type: RType, reverse: bool, length: Value | None = None ) -> None: - assert is_sequence_rprimitive(expr_reg.type), (expr_reg, expr_reg.type) + assert is_sequence_rprimitive(expr_reg.type) or isinstance(expr_reg.type, RVec), ( + expr_reg, expr_reg.type) builder = self.builder # Record a Value indicating the length of the sequence, if known at compile time. self.length = length @@ -1331,7 +1332,7 @@ def get_expr_length_value( builder: IRBuilder, expr: Expression, expr_reg: Value, line: int, use_pyssize_t: bool ) -> Value: rtype = builder.node_type(expr) - assert is_sequence_rprimitive(rtype) or isinstance(rtype, RTuple), rtype + assert is_sequence_rprimitive(rtype) or isinstance(rtype, (RTuple, RVec)), rtype length = get_expr_length(builder, expr) if length is None: # We cannot compute the length at compile time, so we will fetch it. From d852e56fdb54ddce8bbd128dbe4a192b04cece04 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 19 Jan 2026 13:08:31 +0000 Subject: [PATCH 111/180] Update a few vec irbuild test outputs --- mypyc/test-data/irbuild-vec-misc.test | 33 +++++++++++++++------------ 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/mypyc/test-data/irbuild-vec-misc.test b/mypyc/test-data/irbuild-vec-misc.test index 0b44214e9ce1f..394d16c95b3e3 100644 --- a/mypyc/test-data/irbuild-vec-misc.test +++ b/mypyc/test-data/irbuild-vec-misc.test @@ -170,17 +170,19 @@ def pop_float(v, i): v :: vec[float] i :: i64 r0 :: tuple[vec[float], float] - r1, r2 :: vec[float] - r3, r4, x :: float + r1 :: vec[float] + r2 :: float + r3 :: vec[float] + r4, x :: float L0: r0 = VecFloatApi.pop(v, -1) r1 = borrow r0[0] - r2 = unborrow r1 - v = r2 - r3 = borrow r0[1] - r4 = unborrow r3 - x = r4 + r2 = borrow r0[1] keep_alive steal r0 + r3 = unborrow r1 + v = r3 + r4 = unborrow r2 + x = r4 return x [case testVecMiscRemove] @@ -321,8 +323,10 @@ def get_item_nested(v, i): r3, r4 :: VecNestedBufItem{len:native_int, buf:object_nrc} r5 :: vec[i32] r6 :: tuple[vec[vec[i32]], vec[i32]] - r7, r8 :: vec[vec[i32]] - r9, r10, x :: vec[i32] + r7 :: vec[vec[i32]] + r8 :: vec[i32] + r9 :: vec[vec[i32]] + r10, x :: vec[i32] L0: r0 = VecNestedApi.pop(v, i) r1 = borrow r0[0] @@ -333,10 +337,11 @@ L0: r6 = (r2, r5) keep_alive steal r0 r7 = borrow r6[0] - r8 = unborrow r7 - v = r8 - r9 = borrow r6[1] - r10 = unborrow r9 - x = r10 + r8 = borrow r6[1] keep_alive steal r6 + r9 = unborrow r7 + v = r9 + r10 = unborrow r8 + x = r10 return x + From 8da64035d8e4f013d1d6fc7ff117295fb912b1c0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 19 Jan 2026 14:19:12 +0000 Subject: [PATCH 112/180] Update a few vec tests --- mypyc/test-data/refcount.test | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index cc6d75b1f0309..516b9bd1d1431 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -1757,7 +1757,7 @@ def f(vv, v): L0: r0 = v.len r1 = v.buf - r2 = set_element undef, len, r0 + r2 = set_element undef VecNestedBufItem, len, r0 r3 = set_element r2, buf, r1 inc_ref vv r4 = VecNestedApi.append(vv, r3) @@ -1879,8 +1879,10 @@ def f(v): r3, r4 :: VecNestedBufItem{len:native_int, buf:object_nrc} r5 :: vec[str] r6 :: tuple[vec[vec[str]], vec[str]] - r7, r8, vv :: vec[vec[str]] - r9, r10, x :: vec[str] + r7 :: vec[vec[str]] + r8 :: vec[str] + r9, vv :: vec[vec[str]] + r10, x :: vec[str] L0: r0 = VecNestedApi.pop(v, -1) r1 = borrow r0[0] @@ -1890,11 +1892,11 @@ L0: r5 = VecTApi.convert_from_nested(r4) r6 = (r2, r5) r7 = borrow r6[0] - r8 = unborrow r7 - vv = r8 + r8 = borrow r6[1] + r9 = unborrow r7 + vv = r9 dec_ref vv - r9 = borrow r6[1] - r10 = unborrow r9 + r10 = unborrow r8 x = r10 return x From bb9a4b77dcec754fb408cb7da102d9be04212624 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 19 Jan 2026 15:09:57 +0000 Subject: [PATCH 113/180] WIP update symbolic link to vecs.h --- mypyc/lib-rt/vecs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/lib-rt/vecs.h b/mypyc/lib-rt/vecs.h index 03666343e12f4..46a162efedf1c 120000 --- a/mypyc/lib-rt/vecs.h +++ b/mypyc/lib-rt/vecs.h @@ -1 +1 @@ -/home/jukka/src/vecs/vecs.h \ No newline at end of file +../../../vecs/vecs.h \ No newline at end of file From d051c754f3483311717ba3be20d63397434e7093 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 28 Jan 2026 10:22:25 +0000 Subject: [PATCH 114/180] Update vecs.h references to librt_vecs.h --- mypyc/codegen/emitmodule.py | 4 ++-- mypyc/lib-rt/vecs.h | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) delete mode 120000 mypyc/lib-rt/vecs.h diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 9c68667b636ce..f94f1e52171bf 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -646,8 +646,8 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]: for source_dep in sorted(source_deps, key=lambda d: d.path): ext_declarations.emit_line(f'#include "{source_dep.get_header()}"') if self.use_vec_capsule: - ext_declarations.emit_line('#include "vecs.h"') - + ext_declarations.emit_line('#include "vecs/librt_vecs.h"') + declarations = Emitter(self.context) declarations.emit_line(f"#ifndef MYPYC_LIBRT_INTERNAL{self.group_suffix}_H") declarations.emit_line(f"#define MYPYC_LIBRT_INTERNAL{self.group_suffix}_H") diff --git a/mypyc/lib-rt/vecs.h b/mypyc/lib-rt/vecs.h deleted file mode 120000 index 46a162efedf1c..0000000000000 --- a/mypyc/lib-rt/vecs.h +++ /dev/null @@ -1 +0,0 @@ -../../../vecs/vecs.h \ No newline at end of file From bfaf5d755338affa54d51ae37fc06b1c608548c8 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 28 Jan 2026 10:23:11 +0000 Subject: [PATCH 115/180] Remove temporary vecs.pyi --- vecs.pyi | 24 ------------------------ 1 file changed, 24 deletions(-) delete mode 100644 vecs.pyi diff --git a/vecs.pyi b/vecs.pyi deleted file mode 100644 index 2298145ebdd70..0000000000000 --- a/vecs.pyi +++ /dev/null @@ -1,24 +0,0 @@ -from typing import TypeVar, Generic, Type, Iterable, Iterator, overload, Tuple -from mypy_extensions import i64 - -T = TypeVar("T") -S = TypeVar("S") - -class vec(Generic[T]): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, __init: Iterable[T]) -> None: ... - - def __len__(self) -> i64: ... - @overload - def __getitem__(self, i: i64) -> T: ... - @overload - def __getitem__(self, i: slice) -> vec[T]: ... - def __setitem__(self, i: i64, o: T) -> None: ... - def __iter__(self) -> Iterator[T]: ... - - -def append(__v: vec[T], __o: T) -> vec[T]: ... -def remove(__v: vec[T], __x: T) -> vec[T]: ... -def pop(__v: vec[T], __i: i64 = -1) -> Tuple[vec[T], T]: ... From 55e7c2073f30cbaaea6c1ba008e18b859c860814 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 28 Jan 2026 10:28:29 +0000 Subject: [PATCH 116/180] Remove redundant vecs stub --- test-data/unit/lib-stub/vecs.pyi | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 test-data/unit/lib-stub/vecs.pyi diff --git a/test-data/unit/lib-stub/vecs.pyi b/test-data/unit/lib-stub/vecs.pyi deleted file mode 100644 index c181da627d0ff..0000000000000 --- a/test-data/unit/lib-stub/vecs.pyi +++ /dev/null @@ -1,21 +0,0 @@ -from typing import TypeVar, Generic, Type, Iterable, Iterator, overload, Tuple -from mypy_extensions import i64 - -T = TypeVar("T") - -class vec(Generic[T]): - @overload - def __init__(self) -> None: ... - @overload - def __init__(self, __items: Iterable[T]) -> None: ... - def __len__(self) -> i64: ... - @overload - def __getitem__(self, i: i64) -> T: ... - @overload - def __getitem__(self, i: slice) -> vec[T]: ... - def __setitem__(self, i: i64, o: T) -> None: ... - def __iter__(self) -> Iterator[T]: ... - -def append(__v: vec[T], __o: T) -> vec[T]: ... -def remove(__v: vec[T], __o: T) -> vec[T]: ... -def pop(__v: vec[T], __i: i64 = -1) -> Tuple[vec[T], T]: ... From 9ec23443a22951c78abdfaffecbaca3974c4ad94 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 28 Jan 2026 10:34:47 +0000 Subject: [PATCH 117/180] Update vecs references in mypy to librt.vecs --- mypy/semanal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/semanal.py b/mypy/semanal.py index 60e82ed8707cf..5f2bc7a133865 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -6180,7 +6180,7 @@ def analyze_type_application(self, expr: IndexExpr) -> None: expr.analyzed.line = expr.line expr.analyzed.column = expr.column n = self.lookup_type_node(base) - if n and n.fullname == "vecs.vec": + if n and n.fullname == "librt.vecs.vec": check_vec_type_args(types, expr, self) if isinstance(base, RefExpr) and base.fullname == "librt.vecs.vec": From 8c500fa71d58ad8a1ec0c6817880dba54f44441e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 28 Jan 2026 10:35:09 +0000 Subject: [PATCH 118/180] Update vecs references in mypyc to librt.vecs --- mypyc/codegen/emitmodule.py | 4 ++-- mypyc/ir/rtypes.py | 2 +- mypyc/irbuild/expression.py | 2 +- mypyc/irbuild/mapper.py | 2 +- mypyc/irbuild/specialize.py | 6 +++--- mypyc/irbuild/vec.py | 2 +- mypyc/test-data/irbuild-vec-i64.test | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index f94f1e52171bf..13092ace49eeb 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -647,7 +647,7 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]: ext_declarations.emit_line(f'#include "{source_dep.get_header()}"') if self.use_vec_capsule: ext_declarations.emit_line('#include "vecs/librt_vecs.h"') - + declarations = Emitter(self.context) declarations.emit_line(f"#ifndef MYPYC_LIBRT_INTERNAL{self.group_suffix}_H") declarations.emit_line(f"#define MYPYC_LIBRT_INTERNAL{self.group_suffix}_H") @@ -977,7 +977,7 @@ def generate_globals_init(self, emitter: Emitter) -> None: if self.use_vec_capsule: emitter.emit_lines( - 'VecApi = PyCapsule_Import("vecs._C_API", 0);', + 'VecApi = PyCapsule_Import("librt.vecs._C_API", 0);', "if (!VecApi) return -1;", "VecTApi = *VecApi->t;", "VecNestedApi = *VecApi->nested;", diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index f4cc4aa0730fb..2393c726de401 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -1025,7 +1025,7 @@ def serialize(self) -> str: @final class RVec(RType): - """vecs.vec[T]""" + """librt.vecs.vec[T]""" is_unboxed = True diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index 0050ca8940d3f..327f549119795 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -351,7 +351,7 @@ def transform_call_expr(builder: IRBuilder, expr: CallExpr) -> Value: analyzed = callee.analyzed if ( isinstance(analyzed.expr, RefExpr) - and analyzed.expr.fullname == "vecs.vec" + and analyzed.expr.fullname == "librt.vecs.vec" and len(analyzed.types) == 1 ): item_type = builder.type_to_rtype(analyzed.types[0]) diff --git a/mypyc/irbuild/mapper.py b/mypyc/irbuild/mapper.py index 70869a2c16676..523c718c9b867 100644 --- a/mypyc/irbuild/mapper.py +++ b/mypyc/irbuild/mapper.py @@ -124,7 +124,7 @@ def type_to_rtype(self, typ: Type | None) -> RType: return int16_rprimitive elif typ.type.fullname == "mypy_extensions.u8": return uint8_rprimitive - elif typ.type.fullname == "vecs.vec": + elif typ.type.fullname == "librt.vecs.vec": return RVec(self.type_to_rtype(typ.args[0])) elif typ.type.fullname in KNOWN_NATIVE_TYPES: return KNOWN_NATIVE_TYPES[typ.type.fullname] diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index c6988941b8789..4d64ff80e76b7 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -1508,7 +1508,7 @@ def translate_bytes_get_item( ) -@specialize_function("vecs.append") +@specialize_function("librt.vecs.append") def translate_vec_append(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: if len(expr.args) == 2 and expr.arg_kinds == [ARG_POS, ARG_POS]: vec_arg = expr.args[0] @@ -1522,7 +1522,7 @@ def translate_vec_append(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> return None -@specialize_function("vecs.remove") +@specialize_function("librt.vecs.remove") def translate_vec_remove(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: if len(expr.args) == 2 and expr.arg_kinds == [ARG_POS, ARG_POS]: vec_arg = expr.args[0] @@ -1536,7 +1536,7 @@ def translate_vec_remove(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> return None -@specialize_function("vecs.pop") +@specialize_function("librt.vecs.pop") def translate_vec_pop(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: if 1 <= len(expr.args) <= 2 and all(kind == ARG_POS for kind in expr.arg_kinds): vec_arg = expr.args[0] diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 3d8281608718b..c557080f3753b 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -1,4 +1,4 @@ -"""Generate IR for vecs.vec operations""" +"""Generate IR for librt.vecs.vec operations""" from __future__ import annotations diff --git a/mypyc/test-data/irbuild-vec-i64.test b/mypyc/test-data/irbuild-vec-i64.test index 583af33ba0407..0f7e047d495b0 100644 --- a/mypyc/test-data/irbuild-vec-i64.test +++ b/mypyc/test-data/irbuild-vec-i64.test @@ -2,7 +2,7 @@ -- which use a similar runtime representation. [case testVecI64CreateEmpty] -from vecs import vec, append +from librt.vecs import vec, append from mypy_extensions import i64 def f() -> vec[i64]: From 1e0bf8e469a852eb62c2924ea6a94f86eece3042 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 28 Jan 2026 11:18:31 +0000 Subject: [PATCH 119/180] Fix mypyc self check --- mypyc/irbuild/vec.py | 18 ++++++------- mypyc/test-data/irbuild-vec-i64.test | 38 ++++++++++++++-------------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index c557080f3753b..fcb41d1d62ab9 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -168,7 +168,7 @@ def vec_create_initialized( builder.set_mem(for_loop.index, item_type, init) for_loop.finish() - builder.keep_alive([vec]) + builder.keep_alive([vec], line) return vec @@ -182,7 +182,7 @@ def vec_create_from_values( for value in values: builder.set_mem(ptr, item_type, value) ptr = builder.int_add(ptr, step) - builder.keep_alive([vec]) + builder.keep_alive([vec], line) return vec @@ -311,7 +311,7 @@ def vec_get_item_unsafe( vtype = base.type item_addr = vec_item_ptr(builder, base, index) result = builder.load_mem(item_addr, vtype.item_type) - builder.keep_alive([base]) + builder.keep_alive([base], line) return result @@ -332,7 +332,7 @@ def vec_set_item( old_item = builder.load_mem(item_addr, item_type, borrow=True) builder.add(DecRef(old_item)) builder.set_mem(item_addr, item_type, item) - builder.keep_alive([base]) + builder.keep_alive([base], line) def vec_init_item_unsafe( @@ -345,7 +345,7 @@ def vec_init_item_unsafe( item_type = vtype.item_type item = builder.coerce(item, item_type, line) builder.set_mem(item_addr, item_type, item) - builder.keep_alive([base]) + builder.keep_alive([base], line) def convert_to_t_ext_item(builder: LowLevelIRBuilder, item: Value) -> Value: @@ -414,7 +414,7 @@ def vec_append(builder: LowLevelIRBuilder, vec: Value, item: Value, line: int) - ) ) if vec_depth(vec_type) > 0: - builder.keep_alive([item]) + builder.keep_alive([item], line) return call @@ -453,7 +453,7 @@ def vec_pop(builder: LowLevelIRBuilder, base: Value, index: Value, line: int) -> assert isinstance(vec_type.item_type, RVec) z = convert_from_t_ext_item(builder, y, vec_type.item_type) result = builder.add(TupleSet([x, z], line)) - builder.keep_alive([orig], steal=True) + builder.keep_alive([orig], line, steal=True) return result @@ -482,7 +482,7 @@ def vec_remove(builder: LowLevelIRBuilder, vec: Value, item: Value, line: int) - ) ) if vec_depth(vec_type) > 0: - builder.keep_alive([item]) + builder.keep_alive([item], line) return call @@ -509,7 +509,7 @@ def vec_contains(builder: LowLevelIRBuilder, vec: Value, target: Value, line: in builder.activate_block(false) for_loop.finish() - builder.keep_alive([vec]) + builder.keep_alive([vec], line) res = Register(bool_rprimitive) builder.assign(res, Integer(0, bool_rprimitive)) diff --git a/mypyc/test-data/irbuild-vec-i64.test b/mypyc/test-data/irbuild-vec-i64.test index 0f7e047d495b0..c7f7c683aab6f 100644 --- a/mypyc/test-data/irbuild-vec-i64.test +++ b/mypyc/test-data/irbuild-vec-i64.test @@ -15,7 +15,7 @@ L0: return r0 [case testVecI64Len] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64 def f(v: vec[i64]) -> i64: @@ -32,7 +32,7 @@ L0: return l [case testVecI64GetItem] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64 def f(v: vec[i64], i: i64) -> i64: @@ -78,7 +78,7 @@ L5: return r10 [case testVecI64Append] -from vecs import vec, append +from librt.vecs import vec, append from mypy_extensions import i64 def f(v: vec[i64], i: i64) -> vec[i64]: @@ -93,7 +93,7 @@ L0: return r0 [case testVecI64SetItem] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64 def f(v: vec[i64], i: i64, x: i64) -> None: @@ -138,7 +138,7 @@ L5: return 1 [case testVecI64ConstructFromListExpr] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64 def f() -> vec[i64]: @@ -162,7 +162,7 @@ L0: return r0 [case testVecI64ConstructFromListMultiply] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64 def f(n: i64) -> vec[i64]: @@ -197,7 +197,7 @@ L3: return r0 [case testVecI64ConstructFromListMultiply2] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64 def f(n: i64, x: i64) -> vec[i64]: @@ -232,7 +232,7 @@ L3: return r0 [case testVecI64ConstructFromListComprehension] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64 def f(n: i64) -> vec[i64]: @@ -267,7 +267,7 @@ L4: return r1 [case testVecI64FastComprehensionFromList] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64 from typing import List @@ -315,7 +315,7 @@ L4: return r1 [case testVecI64FastComprehensionFromVec] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64 from typing import List @@ -370,7 +370,7 @@ L4: return r1 [case testVecI64ConstructFromRange] -from vecs import vec, pop +from librt.vecs import vec, pop from mypy_extensions import i64 def f() -> vec[i64]: @@ -406,7 +406,7 @@ L4: return r1 [case testVecI64ForLoop] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64 def f(v: vec[i64]) -> i64: @@ -451,7 +451,7 @@ L4: return t [case testVecI64Contains] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64 def contains(v: vec[i64], n: i64) -> bool: @@ -498,7 +498,7 @@ L6: return r10 [case testVecI64GetItemWithInt] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64 def f(v: vec[i64]) -> i64: @@ -543,7 +543,7 @@ L5: return r10 [case testVecI64Slicing] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64 def f(v: vec[i64], n: i64, m: i64) -> None: @@ -571,7 +571,7 @@ L0: return 1 [case testVecI64Remove] -from vecs import vec, remove +from librt.vecs import vec, remove from mypy_extensions import i64 def rem(v: vec[i64], n: i64) -> None: @@ -588,7 +588,7 @@ L0: [case testVecI64PopLast] from typing import Tuple -from vecs import vec, pop +from librt.vecs import vec, pop from mypy_extensions import i64 def pop_last(v: vec[i64]) -> Tuple[vec[i64], i64]: @@ -603,7 +603,7 @@ L0: [case testVecI64PopNth] from typing import Tuple -from vecs import vec, pop +from librt.vecs import vec, pop from mypy_extensions import i64 def pop_nth(v: vec[i64], n: i64) -> Tuple[vec[i64], i64]: @@ -618,7 +618,7 @@ L0: return r0 [case testVecI64InPlaceOp] -from vecs import vec, remove +from librt.vecs import vec, remove from mypy_extensions import i64 def inplace(v: vec[i64], n: i64, m: i64) -> None: From 4419de8be701265ab5d5c564798927db6c81c8a4 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 28 Jan 2026 11:19:50 +0000 Subject: [PATCH 120/180] Fix irbuild tests --- mypyc/test-data/irbuild-vec-misc.test | 22 +++++++++++----------- mypyc/test-data/irbuild-vec-nested.test | 16 ++++++++-------- mypyc/test-data/irbuild-vec-t.test | 24 ++++++++++++------------ 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/mypyc/test-data/irbuild-vec-misc.test b/mypyc/test-data/irbuild-vec-misc.test index 394d16c95b3e3..47f1403065da2 100644 --- a/mypyc/test-data/irbuild-vec-misc.test +++ b/mypyc/test-data/irbuild-vec-misc.test @@ -5,7 +5,7 @@ -- vec[i64] test cases are in irbuild-vec-i64.test. [case testVecMiscCreateEmpty] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i32, i16, u8 @@ -51,7 +51,7 @@ L0: return r0 [case testVecMiscCreateFromList] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64, i32 @@ -74,7 +74,7 @@ L0: return r0 [case testVecMiscLen] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64, i32 @@ -89,7 +89,7 @@ L0: return r0 [case testVecMiscAppend] -from vecs import vec, append +from librt.vecs import vec, append from mypy_extensions import i32 @@ -111,7 +111,7 @@ L0: return r0 [case testVecMiscGetItem] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64 @@ -158,7 +158,7 @@ L5: return r10 [case testVecMiscPop] -from vecs import vec, pop +from librt.vecs import vec, pop from mypy_extensions import i64 @@ -186,7 +186,7 @@ L0: return x [case testVecMiscRemove] -from vecs import vec, remove +from librt.vecs import vec, remove def remove_float(v: vec[float]) -> vec[float]: return remove(v, 1.5) @@ -198,7 +198,7 @@ L0: return r0 [case testVecMiscSlice] -from vecs import vec, remove +from librt.vecs import vec, remove from mypy_extensions import i64 @@ -214,7 +214,7 @@ L0: return r0 [case testVecMiscForLoop] -from vecs import vec, remove +from librt.vecs import vec, remove from mypy_extensions import i64, i16 @@ -260,7 +260,7 @@ L4: return s [case testVecMiscNestedGetItem] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64, i32 @@ -307,7 +307,7 @@ L5: return r10 [case testVecMiscNestedPop] -from vecs import vec, pop +from librt.vecs import vec, pop from mypy_extensions import i64, i32 diff --git a/mypyc/test-data/irbuild-vec-nested.test b/mypyc/test-data/irbuild-vec-nested.test index 975ef614f8066..aa6f79485e97d 100644 --- a/mypyc/test-data/irbuild-vec-nested.test +++ b/mypyc/test-data/irbuild-vec-nested.test @@ -1,7 +1,7 @@ -- Test cases for nested vecs [case testVecNestedCreateEmpty] -from vecs import vec, append +from librt.vecs import vec, append from mypy_extensions import i64 def f() -> vec[vec[str]]: @@ -26,7 +26,7 @@ L0: return r0 [case testVecNestedAppend] -from vecs import vec, append +from librt.vecs import vec, append def f(v: vec[vec[str]], vv: vec[str]) -> vec[vec[str]]: return append(v, vv) @@ -48,7 +48,7 @@ L0: return r4 [case testVecNestedVecI64Append] -from vecs import vec, append +from librt.vecs import vec, append from mypy_extensions import i64 def f(v: vec[vec[i64]], vv: vec[i64]) -> vec[vec[i64]]: @@ -71,7 +71,7 @@ L0: return r4 [case testVecNestedLen] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64 from typing import Optional @@ -95,7 +95,7 @@ L0: return r0 [case testVecNestedGetItem] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64 def f(v: vec[vec[str]], n: i64) -> vec[str]: @@ -141,7 +141,7 @@ L5: return r10 [case testVecNestedI64GetItem] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64 def f(v: vec[vec[i64]], n: i64) -> vec[i64]: @@ -187,7 +187,7 @@ L5: return r10 [case testVecNestedI64GetItemWithBorrow] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64 def f(v: vec[vec[i64]], n: i64) -> i64: @@ -265,7 +265,7 @@ L10: return r21 [case testVecDoublyNestedGetItem] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64 def f(v: vec[vec[vec[str]]], n: i64) -> vec[vec[str]]: diff --git a/mypyc/test-data/irbuild-vec-t.test b/mypyc/test-data/irbuild-vec-t.test index aefa7ed5a6c8f..2670c15d55686 100644 --- a/mypyc/test-data/irbuild-vec-t.test +++ b/mypyc/test-data/irbuild-vec-t.test @@ -2,7 +2,7 @@ -- Also tests for vec[t | None], which uses the same representation. [case testVecTCreateEmpty] -from vecs import vec, append +from librt.vecs import vec, append class C: pass @@ -32,7 +32,7 @@ L0: return r2 [case testVecTAppend] -from vecs import vec, append +from librt.vecs import vec, append def f(v: vec[str]) -> vec[str]: return append(v, 'x') @@ -51,7 +51,7 @@ L0: return r3 [case testVecTOptionalCreateEmpty] -from vecs import vec, append +from librt.vecs import vec, append from typing import Optional class C: pass @@ -84,7 +84,7 @@ L0: return r3 [case testVecTOptionalAppend] -from vecs import vec, append +from librt.vecs import vec, append from typing import Optional def f(v: vec[Optional[str]]) -> vec[Optional[str]]: @@ -115,7 +115,7 @@ L0: return r9 [case testVecTLen] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64 from typing import Optional @@ -139,7 +139,7 @@ L0: return r0 [case testVecTGetItem] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64 def f(v: vec[str], n: i64) -> str: @@ -185,7 +185,7 @@ L5: return r10 [case testVecTOptionalGetItem] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64 from typing import Optional @@ -233,7 +233,7 @@ L5: [case testNewTPopLast] from typing import Tuple -from vecs import vec, pop +from librt.vecs import vec, pop def pop_last(v: vec[str]) -> Tuple[vec[str], str]: return pop(v) @@ -246,7 +246,7 @@ L0: return r0 [case testVecTConstructFromListComprehension] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64 def f(n: i64) -> vec[str]: @@ -289,7 +289,7 @@ L4: return r3 [case testVecTCheckItemType] -from vecs import vec +from librt.vecs import vec from typing import Tuple, Any, List, TypeVar def bad1(v: vec[Tuple[str, str]]) -> None: pass @@ -312,7 +312,7 @@ main:7: error: Invalid item type for "vec" main:9: error: Invalid item type for "vec" [case testVecTCheckItemTypeUnion] -from vecs import vec +from librt.vecs import vec from typing import Tuple, Union, Optional, Any, TypeVar from mypy_extensions import i64 @@ -347,7 +347,7 @@ main:15: error: Invalid item type for "vec" main:17: error: Invalid item type for "vec" [case testVecTCheckItemTypeDuringConstruction] -from vecs import vec +from librt.vecs import vec from typing import Optional, Union def f() -> None: From f1fb682102e32ca2a358966dbda89fa9837259d9 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 28 Jan 2026 11:36:40 +0000 Subject: [PATCH 121/180] Fix vec capsule in generated code (WIP) --- mypyc/codegen/emitmodule.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 13092ace49eeb..d9dcd040f741d 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -588,9 +588,9 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]: if self.use_vec_capsule: self.declare_global("VecCapsule *", "VecApi") for vec_type in sorted(vec_c_types.values()): - self.declare_global(f"{vec_type}Features ", f"{vec_type}Api") - self.declare_global("VecTFeatures ", "VecTApi") - self.declare_global("VecNestedFeatures ", "VecNestedApi") + self.declare_global(f"{vec_type}API ", f"{vec_type}Api") + self.declare_global("VecTAPI ", "VecTApi") + self.declare_global("VecNestedAPI ", "VecNestedApi") for module_name, module in self.modules.items(): if multi_file: @@ -977,6 +977,8 @@ def generate_globals_init(self, emitter: Emitter) -> None: if self.use_vec_capsule: emitter.emit_lines( + 'PyObject *mod = PyImport_ImportModule("librt.vecs");', + 'if (mod == NULL) return -1;', 'VecApi = PyCapsule_Import("librt.vecs._C_API", 0);', "if (!VecApi) return -1;", "VecTApi = *VecApi->t;", From 4a5df85e9c613711716d6636bdf233a84255308a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 28 Jan 2026 11:37:14 +0000 Subject: [PATCH 122/180] Steal first arg of pop and remove --- mypyc/irbuild/vec.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index fcb41d1d62ab9..d4be2a2ec1a61 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -438,7 +438,7 @@ def vec_pop(builder: LowLevelIRBuilder, base: Value, index: Value, line: int) -> name, [base, index], RTuple([vec_type, item_type]), - steals=[False, False], + steals=[True, False], is_borrowed=False, error_kind=ERR_MAGIC, line=line, @@ -475,7 +475,7 @@ def vec_remove(builder: LowLevelIRBuilder, vec: Value, item: Value, line: int) - name, [vec, coerced_item], vec_type, - steals=[False, False], + steals=[True, False], is_borrowed=False, error_kind=ERR_MAGIC, line=line, From 1811d5f0895237835a1c72be3797bdc018a91910 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 28 Jan 2026 11:37:33 +0000 Subject: [PATCH 123/180] Fix vec[i64] run tests using primitives --- mypyc/test-data/run-vec-i64.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypyc/test-data/run-vec-i64.test b/mypyc/test-data/run-vec-i64.test index da165ffdb634d..1cd0b2b352cbd 100644 --- a/mypyc/test-data/run-vec-i64.test +++ b/mypyc/test-data/run-vec-i64.test @@ -1,11 +1,11 @@ -- Test cases for vec[i64]. These also partially cover other unboxed item types, -- which use a similar runtime representation. -[case testVecI64BasicOps] +[case testVecI64BasicOps_librt_experimental] from typing import Final, Any, Iterable, Tuple from mypy_extensions import i64 -from vecs import vec, append, remove, pop +from librt.vecs import vec, append, remove, pop from testutil import assertRaises From 9758bcc1b7730bd3b40b21747ca155a64d2b50eb Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 28 Jan 2026 11:54:20 +0000 Subject: [PATCH 124/180] Fix slicing that returns an empty vec --- mypyc/lib-rt/vecs/vec_t.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 4c2b77f75dd43..8a03ea1a2e644 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -128,6 +128,8 @@ VecT VecT_Slice(VecT vec, int64_t start, int64_t end) { if (end > vec.len) end = vec.len; int64_t slicelength = end - start; + if (slicelength == 0) + return (VecT) { .len = 0, .buf = NULL }; VecT res = vec_alloc(slicelength, vec.buf->item_type); if (VEC_IS_ERROR(res)) return res; From e925a03d7ad9e277fd3231eb66ef0d0b1beb44ea Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 28 Jan 2026 11:54:46 +0000 Subject: [PATCH 125/180] Fix additional vec run tests --- mypyc/test-data/run-vec-misc.test | 4 ++-- mypyc/test-data/run-vec-t.test | 7 ++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/mypyc/test-data/run-vec-misc.test b/mypyc/test-data/run-vec-misc.test index f9b5435191bdf..2e9cc9a4f5f9c 100644 --- a/mypyc/test-data/run-vec-misc.test +++ b/mypyc/test-data/run-vec-misc.test @@ -4,12 +4,12 @@ -- -- vec[i64] test cases are in run-vec-i64.test. -[case testVecMiscBasicOps] +[case testVecMiscBasicOps_librt_experimental] # mypy: allow-redefinition from typing import Any, cast from mypy_extensions import i64, i32, i16, u8 -from vecs import vec, append, remove, pop +from librt.vecs import vec, append, remove, pop from testutil import assertRaises def test_create_empty() -> None: diff --git a/mypyc/test-data/run-vec-t.test b/mypyc/test-data/run-vec-t.test index f7003f1855452..9569d34d9be8e 100644 --- a/mypyc/test-data/run-vec-t.test +++ b/mypyc/test-data/run-vec-t.test @@ -1,11 +1,11 @@ -- Test cases for vec[t] where t is a boxed, non-vec type (PyObject *). -- Also tests for vec[t | None], which uses the same representation. -[case testVecTBasicOps] +[case testVecTBasicOps_librt_experimental] from typing import Final, Any, Iterable, Optional, Tuple from mypy_extensions import i64 -from vecs import vec, append, remove, pop +from librt.vecs import vec, append, remove, pop from testutil import assertRaises @@ -384,9 +384,6 @@ def test_tuple() -> None: with assertRaises(RuntimeError): f_any(True) -[case testVecTBasicOps_python_3_10] -from vecs import vec, append, remove, pop - def test_optional_item_new_syntax() -> None: v = vec[str | None]() assert len(v) == 0 From 4ab45151ffc7f8654e52e9657d8ff03369a9d834 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 28 Jan 2026 11:55:44 +0000 Subject: [PATCH 126/180] Fix nested vec run tests --- mypyc/test-data/run-vec-nested.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypyc/test-data/run-vec-nested.test b/mypyc/test-data/run-vec-nested.test index 89b9556914965..b20100644aff2 100644 --- a/mypyc/test-data/run-vec-nested.test +++ b/mypyc/test-data/run-vec-nested.test @@ -1,9 +1,9 @@ -[case testVecNestedBasicOps] +[case testVecNestedBasicOps_librt_experimental] from typing import Final, Any, Iterable, Optional, Tuple import sys from mypy_extensions import i64 -from vecs import vec, append, remove, pop +from librt.vecs import vec, append, remove, pop from testutil import assertRaises From 5b0c9fd10ab25c0513f0676162fb853411fdefe0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 28 Jan 2026 12:04:45 +0000 Subject: [PATCH 127/180] Fix more tests --- mypyc/test-data/alwaysdefined.test | 2 +- mypyc/test-data/exceptions.test | 2 +- mypyc/test-data/refcount.test | 52 ++++++++++++++++++++++++------ 3 files changed, 44 insertions(+), 12 deletions(-) diff --git a/mypyc/test-data/alwaysdefined.test b/mypyc/test-data/alwaysdefined.test index 49e696ef23096..9c9b15e4e122d 100644 --- a/mypyc/test-data/alwaysdefined.test +++ b/mypyc/test-data/alwaysdefined.test @@ -731,7 +731,7 @@ NestedFunc: [] f___init___NestedFunc_obj: [] [case testAlwaysDefinedWithVecSetItem] -from vecs import vec +from librt.vecs import vec class C: def __init__(self, v: vec[C]) -> None: diff --git a/mypyc/test-data/exceptions.test b/mypyc/test-data/exceptions.test index 8089549c194be..bc2bb5c116886 100644 --- a/mypyc/test-data/exceptions.test +++ b/mypyc/test-data/exceptions.test @@ -721,7 +721,7 @@ L3: return r5 [case testVecNestedRemove] -from vecs import vec, remove +from librt.vecs import vec, remove from mypy_extensions import i64 def f(v: vec[i64], i: i64, n: i64) -> None: diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index 516b9bd1d1431..7c4f238a74b42 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -1501,7 +1501,7 @@ L3: return r4 [case testVecTSetItem] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64 def f(v: vec[str], i: i64, x: str) -> None: @@ -1551,7 +1551,7 @@ L5: [case testVecTConstructFromListMultiply] from typing import Optional -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64 def f(n: i64) -> vec[Optional[str]]: @@ -1592,7 +1592,7 @@ L3: return r4 [case testVecTForLoop] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64 def f(v: vec[str]) -> i64: @@ -1643,7 +1643,7 @@ L0: return 1 [case testVecTConstructFromListExpr] -from vecs import vec +from librt.vecs import vec def f() -> vec[str]: return vec[str](['x', 'y']) @@ -1672,7 +1672,7 @@ L0: return r4 [case testVecI64GetItemBorrowVec] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64 class C: @@ -1722,7 +1722,7 @@ L5: return r11 [case testVecI64LenBorrowVec] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64 class C: @@ -1742,7 +1742,7 @@ L0: return r1 [case testVecNestedAppend] -from vecs import vec, append +from librt.vecs import vec, append def f(vv: vec[vec[str]], v: vec[str]) -> vec[vec[str]]: return append(vv, v) @@ -1764,7 +1764,7 @@ L0: return r4 [case testVecTGetItem] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64 def f(v: vec[str], n: i64) -> str: @@ -1809,7 +1809,7 @@ L5: return r10 [case testVecNestedGetItem] -from vecs import vec +from librt.vecs import vec from mypy_extensions import i64 def f(v: vec[vec[str]], n: i64) -> None: @@ -1865,8 +1865,39 @@ L6: dec_ref r2 goto L2 +[case testVecPop] +from librt.vecs import vec, pop, append +from mypy_extensions import i64 + +def f() -> vec[i64]: + v = vec[i64]() + v = append(v, 7) + v, x = pop(v) + return v +[out] +def f(): + r0, v, r1 :: vec[i64] + r2 :: tuple[vec[i64], i64] + r3 :: vec[i64] + r4 :: i64 + r5 :: vec[i64] + r6, x :: i64 +L0: + r0 = VecI64Api.alloc(0, 0) + v = r0 + r1 = VecI64Api.append(v, 7) + v = r1 + r2 = VecI64Api.pop(v, -1) + r3 = borrow r2[0] + r4 = borrow r2[1] + r5 = unborrow r3 + v = r5 + r6 = unborrow r4 + x = r6 + return v + [case testVecNestedPop] -from vecs import vec, pop +from librt.vecs import vec, pop def f(v: vec[vec[str]]) -> vec[str]: vv, x = pop(v) @@ -1884,6 +1915,7 @@ def f(v): r9, vv :: vec[vec[str]] r10, x :: vec[str] L0: + inc_ref v r0 = VecNestedApi.pop(v, -1) r1 = borrow r0[0] r2 = unborrow r1 From 1016b2f1bbc5214e13d264c0f28b40c91671257d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 28 Jan 2026 12:06:33 +0000 Subject: [PATCH 128/180] Fix more tests --- mypyc/test-data/exceptions.test | 1 + mypyc/test-data/vecusage.test | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/mypyc/test-data/exceptions.test b/mypyc/test-data/exceptions.test index bc2bb5c116886..92e34074618c7 100644 --- a/mypyc/test-data/exceptions.test +++ b/mypyc/test-data/exceptions.test @@ -733,6 +733,7 @@ def f(v, i, n): r0 :: vec[i64] r1 :: None L0: + inc_ref v r0 = VecI64Api.remove(v, n) if is_error(r0) goto L2 (error at f:5) else goto L1 L1: diff --git a/mypyc/test-data/vecusage.test b/mypyc/test-data/vecusage.test index afb357258b046..0d14c049788d2 100644 --- a/mypyc/test-data/vecusage.test +++ b/mypyc/test-data/vecusage.test @@ -10,7 +10,7 @@ def f() -> None: False [case testVecUsedInFunction] -from vecs import vec +from librt.vecs import vec def f() -> None: vec[str]() @@ -18,7 +18,7 @@ def f() -> None: True [case testVecUsedInMethod] -from vecs import vec +from librt.vecs import vec class C: def m(self) -> None: @@ -27,14 +27,14 @@ class C: True [case testVecUsedAtTopLevel] -from vecs import vec +from librt.vecs import vec vec[str]() [out] True [case testVecUsedInArgTypeOnly] -from vecs import vec +from librt.vecs import vec def f(v: vec[str]) -> None: pass @@ -42,7 +42,7 @@ def f(v: vec[str]) -> None: True [case testVecUsedInReturnTypeOnly] -from vecs import vec +from librt.vecs import vec def f() -> vec[str]: assert False @@ -50,7 +50,7 @@ def f() -> vec[str]: True [case testVecUsedInAttributeTypeOnly] -from vecs import vec +from librt.vecs import vec class C: v: vec[str] @@ -58,7 +58,7 @@ class C: True [case testVecUsedInPropertyTypeOnly] -from vecs import vec +from librt.vecs import vec class C: @property From 242ab7cfcf0cc96c7a1a03c586adb26257902420 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 28 Jan 2026 12:16:58 +0000 Subject: [PATCH 129/180] Rename some test files --- mypyc/test-data/{run-vec-i64.test => run-vecs-i64.test} | 0 mypyc/test-data/{run-vec-misc.test => run-vecs-misc.test} | 2 +- .../{run-vec-nested.test => run-vecs-nested.test} | 0 mypyc/test-data/{run-vec-t.test => run-vecs-t.test} | 0 mypyc/test/test_run.py | 8 ++++---- 5 files changed, 5 insertions(+), 5 deletions(-) rename mypyc/test-data/{run-vec-i64.test => run-vecs-i64.test} (100%) rename mypyc/test-data/{run-vec-misc.test => run-vecs-misc.test} (99%) rename mypyc/test-data/{run-vec-nested.test => run-vecs-nested.test} (100%) rename mypyc/test-data/{run-vec-t.test => run-vecs-t.test} (100%) diff --git a/mypyc/test-data/run-vec-i64.test b/mypyc/test-data/run-vecs-i64.test similarity index 100% rename from mypyc/test-data/run-vec-i64.test rename to mypyc/test-data/run-vecs-i64.test diff --git a/mypyc/test-data/run-vec-misc.test b/mypyc/test-data/run-vecs-misc.test similarity index 99% rename from mypyc/test-data/run-vec-misc.test rename to mypyc/test-data/run-vecs-misc.test index 2e9cc9a4f5f9c..0ef4021c4a519 100644 --- a/mypyc/test-data/run-vec-misc.test +++ b/mypyc/test-data/run-vecs-misc.test @@ -2,7 +2,7 @@ -- vec[i32] and vec[float]. Since many of the code paths are the same as for -- vec[i64], only test a subset of functionality. -- --- vec[i64] test cases are in run-vec-i64.test. +-- vec[i64] test cases are in run-vecs-i64.test. [case testVecMiscBasicOps_librt_experimental] # mypy: allow-redefinition diff --git a/mypyc/test-data/run-vec-nested.test b/mypyc/test-data/run-vecs-nested.test similarity index 100% rename from mypyc/test-data/run-vec-nested.test rename to mypyc/test-data/run-vecs-nested.test diff --git a/mypyc/test-data/run-vec-t.test b/mypyc/test-data/run-vecs-t.test similarity index 100% rename from mypyc/test-data/run-vec-t.test rename to mypyc/test-data/run-vecs-t.test diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index ed68c52878f25..b33efed16d187 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -82,10 +82,10 @@ "run-vecs-misc-interp.test", "run-vecs-t-interp.test", "run-vecs-nested-interp.test", - "run-vec-i64.test", - "run-vec-misc.test", - "run-vec-t.test", - "run-vec-nested.test", + "run-vecs-i64.test", + "run-vecs-misc.test", + "run-vecs-t.test", + "run-vecs-nested.test", ] if sys.version_info >= (3, 12): From e892e72d8d08ef3e91b156ebcd9751d165e83f04 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 28 Jan 2026 17:54:15 +0000 Subject: [PATCH 130/180] Fix issues caused by rebase --- mypy/plugins/default.py | 1 - mypy/semanal.py | 4 ---- mypy/typeanal.py | 1 - 3 files changed, 6 deletions(-) diff --git a/mypy/plugins/default.py b/mypy/plugins/default.py index 71cd6fd14efa5..57909ce7c7309 100644 --- a/mypy/plugins/default.py +++ b/mypy/plugins/default.py @@ -227,7 +227,6 @@ def len_callback(ctx: FunctionContext) -> Type: return ctx.default_return_type - def typed_dict_get_signature_callback(ctx: MethodSigContext) -> CallableType: """Try to infer a better signature type for TypedDict.get. diff --git a/mypy/semanal.py b/mypy/semanal.py index 5f2bc7a133865..f38a71cb16e30 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -251,7 +251,6 @@ fix_instance, has_any_from_unimported_type, type_constructors, - check_vec_type_args, validate_instance, ) from mypy.typeops import function_type, get_type_vars, try_getting_str_literals_from_type @@ -6179,9 +6178,6 @@ def analyze_type_application(self, expr: IndexExpr) -> None: expr.analyzed = TypeApplication(base, types) expr.analyzed.line = expr.line expr.analyzed.column = expr.column - n = self.lookup_type_node(base) - if n and n.fullname == "librt.vecs.vec": - check_vec_type_args(types, expr, self) if isinstance(base, RefExpr) and base.fullname == "librt.vecs.vec": # Apply restrictions specific to vec diff --git a/mypy/typeanal.py b/mypy/typeanal.py index cc4aa5a9ec9df..5190eeb2df2c5 100644 --- a/mypy/typeanal.py +++ b/mypy/typeanal.py @@ -883,7 +883,6 @@ def analyze_type_with_type_info( ): return AnyType(TypeOfAny.from_error) - # Check type argument count. instance.args = tuple(flatten_nested_tuples(instance.args)) if not (self.defining_alias and self.nesting_level == 0) and not validate_instance( From f889502a844302332213107a7680378f1afc7cb5 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 28 Jan 2026 17:56:03 +0000 Subject: [PATCH 131/180] Revert test fixture change --- test-data/unit/fixtures/dict.pyi | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/test-data/unit/fixtures/dict.pyi b/test-data/unit/fixtures/dict.pyi index 012d6c7eb2842..ed2287511161b 100644 --- a/test-data/unit/fixtures/dict.pyi +++ b/test-data/unit/fixtures/dict.pyi @@ -7,7 +7,7 @@ from _typeshed import SupportsKeysAndGetItem import _typeshed from typing import ( TypeVar, Generic, Iterable, Iterator, Mapping, Tuple, overload, Optional, Union, Sequence, - Self, Protocol + Self, ) T = TypeVar('T') @@ -62,8 +62,3 @@ class BaseException: pass def isinstance(x: object, t: Union[type, Tuple[type, ...]]) -> bool: pass def iter(__iterable: Iterable[T]) -> Iterator[T]: pass - -class Sized(Protocol): - def __len__(self) -> int: ... - -def len(s: Sized) -> int: ... From e730c3267876c7754ac4d6cf64ec0b761ae36eaa Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 29 Jan 2026 11:22:46 +0000 Subject: [PATCH 132/180] Lint --- mypyc/analysis/vecusage.py | 4 +-- mypyc/codegen/emit.py | 4 +-- mypyc/codegen/emitfunc.py | 12 +------ mypyc/codegen/emitmodule.py | 6 ++-- mypyc/ir/ops.py | 1 - mypyc/ir/rtypes.py | 10 ++---- mypyc/irbuild/builder.py | 1 - mypyc/irbuild/expression.py | 13 ++++---- mypyc/irbuild/for_helpers.py | 18 ++++++---- mypyc/irbuild/ll_builder.py | 12 +++---- mypyc/irbuild/specialize.py | 17 ++++------ mypyc/irbuild/vec.py | 64 +++++++++++++++--------------------- mypyc/test/test_emitfunc.py | 24 ++++++++------ mypyc/test/test_vecusage.py | 2 +- 14 files changed, 82 insertions(+), 106 deletions(-) diff --git a/mypyc/analysis/vecusage.py b/mypyc/analysis/vecusage.py index 6950149aab32a..753faade84de7 100644 --- a/mypyc/analysis/vecusage.py +++ b/mypyc/analysis/vecusage.py @@ -1,8 +1,8 @@ """Analysis to decide whether a module needs the vec capsule.""" -from mypyc.ir.module_ir import ModuleIR from mypyc.ir.func_ir import FuncIR -from mypyc.ir.rtypes import RType, RVec, RUnion, RTuple, RStruct +from mypyc.ir.module_ir import ModuleIR +from mypyc.ir.rtypes import RStruct, RTuple, RType, RUnion, RVec def needs_vec_capsule(module: ModuleIR) -> bool: diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index 6c067904dc13b..f75f8d8a6bde6 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -65,7 +65,6 @@ is_uint8_rprimitive, object_rprimitive, optional_value_type, - vec_depth, vec_api_by_item_type, vec_item_type_tags, ) @@ -1106,8 +1105,7 @@ def emit_unbox( if depth == 0: self.emit_line(f"{dest} = VecTApi.unbox({src}, {type_value});") else: - self.emit_line( - f"{dest} = VecNestedApi.unbox({src}, {type_value}, {depth});") + self.emit_line(f"{dest} = VecNestedApi.unbox({src}, {type_value}, {depth});") self.emit_line(f"if (VEC_IS_ERROR({dest})) {{") self.emit_line(failure) diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index 5939782b81288..c1202d1c928ca 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py @@ -12,22 +12,12 @@ TracebackAndGotoHandler, c_array_initializer, ) -from mypyc.common import ( - GENERATOR_ATTRIBUTE_PREFIX, - HAVE_IMMORTAL, - MODULE_PREFIX, - NATIVE_PREFIX, - REG_PREFIX, - STATIC_PREFIX, - TYPE_PREFIX, - TYPE_VAR_PREFIX, -) +from mypyc.common import GENERATOR_ATTRIBUTE_PREFIX, HAVE_IMMORTAL, NATIVE_PREFIX, REG_PREFIX from mypyc.ir.class_ir import ClassIR from mypyc.ir.func_ir import FUNC_CLASSMETHOD, FUNC_STATICMETHOD, FuncDecl, FuncIR, all_values from mypyc.ir.ops import ( ERR_FALSE, NAMESPACE_TYPE, - NAMESPACE_TYPE_VAR, Assign, AssignMulti, BasicBlock, diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index d9dcd040f741d..b11d7ce546689 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -28,6 +28,7 @@ from mypy.plugin import Plugin, ReportConfigContext from mypy.util import hash_digest, json_dumps from mypyc.analysis.capsule_deps import find_implicit_op_dependencies +from mypyc.analysis.vecusage import needs_vec_capsule from mypyc.codegen.cstring import c_string_initializer from mypyc.codegen.emit import ( Emitter, @@ -60,7 +61,7 @@ from mypyc.ir.func_ir import FuncIR from mypyc.ir.module_ir import ModuleIR, ModuleIRs, deserialize_modules from mypyc.ir.ops import DeserMaps, LoadLiteral -from mypyc.ir.rtypes import RType, vec_c_types, vec_api_fields, vec_api_by_item_type +from mypyc.ir.rtypes import RType, vec_api_by_item_type, vec_api_fields, vec_c_types from mypyc.irbuild.main import build_ir from mypyc.irbuild.mapper import Mapper from mypyc.irbuild.prepare import load_type_map @@ -74,7 +75,6 @@ from mypyc.transform.refcount import insert_ref_count_opcodes from mypyc.transform.spill import insert_spills from mypyc.transform.uninit import insert_uninit_checks -from mypyc.analysis.vecusage import needs_vec_capsule # All the modules being compiled are divided into "groups". A group # is a set of modules that are placed into the same shared library. @@ -978,7 +978,7 @@ def generate_globals_init(self, emitter: Emitter) -> None: if self.use_vec_capsule: emitter.emit_lines( 'PyObject *mod = PyImport_ImportModule("librt.vecs");', - 'if (mod == NULL) return -1;', + "if (mod == NULL) return -1;", 'VecApi = PyCapsule_Import("librt.vecs._C_API", 0);', "if (!VecApi) return -1;", "VecTApi = *VecApi->t;", diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py index d64d9add5905b..cd1831a1aa18b 100644 --- a/mypyc/ir/ops.py +++ b/mypyc/ir/ops.py @@ -269,7 +269,6 @@ def __init__(self, rtype: RType) -> None: self.type = rtype - class Op(Value): """Abstract base class for all IR operations. diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 2393c726de401..d8d2420d06b4c 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -1097,7 +1097,7 @@ def field_type(self, name: str) -> RType: return object_rprimitive assert False, f"RVec has no field '{name}'" - def accept(self, visitor: "RTypeVisitor[T]") -> T: + def accept(self, visitor: RTypeVisitor[T]) -> T: return visitor.visit_rvec(self) def __str__(self) -> str: @@ -1365,9 +1365,7 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: ) VecU8BufObject = RStruct( - name="VecU8BufObject", - names=["ob_base", "len", "items"], - types=[PyVarObject, int64_rprimitive], + name="VecU8BufObject", names=["ob_base", "len", "items"], types=[PyVarObject, int64_rprimitive] ) VecFloatBufObject = RStruct( @@ -1438,9 +1436,7 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: ) VecNestedPopResult = RStruct( - name="VecNestedPopResult", - names=["vec", "item"], - types=[VecNested, VecNestedBufItem], + name="VecNestedPopResult", names=["vec", "item"], types=[VecNested, VecNestedBufItem] ) diff --git a/mypyc/irbuild/builder.py b/mypyc/irbuild/builder.py index ef127a1f3c2ee..e76e6a30cd4d0 100644 --- a/mypyc/irbuild/builder.py +++ b/mypyc/irbuild/builder.py @@ -96,7 +96,6 @@ bitmap_rprimitive, bool_rprimitive, bytes_rprimitive, - c_int_rprimitive, c_pyssize_t_rprimitive, dict_rprimitive, int_rprimitive, diff --git a/mypyc/irbuild/expression.py b/mypyc/irbuild/expression.py index 327f549119795..1e7ece6eeacf1 100644 --- a/mypyc/irbuild/expression.py +++ b/mypyc/irbuild/expression.py @@ -80,7 +80,6 @@ RTuple, RVec, bool_rprimitive, - c_pyssize_t_rprimitive, int_rprimitive, is_any_int, is_fixed_width_rtype, @@ -117,13 +116,11 @@ translate_object_setattr, ) from mypyc.irbuild.vec import ( + vec_append, vec_create, vec_create_from_values, vec_create_initialized, - vec_pop, - vec_remove, vec_slice, - vec_append, ) from mypyc.primitives.bytes_ops import bytes_slice_op from mypyc.primitives.dict_ops import dict_get_item_op, dict_new_op, exact_dict_set_item_op @@ -586,8 +583,9 @@ def translate_vec_create_from_iterable( return vec_from_iterable(builder, vec_type, arg, line) -def vec_from_iterable(builder: IRBuilder, vec_type: RVec, iterable: Expression, - line: int) -> Value: +def vec_from_iterable( + builder: IRBuilder, vec_type: RVec, iterable: Expression, line: int +) -> Value: """Construct a vec from an arbitrary iterable.""" # Translate it as a vec comprehension vec[t]([ for in # iterable]). This way we can use various special casing supported @@ -601,7 +599,8 @@ def vec_from_iterable(builder: IRBuilder, vec_type: RVec, iterable: Expression, index.kind = LDEF index.node = var loop_params: list[tuple[Expression, Expression, list[Expression], bool]] = [ - (index, iterable, [], False)] + (index, iterable, [], False) + ] def gen_inner_stmts() -> None: builder.assign(vec, vec_append(builder.builder, vec, reg, line), line) diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index 84c34a7ebc8ae..030590cb3c474 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -174,7 +174,10 @@ def for_loop_helper_with_index( body_insts: a function that generates the body of the loop. It needs a index as parameter. """ - assert is_sequence_rprimitive(expr_reg.type) or isinstance(expr_reg.type, RVec), (expr_reg, expr_reg.type) + assert is_sequence_rprimitive(expr_reg.type) or isinstance(expr_reg.type, RVec), ( + expr_reg, + expr_reg.type, + ) target_type = builder.get_sequence_type(expr) body_block = BasicBlock() @@ -213,7 +216,7 @@ def sequence_from_generator_preallocate_helper( builder: IRBuilder, gen: GeneratorExpr, empty_op_llbuilder: Callable[[Value, int], Value], - set_item_op: Callable[[Value, Value, Value, int], None], # CFunctionDescription, + set_item_op: Callable[[Value, Value, Value, int], None], # CFunctionDescription, ) -> Value | None: """Generate a new tuple or list from a simple generator expression. @@ -241,7 +244,7 @@ def sequence_from_generator_preallocate_helper( line = gen.line sequence_expr = gen.sequences[0] rtype = builder.node_type(sequence_expr) - if not (is_sequence_rprimitive(rtype) or isinstance(rtype, RTuple) or isinstance(rtype, RVec)): + if not (is_sequence_rprimitive(rtype) or isinstance(rtype, (RTuple, RVec))): return None if isinstance(rtype, RTuple): @@ -368,8 +371,9 @@ def set_item(x: Value, y: Value, z: Value, line: int) -> None: val = sequence_from_generator_preallocate_helper( builder, gen, - empty_op_llbuilder=lambda length, line: vec_create(builder.builder, - vec_type, length, line), + empty_op_llbuilder=lambda length, line: vec_create( + builder.builder, vec_type, length, line + ), set_item_op=set_item, ) if val is not None: @@ -881,7 +885,9 @@ def init( self, expr_reg: Value, target_type: RType, reverse: bool, length: Value | None = None ) -> None: assert is_sequence_rprimitive(expr_reg.type) or isinstance(expr_reg.type, RVec), ( - expr_reg, expr_reg.type) + expr_reg, + expr_reg.type, + ) builder = self.builder # Record a Value indicating the length of the sequence, if known at compile time. self.length = length diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 2b43a8e92a524..208ff763dfeec 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -2007,7 +2007,6 @@ def get_item( base, "__getitem__", [item], result_type, line, can_borrow=can_borrow ) - def make_dict(self, key_value_pairs: Sequence[DictEntry], line: int) -> Value: result: Value | None = None keys: list[Value] = [] @@ -2734,7 +2733,7 @@ def set_immortal_if_free_threaded(self, v: Value, line: int) -> None: # Loop helpers - def begin_for(self, start: Value, end: Value, step: Value, *, signed: bool) -> "ForBuilder": + def begin_for(self, start: Value, end: Value, step: Value, *, signed: bool) -> ForBuilder: """Generate for loop over a pointer or native int range. Roughly corresponds to "for i in range(start, end, step): ....". Only @@ -2756,7 +2755,6 @@ def begin_for(self, start: Value, end: Value, step: Value, *, signed: bool) -> " b.begin() return b - # Internal helpers def decompose_union_helper( @@ -2840,7 +2838,8 @@ def translate_special_method_call( Return None if no translation found; otherwise return the target register. """ low_level_op = self._translate_special_low_level_method_call( - base_reg, name, args, result_type, line, can_borrow) + base_reg, name, args, result_type, line, can_borrow + ) if low_level_op is not None: return low_level_op primitive_ops_candidates = method_call_ops.get(name, []) @@ -2860,8 +2859,9 @@ def _translate_special_low_level_method_call( ) -> Value | None: if name == "__getitem__" and len(args) == 1: arg = args[0] - if isinstance(base_reg.type, RVec) and (is_int64_rprimitive(arg.type) or - is_tagged(arg.type)): + if isinstance(base_reg.type, RVec) and ( + is_int64_rprimitive(arg.type) or is_tagged(arg.type) + ): if is_tagged(arg.type): arg = self.coerce(arg, int64_rprimitive, line) return vec_get_item(self, base_reg, arg, line) diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index 4d64ff80e76b7..9b22d52c6e148 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -37,10 +37,8 @@ ) from mypy.types import AnyType, TypeOfAny from mypyc.ir.ops import ( - ERR_MAGIC, BasicBlock, Call, - CallC, Extend, Integer, PrimitiveDescription, @@ -56,7 +54,6 @@ RPrimitive, RTuple, RType, - RUnion, RVec, bool_rprimitive, bytes_rprimitive, @@ -86,8 +83,6 @@ str_rprimitive, string_writer_rprimitive, uint8_rprimitive, - is_float_rprimitive, - vec_depth, ) from mypyc.irbuild.builder import IRBuilder from mypyc.irbuild.constant_fold import constant_fold_expr @@ -152,7 +147,6 @@ str_range_check_op, ) from mypyc.primitives.tuple_ops import isinstance_tuple, new_tuple_set_item_op -from mypyc.rt_subtype import is_runtime_subtype # Specializers are attempted before compiling the arguments to the # function. Specializers can return None to indicate that they failed @@ -321,7 +315,9 @@ def translate_len(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value arg = expr.args[0] expr_rtype = builder.node_type(arg) # NOTE (?) I'm not sure if my handling of can_borrow is correct here - obj = builder.accept(arg, can_borrow=is_list_rprimitive(expr_rtype) or isinstance(expr_rtype, RVec)) + obj = builder.accept( + arg, can_borrow=is_list_rprimitive(expr_rtype) or isinstance(expr_rtype, RVec) + ) if is_sequence_rprimitive(expr_rtype) or isinstance(expr_rtype, RTuple): return get_expr_length_value(builder, arg, obj, expr.line, use_pyssize_t=False) else: @@ -380,6 +376,7 @@ def translate_list_from_generator_call( and expr.arg_kinds[0] == ARG_POS and isinstance(expr.args[0], GeneratorExpr) ): + def set_item(x: Value, y: Value, z: Value, line: int) -> None: builder.call_c(new_list_set_item_op, [x, y, z], line) @@ -408,6 +405,7 @@ def translate_tuple_from_generator_call( and expr.arg_kinds[0] == ARG_POS and isinstance(expr.args[0], GeneratorExpr) ): + def set_item(x: Value, y: Value, z: Value, line: int) -> None: builder.call_c(new_tuple_set_item_op, [x, y, z], line) @@ -1195,7 +1193,6 @@ def translate_ord(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value return None - def is_object(callee: RefExpr) -> bool: """Returns True for object. calls.""" return ( @@ -1508,13 +1505,12 @@ def translate_bytes_get_item( ) -@specialize_function("librt.vecs.append") +@specialize_function("librt.vecs.append") def translate_vec_append(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> Value | None: if len(expr.args) == 2 and expr.arg_kinds == [ARG_POS, ARG_POS]: vec_arg = expr.args[0] item_arg = expr.args[1] vec_type = builder.node_type(vec_arg) - item_type = builder.node_type(item_arg) if isinstance(vec_type, RVec): vec_value = builder.accept(vec_arg) arg_value = builder.accept(item_arg) @@ -1528,7 +1524,6 @@ def translate_vec_remove(builder: IRBuilder, expr: CallExpr, callee: RefExpr) -> vec_arg = expr.args[0] item_arg = expr.args[1] vec_type = builder.node_type(vec_arg) - item_type = builder.node_type(item_arg) if isinstance(vec_type, RVec): vec_value = builder.accept(vec_arg) arg_value = builder.accept(item_arg) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index d4be2a2ec1a61..146fde0ea519b 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -2,12 +2,10 @@ from __future__ import annotations -from typing import TYPE_CHECKING, List, Optional, Tuple, Union, cast -from typing_extensions import Final +from typing import TYPE_CHECKING, Final, cast from mypyc.common import PLATFORM_SIZE from mypyc.ir.ops import ( - ERR_FALSE, ERR_MAGIC, ERR_NEVER, Assign, @@ -20,17 +18,15 @@ GetElementPtr, Integer, IntOp, - KeepAlive, - LoadAddress, RaiseStandardError, Register, SetElement, - Undef, - Unreachable, - Value, TupleGet, TupleSet, Unborrow, + Undef, + Unreachable, + Value, ) from mypyc.ir.rtypes import ( RInstance, @@ -41,26 +37,20 @@ RVec, VecNestedBufItem, bool_rprimitive, - c_int_rprimitive, c_pyssize_t_rprimitive, c_size_t_rprimitive, - int64_rprimitive, int32_rprimitive, - int16_rprimitive, - uint8_rprimitive, - float_rprimitive, + int64_rprimitive, is_c_py_ssize_t_rprimitive, is_int32_rprimitive, is_int64_rprimitive, is_int_rprimitive, - is_none_rprimitive, is_short_int_rprimitive, - object_pointer_rprimitive, object_rprimitive, optional_value_type, pointer_rprimitive, - vec_depth, vec_api_by_item_type, + vec_depth, vec_item_type_tags, ) from mypyc.primitives.registry import builtin_names @@ -85,9 +75,7 @@ def as_platform_int(builder: LowLevelIRBuilder, v: Value, line: int) -> Value: return builder.coerce(v, c_pyssize_t_rprimitive, line) -def vec_create( - builder: LowLevelIRBuilder, vtype: RVec, length: Union[int, Value], line: int -) -> Value: +def vec_create(builder: LowLevelIRBuilder, vtype: RVec, length: int | Value, line: int) -> Value: if isinstance(length, int): length = Integer(length, c_pyssize_t_rprimitive) length = as_platform_int(builder, length, line) @@ -96,8 +84,13 @@ def vec_create( api_name = vec_api_by_item_type.get(item_type) if api_name is not None: call = CallC( - f"{api_name}.alloc", [length, length], vtype, False, False, error_kind=ERR_MAGIC, - line=line + f"{api_name}.alloc", + [length, length], + vtype, + False, + False, + error_kind=ERR_MAGIC, + line=line, ) return builder.add(call) @@ -129,12 +122,7 @@ def vec_create( else: call = CallC( "VecNestedApi.alloc", - [ - length, - length, - typeval, - Integer(depth, int32_rprimitive), - ], + [length, length, typeval, Integer(depth, int32_rprimitive)], vtype, False, False, @@ -147,7 +135,7 @@ def vec_create( def vec_create_initialized( - builder: LowLevelIRBuilder, vtype: RVec, length: Union[int, Value], init: Value, line: int + builder: LowLevelIRBuilder, vtype: RVec, length: int | Value, init: Value, line: int ) -> Value: """Create vec with items initialized to the given value.""" if isinstance(length, int): @@ -173,7 +161,7 @@ def vec_create_initialized( def vec_create_from_values( - builder: LowLevelIRBuilder, vtype: RVec, values: List[Value], line: int + builder: LowLevelIRBuilder, vtype: RVec, values: list[Value], line: int ) -> Value: vec = vec_create(builder, vtype, len(values), line) ptr = vec_items(builder, vec) @@ -200,7 +188,7 @@ def step_size(item_type: RType) -> int: def vec_item_type_info( builder: LowLevelIRBuilder, typ: RType, line: int -) -> Tuple[Optional[Value], bool, int]: +) -> tuple[Value | None, bool, int]: if isinstance(typ, RPrimitive) and typ.is_refcounted: typ, src = builtin_names[typ.name] return builder.load_address(src, typ), False, 0 @@ -254,7 +242,8 @@ def vec_item_ptr(builder: LowLevelIRBuilder, vecobj: Value, index: Value) -> Val def vec_check_and_adjust_index( - builder: LowLevelIRBuilder, lenv: Value, index: Value, line: int) -> Value: + builder: LowLevelIRBuilder, lenv: Value, index: Value, line: int +) -> Value: r = Register(int64_rprimitive) ok, ok2, ok3 = BasicBlock(), BasicBlock(), BasicBlock() fail, fail2 = BasicBlock(), BasicBlock() @@ -302,9 +291,7 @@ def vec_get_item( return result -def vec_get_item_unsafe( - builder: LowLevelIRBuilder, base: Value, index: Value, line: int -) -> Value: +def vec_get_item_unsafe(builder: LowLevelIRBuilder, base: Value, index: Value, line: int) -> Value: """Get vec item, assuming index is non-negative and within bounds.""" assert isinstance(base.type, RVec) index = as_platform_int(builder, index, line) @@ -365,8 +352,11 @@ def convert_from_t_ext_item(builder: LowLevelIRBuilder, item: Value, vec_type: R else: name = "VecTApi.convert_from_nested" - return builder.add(CallC( - name, [item], vec_type, steals=[True], is_borrowed=False, error_kind=ERR_NEVER, line=-1)) + return builder.add( + CallC( + name, [item], vec_type, steals=[True], is_borrowed=False, error_kind=ERR_NEVER, line=-1 + ) + ) def vec_item_type(builder: LowLevelIRBuilder, item_type: RType, line: int) -> Value: @@ -427,7 +417,7 @@ def vec_pop(builder: LowLevelIRBuilder, base: Value, index: Value, line: int) -> api_name = vec_api_by_item_type.get(item_type) if api_name is not None: name = f"{api_name}.pop" - elif vec_depth(vec_type) == 0 and not isinstance(item_type, RUnion): # TODO fix union + elif vec_depth(vec_type) == 0 and not isinstance(item_type, RUnion): # TODO fix union name = "VecTApi.pop" else: name = "VecNestedApi.pop" diff --git a/mypyc/test/test_emitfunc.py b/mypyc/test/test_emitfunc.py index 00e206bb73492..59ab4cb8a0e5d 100644 --- a/mypyc/test/test_emitfunc.py +++ b/mypyc/test/test_emitfunc.py @@ -374,10 +374,12 @@ def test_unbox_i64(self) -> None: def test_box_vec(self) -> None: self.assert_emit(Box(self.vi64), """cpy_r_r0 = VecI64Api.box(cpy_r_vi64);""") self.assert_emit(Box(self.vi32), """cpy_r_r0 = VecI32Api.box(cpy_r_vi32);""") - self.assert_emit(Box(self.vs), - """cpy_r_r0 = VecTApi.box(cpy_r_vs, (size_t)&PyUnicode_Type);""") - self.assert_emit(Box(self.vs_opt), - """cpy_r_r0 = VecTApi.box(cpy_r_vs, (size_t)&PyUnicode_Type | 1);""") + self.assert_emit( + Box(self.vs), """cpy_r_r0 = VecTApi.box(cpy_r_vs, (size_t)&PyUnicode_Type);""" + ) + self.assert_emit( + Box(self.vs_opt), """cpy_r_r0 = VecTApi.box(cpy_r_vs, (size_t)&PyUnicode_Type | 1);""" + ) self.assert_emit(Box(self.vvs), """cpy_r_r0 = VecNestedApi.box(cpy_r_vvs);""") def test_unbox_vec(self) -> None: @@ -387,7 +389,7 @@ def test_unbox_vec(self) -> None: if (VEC_IS_ERROR(cpy_r_r0)) { CPy_TypeError("vec[i64]", cpy_r_o); cpy_r_r0 = (VecI64) { -1, NULL }; } - """ + """, ) self.assert_emit( Unbox(self.o, RVec(int32_rprimitive), 55), @@ -395,7 +397,7 @@ def test_unbox_vec(self) -> None: if (VEC_IS_ERROR(cpy_r_r0)) { CPy_TypeError("vec[i32]", cpy_r_o); cpy_r_r0 = (VecI32) { -1, NULL }; } - """ + """, ) self.assert_emit( Unbox(self.o, RVec(str_rprimitive), 55), @@ -429,21 +431,24 @@ def test_unbox_vec_nested(self) -> None: if (VEC_IS_ERROR(cpy_r_r0)) { CPy_TypeError("vec[vec[str]]", cpy_r_o); cpy_r_r0 = (VecNested) { -1, NULL }; } - """) + """, + ) self.assert_emit( Unbox(self.o, RVec(RVec(RUnion([str_rprimitive, none_rprimitive]))), 55), """cpy_r_r0 = VecNestedApi.unbox(cpy_r_o, (size_t)&PyUnicode_Type | 1, 1); if (VEC_IS_ERROR(cpy_r_r0)) { CPy_TypeError("vec[vec[str | None]]", cpy_r_o); cpy_r_r0 = (VecNested) { -1, NULL }; } - """) + """, + ) self.assert_emit( Unbox(self.o, RVec(RVec(int64_rprimitive)), 55), """cpy_r_r0 = VecNestedApi.unbox(cpy_r_o, 2, 1); if (VEC_IS_ERROR(cpy_r_r0)) { CPy_TypeError("vec[vec[i64]]", cpy_r_o); cpy_r_r0 = (VecNested) { -1, NULL }; } - """) + """, + ) def test_list_append(self) -> None: self.assert_emit( @@ -774,7 +779,6 @@ def test_set_element(self) -> None: """cpy_r_r0 = (Foo) { cpy_r_st.b, cpy_r_i32, cpy_r_st.y };""", ) - def test_load_address(self) -> None: self.assert_emit( LoadAddress(object_rprimitive, "PyDict_Type"), diff --git a/mypyc/test/test_vecusage.py b/mypyc/test/test_vecusage.py index 676dab5919b06..77e5035a5ab34 100644 --- a/mypyc/test/test_vecusage.py +++ b/mypyc/test/test_vecusage.py @@ -7,6 +7,7 @@ from mypy.errors import CompileError from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase +from mypyc.analysis.vecusage import needs_vec_capsule from mypyc.test.testutil import ( ICODE_GEN_BUILTINS, MypycDataSuite, @@ -15,7 +16,6 @@ infer_ir_build_options_from_test_name, use_custom_builtins, ) -from mypyc.analysis.vecusage import needs_vec_capsule files = ["vecusage.test"] From 0859c403d8799e878081ae6a81a7f576de7df34e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 29 Jan 2026 11:49:39 +0000 Subject: [PATCH 133/180] Move to a more standard capsule mechanism --- mypyc/analysis/capsule_deps.py | 4 ++- mypyc/codegen/emitmodule.py | 34 +++++---------------- mypyc/ir/rtypes.py | 3 +- mypyc/lib-rt/vecs/librt_vecs.h | 40 ++++++++++++++++++++++--- mypyc/test-data/irbuild-dict.test | 1 - mypyc/test-data/irbuild-lists.test | 1 - mypyc/test-data/irbuild-vec-misc.test | 1 - mypyc/test-data/irbuild-vec-nested.test | 1 - mypyc/test-data/refcount.test | 1 - 9 files changed, 49 insertions(+), 37 deletions(-) diff --git a/mypyc/analysis/capsule_deps.py b/mypyc/analysis/capsule_deps.py index bcfa40047ce08..d584fdfa896df 100644 --- a/mypyc/analysis/capsule_deps.py +++ b/mypyc/analysis/capsule_deps.py @@ -3,7 +3,7 @@ from mypyc.ir.deps import Dependency from mypyc.ir.func_ir import FuncIR from mypyc.ir.ops import Assign, CallC, PrimitiveOp -from mypyc.ir.rtypes import RStruct, RTuple, RType, RUnion +from mypyc.ir.rtypes import RStruct, RTuple, RType, RUnion, RVec def find_implicit_op_dependencies(fn: FuncIR) -> set[Dependency] | None: @@ -64,4 +64,6 @@ def collect_type_deps(typ: RType, deps: set[Dependency] | None) -> set[Dependenc elif isinstance(typ, RStruct): for item in typ.types: deps = collect_type_deps(item, deps) + elif isinstance(typ, RVec): + deps = collect_type_deps(typ.item_type, deps) return deps diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index b11d7ce546689..b7e4e49ad80aa 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -28,7 +28,6 @@ from mypy.plugin import Plugin, ReportConfigContext from mypy.util import hash_digest, json_dumps from mypyc.analysis.capsule_deps import find_implicit_op_dependencies -from mypyc.analysis.vecusage import needs_vec_capsule from mypyc.codegen.cstring import c_string_initializer from mypyc.codegen.emit import ( Emitter, @@ -57,11 +56,11 @@ short_id_from_name, ) from mypyc.errors import Errors -from mypyc.ir.deps import LIBRT_BASE64, LIBRT_STRINGS, SourceDep +from mypyc.ir.deps import LIBRT_BASE64, LIBRT_STRINGS, LIBRT_VECS, SourceDep from mypyc.ir.func_ir import FuncIR from mypyc.ir.module_ir import ModuleIR, ModuleIRs, deserialize_modules from mypyc.ir.ops import DeserMaps, LoadLiteral -from mypyc.ir.rtypes import RType, vec_api_by_item_type, vec_api_fields, vec_c_types +from mypyc.ir.rtypes import RType from mypyc.irbuild.main import build_ir from mypyc.irbuild.mapper import Mapper from mypyc.irbuild.prepare import load_type_map @@ -550,7 +549,6 @@ def __init__( # Multi-phase init is needed to enable free-threading. In the future we'll # probably want to enable it always, but we'll wait until it's stable. self.multi_phase_init = IS_FREE_THREADED - self.use_vec_capsule = any(needs_vec_capsule(m) for m in self.modules.values()) @property def group_suffix(self) -> str: @@ -585,13 +583,6 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]: self.generate_literal_tables() - if self.use_vec_capsule: - self.declare_global("VecCapsule *", "VecApi") - for vec_type in sorted(vec_c_types.values()): - self.declare_global(f"{vec_type}API ", f"{vec_type}Api") - self.declare_global("VecTAPI ", "VecTApi") - self.declare_global("VecNestedAPI ", "VecNestedApi") - for module_name, module in self.modules.items(): if multi_file: emitter = Emitter(self.context, filepath=self.source_paths[module_name]) @@ -641,12 +632,12 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]: ext_declarations.emit_line("#include ") if any(LIBRT_STRINGS in mod.dependencies for mod in self.modules.values()): ext_declarations.emit_line("#include ") + if any(LIBRT_VECS in mod.dependencies for mod in self.modules.values()): + ext_declarations.emit_line('#include "vecs/librt_vecs.h"') # Include headers for conditional source files source_deps = collect_source_dependencies(self.modules) for source_dep in sorted(source_deps, key=lambda d: d.path): ext_declarations.emit_line(f'#include "{source_dep.get_header()}"') - if self.use_vec_capsule: - ext_declarations.emit_line('#include "vecs/librt_vecs.h"') declarations = Emitter(self.context) declarations.emit_line(f"#ifndef MYPYC_LIBRT_INTERNAL{self.group_suffix}_H") @@ -975,19 +966,6 @@ def generate_globals_init(self, emitter: Emitter) -> None: f"if (CPyStatics_Initialize(CPyStatics, {values}) < 0) {{", "return -1;", "}" ) - if self.use_vec_capsule: - emitter.emit_lines( - 'PyObject *mod = PyImport_ImportModule("librt.vecs");', - "if (mod == NULL) return -1;", - 'VecApi = PyCapsule_Import("librt.vecs._C_API", 0);', - "if (!VecApi) return -1;", - "VecTApi = *VecApi->t;", - "VecNestedApi = *VecApi->nested;", - ) - for item_type, api_name in vec_api_by_item_type.items(): - field_name = vec_api_fields[item_type] - emitter.emit_line(f"{api_name} = *VecApi->{field_name};") - emitter.emit_lines("is_initialized = 1;", "return 0;", "}") def generate_module_def(self, emitter: Emitter, module_name: str, module: ModuleIR) -> None: @@ -1126,6 +1104,10 @@ def emit_module_exec_func( emitter.emit_line("if (import_librt_strings() < 0) {") emitter.emit_line("return -1;") emitter.emit_line("}") + if LIBRT_VECS in module.dependencies: + emitter.emit_line("if (import_librt_vecs() < 0) {") + emitter.emit_line("return -1;") + emitter.emit_line("}") emitter.emit_line("PyObject* modname = NULL;") if self.multi_phase_init: emitter.emit_line(f"{module_static} = module;") diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index d8d2420d06b4c..4a7a913093df0 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -41,7 +41,7 @@ class to enable the new behavior. In rare cases, adding a new from typing import TYPE_CHECKING, ClassVar, Final, Generic, TypeGuard, TypeVar, Union, final from mypyc.common import HAVE_IMMORTAL, IS_32_BIT_PLATFORM, PLATFORM_SIZE, JsonDict, short_name -from mypyc.ir.deps import LIBRT_STRINGS, Dependency +from mypyc.ir.deps import LIBRT_STRINGS, LIBRT_VECS, Dependency from mypyc.namegen import NameGenerator if TYPE_CHECKING: @@ -1033,6 +1033,7 @@ def __init__(self, item_type: RType) -> None: self.name = "vec[%s]" % item_type self.item_type = item_type self.names = ["len", "buf"] + self.dependencies = (LIBRT_VECS,) if isinstance(item_type, RUnion): non_opt = optional_value_type(item_type) else: diff --git a/mypyc/lib-rt/vecs/librt_vecs.h b/mypyc/lib-rt/vecs/librt_vecs.h index 2f88dc51201b1..06cda24104bc2 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.h +++ b/mypyc/lib-rt/vecs/librt_vecs.h @@ -4,6 +4,10 @@ // Header for the implementation of librt.vecs, which defines the 'vec' type. // Refer to librt_vecs.c for more detailed information. +#define PY_SSIZE_T_CLEAN +#include +#include + #ifndef MYPYC_EXPERIMENTAL static int @@ -15,10 +19,6 @@ import_librt_vecs(void) #else // MYPYC_EXPERIMENTAL -#define PY_SSIZE_T_CLEAN -#include -#include - // Magic (native) integer return value on exception. Caller must also // use PyErr_Occurred() since this overlaps with valid integer values. #define MYPYC_INT_ERROR -113 @@ -835,6 +835,38 @@ int Vec_GenericRemove(Py_ssize_t *len, PyObject **items, PyObject *item); PyObject *Vec_GenericPopWrapper(Py_ssize_t *len, PyObject **items, PyObject *args); PyObject *Vec_GenericPop(Py_ssize_t *len, PyObject **items, Py_ssize_t index); +// Global API pointers initialized by import_librt_vecs() +static VecCapsule *VecApi; +static VecI64API VecI64Api; +static VecI32API VecI32Api; +static VecI16API VecI16Api; +static VecU8API VecU8Api; +static VecFloatAPI VecFloatApi; +static VecBoolAPI VecBoolApi; +static VecTAPI VecTApi; +static VecNestedAPI VecNestedApi; + +static int +import_librt_vecs(void) +{ + PyObject *mod = PyImport_ImportModule("librt.vecs"); + if (mod == NULL) + return -1; + Py_DECREF(mod); // we import just for the side effect of making the below work. + VecApi = PyCapsule_Import("librt.vecs._C_API", 0); + if (!VecApi) + return -1; + VecI64Api = *VecApi->i64; + VecI32Api = *VecApi->i32; + VecI16Api = *VecApi->i16; + VecU8Api = *VecApi->u8; + VecFloatApi = *VecApi->float_; + VecBoolApi = *VecApi->bool_; + VecTApi = *VecApi->t; + VecNestedApi = *VecApi->nested; + return 0; +} + #endif // MYPYC_EXPERIMENTAL #endif // VEC_H_INCL diff --git a/mypyc/test-data/irbuild-dict.test b/mypyc/test-data/irbuild-dict.test index 89e19710ac2e6..2e3cbeba60519 100644 --- a/mypyc/test-data/irbuild-dict.test +++ b/mypyc/test-data/irbuild-dict.test @@ -565,4 +565,3 @@ L2: L3: r7 = box(None, 1) return r7 - diff --git a/mypyc/test-data/irbuild-lists.test b/mypyc/test-data/irbuild-lists.test index 9a07749f780d1..073d0c99de04f 100644 --- a/mypyc/test-data/irbuild-lists.test +++ b/mypyc/test-data/irbuild-lists.test @@ -488,4 +488,3 @@ L3: goto L1 L4: return 1 - diff --git a/mypyc/test-data/irbuild-vec-misc.test b/mypyc/test-data/irbuild-vec-misc.test index 47f1403065da2..b50411da7f5b1 100644 --- a/mypyc/test-data/irbuild-vec-misc.test +++ b/mypyc/test-data/irbuild-vec-misc.test @@ -344,4 +344,3 @@ L0: r10 = unborrow r8 x = r10 return x - diff --git a/mypyc/test-data/irbuild-vec-nested.test b/mypyc/test-data/irbuild-vec-nested.test index aa6f79485e97d..4d55d84f25749 100644 --- a/mypyc/test-data/irbuild-vec-nested.test +++ b/mypyc/test-data/irbuild-vec-nested.test @@ -309,4 +309,3 @@ L5: r10 = load_mem r9 :: vec[vec[str]]* keep_alive v return r10 - diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index 7c4f238a74b42..d0b08a1421ddc 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -1931,4 +1931,3 @@ L0: r10 = unborrow r8 x = r10 return x - From 2c9ddbc7eb6f93c9a60b192cf0e1636130d05525 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 29 Jan 2026 11:51:00 +0000 Subject: [PATCH 134/180] Delete unused vecusage module and tests --- mypyc/analysis/vecusage.py | 43 ---------------------- mypyc/test-data/vecusage.test | 68 ----------------------------------- mypyc/test/test_vecusage.py | 40 --------------------- 3 files changed, 151 deletions(-) delete mode 100644 mypyc/analysis/vecusage.py delete mode 100644 mypyc/test-data/vecusage.test delete mode 100644 mypyc/test/test_vecusage.py diff --git a/mypyc/analysis/vecusage.py b/mypyc/analysis/vecusage.py deleted file mode 100644 index 753faade84de7..0000000000000 --- a/mypyc/analysis/vecusage.py +++ /dev/null @@ -1,43 +0,0 @@ -"""Analysis to decide whether a module needs the vec capsule.""" - -from mypyc.ir.func_ir import FuncIR -from mypyc.ir.module_ir import ModuleIR -from mypyc.ir.rtypes import RStruct, RTuple, RType, RUnion, RVec - - -def needs_vec_capsule(module: ModuleIR) -> bool: - for f in module.functions: - if func_needs_vec(f): - return True - for cl in module.classes: - for base in cl.mro: - for f in base.methods.values(): - if func_needs_vec(f): - return True - for t in base.attributes.values(): - if uses_vec_type(t): - return True - return False - - -def func_needs_vec(func: FuncIR) -> bool: - for arg in func.arg_regs: - if uses_vec_type(arg.type): - return True - if uses_vec_type(func.decl.sig.ret_type): - return True - for block in func.blocks: - for op in block.ops: - if uses_vec_type(op.type) or any(uses_vec_type(s.type) for s in op.sources()): - return True - return False - - -def uses_vec_type(typ: RType) -> bool: - if isinstance(typ, RVec): - return True - if isinstance(typ, RUnion) and any(uses_vec_type(t) for t in typ.items): - return True - if isinstance(typ, (RTuple, RStruct)) and any(uses_vec_type(t) for t in typ.types): - return True - return False diff --git a/mypyc/test-data/vecusage.test b/mypyc/test-data/vecusage.test deleted file mode 100644 index 0d14c049788d2..0000000000000 --- a/mypyc/test-data/vecusage.test +++ /dev/null @@ -1,68 +0,0 @@ -[case testNoVecNeeded] -class vec: pass -class RVec: pass -def append(v: vec) -> None: - pass - -def f() -> None: - append(vec()) -[out] -False - -[case testVecUsedInFunction] -from librt.vecs import vec - -def f() -> None: - vec[str]() -[out] -True - -[case testVecUsedInMethod] -from librt.vecs import vec - -class C: - def m(self) -> None: - vec[str]() -[out] -True - -[case testVecUsedAtTopLevel] -from librt.vecs import vec - -vec[str]() -[out] -True - -[case testVecUsedInArgTypeOnly] -from librt.vecs import vec - -def f(v: vec[str]) -> None: - pass -[out] -True - -[case testVecUsedInReturnTypeOnly] -from librt.vecs import vec - -def f() -> vec[str]: - assert False -[out] -True - -[case testVecUsedInAttributeTypeOnly] -from librt.vecs import vec - -class C: - v: vec[str] -[out] -True - -[case testVecUsedInPropertyTypeOnly] -from librt.vecs import vec - -class C: - @property - def f(self) -> vec[str]: - assert False -[out] -True diff --git a/mypyc/test/test_vecusage.py b/mypyc/test/test_vecusage.py deleted file mode 100644 index 77e5035a5ab34..0000000000000 --- a/mypyc/test/test_vecusage.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Test cases for analysing if vec capsule is needed.""" - -from __future__ import annotations - -import os.path - -from mypy.errors import CompileError -from mypy.test.config import test_temp_dir -from mypy.test.data import DataDrivenTestCase -from mypyc.analysis.vecusage import needs_vec_capsule -from mypyc.test.testutil import ( - ICODE_GEN_BUILTINS, - MypycDataSuite, - assert_test_output, - build_ir_for_single_file2, - infer_ir_build_options_from_test_name, - use_custom_builtins, -) - -files = ["vecusage.test"] - - -class TestVecUsage(MypycDataSuite): - files = files - base_path = test_temp_dir - - def run_case(self, testcase: DataDrivenTestCase) -> None: - options = infer_ir_build_options_from_test_name(testcase.name) - if options is None: - # Skipped test case - return - with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase): - try: - ir, _, _, _ = build_ir_for_single_file2(testcase.input, options) - except CompileError as e: - actual = e.messages - else: - actual = [str(needs_vec_capsule(ir))] - - assert_test_output(testcase, actual, "Invalid test output", testcase.output) From 536591b8b6818788a057c071f2d580de1d7ec696 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 29 Jan 2026 12:03:41 +0000 Subject: [PATCH 135/180] Fix attribute type capsule deps --- mypyc/analysis/capsule_deps.py | 10 ++++++++++ mypyc/codegen/emitmodule.py | 8 +++++++- mypyc/test-data/capsule-deps.test | 32 +++++++++++++++++++++++++++++++ mypyc/test/test_capsule_deps.py | 16 +++++++++------- 4 files changed, 58 insertions(+), 8 deletions(-) diff --git a/mypyc/analysis/capsule_deps.py b/mypyc/analysis/capsule_deps.py index d584fdfa896df..5b39b8bd2e101 100644 --- a/mypyc/analysis/capsule_deps.py +++ b/mypyc/analysis/capsule_deps.py @@ -1,5 +1,6 @@ from __future__ import annotations +from mypyc.ir.class_ir import ClassIR from mypyc.ir.deps import Dependency from mypyc.ir.func_ir import FuncIR from mypyc.ir.ops import Assign, CallC, PrimitiveOp @@ -48,6 +49,15 @@ def find_type_dependencies(fn: FuncIR, deps: set[Dependency] | None) -> set[Depe return deps +def find_class_dependencies(cl: ClassIR) -> set[Dependency] | None: + """Find dependencies from class attribute types.""" + deps: set[Dependency] | None = None + for base in cl.mro: + for attr_type in base.attributes.values(): + deps = collect_type_deps(attr_type, deps) + return deps + + def collect_type_deps(typ: RType, deps: set[Dependency] | None) -> set[Dependency] | None: """Collect dependencies from an RType, recursively checking compound types.""" if typ.dependencies is not None: diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index b7e4e49ad80aa..5589fb987aef6 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -27,7 +27,7 @@ from mypy.options import Options from mypy.plugin import Plugin, ReportConfigContext from mypy.util import hash_digest, json_dumps -from mypyc.analysis.capsule_deps import find_implicit_op_dependencies +from mypyc.analysis.capsule_deps import find_class_dependencies, find_implicit_op_dependencies from mypyc.codegen.cstring import c_string_initializer from mypyc.codegen.emit import ( Emitter, @@ -271,6 +271,12 @@ def compile_scc_to_ir( do_copy_propagation(fn, compiler_options) do_flag_elimination(fn, compiler_options) + # Calculate implicit dependencies from class attribute types + for cl in module.classes: + deps = find_class_dependencies(cl) + if deps is not None: + module.dependencies.update(deps) + return modules diff --git a/mypyc/test-data/capsule-deps.test b/mypyc/test-data/capsule-deps.test index 063be3d6261d2..6e70927538bfb 100644 --- a/mypyc/test-data/capsule-deps.test +++ b/mypyc/test-data/capsule-deps.test @@ -87,3 +87,35 @@ def f() -> bytes: [out] Capsule(name='librt.base64') Capsule(name='librt.strings') + +[case testVecCapsuleDepInFunction_experimental] +from librt.vecs import vec + +def f() -> None: + vec[str]() +[out] +Capsule(name='librt.vecs') + +[case testVecCapsuleDepInMethod_experimental] +from librt.vecs import vec + +class C: + def m(self) -> None: + vec[str]() +[out] +Capsule(name='librt.vecs') + +[case testVecCapsuleDepAtTopLevel_experimental] +from librt.vecs import vec + +vec[str]() +[out] +Capsule(name='librt.vecs') + +[case testVecCapsuleDepInAttributeType_experimental] +from librt.vecs import vec + +class C: + v: vec[str] +[out] +Capsule(name='librt.vecs') diff --git a/mypyc/test/test_capsule_deps.py b/mypyc/test/test_capsule_deps.py index 84ba221dc7664..e8b1c4a790571 100644 --- a/mypyc/test/test_capsule_deps.py +++ b/mypyc/test/test_capsule_deps.py @@ -7,14 +7,13 @@ from mypy.errors import CompileError from mypy.test.config import test_temp_dir from mypy.test.data import DataDrivenTestCase -from mypyc.analysis.capsule_deps import find_implicit_op_dependencies -from mypyc.common import TOP_LEVEL_NAME +from mypyc.analysis.capsule_deps import find_class_dependencies, find_implicit_op_dependencies from mypyc.options import CompilerOptions from mypyc.test.testutil import ( ICODE_GEN_BUILTINS, MypycDataSuite, assert_test_output, - build_ir_for_single_file, + build_ir_for_single_file2, infer_ir_build_options_from_test_name, use_custom_builtins, ) @@ -34,20 +33,23 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: return with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase): try: - ir = build_ir_for_single_file(testcase.input, options) + module_ir, _, _, _ = build_ir_for_single_file2(testcase.input, options) except CompileError as e: actual = e.messages else: all_deps: set[str] = set() - for fn in ir: - if fn.name == TOP_LEVEL_NAME and not testcase.name.endswith("_toplevel"): - continue + for fn in module_ir.functions: compiler_options = CompilerOptions() lower_ir(fn, compiler_options) deps = find_implicit_op_dependencies(fn) if deps: for dep in deps: all_deps.add(repr(dep)) + for cl in module_ir.classes: + deps = find_class_dependencies(cl) + if deps: + for dep in deps: + all_deps.add(repr(dep)) actual = sorted(all_deps) if all_deps else ["No deps"] assert_test_output(testcase, actual, "Invalid test output", testcase.output) From 9d217357ee7665bdae617f4215bde5b27b8c7437 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 29 Jan 2026 13:39:57 +0000 Subject: [PATCH 136/180] WIP fix 32-bit --- mypyc/irbuild/ll_builder.py | 2 +- mypyc/irbuild/vec.py | 13 ++++-- mypyc/test-data/irbuild-vec-i64.test | 70 +++++++++++++++++++++++++++- mypyc/test-data/refcount.test | 2 +- 4 files changed, 78 insertions(+), 9 deletions(-) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 208ff763dfeec..616f2be4f2936 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -128,7 +128,7 @@ optional_value_type, pointer_rprimitive, short_int_rprimitive, - str_rprimitive, + str_rprimitive, is_c_py_ssize_t_rprimitive, ) from mypyc.irbuild.util import concrete_arg_kind from mypyc.irbuild.vec import vec_contains, vec_get_item, vec_len diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 146fde0ea519b..9f5b7b18a4a00 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Final, cast -from mypyc.common import PLATFORM_SIZE +from mypyc.common import PLATFORM_SIZE, IS_32_BIT_PLATFORM from mypyc.ir.ops import ( ERR_MAGIC, ERR_NEVER, @@ -210,12 +210,15 @@ def vec_item_type_info( def vec_len(builder: LowLevelIRBuilder, val: Value) -> Value: - # TODO: what about 32-bit archs? - # TODO: merge vec_len and vec_len_native - return vec_len_native(builder, val) + """Return len() as i64.""" + len_val = vec_len_native(builder, val) + if IS_32_BIT_PLATFORM: + return builder.coerce(len_val, int64_rprimitive, -1) + return len_val def vec_len_native(builder: LowLevelIRBuilder, val: Value) -> Value: + """Return len() as platform integer type (32-bit/64-bit).""" return builder.get_element(val, "len") @@ -283,7 +286,7 @@ def vec_get_item( vtype = base.type # TODO: Support more item types # TODO: Support more index types - len_val = vec_len_native(builder, base) + len_val = vec_len(builder, base) index = vec_check_and_adjust_index(builder, len_val, index, line) item_addr = vec_item_ptr(builder, base, index) result = builder.load_mem(item_addr, vtype.item_type, borrow=can_borrow) diff --git a/mypyc/test-data/irbuild-vec-i64.test b/mypyc/test-data/irbuild-vec-i64.test index c7f7c683aab6f..5d9c53079a74e 100644 --- a/mypyc/test-data/irbuild-vec-i64.test +++ b/mypyc/test-data/irbuild-vec-i64.test @@ -14,7 +14,7 @@ L0: r0 = VecI64Api.alloc(0, 0) return r0 -[case testVecI64Len] +[case testVecI64Len_64bit] from librt.vecs import vec from mypy_extensions import i64 @@ -31,7 +31,25 @@ L0: l = r0 return l -[case testVecI64GetItem] +[case testVecI64Len_32bit] +from librt.vecs import vec +from mypy_extensions import i64 + +def f(v: vec[i64]) -> i64: + l = len(v) + return l +[out] +def f(v): + v :: vec[i64] + r0 :: native_int + r1, l :: i64 +L0: + r0 = v.len + r1 = extend signed r0: native_int to i64 + l = r1 + return l + +[case testVecI64GetItem_64bit] from librt.vecs import vec from mypy_extensions import i64 @@ -77,6 +95,54 @@ L5: keep_alive v return r10 +[case testVecI64GetItem_32bit] +from librt.vecs import vec +from mypy_extensions import i64 + +def f(v: vec[i64], i: i64) -> i64: + return v[i] +[out] +def f(v, i): + v :: vec[i64] + i :: i64 + r0 :: native_int + r1 :: i64 + r2 :: bit + r3 :: i64 + r4 :: bit + r5 :: bool + r6 :: i64 + r7 :: object + r8 :: ptr + r9 :: i64 + r10 :: ptr + r11 :: i64 +L0: + r0 = v.len + r1 = extend signed r0: native_int to i64 + r2 = i < r1 :: unsigned + if r2 goto L4 else goto L1 :: bool +L1: + r3 = i + r1 + r4 = r3 < r1 :: unsigned + if r4 goto L3 else goto L2 :: bool +L2: + r5 = raise IndexError + unreachable +L3: + r6 = r3 + goto L5 +L4: + r6 = i +L5: + r7 = v.buf + r8 = get_element_ptr r7 items :: VecI64BufObject + r9 = r6 * 8 + r10 = r8 + r9 + r11 = load_mem r10 :: i64* + keep_alive v + return r11 + [case testVecI64Append] from librt.vecs import vec, append from mypy_extensions import i64 diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index d0b08a1421ddc..bd0732f52e1e3 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -1721,7 +1721,7 @@ L5: r11 = load_mem r10 :: i64* return r11 -[case testVecI64LenBorrowVec] +[case testVecI64LenBorrowVec_64bit] from librt.vecs import vec from mypy_extensions import i64 From c5fd68c92d0046ec56427adc73bfbd935315eef2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 29 Jan 2026 15:16:34 +0000 Subject: [PATCH 137/180] WIP --- mypyc/irbuild/vec.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 9f5b7b18a4a00..a4136174fd4cb 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -309,9 +309,8 @@ def vec_set_item( builder: LowLevelIRBuilder, base: Value, index: Value, item: Value, line: int ) -> None: assert isinstance(base.type, RVec) - index = as_platform_int(builder, index, line) vtype = base.type - len_val = vec_len_native(builder, base) + len_val = vec_len(builder, base) index = vec_check_and_adjust_index(builder, len_val, index, line) item_addr = vec_item_ptr(builder, base, index) item_type = vtype.item_type From f3ee70bd023a2b5da4b8189ba879580e3d741ce1 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 29 Jan 2026 15:29:29 +0000 Subject: [PATCH 138/180] WIP --- mypyc/test-data/irbuild-vec-i64.test | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/mypyc/test-data/irbuild-vec-i64.test b/mypyc/test-data/irbuild-vec-i64.test index 5d9c53079a74e..9bac1228b2df9 100644 --- a/mypyc/test-data/irbuild-vec-i64.test +++ b/mypyc/test-data/irbuild-vec-i64.test @@ -158,7 +158,7 @@ L0: r0 = VecI64Api.append(v, i) return r0 -[case testVecI64SetItem] +[case testVecI64SetItem_64bit] from librt.vecs import vec from mypy_extensions import i64 @@ -203,6 +203,14 @@ L5: keep_alive v return 1 +[case testVecI64SetItem_32bit] +from librt.vecs import vec +from mypy_extensions import i64 + +def f(v: vec[i64], i: i64, x: i64) -> None: + v[i] = x +[out] + [case testVecI64ConstructFromListExpr] from librt.vecs import vec from mypy_extensions import i64 From a79ef032328b015c9e349a1a02bcc2096275f298 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 29 Jan 2026 18:28:47 +0000 Subject: [PATCH 139/180] ircheck: Allow pointer arithmetic even though it's unsigned + signed --- mypyc/analysis/ircheck.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/mypyc/analysis/ircheck.py b/mypyc/analysis/ircheck.py index 238d416729241..a473a167955fc 100644 --- a/mypyc/analysis/ircheck.py +++ b/mypyc/analysis/ircheck.py @@ -66,9 +66,13 @@ bytes_rprimitive, dict_rprimitive, int_rprimitive, + is_c_py_ssize_t_rprimitive, + is_fixed_width_rtype, is_float_rprimitive, is_object_rprimitive, + is_pointer_rprimitive, list_rprimitive, + pointer_rprimitive, range_rprimitive, set_rprimitive, str_rprimitive, @@ -211,6 +215,29 @@ def can_coerce_to(src: RType, dest: RType) -> bool: return True +def is_valid_ptr_displacement_type(rtype: RType) -> bool: + """Check if rtype is a valid displacement type for pointer arithmetic.""" + if not (is_fixed_width_rtype(rtype) or is_c_py_ssize_t_rprimitive(rtype)): + return False + assert isinstance(rtype, RPrimitive) + return rtype.size == pointer_rprimitive.size + + +def is_pointer_arithmetic(op: IntOp) -> bool: + """Check if op is add/subtract targeting pointer_rprimtive and integer of the same size.""" + if op.op not in (IntOp.ADD, IntOp.SUB): + return False + if not is_pointer_rprimitive(op.type): + return False + left = op.lhs.type + right = op.rhs.type + if is_pointer_rprimitive(left): + return is_valid_ptr_displacement_type(right) + if is_pointer_rprimitive(right): + return is_valid_ptr_displacement_type(left) + return False + + class OpChecker(OpVisitor[None]): def __init__(self, parent_fn: FuncIR) -> None: self.parent_fn = parent_fn @@ -417,6 +444,7 @@ def visit_int_op(self, op: IntOp) -> None: op_str in ("+", "-", "*", "/", "%") or (op_str not in ("<<", ">>") and left.size != right.size) ) + and not is_pointer_arithmetic(op) ): self.fail(op, f"Operand types have incompatible signs: {left}, {right}") From 06f31333573134f5a6198de4630441dcfde30b94 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 29 Jan 2026 18:29:19 +0000 Subject: [PATCH 140/180] Lint --- mypyc/irbuild/ll_builder.py | 2 +- mypyc/irbuild/vec.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 616f2be4f2936..208ff763dfeec 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -128,7 +128,7 @@ optional_value_type, pointer_rprimitive, short_int_rprimitive, - str_rprimitive, is_c_py_ssize_t_rprimitive, + str_rprimitive, ) from mypyc.irbuild.util import concrete_arg_kind from mypyc.irbuild.vec import vec_contains, vec_get_item, vec_len diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index a4136174fd4cb..38cde70b95587 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Final, cast -from mypyc.common import PLATFORM_SIZE, IS_32_BIT_PLATFORM +from mypyc.common import IS_32_BIT_PLATFORM, PLATFORM_SIZE from mypyc.ir.ops import ( ERR_MAGIC, ERR_NEVER, From 7c1fbc737a05688c19957d780ffb24ad75cdf142 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 29 Jan 2026 18:44:29 +0000 Subject: [PATCH 141/180] Add range check helper and int coercion to ssize_t --- mypyc/irbuild/ll_builder.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 208ff763dfeec..a9a57a4288557 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -682,6 +682,32 @@ def coerce_short_int_to_fixed_width(self, src: Value, target_type: RType, line: self.activate_block(end) return res + def coerce_i64_to_ssize_t(self, src: Value, line: int) -> Value: + """Coerce int64 to c_pyssize_t with range check on 32-bit platforms. + + On 64-bit platforms, this is a no-op since the types have the same size. + On 32-bit platforms, raises OverflowError if the value doesn't fit in 32 bits. + """ + if PLATFORM_SIZE == 8: + return src + + # 32-bit platform: need range check + overflow, end = BasicBlock(), BasicBlock() + + self.check_fixed_width_range(src, c_pyssize_t_rprimitive, overflow, line) + + # Value is in range - truncate to 32 bits + res = self.add(Truncate(src, c_pyssize_t_rprimitive, line)) + self.goto(end) + + # Handle overflow case + self.activate_block(overflow) + self.call_c(int32_overflow, [], line) + self.add(Unreachable()) + + self.activate_block(end) + return res + def coerce_fixed_width_to_int(self, src: Value, line: int) -> Value: if ( (is_int32_rprimitive(src.type) and PLATFORM_SIZE == 8) From d2a9420b46aa1958100eeca7115f83f8e1c398a0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 29 Jan 2026 18:44:48 +0000 Subject: [PATCH 142/180] Coerce set item index to ssize_t --- mypyc/irbuild/vec.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 38cde70b95587..ebab360f7101b 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -312,6 +312,7 @@ def vec_set_item( vtype = base.type len_val = vec_len(builder, base) index = vec_check_and_adjust_index(builder, len_val, index, line) + index = builder.coerce(index, c_pyssize_t_rprimitive, line) item_addr = vec_item_ptr(builder, base, index) item_type = vtype.item_type item = builder.coerce(item, item_type, line) From 4bd48857acda8192ef2670b84f60fb518172278f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 29 Jan 2026 18:46:05 +0000 Subject: [PATCH 143/180] Remove temporary test case --- mypyc/test-data/irbuild-vec-i64.test | 8 -------- 1 file changed, 8 deletions(-) diff --git a/mypyc/test-data/irbuild-vec-i64.test b/mypyc/test-data/irbuild-vec-i64.test index 9bac1228b2df9..0afcf43a5cadb 100644 --- a/mypyc/test-data/irbuild-vec-i64.test +++ b/mypyc/test-data/irbuild-vec-i64.test @@ -203,14 +203,6 @@ L5: keep_alive v return 1 -[case testVecI64SetItem_32bit] -from librt.vecs import vec -from mypy_extensions import i64 - -def f(v: vec[i64], i: i64, x: i64) -> None: - v[i] = x -[out] - [case testVecI64ConstructFromListExpr] from librt.vecs import vec from mypy_extensions import i64 From 6e5c3a496ab59bbee06c3db8dd0c242353a50ad5 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 29 Jan 2026 18:47:39 +0000 Subject: [PATCH 144/180] Also coerce on get item --- mypyc/irbuild/vec.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index ebab360f7101b..8d0da6b1165da 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -288,6 +288,7 @@ def vec_get_item( # TODO: Support more index types len_val = vec_len(builder, base) index = vec_check_and_adjust_index(builder, len_val, index, line) + index = builder.coerce(index, c_pyssize_t_rprimitive, line) item_addr = vec_item_ptr(builder, base, index) result = builder.load_mem(item_addr, vtype.item_type, borrow=can_borrow) builder.keep_alives.append(base) From fb1d828e668da5610492f85079da3a0c7755c418 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 30 Jan 2026 10:52:31 +0000 Subject: [PATCH 145/180] WIP --- mypyc/test-data/irbuild-vec-i64.test | 37 +++++++++++++++++++--------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/mypyc/test-data/irbuild-vec-i64.test b/mypyc/test-data/irbuild-vec-i64.test index 0afcf43a5cadb..a6f252582d2ec 100644 --- a/mypyc/test-data/irbuild-vec-i64.test +++ b/mypyc/test-data/irbuild-vec-i64.test @@ -96,6 +96,7 @@ L5: return r10 [case testVecI64GetItem_32bit] +# The IR is quite verbose, but it's acceptable since 32-bit targets are not common any more from librt.vecs import vec from mypy_extensions import i64 @@ -112,11 +113,13 @@ def f(v, i): r4 :: bit r5 :: bool r6 :: i64 - r7 :: object - r8 :: ptr - r9 :: i64 - r10 :: ptr - r11 :: i64 + r7, r8 :: bit + r9 :: native_int + r10 :: object + r11 :: ptr + r12 :: native_int + r13 :: ptr + r14 :: i64 L0: r0 = v.len r1 = extend signed r0: native_int to i64 @@ -135,13 +138,25 @@ L3: L4: r6 = i L5: - r7 = v.buf - r8 = get_element_ptr r7 items :: VecI64BufObject - r9 = r6 * 8 - r10 = r8 + r9 - r11 = load_mem r10 :: i64* + r7 = r6 < 2147483648 :: signed + if r7 goto L6 else goto L8 :: bool +L6: + r8 = r6 >= -2147483648 :: signed + if r8 goto L7 else goto L8 :: bool +L7: + r9 = truncate r6: i64 to native_int + goto L9 +L8: + CPyInt32_Overflow() + unreachable +L9: + r10 = v.buf + r11 = get_element_ptr r10 items :: VecI64BufObject + r12 = r9 * 8 + r13 = r11 + r12 + r14 = load_mem r13 :: i64* keep_alive v - return r11 + return r14 [case testVecI64Append] from librt.vecs import vec, append From 025fc695f5da0f4e6ae46f1378b5ea6b6a26ebc1 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 30 Jan 2026 11:15:30 +0000 Subject: [PATCH 146/180] WIP --- mypyc/test-data/irbuild-vec-i64.test | 10 +++++----- mypyc/test-data/refcount.test | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mypyc/test-data/irbuild-vec-i64.test b/mypyc/test-data/irbuild-vec-i64.test index a6f252582d2ec..53a8077b3446a 100644 --- a/mypyc/test-data/irbuild-vec-i64.test +++ b/mypyc/test-data/irbuild-vec-i64.test @@ -242,7 +242,7 @@ L0: keep_alive r0 return r0 -[case testVecI64ConstructFromListMultiply] +[case testVecI64ConstructFromListMultiply_64bit] from librt.vecs import vec from mypy_extensions import i64 @@ -578,7 +578,7 @@ L5: L6: return r10 -[case testVecI64GetItemWithInt] +[case testVecI64GetItemWithInt_64bit] from librt.vecs import vec from mypy_extensions import i64 @@ -623,7 +623,7 @@ L5: keep_alive v return r10 -[case testVecI64Slicing] +[case testVecI64Slicing_64bit] from librt.vecs import vec from mypy_extensions import i64 @@ -682,7 +682,7 @@ L0: r0 = VecI64Api.pop(v, -1) return r0 -[case testVecI64PopNth] +[case testVecI64PopNth_64bit] from typing import Tuple from librt.vecs import vec, pop from mypy_extensions import i64 @@ -698,7 +698,7 @@ L0: r0 = VecI64Api.pop(v, n) return r0 -[case testVecI64InPlaceOp] +[case testVecI64InPlaceOp_64bit] from librt.vecs import vec, remove from mypy_extensions import i64 diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index bd0732f52e1e3..a3f09d6d8e41a 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -1671,7 +1671,7 @@ L0: r8 = r7 + 8 return r4 -[case testVecI64GetItemBorrowVec] +[case testVecI64GetItemBorrowVec_64bit] from librt.vecs import vec from mypy_extensions import i64 From 830e510b862d7681f2744d4259d8322f07627f22 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 30 Jan 2026 11:44:16 +0000 Subject: [PATCH 147/180] WIP --- mypyc/irbuild/ll_builder.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index a9a57a4288557..2f88aaf00a459 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -131,7 +131,7 @@ str_rprimitive, ) from mypyc.irbuild.util import concrete_arg_kind -from mypyc.irbuild.vec import vec_contains, vec_get_item, vec_len +from mypyc.irbuild.vec import vec_contains, vec_get_item, vec_len, vec_len_native from mypyc.options import CompilerOptions from mypyc.primitives.bytes_ops import bytes_compare from mypyc.primitives.dict_ops import ( @@ -2712,10 +2712,10 @@ def builtin_len(self, val: Value, line: int, use_pyssize_t: bool = False) -> Val return length elif isinstance(typ, RVec): - len_value = vec_len(self, val) + len_value = vec_len_native(self, val) if use_pyssize_t: return len_value - count = Integer(1, int64_rprimitive, line) + count = Integer(1, c_pyssize_t_rprimitive, line) return self.int_op(short_int_rprimitive, len_value, count, IntOp.LEFT_SHIFT, line) op = self.matching_primitive_op(function_ops["builtins.len"], [val], line) From e18998559c3a464e290a5c522f58be087332bf3a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 30 Jan 2026 11:45:39 +0000 Subject: [PATCH 148/180] WIP --- mypyc/test-data/irbuild-vec-t.test | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mypyc/test-data/irbuild-vec-t.test b/mypyc/test-data/irbuild-vec-t.test index 2670c15d55686..36132a5cac4cb 100644 --- a/mypyc/test-data/irbuild-vec-t.test +++ b/mypyc/test-data/irbuild-vec-t.test @@ -114,7 +114,7 @@ L0: r9 = VecTApi.append(v, r5, r8) return r9 -[case testVecTLen] +[case testVecTLen_64bit] from librt.vecs import vec from mypy_extensions import i64 from typing import Optional @@ -138,7 +138,7 @@ L0: r0 = v.len return r0 -[case testVecTGetItem] +[case testVecTGetItem_64bit] from librt.vecs import vec from mypy_extensions import i64 @@ -184,7 +184,7 @@ L5: keep_alive v return r10 -[case testVecTOptionalGetItem] +[case testVecTOptionalGetItem_64bit] from librt.vecs import vec from mypy_extensions import i64 from typing import Optional From 4c936632a0fb8e1b9d49e8cec6926b9c4a138760 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 30 Jan 2026 11:46:59 +0000 Subject: [PATCH 149/180] WIP --- mypyc/test-data/irbuild-vec-misc.test | 8 ++++---- mypyc/test-data/irbuild-vec-nested.test | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/mypyc/test-data/irbuild-vec-misc.test b/mypyc/test-data/irbuild-vec-misc.test index b50411da7f5b1..d48162e1e8300 100644 --- a/mypyc/test-data/irbuild-vec-misc.test +++ b/mypyc/test-data/irbuild-vec-misc.test @@ -73,7 +73,7 @@ L0: keep_alive r0 return r0 -[case testVecMiscLen] +[case testVecMiscLen_64bit] from librt.vecs import vec from mypy_extensions import i64, i32 @@ -110,7 +110,7 @@ L0: r0 = VecI32Api.append(v, 123) return r0 -[case testVecMiscGetItem] +[case testVecMiscGetItem_64bit] from librt.vecs import vec from mypy_extensions import i64 @@ -259,7 +259,7 @@ L3: L4: return s -[case testVecMiscNestedGetItem] +[case testVecMiscNestedGetItem_64bit] from librt.vecs import vec from mypy_extensions import i64, i32 @@ -306,7 +306,7 @@ L5: keep_alive v return r10 -[case testVecMiscNestedPop] +[case testVecMiscNestedPop_64bit] from librt.vecs import vec, pop from mypy_extensions import i64, i32 diff --git a/mypyc/test-data/irbuild-vec-nested.test b/mypyc/test-data/irbuild-vec-nested.test index 4d55d84f25749..6b3389394d93d 100644 --- a/mypyc/test-data/irbuild-vec-nested.test +++ b/mypyc/test-data/irbuild-vec-nested.test @@ -70,7 +70,7 @@ L0: keep_alive vv return r4 -[case testVecNestedLen] +[case testVecNestedLen_64bit] from librt.vecs import vec from mypy_extensions import i64 from typing import Optional @@ -94,7 +94,7 @@ L0: r0 = v.len return r0 -[case testVecNestedGetItem] +[case testVecNestedGetItem_64bit] from librt.vecs import vec from mypy_extensions import i64 @@ -140,7 +140,7 @@ L5: keep_alive v return r10 -[case testVecNestedI64GetItem] +[case testVecNestedI64GetItem_64bit] from librt.vecs import vec from mypy_extensions import i64 @@ -186,7 +186,7 @@ L5: keep_alive v return r10 -[case testVecNestedI64GetItemWithBorrow] +[case testVecNestedI64GetItemWithBorrow_64bit] from librt.vecs import vec from mypy_extensions import i64 @@ -264,7 +264,7 @@ L10: keep_alive v, r10 return r21 -[case testVecDoublyNestedGetItem] +[case testVecDoublyNestedGetItem_64bit] from librt.vecs import vec from mypy_extensions import i64 From 91d92f5ce3fbf2ef0ce84dc4f9c655fe01f0fef8 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 30 Jan 2026 11:58:21 +0000 Subject: [PATCH 150/180] WIP --- mypyc/test-data/refcount.test | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index a3f09d6d8e41a..230b9b2d7282b 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -1500,7 +1500,7 @@ L2: L3: return r4 -[case testVecTSetItem] +[case testVecTSetItem_64bit] from librt.vecs import vec from mypy_extensions import i64 @@ -1549,7 +1549,7 @@ L5: set_mem r9, x :: builtins.str* return 1 -[case testVecTConstructFromListMultiply] +[case testVecTConstructFromListMultiply_64bit] from typing import Optional from librt.vecs import vec from mypy_extensions import i64 @@ -1591,7 +1591,7 @@ L2: L3: return r4 -[case testVecTForLoop] +[case testVecTForLoop_64bit] from librt.vecs import vec from mypy_extensions import i64 @@ -1642,7 +1642,7 @@ def g(s): L0: return 1 -[case testVecTConstructFromListExpr] +[case testVecTConstructFromListExpr_64bit] from librt.vecs import vec def f() -> vec[str]: @@ -1763,7 +1763,7 @@ L0: r4 = VecNestedApi.append(vv, r3) return r4 -[case testVecTGetItem] +[case testVecTGetItem_64bit] from librt.vecs import vec from mypy_extensions import i64 @@ -1808,7 +1808,7 @@ L5: r10 = load_mem r9 :: builtins.str* return r10 -[case testVecNestedGetItem] +[case testVecNestedGetItem_64bit] from librt.vecs import vec from mypy_extensions import i64 From 5548217a734ca50284284e7e4c70146f7b1879ee Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 30 Jan 2026 12:03:21 +0000 Subject: [PATCH 151/180] WIP attempt at fixing 32-bit issue --- mypyc/irbuild/vec.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 8d0da6b1165da..9a5cb7ac559b3 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -248,6 +248,8 @@ def vec_check_and_adjust_index( builder: LowLevelIRBuilder, lenv: Value, index: Value, line: int ) -> Value: r = Register(int64_rprimitive) + index = builder.coerce(index, int64_rprimitive, line) + lenv = builder.coerce(lenv, int64_rprimitive, line) ok, ok2, ok3 = BasicBlock(), BasicBlock(), BasicBlock() fail, fail2 = BasicBlock(), BasicBlock() is_less = builder.comparison_op(index, lenv, ComparisonOp.ULT, line) From 4c26e03a851d058da5a988a3e7d2277ca9d3e355 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 30 Jan 2026 12:05:17 +0000 Subject: [PATCH 152/180] WIP Fix 32-bit --- mypyc/irbuild/vec.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 9a5cb7ac559b3..4b312267aeb6f 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -236,8 +236,7 @@ def vec_item_ptr(builder: LowLevelIRBuilder, vecobj: Value, index: Value) -> Val if isinstance(item_type, RPrimitive): item_size = item_type.size elif isinstance(item_type, RVec): - # TODO: Support 32-bit platforms - item_size = 16 + item_size = 2 * PLATFORM_SIZE else: item_size = object_rprimitive.size delta = builder.int_mul(index, item_size) From 81b2356a7da6ab526982714b98d01ad50d4a1250 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 30 Jan 2026 13:41:06 +0000 Subject: [PATCH 153/180] [mypyc] Support isinstance with librt.vecs.vec objects Use a common base class 'vec' for isinstance support. The base class is currently only used for isinstance, but we could add some other basic functionality there in the future. --- mypyc/lib-rt/vecs/vec_nested.c | 3 +++ mypyc/lib-rt/vecs/vec_t.c | 3 +++ mypyc/lib-rt/vecs/vec_template.c | 3 +++ 3 files changed, 9 insertions(+) diff --git a/mypyc/lib-rt/vecs/vec_nested.c b/mypyc/lib-rt/vecs/vec_nested.c index 0034d9e9bfff5..02616ce7a4d9b 100644 --- a/mypyc/lib-rt/vecs/vec_nested.c +++ b/mypyc/lib-rt/vecs/vec_nested.c @@ -11,6 +11,9 @@ #include "librt_vecs.h" #include "vecs_internal.h" +// Forward declaration of base vec type (defined in librt_vecs.c) +extern PyTypeObject VecType; + static inline VecNested vec_error() { VecNested v = { .len = -1 }; return v; diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 8a03ea1a2e644..ac20da14a832e 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -13,6 +13,9 @@ #include "librt_vecs.h" #include "vecs_internal.h" +// Forward declaration of base vec type (defined in librt_vecs.c) +extern PyTypeObject VecType; + static inline VecT vec_error() { VecT v = { .len = -1 }; return v; diff --git a/mypyc/lib-rt/vecs/vec_template.c b/mypyc/lib-rt/vecs/vec_template.c index dc1e9df3609fc..0091f8c07a757 100644 --- a/mypyc/lib-rt/vecs/vec_template.c +++ b/mypyc/lib-rt/vecs/vec_template.c @@ -365,6 +365,9 @@ PyTypeObject BUF_TYPE = { .tp_free = PyObject_Del, }; +// Forward declaration of base vec type (defined in librt_vecs.c) +extern PyTypeObject VecType; + PyTypeObject VEC_TYPE = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "vec[" ITEM_TYPE_STR "]", From d08017150ce41920a9e4effa9a01073ffd2e6f3a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 30 Jan 2026 14:03:08 +0000 Subject: [PATCH 154/180] Test unions with vec[i64] --- mypyc/test-data/run-vecs-i64.test | 67 ++++++++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/mypyc/test-data/run-vecs-i64.test b/mypyc/test-data/run-vecs-i64.test index 1cd0b2b352cbd..326561d25a648 100644 --- a/mypyc/test-data/run-vecs-i64.test +++ b/mypyc/test-data/run-vecs-i64.test @@ -4,7 +4,7 @@ [case testVecI64BasicOps_librt_experimental] from typing import Final, Any, Iterable, Tuple -from mypy_extensions import i64 +from mypy_extensions import i64, i32 from librt.vecs import vec, append, remove, pop from testutil import assertRaises @@ -163,6 +163,71 @@ def test_box_and_unbox() -> None: with assertRaises(TypeError, "vec[i64] object expected; got vec"): v2 = o3 +def check_opt(x: vec[i64] | None, is_none: bool) -> None: + if is_none: + assert x is None + else: + if int() + 1: + assert x is not None + assert len(x) == 1 + assert x[0] == 5 + if int() + 2: + assert isinstance(x, vec) + assert len(x) == 1 + assert x[0] == 5 + if isinstance(x, vec): + assert x[0] == 5 + assert not is_none + else: + assert is_none + +def test_vec_optional() -> None: + v = vec[i64]([5]) + check_opt(None, True) + check_opt(v, False) + a1: Any = None + check_opt(a1, True) + a2: Any = v + check_opt(a2, False) + a3: Any = str + with assertRaises(TypeError): + check_opt(a3, False) + +def check_union(x: vec[i64] | str, is_str: bool) -> None: + if is_str: + assert isinstance(x, str) + assert x == "ss" + else: + if int() + 1: + assert not isinstance(x, str) + assert len(x) == 1 + assert x[0] == 5 + if int() + 2: + assert isinstance(x, vec) + assert len(x) == 1 + assert x[0] == 5 + if isinstance(x, vec): + assert x[0] == 5 + assert not is_str + else: + assert is_str + assert x == "ss" + +def test_vec_union() -> None: + v = vec[i64]([5]) + check_union("ss", True) + check_union(v, False) + a1: Any = "ss" + check_union(a1, True) + a2: Any = v + check_union(a2, False) + a3: Any = b"ss" + with assertRaises(TypeError): + check_union(a3, False) + a4: Any = vec[i32]() + with assertRaises(TypeError, 'vec[i64] object expected; got vec[i32]'): + check_union(a4, False) + def test_construct_from_list_multiply() -> None: for i in range(50): v = vec[i64]([i + 1] * i) From 3723b4d63480a663d0b4652a94c279213e6a5ab7 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 30 Jan 2026 14:09:02 +0000 Subject: [PATCH 155/180] Remove unused 'struct_type' attr --- mypyc/ir/rtypes.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 4a7a913093df0..4c10a1ec1fe4b 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -1042,16 +1042,13 @@ def __init__(self, item_type: RType) -> None: self._ctype = vec_c_types[item_type] self.buf_type = vec_buf_types[item_type] self.types = [c_pyssize_t_rprimitive, self.buf_type] - self.struct_type = VecI64 # TODO elif isinstance(non_opt, RVec): self._ctype = "VecNested" self.types = [c_pyssize_t_rprimitive, VecTBufObject] - self.struct_type = VecNested self.buf_type = VecNestedBufObject else: self._ctype = "VecT" self.types = [c_pyssize_t_rprimitive, VecTBufObject] - self.struct_type = VecT self.buf_type = VecTBufObject @property From 0d9611810eba3e99391722cab68a85e2b0f5316c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 30 Jan 2026 18:22:28 +0000 Subject: [PATCH 156/180] Add primitive for isinstance(x, vec) --- mypyc/analysis/ircheck.py | 3 +- mypyc/irbuild/specialize.py | 2 ++ mypyc/lib-rt/vecs/librt_vecs.c | 6 ++++ mypyc/lib-rt/vecs/librt_vecs.h | 6 ++++ mypyc/test-data/irbuild-vec-i64.test | 42 ++++++++++++++++++++++++++++ 5 files changed, 58 insertions(+), 1 deletion(-) diff --git a/mypyc/analysis/ircheck.py b/mypyc/analysis/ircheck.py index a473a167955fc..a49aab26ce074 100644 --- a/mypyc/analysis/ircheck.py +++ b/mypyc/analysis/ircheck.py @@ -63,6 +63,7 @@ RPrimitive, RType, RUnion, + RVec, bytes_rprimitive, dict_rprimitive, int_rprimitive, @@ -204,7 +205,7 @@ def can_coerce_to(src: RType, dest: RType) -> bool: if src.name in disjoint_types and dest.name in disjoint_types: return src.name == dest.name return src.size == dest.size - if isinstance(src, RInstance): + if isinstance(src, (RInstance, RVec)): return is_object_rprimitive(dest) if isinstance(src, RUnion): # IR doesn't have the ability to narrow unions based on diff --git a/mypyc/irbuild/specialize.py b/mypyc/irbuild/specialize.py index 9b22d52c6e148..cf460b94e0827 100644 --- a/mypyc/irbuild/specialize.py +++ b/mypyc/irbuild/specialize.py @@ -131,6 +131,7 @@ string_writer_get_item_unsafe_op, string_writer_range_check_op, ) +from mypyc.primitives.librt_vecs_ops import isinstance_vec from mypyc.primitives.list_ops import isinstance_list, new_list_set_item_op from mypyc.primitives.misc_ops import isinstance_bool from mypyc.primitives.set_ops import isinstance_frozenset, isinstance_set @@ -683,6 +684,7 @@ def gen_inner_stmts() -> None: "builtins.set": isinstance_set, "builtins.str": isinstance_str, "builtins.tuple": isinstance_tuple, + "librt.vecs.vec": isinstance_vec, } diff --git a/mypyc/lib-rt/vecs/librt_vecs.c b/mypyc/lib-rt/vecs/librt_vecs.c index c32ca1e349d18..3696e696d5c3e 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.c +++ b/mypyc/lib-rt/vecs/librt_vecs.c @@ -871,6 +871,11 @@ static PyObject *vec_pop(PyObject *self, PyObject *args) return res; } +// Return the base VecType for isinstance checks +static PyTypeObject *get_vec_type(void) { + return &VecType; +} + static VecCapsule Capsule = { &Vec_TAPI, &Vec_NestedAPI, @@ -880,6 +885,7 @@ static VecCapsule Capsule = { &Vec_U8API, &Vec_FloatAPI, &Vec_BoolAPI, + get_vec_type, }; #endif // MYPYC_EXPERIMENTAL diff --git a/mypyc/lib-rt/vecs/librt_vecs.h b/mypyc/lib-rt/vecs/librt_vecs.h index 06cda24104bc2..90851ada935c8 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.h +++ b/mypyc/lib-rt/vecs/librt_vecs.h @@ -460,6 +460,7 @@ typedef struct { VecU8API *u8; VecFloatAPI *float_; VecBoolAPI *bool_; + PyTypeObject *(*get_vec_type)(void); // Function to get base VecType for isinstance checks } VecCapsule; #define VEC_BUF_SIZE(b) ((b)->ob_base.ob_size) @@ -846,6 +847,11 @@ static VecBoolAPI VecBoolApi; static VecTAPI VecTApi; static VecNestedAPI VecNestedApi; +// Check if obj is an instance of vec (any vec type) +static inline int CPyVec_Check(PyObject *obj) { + return PyObject_TypeCheck(obj, VecApi->get_vec_type()); +} + static int import_librt_vecs(void) { diff --git a/mypyc/test-data/irbuild-vec-i64.test b/mypyc/test-data/irbuild-vec-i64.test index 53a8077b3446a..2b22ecbdc4915 100644 --- a/mypyc/test-data/irbuild-vec-i64.test +++ b/mypyc/test-data/irbuild-vec-i64.test @@ -776,3 +776,45 @@ L10: set_mem r21, r11 :: i64* keep_alive v return 1 + +[case testVecI64Narrow] +from librt.vecs import vec +from mypy_extensions import i64 +from typing import Optional, Union + +def f(x: Optional[vec[i64]]) -> None: + if x is not None: + len(x) + +def g(x: Union[vec[i64], str]) -> None: + if isinstance(x, vec): + len(x) +[out] +def f(x): + x :: union[vec[i64], None] + r0 :: object + r1 :: bit + r2 :: vec[i64] + r3 :: native_int +L0: + r0 = load_address _Py_NoneStruct + r1 = x != r0 + if r1 goto L1 else goto L2 :: bool +L1: + r2 = unbox(vec[i64], x) + r3 = r2.len +L2: + return 1 +def g(x): + x :: union[vec[i64], str] + r0 :: bit + r1 :: vec[i64] + r2 :: native_int +L0: + r0 = CPyVec_Check(x) + if r0 goto L1 else goto L2 :: bool +L1: + r1 = unbox(vec[i64], x) + r2 = r1.len +L2: + return 1 From 86b17da4a66c9784b9af47a86d77df1a10d6d923 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 2 Feb 2026 10:47:42 +0000 Subject: [PATCH 157/180] Minor cleanup --- mypyc/ir/rtypes.py | 9 --------- mypyc/irbuild/for_helpers.py | 2 +- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 4c10a1ec1fe4b..532044c66cc78 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -1333,15 +1333,6 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: return -limit <= n < limit -""" - -# vec (common fields) -VecObject = RStruct( - name="VecObject", names=["ob_base", "len"], types=[PyObject, c_pyssize_t_rprimitive] -) -""" - - # Buffers for vec item types that have a packed representation VecI64BufObject = RStruct( diff --git a/mypyc/irbuild/for_helpers.py b/mypyc/irbuild/for_helpers.py index 030590cb3c474..949ae458a9ce7 100644 --- a/mypyc/irbuild/for_helpers.py +++ b/mypyc/irbuild/for_helpers.py @@ -216,7 +216,7 @@ def sequence_from_generator_preallocate_helper( builder: IRBuilder, gen: GeneratorExpr, empty_op_llbuilder: Callable[[Value, int], Value], - set_item_op: Callable[[Value, Value, Value, int], None], # CFunctionDescription, + set_item_op: Callable[[Value, Value, Value, int], None], ) -> Value | None: """Generate a new tuple or list from a simple generator expression. From 9277ae507dd8f97dc8107c7601ade54647fb8270 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 2 Feb 2026 10:50:11 +0000 Subject: [PATCH 158/180] Restore deleted tests due to bad merge --- mypyc/test-data/irbuild-lists.test | 484 +++++++++++++++++++++++++++++ 1 file changed, 484 insertions(+) diff --git a/mypyc/test-data/irbuild-lists.test b/mypyc/test-data/irbuild-lists.test index 073d0c99de04f..d864bfd19df26 100644 --- a/mypyc/test-data/irbuild-lists.test +++ b/mypyc/test-data/irbuild-lists.test @@ -145,6 +145,32 @@ L0: x = r10 return 1 +[case testListAdd] +from typing import List +def f(a: List[int], b: List[int]) -> None: + c = a + b +[out] +def f(a, b): + a, b, r0, c :: list +L0: + r0 = PySequence_Concat(a, b) + c = r0 + return 1 + +[case testListIAdd] +from typing import List, Any +def f(a: List[int], b: Any) -> None: + a += b +[out] +def f(a, b): + a :: list + b :: object + r0 :: list +L0: + r0 = PySequence_InPlaceConcat(a, b) + a = r0 + return 1 + [case testListMultiply] from typing import List def f(a: List[int]) -> None: @@ -168,6 +194,18 @@ L0: b = r4 return 1 +[case testListIMultiply] +from typing import List +def f(a: List[int]) -> None: + a *= 2 +[out] +def f(a): + a, r0 :: list +L0: + r0 = CPySequence_InPlaceMultiply(a, 4) + a = r0 + return 1 + [case testListLen] from typing import List def f(a: List[int]) -> int: @@ -182,6 +220,30 @@ L0: r1 = r0 << 1 return r1 +[case testListClear] +from typing import List +def f(l: List[int]) -> None: + return l.clear() +[out] +def f(l): + l :: list + r0 :: bit +L0: + r0 = CPyList_Clear(l) + return 1 + +[case testListCopy] +from typing import List +from typing import Any +def f(a: List[Any]) -> List[Any]: + return a.copy() +[out] +def f(a): + a, r0 :: list +L0: + r0 = CPyList_Copy(a) + return r0 + [case testListAppend] from typing import List def f(a: List[int], x: int) -> None: @@ -488,3 +550,425 @@ L3: goto L1 L4: return 1 + +[case testSorted] +from typing import List, Any +def list_sort(a: List[int]) -> None: + a.sort() +def sort_iterable(a: Any) -> None: + sorted(a) +[out] +def list_sort(a): + a :: list + r0 :: i32 + r1 :: bit +L0: + r0 = PyList_Sort(a) + r1 = r0 >= 0 :: signed + return 1 +def sort_iterable(a): + a :: object + r0 :: list +L0: + r0 = CPySequence_Sort(a) + return 1 + +[case testListBuiltFromStr] +def f2(val: str) -> str: + return val + "f2" + +def test() -> None: + source = "abc" + a = [f2(x) for x in source] +[out] +def f2(val): + val, r0, r1 :: str +L0: + r0 = 'f2' + r1 = PyUnicode_Concat(val, r0) + return r1 +def test(): + r0, source :: str + r1 :: native_int + r2 :: bit + r3 :: list + r4 :: native_int + r5 :: bit + r6, x, r7 :: str + r8 :: native_int + a :: list +L0: + r0 = 'abc' + source = r0 + r1 = CPyStr_Size_size_t(source) + r2 = r1 >= 0 :: signed + r3 = PyList_New(r1) + r4 = 0 +L1: + r5 = r4 < r1 :: signed + if r5 goto L2 else goto L4 :: bool +L2: + r6 = CPyStr_GetItemUnsafe(source, r4) + x = r6 + r7 = f2(x) + CPyList_SetItemUnsafe(r3, r4, r7) +L3: + r8 = r4 + 1 + r4 = r8 + goto L1 +L4: + a = r3 + return 1 + +[case testListBuiltFromStrExpr] +def f2(val: str) -> str: + return val + "f2" + +def test() -> None: + a = [f2(x) for x in "abc"] +[out] +def f2(val): + val, r0, r1 :: str +L0: + r0 = 'f2' + r1 = PyUnicode_Concat(val, r0) + return r1 +def test(): + r0 :: str + r1 :: list + r2 :: native_int + r3 :: bit + r4, x, r5 :: str + r6 :: native_int + a :: list +L0: + r0 = 'abc' + r1 = PyList_New(3) + r2 = 0 + goto L2 +L1: + r3 = r2 < 3 :: signed + if r3 goto L2 else goto L4 :: bool +L2: + r4 = CPyStr_GetItemUnsafe(r0, r2) + x = r4 + r5 = f2(x) + CPyList_SetItemUnsafe(r1, r2, r5) +L3: + r6 = r2 + 1 + r2 = r6 + goto L1 +L4: + a = r1 + return 1 + +[case testListBuiltFromFinalStr] +from typing import Final + +source: Final = "abc" + +def f2(val: str) -> str: + return val + "f2" + +def test() -> None: + a = [f2(x) for x in source] +[out] +def f2(val): + val, r0, r1 :: str +L0: + r0 = 'f2' + r1 = PyUnicode_Concat(val, r0) + return r1 +def test(): + r0 :: str + r1 :: list + r2 :: native_int + r3 :: bit + r4, x, r5 :: str + r6 :: native_int + a :: list +L0: + r0 = 'abc' + r1 = PyList_New(3) + r2 = 0 + goto L2 +L1: + r3 = r2 < 3 :: signed + if r3 goto L2 else goto L4 :: bool +L2: + r4 = CPyStr_GetItemUnsafe(r0, r2) + x = r4 + r5 = f2(x) + CPyList_SetItemUnsafe(r1, r2, r5) +L3: + r6 = r2 + 1 + r2 = r6 + goto L1 +L4: + a = r1 + return 1 + +[case testListBuiltFromBytes_64bit] +def f2(val: int) -> int: + return val + 2 + +def test() -> None: + source = b"abc" + a = [f2(x) for x in source] + +[out] +def f2(val): + val, r0 :: int +L0: + r0 = CPyTagged_Add(val, 4) + return r0 +def test(): + r0, source :: bytes + r1 :: native_int + r2 :: list + r3 :: native_int + r4, r5, r6 :: bit + r7, r8, r9, r10 :: int + r11 :: object + r12, x, r13 :: int + r14 :: object + r15 :: native_int + a :: list +L0: + r0 = b'abc' + source = r0 + r1 = var_object_size source + r2 = PyList_New(r1) + r3 = 0 +L1: + r4 = r3 < r1 :: signed + if r4 goto L2 else goto L8 :: bool +L2: + r5 = r3 <= 4611686018427387903 :: signed + if r5 goto L3 else goto L4 :: bool +L3: + r6 = r3 >= -4611686018427387904 :: signed + if r6 goto L5 else goto L4 :: bool +L4: + r7 = CPyTagged_FromInt64(r3) + r8 = r7 + goto L6 +L5: + r9 = r3 << 1 + r8 = r9 +L6: + r10 = CPyBytes_GetItem(source, r8) + r11 = box(int, r10) + r12 = unbox(int, r11) + x = r12 + r13 = f2(x) + r14 = box(int, r13) + CPyList_SetItemUnsafe(r2, r3, r14) +L7: + r15 = r3 + 1 + r3 = r15 + goto L1 +L8: + a = r2 + return 1 + +[case testListBuiltFromBytesExpr_64bit] +def f2(val: int) -> int: + return val + 2 + +def test() -> None: + a = [f2(x) for x in b"abc"] + +[out] +def f2(val): + val, r0 :: int +L0: + r0 = CPyTagged_Add(val, 4) + return r0 +def test(): + r0 :: bytes + r1 :: list + r2 :: native_int + r3, r4, r5 :: bit + r6, r7, r8, r9 :: int + r10 :: object + r11, x, r12 :: int + r13 :: object + r14 :: native_int + a :: list +L0: + r0 = b'abc' + r1 = PyList_New(3) + r2 = 0 + goto L2 +L1: + r3 = r2 < 3 :: signed + if r3 goto L2 else goto L8 :: bool +L2: + r4 = r2 <= 4611686018427387903 :: signed + if r4 goto L3 else goto L4 :: bool +L3: + r5 = r2 >= -4611686018427387904 :: signed + if r5 goto L5 else goto L4 :: bool +L4: + r6 = CPyTagged_FromInt64(r2) + r7 = r6 + goto L6 +L5: + r8 = r2 << 1 + r7 = r8 +L6: + r9 = CPyBytes_GetItem(r0, r7) + r10 = box(int, r9) + r11 = unbox(int, r10) + x = r11 + r12 = f2(x) + r13 = box(int, r12) + CPyList_SetItemUnsafe(r1, r2, r13) +L7: + r14 = r2 + 1 + r2 = r14 + goto L1 +L8: + a = r1 + return 1 + +[case testListBuiltFromFinalBytes_64bit] +from typing import Final + +source: Final = b"abc" + +def f2(val: int) -> int: + return val + 2 + +def test() -> None: + a = [f2(x) for x in source] + +[out] +def f2(val): + val, r0 :: int +L0: + r0 = CPyTagged_Add(val, 4) + return r0 +def test(): + r0 :: bytes + r1 :: bool + r2 :: native_int + r3 :: list + r4 :: native_int + r5, r6, r7 :: bit + r8, r9, r10, r11 :: int + r12 :: object + r13, x, r14 :: int + r15 :: object + r16 :: native_int + a :: list +L0: + r0 = __main__.source :: static + if is_error(r0) goto L1 else goto L2 +L1: + r1 = raise NameError('value for final name "source" was not set') + unreachable +L2: + r2 = var_object_size r0 + r3 = PyList_New(r2) + r4 = 0 +L3: + r5 = r4 < r2 :: signed + if r5 goto L4 else goto L10 :: bool +L4: + r6 = r4 <= 4611686018427387903 :: signed + if r6 goto L5 else goto L6 :: bool +L5: + r7 = r4 >= -4611686018427387904 :: signed + if r7 goto L7 else goto L6 :: bool +L6: + r8 = CPyTagged_FromInt64(r4) + r9 = r8 + goto L8 +L7: + r10 = r4 << 1 + r9 = r10 +L8: + r11 = CPyBytes_GetItem(r0, r9) + r12 = box(int, r11) + r13 = unbox(int, r12) + x = r13 + r14 = f2(x) + r15 = box(int, r14) + CPyList_SetItemUnsafe(r3, r4, r15) +L9: + r16 = r4 + 1 + r4 = r16 + goto L3 +L10: + a = r3 + return 1 + +[case testListBuiltFromStars] +from typing import Final + +abc: Final = "abc" + +def test() -> None: + a = [str(x) for x in [*abc, *"def", *b"ghi", ("j", "k"), *("l", "m", "n")]] + +[out] +def test(): + r0, r1 :: str + r2 :: bytes + r3, r4 :: str + r5 :: tuple[str, str] + r6, r7, r8 :: str + r9 :: tuple[str, str, str] + r10 :: list + r11, r12, r13, r14 :: object + r15 :: i32 + r16 :: bit + r17, r18 :: object + r19 :: list + r20, r21 :: native_int + r22 :: bit + r23, x :: object + r24 :: str + r25 :: native_int + a :: list +L0: + r0 = 'abc' + r1 = 'def' + r2 = b'ghi' + r3 = 'j' + r4 = 'k' + r5 = (r3, r4) + r6 = 'l' + r7 = 'm' + r8 = 'n' + r9 = (r6, r7, r8) + r10 = PyList_New(0) + r11 = CPyList_Extend(r10, r0) + r12 = CPyList_Extend(r10, r1) + r13 = CPyList_Extend(r10, r2) + r14 = box(tuple[str, str], r5) + r15 = PyList_Append(r10, r14) + r16 = r15 >= 0 :: signed + r17 = box(tuple[str, str, str], r9) + r18 = CPyList_Extend(r10, r17) + r19 = PyList_New(13) + r20 = 0 + goto L2 +L1: + r21 = var_object_size r10 + r22 = r20 < r21 :: signed + if r22 goto L2 else goto L4 :: bool +L2: + r23 = list_get_item_unsafe r10, r20 + x = r23 + r24 = PyObject_Str(x) + CPyList_SetItemUnsafe(r19, r20, r24) +L3: + r25 = r20 + 1 + r20 = r25 + goto L1 +L4: + a = r19 + return 1 From bafca4080b787fb86c8a52d8c3f63f30b143fba6 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 2 Feb 2026 11:49:55 +0000 Subject: [PATCH 159/180] Minor clarifications --- mypyc/ir/ops.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/ir/ops.py b/mypyc/ir/ops.py index cd1831a1aa18b..f1b5b746770f4 100644 --- a/mypyc/ir/ops.py +++ b/mypyc/ir/ops.py @@ -1721,7 +1721,7 @@ class GetElementPtr(RegisterOp): def __init__(self, src: Value, src_type: RType, field: str, line: int = -1) -> None: super().__init__(line) - assert not isinstance(src.type, RStruct) + assert not isinstance(src.type, (RStruct, RVec)) self.type = pointer_rprimitive self.src = src self.src_type = src_type From 7df1be72a33345a8dc92410c7af955bbce2469f7 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 2 Feb 2026 11:50:58 +0000 Subject: [PATCH 160/180] Add missing librt_vercs_ops.py file --- mypyc/primitives/librt_vecs_ops.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 mypyc/primitives/librt_vecs_ops.py diff --git a/mypyc/primitives/librt_vecs_ops.py b/mypyc/primitives/librt_vecs_ops.py new file mode 100644 index 0000000000000..f6448c8c7ab36 --- /dev/null +++ b/mypyc/primitives/librt_vecs_ops.py @@ -0,0 +1,13 @@ +from mypyc.ir.ops import ERR_NEVER +from mypyc.ir.rtypes import bit_rprimitive, object_rprimitive +from mypyc.primitives.registry import function_op + +# isinstance(obj, vec) +isinstance_vec = function_op( + name="builtins.isinstance", + arg_types=[object_rprimitive], + return_type=bit_rprimitive, + c_function_name="CPyVec_Check", + error_kind=ERR_NEVER, + experimental=True, +) From 77c0646ac6516bb44445b6ccf0a740f4fe218001 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 2 Feb 2026 11:53:40 +0000 Subject: [PATCH 161/180] Fix primitive dep --- mypyc/primitives/librt_vecs_ops.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mypyc/primitives/librt_vecs_ops.py b/mypyc/primitives/librt_vecs_ops.py index f6448c8c7ab36..6b9ca9ef42247 100644 --- a/mypyc/primitives/librt_vecs_ops.py +++ b/mypyc/primitives/librt_vecs_ops.py @@ -1,3 +1,4 @@ +from mypyc.ir.deps import LIBRT_VECS from mypyc.ir.ops import ERR_NEVER from mypyc.ir.rtypes import bit_rprimitive, object_rprimitive from mypyc.primitives.registry import function_op @@ -10,4 +11,5 @@ c_function_name="CPyVec_Check", error_kind=ERR_NEVER, experimental=True, + dependencies=[LIBRT_VECS], ) From e999af24e83d4a6a2f038bf32dd33e7db258e938 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 2 Feb 2026 11:59:27 +0000 Subject: [PATCH 162/180] Move function to extra ops header --- mypyc/ir/deps.py | 1 + mypyc/lib-rt/vecs/librt_vecs.h | 5 ----- mypyc/lib-rt/vecs_extra_ops.c | 10 ++++++++++ mypyc/lib-rt/vecs_extra_ops.h | 15 +++++++++++++++ mypyc/primitives/librt_vecs_ops.py | 4 ++-- mypyc/test/test_cheader.py | 4 ++++ 6 files changed, 32 insertions(+), 7 deletions(-) create mode 100644 mypyc/lib-rt/vecs_extra_ops.c create mode 100644 mypyc/lib-rt/vecs_extra_ops.h diff --git a/mypyc/ir/deps.py b/mypyc/ir/deps.py index 7c52520e274b9..933ff72e40c2c 100644 --- a/mypyc/ir/deps.py +++ b/mypyc/ir/deps.py @@ -55,3 +55,4 @@ def get_header(self) -> str: STRING_WRITER_EXTRA_OPS: Final = SourceDep("stringwriter_extra_ops.c") BYTEARRAY_EXTRA_OPS: Final = SourceDep("bytearray_extra_ops.c") STR_EXTRA_OPS: Final = SourceDep("str_extra_ops.c") +VECS_EXTRA_OPS: Final = SourceDep("vecs_extra_ops.c") diff --git a/mypyc/lib-rt/vecs/librt_vecs.h b/mypyc/lib-rt/vecs/librt_vecs.h index 90851ada935c8..80dc977c19146 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.h +++ b/mypyc/lib-rt/vecs/librt_vecs.h @@ -847,11 +847,6 @@ static VecBoolAPI VecBoolApi; static VecTAPI VecTApi; static VecNestedAPI VecNestedApi; -// Check if obj is an instance of vec (any vec type) -static inline int CPyVec_Check(PyObject *obj) { - return PyObject_TypeCheck(obj, VecApi->get_vec_type()); -} - static int import_librt_vecs(void) { diff --git a/mypyc/lib-rt/vecs_extra_ops.c b/mypyc/lib-rt/vecs_extra_ops.c new file mode 100644 index 0000000000000..564eae2c9fcb5 --- /dev/null +++ b/mypyc/lib-rt/vecs_extra_ops.c @@ -0,0 +1,10 @@ +// Primitives related to librt.vecs that get linked statically +// with compiled modules, instead of being called via a capsule. + +#include "vecs_extra_ops.h" + +#ifdef MYPYC_EXPERIMENTAL + +// All operations are currently implemented as inline functions in vecs_extra_ops.h + +#endif // MYPYC_EXPERIMENTAL diff --git a/mypyc/lib-rt/vecs_extra_ops.h b/mypyc/lib-rt/vecs_extra_ops.h new file mode 100644 index 0000000000000..b8f72876b129e --- /dev/null +++ b/mypyc/lib-rt/vecs_extra_ops.h @@ -0,0 +1,15 @@ +#ifndef BYTESWRITER_EXTRA_OPS_H +#define BYTESWRITER_EXTRA_OPS_H + +#ifdef MYPYC_EXPERIMENTAL + +#include "vecs/librt_vecs.h" + +// Check if obj is an instance of vec (any vec type) +static inline int CPyVec_Check(PyObject *obj) { + return PyObject_TypeCheck(obj, VecApi->get_vec_type()); +} + +#endif // MYPYC_EXPERIMENTAL + +#endif diff --git a/mypyc/primitives/librt_vecs_ops.py b/mypyc/primitives/librt_vecs_ops.py index 6b9ca9ef42247..4688aea14aaea 100644 --- a/mypyc/primitives/librt_vecs_ops.py +++ b/mypyc/primitives/librt_vecs_ops.py @@ -1,4 +1,4 @@ -from mypyc.ir.deps import LIBRT_VECS +from mypyc.ir.deps import LIBRT_VECS, VECS_EXTRA_OPS from mypyc.ir.ops import ERR_NEVER from mypyc.ir.rtypes import bit_rprimitive, object_rprimitive from mypyc.primitives.registry import function_op @@ -11,5 +11,5 @@ c_function_name="CPyVec_Check", error_kind=ERR_NEVER, experimental=True, - dependencies=[LIBRT_VECS], + dependencies=[LIBRT_VECS, VECS_EXTRA_OPS], ) diff --git a/mypyc/test/test_cheader.py b/mypyc/test/test_cheader.py index 82223a0c451f2..abbd0aa57b47c 100644 --- a/mypyc/test/test_cheader.py +++ b/mypyc/test/test_cheader.py @@ -24,6 +24,8 @@ str_ops, tuple_ops, weakref_ops, + librt_vecs_ops, + librt_strings_ops, ) @@ -65,6 +67,8 @@ def check_name(name: str) -> None: float_ops, set_ops, weakref_ops, + librt_vecs_ops, + librt_strings_ops, ]: for name in dir(module): val = getattr(module, name, None) From 79455a58696e851ab7e1fc8afdf9432deac5f9bc Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 2 Feb 2026 12:59:26 +0000 Subject: [PATCH 163/180] Fix vec pop with optional item type --- mypyc/irbuild/vec.py | 2 +- mypyc/test-data/run-vecs-t.test | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 4b312267aeb6f..fdd77d3374b86 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -422,7 +422,7 @@ def vec_pop(builder: LowLevelIRBuilder, base: Value, index: Value, line: int) -> api_name = vec_api_by_item_type.get(item_type) if api_name is not None: name = f"{api_name}.pop" - elif vec_depth(vec_type) == 0 and not isinstance(item_type, RUnion): # TODO fix union + elif vec_depth(vec_type) == 0: name = "VecTApi.pop" else: name = "VecNestedApi.pop" diff --git a/mypyc/test-data/run-vecs-t.test b/mypyc/test-data/run-vecs-t.test index 9569d34d9be8e..b9c6a6b0710c1 100644 --- a/mypyc/test-data/run-vecs-t.test +++ b/mypyc/test-data/run-vecs-t.test @@ -340,6 +340,19 @@ def test_pop_index() -> None: assert n == "15" assert v == vec[str]() +def test_pop_optional() -> None: + v = vec[Optional[str]](["x" + str(), None]) + v, s = pop(v) + assert s is None + v, s = pop(v) + assert s == "x" + assert len(v) == 0 + v = vec[Optional[str]](["x" + str(), None]) + v, s = pop(v, 0) + assert s == "x" + v, s = pop(v, -1) + assert s is None + def f(x: vec[str]) -> None: pass From d1ee773f346662aca5d949bc746c68ae13ca52e1 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 2 Feb 2026 15:46:07 +0000 Subject: [PATCH 164/180] Fix after rebase --- mypyc/irbuild/ll_builder.py | 52 +------------------------------------ 1 file changed, 1 insertion(+), 51 deletions(-) diff --git a/mypyc/irbuild/ll_builder.py b/mypyc/irbuild/ll_builder.py index 2f88aaf00a459..acc8acbe91305 100644 --- a/mypyc/irbuild/ll_builder.py +++ b/mypyc/irbuild/ll_builder.py @@ -131,7 +131,7 @@ str_rprimitive, ) from mypyc.irbuild.util import concrete_arg_kind -from mypyc.irbuild.vec import vec_contains, vec_get_item, vec_len, vec_len_native +from mypyc.irbuild.vec import vec_contains, vec_get_item, vec_len_native from mypyc.options import CompilerOptions from mypyc.primitives.bytes_ops import bytes_compare from mypyc.primitives.dict_ops import ( @@ -682,32 +682,6 @@ def coerce_short_int_to_fixed_width(self, src: Value, target_type: RType, line: self.activate_block(end) return res - def coerce_i64_to_ssize_t(self, src: Value, line: int) -> Value: - """Coerce int64 to c_pyssize_t with range check on 32-bit platforms. - - On 64-bit platforms, this is a no-op since the types have the same size. - On 32-bit platforms, raises OverflowError if the value doesn't fit in 32 bits. - """ - if PLATFORM_SIZE == 8: - return src - - # 32-bit platform: need range check - overflow, end = BasicBlock(), BasicBlock() - - self.check_fixed_width_range(src, c_pyssize_t_rprimitive, overflow, line) - - # Value is in range - truncate to 32 bits - res = self.add(Truncate(src, c_pyssize_t_rprimitive, line)) - self.goto(end) - - # Handle overflow case - self.activate_block(overflow) - self.call_c(int32_overflow, [], line) - self.add(Unreachable()) - - self.activate_block(end) - return res - def coerce_fixed_width_to_int(self, src: Value, line: int) -> Value: if ( (is_int32_rprimitive(src.type) and PLATFORM_SIZE == 8) @@ -2757,30 +2731,6 @@ def set_immortal_if_free_threaded(self, v: Value, line: int) -> None: if IS_FREE_THREADED and sys.version_info >= (3, 14): self.primitive_op(set_immortal_op, [v], line) - # Loop helpers - - def begin_for(self, start: Value, end: Value, step: Value, *, signed: bool) -> ForBuilder: - """Generate for loop over a pointer or native int range. - - Roughly corresponds to "for i in range(start, end, step): ....". Only - positive values for step are supported. - - The loop index register is generated automatically and is available - via the return value. Do not modify it! - - Example usage that clears a memory area: - - for_loop = builder.begin_for(start_ptr, end_ptr, int32_rprimitive.size) - - # For loop body - builder.set_mem(for_loop.index, int32_rprimitive, Integer(0, int32_rprimitive)) - - for_loop.finish() - """ - b = ForBuilder(self, start, end, step, signed=signed) - b.begin() - return b - # Internal helpers def decompose_union_helper( From 2893618b2e73d0896641a592b97ed48de11dbe9c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 2 Feb 2026 15:46:15 +0000 Subject: [PATCH 165/180] Lint --- mypyc/test/test_cheader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypyc/test/test_cheader.py b/mypyc/test/test_cheader.py index abbd0aa57b47c..d955e51e33a69 100644 --- a/mypyc/test/test_cheader.py +++ b/mypyc/test/test_cheader.py @@ -17,6 +17,8 @@ float_ops, generic_ops, int_ops, + librt_strings_ops, + librt_vecs_ops, list_ops, misc_ops, registry, @@ -24,8 +26,6 @@ str_ops, tuple_ops, weakref_ops, - librt_vecs_ops, - librt_strings_ops, ) From 68763a41ef13785a3b0fbdc932ac6223bdc022ac Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 2 Feb 2026 16:33:05 +0000 Subject: [PATCH 166/180] Remove commented out code --- mypyc/ir/rtypes.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 532044c66cc78..ec3531b80662c 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -1370,10 +1370,6 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: ) -# vecbuf_i64_rprimitive: Final = RPrimitive( -# "VecI64BufObject", is_unboxed=False, is_refcounted=True, ctype="VecI64BufObject *" -# ) - # Struct type for vec[i64] (in most cases use RVec instead). VecI64 = RStruct( name="VecI64", names=["len", "buf"], types=[c_pyssize_t_rprimitive, object_rprimitive] From fbaa99faf1a8cb572f21701cec48b57964eefed2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 2 Feb 2026 16:37:12 +0000 Subject: [PATCH 167/180] Remove unused param --- mypyc/ir/rtypes.py | 43 ++++++++++++------------------------------- 1 file changed, 12 insertions(+), 31 deletions(-) diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index ec3531b80662c..3a8f046ab9426 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -240,7 +240,6 @@ def __init__( is_refcounted: bool, is_native_int: bool = False, is_signed: bool = False, - is_pointer: bool = False, ctype: str = "PyObject *", size: int = PLATFORM_SIZE, error_overlap: bool = False, @@ -316,19 +315,17 @@ def __hash__(self) -> int: # little as possible, as generic ops are typically slow. Other types, # including other primitive types and RInstance, are usually much # faster. -object_rprimitive: Final = RPrimitive( - "builtins.object", is_unboxed=False, is_refcounted=True, is_pointer=True -) +object_rprimitive: Final = RPrimitive("builtins.object", is_unboxed=False, is_refcounted=True) # represents a low level pointer of an object object_pointer_rprimitive: Final = RPrimitive( - "object_ptr", is_unboxed=False, is_refcounted=False, ctype="PyObject **", is_pointer=True + "object_ptr", is_unboxed=False, is_refcounted=False, ctype="PyObject **" ) # Similar to object_primitive, but doesn not use automatic reference # counting. Useful for temporaries. object_non_refcounted_rprimitive: Final = RPrimitive( - "builtins.object_nrc", is_unboxed=False, is_refcounted=False, is_pointer=True + "builtins.object_nrc", is_unboxed=False, is_refcounted=False ) # Arbitrary-precision integer (corresponds to Python 'int'). Small @@ -462,7 +459,7 @@ def __hash__(self) -> int: # Untyped pointer, represented as void * in the C backend c_pointer_rprimitive: Final = RPrimitive( - "c_ptr", is_unboxed=False, is_refcounted=False, ctype="void *", is_pointer=True + "c_ptr", is_unboxed=False, is_refcounted=False, ctype="void *" ) cstring_rprimitive: Final = RPrimitive( @@ -506,18 +503,14 @@ def __hash__(self) -> int: # immortal, but since this is expected to be very rare, and the immortality checks # can be pretty expensive for lists, we treat lists as non-immortal. list_rprimitive: Final = RPrimitive( - "builtins.list", is_unboxed=False, is_refcounted=True, is_pointer=True, may_be_immortal=False + "builtins.list", is_unboxed=False, is_refcounted=True, may_be_immortal=False ) # Python dict object (or an instance of a subclass of dict). -dict_rprimitive: Final = RPrimitive( - "builtins.dict", is_unboxed=False, is_refcounted=True, is_pointer=True -) +dict_rprimitive: Final = RPrimitive("builtins.dict", is_unboxed=False, is_refcounted=True) # Python set object (or an instance of a subclass of set). -set_rprimitive: Final = RPrimitive( - "builtins.set", is_unboxed=False, is_refcounted=True, is_pointer=True -) +set_rprimitive: Final = RPrimitive("builtins.set", is_unboxed=False, is_refcounted=True) # Python frozenset object (or an instance of a subclass of frozenset). frozenset_rprimitive: Final = RPrimitive( @@ -526,14 +519,10 @@ def __hash__(self) -> int: # Python str object. At the C layer, str is referred to as unicode # (PyUnicode). -str_rprimitive: Final = RPrimitive( - "builtins.str", is_unboxed=False, is_refcounted=True, is_pointer=True -) +str_rprimitive: Final = RPrimitive("builtins.str", is_unboxed=False, is_refcounted=True) # Python bytes object. -bytes_rprimitive: Final = RPrimitive( - "builtins.bytes", is_unboxed=False, is_refcounted=True, is_pointer=True -) +bytes_rprimitive: Final = RPrimitive("builtins.bytes", is_unboxed=False, is_refcounted=True) # Python bytearray object. bytearray_rprimitive: Final = RPrimitive( @@ -542,14 +531,10 @@ def __hash__(self) -> int: # Tuple of an arbitrary length (corresponds to Tuple[t, ...], with # explicit '...'). -tuple_rprimitive: Final = RPrimitive( - "builtins.tuple", is_unboxed=False, is_refcounted=True, is_pointer=True -) +tuple_rprimitive: Final = RPrimitive("builtins.tuple", is_unboxed=False, is_refcounted=True) # Python range object. -range_rprimitive: Final = RPrimitive( - "builtins.range", is_unboxed=False, is_refcounted=True, is_pointer=True -) +range_rprimitive: Final = RPrimitive("builtins.range", is_unboxed=False, is_refcounted=True) KNOWN_NATIVE_TYPES: Final = { name: RPrimitive(name, is_unboxed=False, is_refcounted=True, dependencies=(LIBRT_STRINGS,)) @@ -1413,11 +1398,7 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: ) VecNestedBufObject_rprimitive = RPrimitive( - "VecNestedBufObject_ptr", - is_unboxed=False, - is_pointer=True, - is_refcounted=True, - ctype="VecNestedBufObject *", + "VecNestedBufObject_ptr", is_unboxed=False, is_refcounted=True, ctype="VecNestedBufObject *" ) VecNestedPopResult = RStruct( From 8484cf89743d1cf7f005a68badbb6dc1625e6381 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 2 Feb 2026 16:38:55 +0000 Subject: [PATCH 168/180] WIP temporarily delete vec run tests --- mypyc/test-data/run-vecs-i64.test | 466 -------------------------- mypyc/test-data/run-vecs-misc.test | 222 ------------- mypyc/test-data/run-vecs-nested.test | 468 --------------------------- mypyc/test-data/run-vecs-t.test | 406 ----------------------- 4 files changed, 1562 deletions(-) delete mode 100644 mypyc/test-data/run-vecs-i64.test delete mode 100644 mypyc/test-data/run-vecs-misc.test delete mode 100644 mypyc/test-data/run-vecs-nested.test delete mode 100644 mypyc/test-data/run-vecs-t.test diff --git a/mypyc/test-data/run-vecs-i64.test b/mypyc/test-data/run-vecs-i64.test deleted file mode 100644 index 326561d25a648..0000000000000 --- a/mypyc/test-data/run-vecs-i64.test +++ /dev/null @@ -1,466 +0,0 @@ --- Test cases for vec[i64]. These also partially cover other unboxed item types, --- which use a similar runtime representation. - -[case testVecI64BasicOps_librt_experimental] -from typing import Final, Any, Iterable, Tuple - -from mypy_extensions import i64, i32 -from librt.vecs import vec, append, remove, pop - -from testutil import assertRaises - -ERROR: Final = -113 - -def test_create_empty() -> None: - v = vec[i64]() - assert len(v) == 0 - -def test_create_from_list_and_get_item() -> None: - v = vec[i64]([]) - assert len(v) == 0 - - v = vec[i64]([ERROR, 0, 255, int() + (2**63 - 1)]) - assert len(v) == 4 - assert v[0] == -113 - assert v[1] == 0 - assert v[2] == 255 - assert v[3] == 2**63 - 1 - - v = vec[i64]([1, 2, 3, 4, 5, 6, 7, 8, 9]) - assert len(v) == 9 - for i in range(i64(9)): - assert v[i] == i + 1 - -def test_append() -> None: - v = vec[i64]() - v = append(v, int() + 3) - assert len(v) == 1 - x: i64 = ERROR + int() - v = append(v, x) - assert len(v) == 2 - assert v[0] == 3 - assert v[1] == ERROR - v = vec[i64]() - for i in range(1024): - v = append(v, i * 3) - assert len(v) == 1024 - for i in range(1024): - assert v[i] == i * 3 - -def test_get_item() -> None: - v = vec[i64]([3, 4]) - x: i64 = int() - assert v[x] == 3 - assert v[int()] == 3 - x += 1 - assert v[x] == 4 - assert v[int() + 1] == 4 - with assertRaises(IndexError): - v[x + 1] - with assertRaises(IndexError): - v[2] - with assertRaises(IndexError): - v[2**63 - 1] - -def test_get_item_negative() -> None: - v = vec[i64]([3, 4]) - x: i64 = int() - 1 - assert v[x] == 4 - assert v[int() - 2] == 3 - x -= 1 - assert v[x] == 3 - assert v[int() - 1] == 4 - with assertRaises(IndexError): - v[x - 1] - with assertRaises(IndexError): - v[-3] - with assertRaises(IndexError): - v[-2**63] - with assertRaises(IndexError): - v[int() - 3] - -def test_set_item() -> None: - v = vec[i64]([3, 4]) - v[0] = 0 - assert v[0] == 0 - - x: i64 = int() - - v[x] = 4 - assert v[x] == 4 - - v[int()] = 5 - assert v[int()] == 5 - - x += 1 - - v[x] = -5 - assert v[x] == -5 - - v[1 + int()] = ERROR - assert v[1] == ERROR - - with assertRaises(IndexError): - v[1 + x] = 6 - with assertRaises(IndexError): - v[2 + int()] = 6 - -def test_set_item_negative() -> None: - v = vec[i64]([3, 4]) - v[-1] = 0 - assert v[-1] == 0 - - x: i64 = int() - 1 - - v[x] = 4 - assert v[x] == 4 - - v[int() - 2] = 5 - assert v[int() - 2] == 5 - - x -= 1 - - v[x] = -5 - assert v[x] == -5 - - v[-1 + int()] = ERROR - assert v[-1] == ERROR - - with assertRaises(IndexError): - v[x - 1] = 6 - with assertRaises(IndexError): - v[int() - 3] = 6 - -def test_operator_assignment() -> None: - v = vec[i64]([3, 4]) - x: i64 = int() - v[x] += 2 - assert v[0] == 5 - v[x + 1] -= 10 - assert v[1] == -6 - y = int() - v[y] += 6 - assert v[y] == 11 - v[y + 1] *= 2 - assert v[1 + y] == -12 - -def test_box_and_unbox() -> None: - v = vec[i64]([3, 5]) - o: Any = v - assert len(o) == 2 - assert o[0] == 3 - assert o[1] == 5 - o[1] = 6 - v2: vec[i64] = o - assert len(v2) == 2 - assert v2[0] == 3 - assert v2[1] == 6 - o2: Any = "x" - with assertRaises(TypeError, "vec[i64] object expected; got str"): - v2 = o2 - o3: Any = vec[str]([]) - # TODO: Better message - with assertRaises(TypeError, "vec[i64] object expected; got vec"): - v2 = o3 - -def check_opt(x: vec[i64] | None, is_none: bool) -> None: - if is_none: - assert x is None - else: - if int() + 1: - assert x is not None - assert len(x) == 1 - assert x[0] == 5 - if int() + 2: - assert isinstance(x, vec) - assert len(x) == 1 - assert x[0] == 5 - if isinstance(x, vec): - assert x[0] == 5 - assert not is_none - else: - assert is_none - -def test_vec_optional() -> None: - v = vec[i64]([5]) - check_opt(None, True) - check_opt(v, False) - a1: Any = None - check_opt(a1, True) - a2: Any = v - check_opt(a2, False) - a3: Any = str - with assertRaises(TypeError): - check_opt(a3, False) - -def check_union(x: vec[i64] | str, is_str: bool) -> None: - if is_str: - assert isinstance(x, str) - assert x == "ss" - else: - if int() + 1: - assert not isinstance(x, str) - assert len(x) == 1 - assert x[0] == 5 - if int() + 2: - assert isinstance(x, vec) - assert len(x) == 1 - assert x[0] == 5 - if isinstance(x, vec): - assert x[0] == 5 - assert not is_str - else: - assert is_str - assert x == "ss" - -def test_vec_union() -> None: - v = vec[i64]([5]) - check_union("ss", True) - check_union(v, False) - a1: Any = "ss" - check_union(a1, True) - a2: Any = v - check_union(a2, False) - a3: Any = b"ss" - with assertRaises(TypeError): - check_union(a3, False) - a4: Any = vec[i32]() - with assertRaises(TypeError, 'vec[i64] object expected; got vec[i32]'): - check_union(a4, False) - -def test_construct_from_list_multiply() -> None: - for i in range(50): - v = vec[i64]([i + 1] * i) - assert len(v) == i - for j in range(i): - assert v[j] == i + 1 - for i in range(50): - v = vec[i64](i * [i - 1]) - assert len(v) == i - for j in range(i): - assert v[j] == i - 1 - -def test_construct_from_list_comprehension() -> None: - for i in range(50): - l = [i * i for i in range(i)] - v = vec[i64]([n + 5 for n in l]) - assert len(v) == i - for j in range(i): - assert v[j] == j * j + 5 - -def test_construct_from_range() -> None: - v = vec[i64](range(0)) - assert v == vec[i64]() - v = vec[i64](range(5)) - assert v == vec[i64]([0, 1, 2, 3, 4]) - v = vec[i64](range(2, 5)) - assert v == vec[i64]([2, 3, 4]) - -def test_construct_from_iterable() -> None: - for i in range(50): - it: Iterable[i64] = iter([i * i for i in range(i)]) - v = vec[i64](it) - assert len(v) == i - for j in range(i): - assert v[j] == j * j - for i in range(50): - it2: Iterable[int] = iter([i * i for i in range(i)]) - v = vec[i64](it2) - assert len(v) == i - for j in range(i): - assert v[j] == j * j - -def test_equality() -> None: - v0 = vec[i64]() - v0b = vec[i64]() - v1 = vec[i64]([1]) - v1b = vec[i64]([3]) - assert v0 == v0 - assert v1 == v1 - assert v0 == v0b - assert v0 != v1 - assert v1 != v1b - assert v1 != v0 - -def test_str_conversion() -> None: - v = vec[i64]() - assert str(v) == "vec[i64]([])" - assert repr(v) == "vec[i64]([])" - v = vec[i64]([126]) - assert str(v) == "vec[i64]([126])" - v = append(v, -5) - assert str(v) == "vec[i64]([126, -5])" - v = append(v, ERROR) - assert str(v) == "vec[i64]([126, -5, -113])" - -def test_for_loop() -> None: - for n in vec[i64](): - assert False - a = [] - for n in vec[i64]([5]): - a.append(n) - assert a == [5] - v = vec[i64]([ERROR, 9, 8]) - a = [] - for n in v: - a.append(n) - assert a == [ERROR, 9, 8] - assert len(v) == 3 - -def test_contains() -> None: - v = vec[i64]() - x: i64 = int() - assert x not in v - v = vec[i64]([0]) - assert x in v - assert x + 1 not in v - v2 = vec[i64]([ERROR, 7, 9]) - assert x + ERROR in v2 - assert x + 7 in v2 - assert int() + 9 in v2 - assert x not in v2 - assert int() not in v2 - -def test_slicing() -> None: - v = vec[i64]() - assert v[:] == vec[i64]([]) - assert v[1:] == vec[i64]([]) - assert v[:-5] == vec[i64]([]) - v = vec[i64]([0, 1, 2, 3, 4]) - assert v[1:4] == vec[i64]([1, 2, 3]) - assert v[2:-1] == vec[i64]([2, 3]) - assert v[-2:-1] == vec[i64]([3]) - assert v[1:] == vec[i64]([1, 2, 3, 4]) - assert v[:-1] == vec[i64]([0, 1, 2, 3]) - assert v[:] == v - assert v[:] is not v - assert v[5:] == vec[i64]() - assert v[100:] == vec[i64]() - assert v[0:5] ==v - assert v[0:5] is not v - assert v[2:100] == vec[i64]([2, 3, 4]) - assert v[-100:2] == vec[i64]([0, 1]) - assert v[5:100] == vec[i64]([]) - assert v[50:100] == vec[i64]([]) - assert v[-100:-50] == vec[i64]([]) - -def test_slicing_with_step() -> None: - v = vec[i64]([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) - assert v[1:5:2] == vec[i64]([1, 3]) - assert v[1:6:2] == vec[i64]([1, 3, 5]) - assert v[5:1:-2] == vec[i64]([5, 3]) - assert v[6:1:-2] == vec[i64]([6, 4, 2]) - v = vec[i64]([0, 1, 2, 3, 4]) - assert v[::-1] == vec[i64]([4, 3, 2, 1, 0]) - with assertRaises(ValueError): - v[1:3:0] - -def test_remove() -> None: - a = [4, 7, ERROR] - for i in a: - v = vec[i64](a) - v = remove(v, i) - assert v == vec[i64]([j for j in a if j != i]) - v = vec[i64](a) - v = remove(v, 4) - v = remove(v, 7) - v = remove(v, ERROR) - assert v == vec[i64]() - with assertRaises(ValueError): - remove(v, 4) - v = append(v, 5) - with assertRaises(ValueError): - remove(v, 7) - v = remove(v, 5) - assert len(v) == 0 - v = vec[i64]([1, 1, 1]) - v = remove(v, 1) - assert v == vec[i64]([1, 1]) - v = remove(v, 1) - assert v == vec[i64]([1]) - v = remove(v, 1) - assert v == vec[i64]() - f: Any = 1.1 - with assertRaises(TypeError): - remove(v, f) - s: Any = 'x' - with assertRaises(TypeError): - remove(v, s) - -def test_pop_last() -> None: - v = vec[i64]([4, 7, ERROR]) - v, n = pop(v) - assert n == ERROR - assert v == vec[i64]([4, 7]) - v, n = pop(v) - assert n == 7 - assert v == vec[i64]([4]) - v, n = pop(v) - assert n == 4 - assert v == vec[i64]() - with assertRaises(IndexError): - pop(v) - -def test_pop_index() -> None: - v = vec[i64]([4, 7, 9, 15, 22]) - v, n = pop(v, 0) - assert n == 4 - assert v == vec[i64]([7, 9, 15, 22]) - v, n = pop(v, -1) - assert n == 22 - assert v == vec[i64]([7, 9, 15]) - v, n = pop(v, 1) - assert n == 9 - assert v == vec[i64]([7, 15]) - - with assertRaises(IndexError): - pop(v, 2) - - with assertRaises(IndexError): - pop(v, -3) - - v, n = pop(v, -2) - assert n == 7 - assert v == vec[i64]([15]) - v, n = pop(v, 0) - assert n == 15 - assert v == vec[i64]() - -def f(x: vec[i64]) -> None: - pass - -def test_wrapper_arg_check() -> None: - f_any: Any = f - with assertRaises(TypeError): - f_any([]) - -v: Final = vec[i64]([5, 6]) - -def test_final() -> None: - assert v == vec[i64]([5, 6]) - -def fdefault(v: vec[i64] = vec[i64]([5])) -> vec[i64]: - return v - -def test_default_arg() -> None: - assert fdefault() == vec[i64]([5]) - assert fdefault(vec[i64]()) == vec[i64]() - f_any: Any = fdefault - assert f_any() == vec[i64]([5]) - assert f_any(vec[i64]([6])) == vec[i64]([6]) - -def ftuple(b: bool) -> Tuple[vec[i64], vec[i64]]: - if b: - raise RuntimeError() - return vec[i64]([5]), vec[i64]() - -def test_tuple() -> None: - t = ftuple(False) - assert len(t) == 2 - assert t[0] == vec[i64]([5]) - assert t[1] == vec[i64]() - a, b = t - assert a == t[0] - assert b == t[1] - with assertRaises(RuntimeError): - ftuple(True) diff --git a/mypyc/test-data/run-vecs-misc.test b/mypyc/test-data/run-vecs-misc.test deleted file mode 100644 index 0ef4021c4a519..0000000000000 --- a/mypyc/test-data/run-vecs-misc.test +++ /dev/null @@ -1,222 +0,0 @@ --- Test cases for packed/specialized vec item types other than i64, such as --- vec[i32] and vec[float]. Since many of the code paths are the same as for --- vec[i64], only test a subset of functionality. --- --- vec[i64] test cases are in run-vecs-i64.test. - -[case testVecMiscBasicOps_librt_experimental] -# mypy: allow-redefinition - -from typing import Any, cast -from mypy_extensions import i64, i32, i16, u8 -from librt.vecs import vec, append, remove, pop -from testutil import assertRaises - -def test_create_empty() -> None: - assert str(vec[i32]()) == "vec[i32]([])" - assert str(vec[i16]()) == "vec[i16]([])" - assert str(vec[u8]()) == "vec[u8]([])" - assert str(vec[float]()) == "vec[float]([])" - assert str(vec[bool]()) == "vec[bool]([])" - -def test_create_from_values() -> None: - assert str(vec[i32]([5, -12])) == "vec[i32]([5, -12])" - assert str(vec[i16]([6, -8])) == "vec[i16]([6, -8])" - assert str(vec[u8]([0, 123, 255])) == "vec[u8]([0, 123, 255])" - assert str(vec[float]([1.5, -3.5])) == "vec[float]([1.5, -3.5])" - assert str(vec[bool]([False, True])) == "vec[bool]([False, True])" - -def test_get_item() -> None: - assert vec[i32]([5, -12])[1] == -12 - assert vec[i16]([6, -8])[1] == -8 - assert vec[u8]([0, 123, 255])[2] == 255 - assert vec[float]([1.5, -3.5])[1] == -3.5 - assert vec[bool]([False, True])[1] == True - -def test_get_item_negative() -> None: - assert vec[i32]([5, -12])[-2] == 5 - assert vec[i16]([6, -8])[-2] == 6 - assert vec[u8]([0, 123, 255])[-2] == 123 - assert vec[float]([1.5, -3.5])[-2] == 1.5 - assert vec[bool]([False, True])[-2] == False - -def test_append() -> None: - assert append(vec[i32](), 123) == vec[i32]([123]) - assert append(vec[i16](), 123) == vec[i16]([123]) - assert append(vec[u8](), 123) == vec[u8]([123]) - assert append(vec[float](), 123.5) == vec[float]([123.5]) - assert append(vec[bool](), True) == vec[bool]([True]) - -def test_unbox() -> None: - v: vec[i64] - - a = cast(Any, vec[i32]([1, 2])) - v_i32: vec[i32] = a - assert v_i32 == vec[i32]([1, 2]) - with assertRaises(TypeError): - v = a - - a = cast(Any, vec[i16]([1, 2])) - v_i16: vec[i16] = a - assert v_i16 == vec[i16]([1, 2]) - with assertRaises(TypeError): - v = a - - a = cast(Any, vec[u8]([1, 2])) - v_u8: vec[u8] = a - assert v_u8 == vec[u8]([1, 2]) - with assertRaises(TypeError): - v = a - - a = cast(Any, vec[float]([1, 2])) - v_float: vec[float] = a - assert v_float == vec[float]([1, 2]) - with assertRaises(TypeError): - v = a - - a = cast(Any, vec[bool]([True, False])) - v_bool: vec[bool] = a - assert v_bool == vec[bool]([True, False]) - with assertRaises(TypeError): - v = a - -def test_set_item() -> None: - v = vec[i32]([5, -12]) - v[1] = 55 - assert v[0] == 5 and v[1] == 55 - - v = vec[i16]([5, -12]) - v[1] = 55 - assert v[0] == 5 and v[1] == 55 - - v = vec[u8]([5, 237]) - v[1] = 55 - assert v[0] == 5 and v[1] == 55 - - v = vec[float]([1.5, -3.5]) - v[1] = 5.5 - assert v[0] == 1.5 and v[1] == 5.5 - - v = vec[bool]([True, False]) - v[1] = True - assert v[0] and v[1] - -def test_set_item_negative() -> None: - v = vec[i32]([5, -12]) - v[-1] = 55 - assert v[0] == 5 and v[1] == 55 - - v = vec[i16]([5, -12]) - v[-1] = 55 - assert v[0] == 5 and v[1] == 55 - - v = vec[u8]([5, 237]) - v[-1] = 55 - assert v[0] == 5 and v[1] == 55 - - v = vec[float]([1.5, -3.5]) - v[-1] = 5.5 - assert v[0] == 1.5 and v[1] == 5.5 - - v = vec[bool]([True, False]) - v[-1] = True - assert v[0] and v[1] - -def test_for_loop() -> None: - a: Any = [] - for x in vec[float]([1.5, -3.5]): - a.append(x) - assert a == [1.5, -3.5] - - a = [] - for x in vec[bool]([True, False, False, True]): - a.append(x) - assert a == [True, False, False, True] - - a = [] - for x in vec[u8]([0, 5, 237, 255]): - a.append(x) - assert a == [0, 5, 237, 255] - -def test_comprehension() -> None: - v = vec[i32]([x + 1 for x in [1, 4, -5]]) - assert list(v) == [2, 5, -4] - - v = vec[float]([x + 1.5 for x in range(4)]) - assert list(v) == [1.5, 2.5, 3.5, 4.5] - -def test_nested_basics() -> None: - v = vec[vec[i32]]([vec[i32]([5, 6]), - vec[i32]([-12])]) - assert str(v) == "vec[vec[i32]]([[5, 6], [-12]])" - assert v[0][1] == 6 - assert v[1][0] == -12 - - v = vec[vec[i16]]([vec[i16]([5, 6]), - vec[i16]([-12])]) - assert str(v) == "vec[vec[i16]]([[5, 6], [-12]])" - assert v[0][1] == 6 - assert v[1][0] == -12 - - v = vec[vec[u8]]([vec[u8]([5, 6]), - vec[u8]([237])]) - assert str(v) == "vec[vec[u8]]([[5, 6], [237]])" - assert v[0][1] == 6 - assert v[1][0] == 237 - - v = vec[vec[float]]([vec[float]([1.5, -3.5]), - vec[float]([3.0])]) - assert str(v) == "vec[vec[float]]([[1.5, -3.5], [3.0]])" - assert v[0][1] == -3.5 - assert v[1][0] == 3.0 - - v = vec[vec[bool]]([vec[bool]([False, True]), - vec[bool]([False])]) - assert str(v) == "vec[vec[bool]]([[False, True], [False]])" - assert v[0][1] == True - assert v[1][0] == False - -def test_nested_unbox() -> None: - vv: vec[vec[i64]] - - a = cast(Any, vec[vec[i32]]([vec[i32]([5])])) - v_i32: vec[vec[i32]] = a - assert str(v_i32) == "vec[vec[i32]]([[5]])" - with assertRaises(TypeError): - vv = a - - a = cast(Any, vec[vec[i16]]([vec[i16]([5])])) - v_i16: vec[vec[i16]] = a - with assertRaises(TypeError): - vv = a - - a = cast(Any, vec[vec[u8]]([vec[u8]([5])])) - v_u8: vec[vec[u8]] = a - with assertRaises(TypeError): - vv = a - - a = cast(Any, vec[vec[float]]([vec[float]([5])])) - v_float: vec[vec[float]] = a - with assertRaises(TypeError): - vv = a - - a = cast(Any, vec[vec[bool]]([vec[bool]([True])])) - v_bool: vec[vec[bool]] = a - with assertRaises(TypeError): - vv = a - -def test_nested_misc() -> None: - v = vec[vec[float]]() - assert len(v) == 0 - v = append(v, vec[float]([1.5])) - assert len(v) == 1 - for x in v: - assert x == vec[float]([1.5]) - vv, x = pop(v) - assert vv == vec[vec[float]]() - assert x == vec[float]([1.5]) - v[0] = vec[float]([-2.5]) - assert v[0] == vec[float]([-2.5]) - v[0][0] = 3.5 - assert v[0] == vec[float]([3.5]) - assert v[:5] == vec[vec[float]]([vec[float]([3.5])]) diff --git a/mypyc/test-data/run-vecs-nested.test b/mypyc/test-data/run-vecs-nested.test deleted file mode 100644 index b20100644aff2..0000000000000 --- a/mypyc/test-data/run-vecs-nested.test +++ /dev/null @@ -1,468 +0,0 @@ -[case testVecNestedBasicOps_librt_experimental] -from typing import Final, Any, Iterable, Optional, Tuple -import sys - -from mypy_extensions import i64 -from librt.vecs import vec, append, remove, pop - -from testutil import assertRaises - -def test_construct_empty_nested() -> None: - v = vec[vec[str]]() - assert len(v) == 0 - v2 = vec[vec[vec[vec[vec[str]]]]]() - assert len(v2) == 0 - v3 = vec[vec[i64]]() - assert len(v3) == 0 - -def test_construct_empty_nested_optional() -> None: - v = vec[vec[Optional[str]]]() - assert len(v) == 0 - -def test_construct_from_initializer_nested() -> None: - v = vec[vec[str]]([vec[str](['xyz', 'a'])]) - assert len(v) == 1 - assert len(v[0]) == 2 - assert v[0][0] == 'xyz' - assert v[0][1] == 'a' - -def test_repr() -> None: - assert str(vec[vec[str]]()) == "vec[vec[str]]([])" - assert str(vec[vec[Optional[str]]]()) == "vec[vec[str | None]]([])" - if sys.version_info[1] >= 10: - assert str(vec[vec[str | None]]()) == "vec[vec[str | None]]([])" - -def test_append_nested() -> None: - v = vec[vec[str]]() - vv = append(vec[str](), '1') - v = append(v, vv) - assert str(v) == "vec[vec[str]]([['1']])" - for n in range(2, 10): - vv = append(vec[str](), str(n)) - v = append(v, vv) - assert str(v) == ( - "vec[vec[str]]([['1'], ['2'], ['3'], ['4'], ['5'], ['6'], ['7'], ['8'], ['9']])" - ) - -def test_append_i64() -> None: - v = vec[vec[i64]]() - vv = append(vec[i64](), 1) - v = append(v, vv) - assert str(v) == "vec[vec[i64]]([[1]])" - for n in range(2, 10): - vv = append(vec[i64](), n) - v = append(v, vv) - assert str(v) == ( - "vec[vec[i64]]([[1], [2], [3], [4], [5], [6], [7], [8], [9]])" - ) - -def test_append_nested_optional() -> None: - v = vec[vec[Optional[str]]]() - v = append(v, vec[Optional[str]]()) - v[0] = append(v[0], None) - v[0] = append(v[0], 'x') - assert str(v) == "vec[vec[str | None]]([[None, 'x']])" - -def test_get_item() -> None: - v = append(vec[vec[str]](), append(vec[str](), 'x')) - vv = v[0] - assert vv == vec[str](['x']) - v = append(v, append(vec[str](), 'y')) - assert repr(v[0]) == "vec[str](['x'])" - assert repr(v[1]) == "vec[str](['y'])" - -def test_get_item_error() -> None: - v = append(vec[vec[str]](), vec[str](['1'])) - v = append(v, vec[str](['3'])) - with assertRaises(IndexError): - v[2] - with assertRaises(IndexError): - v[-3] - -def test_len() -> None: - v = vec[vec[str]]() - assert len(v) == 0 - v = append(v, vec[str](['1'])) - assert len(v) == 1 - v = append(v, vec[str]()) - assert len(v) == 2 - for i in range(100): - v = append(v, vec[str](['i'])) - assert len(v) == i + 3 - -def test_set_item_error_nested() -> None: - v = append(vec[vec[str]](), vec[str]()) - v = append(v, vec[str]()) - v = append(v, vec[str]()) - with assertRaises(IndexError): - v[3] = vec[str]() - with assertRaises(IndexError): - v[-4] = vec[str]() - assert repr(v[0]) == 'vec[str]([])' - -def test_equality() -> None: - assert vec[vec[str]]() == vec[vec[str]]() - assert vec[vec[str]]([vec[str](['x'])]) == vec[vec[str]]([vec[str](['x'])]) - assert vec[vec[str]]([vec[str](['x'])]) != vec[vec[str]]() - assert vec[vec[str]]([vec[str](['x'])]) != vec[vec[str]]([vec[str](['y'])]) - -def test_equality_different_types() -> None: - a = vec[vec[str]]() - b = vec[str]() - assert a == a - assert b == b - assert b != a - assert a != b - - c = vec[vec[vec[str]]]() - assert c != a - assert a != c - assert c == c - - d = vec[vec[Optional[str]]]() - assert d != a - assert a != d - assert d == d - - assert vec[vec[str]]() != vec[vec[bytes]]() - assert vec[vec[str]]() != vec[vec[i64]]() - -def test_slicing() -> None: - v = vec[vec[str]](vec[str]([str(i)]) for i in range(5)) - assert v[1:4] == vec[vec[str]]([v[1], v[2], v[3]]) - assert v[2:-1] == vec[vec[str]]([v[2], v[3]]) - assert v[-2:-1] == vec[vec[str]]([v[3]]) - assert v[1:] == vec[vec[str]]([v[1], v[2], v[3], v[4]]) - assert v[:-1] == vec[vec[str]]([v[0], v[1], v[2], v[3]]) - assert v[:] == v - assert v[:] is not v - assert v[0:5] ==v - assert v[0:5] is not v - assert v[2:100] == vec[vec[str]]([v[2], v[3], v[4]]) - assert v[-100:2] == vec[vec[str]]([v[0], v[1]]) - assert v[5:100] == vec[vec[str]]([]) - assert v[50:100] == vec[vec[str]]([]) - assert v[-100:-50] == vec[vec[str]]([]) - -def test_slicing_with_step() -> None: - v = vec[vec[str]](vec[str]([str(i)]) for i in range(10)) - assert v[1:5:2] == vec[vec[str]]([v[1], v[3]]) - assert v[1:6:2] == vec[vec[str]]([v[1], v[3], v[5]]) - assert v[5:1:-2] == vec[vec[str]]([v[5], v[3]]) - assert v[6:1:-2] == vec[vec[str]]([v[6], v[4], v[2]]) - v = vec[vec[str]](vec[str]([str(i)]) for i in range(5)) - assert v[::-1] == vec[vec[str]]([v[4], v[3], v[2], v[1], v[0]]) - with assertRaises(ValueError): - v[1:3:0] - -def test_slicing_with_different_item_types() -> None: - v = vec[vec[i64]]([vec[i64]([11]), vec[i64]([22])]) - assert v[1:] == vec[vec[i64]]([vec[i64]([22])]) - -def test_remove() -> None: - a = ['4x', '7x', '9x'] - vv = [vec[str]([s]) for s in a] - for i, s in enumerate(a): - v = vec[vec[str]](vv) - v = remove(v, vv[i]) - assert v == vec[vec[str]]([j for j in vv if j != vv[i]]) - v = vec[vec[str]](vv) - # Make sure we have a different object identity - v = remove(v, vec[str]([a[i]])) - assert v == vec[vec[str]]([j for j in vv if j != vv[i]]) - v = vec[vec[str]](vv) - v = remove(v, vv[0]) - v = remove(v, vv[1]) - v = remove(v, vv[2]) - assert v == vec[vec[str]]() - with assertRaises(ValueError): - remove(v, vec[str](['4'])) - v = append(v, vec[str](['5'])) - with assertRaises(ValueError): - remove(v, vec[str](['7'])) - v = remove(v, vec[str](['5'])) - assert len(v) == 0 - v = v0 = vec[vec[str]]([vec[str](['x']) for _ in range(3)]) - v = remove(v, vec[str](['x'])) - assert v == v0[1:] - v = remove(v, vec[str](['x'])) - assert v == v0[2:] - v = remove(v, vec[str](['x'])) - assert v == vec[vec[str]]() - vb: Any = vec[bytes]([b'x']) - with assertRaises(TypeError): - remove(v, vb) - -def test_pop_last() -> None: - v = vec[vec[str]]([vec[str](['4']), vec[str](['6']), vec[str](['9'])]) - v, item = pop(v) - assert item == vec[str](['9']) - assert v == vec[vec[str]]([vec[str](['4']), vec[str](['6'])]) - v, item = pop(v) - assert item == vec[str](['6']) - assert v == vec[vec[str]]([vec[str](['4'])]) - v, item = pop(v) - assert item == vec[str](['4']) - assert v == vec[vec[str]]() - with assertRaises(IndexError): - pop(v) - -def test_pop_index() -> None: - v = vec[vec[str]](vec[str]([s]) for s in ['4', '7', '9', '15', '22']) - v, item = pop(v, 0) - assert item == vec[str](['4']) - assert v == vec[vec[str]](vec[str]([s]) for s in ['7', '9', '15', '22']) - v, item = pop(v, -1) - assert item == vec[str](['22']) - assert v == vec[vec[str]](vec[str]([s]) for s in ['7', '9', '15']) - v, item = pop(v, 1) - assert item == vec[str](['9']) - assert v == vec[vec[str]](vec[str]([s]) for s in ['7', '15']) - - with assertRaises(IndexError): - pop(v, 2) - - with assertRaises(IndexError): - pop(v, -3) - - v, item = pop(v, -2) - assert item == vec[str](['7']) - assert v == vec[vec[str]]([vec[str](['15'])]) - v, item = pop(v, 0) - assert item == vec[str](['15']) - assert v == vec[vec[str]]() - -def test_doubly_nested_operations() -> None: - a = vec[vec[str]]([vec[str](["x"])]) - b = vec[vec[str]]([vec[str]()]) - v = vec[vec[vec[str]]]() - - # append - v = append(v, a) - assert repr(v) == "vec[vec[vec[str]]]([[['x']]])" - v = append(v, b) - assert len(v) == 2 - - # get item - assert v[0] == a - assert v[1] == b - - # set item - v[0] = b - assert v[0] == b - v[0] = a - with assertRaises(IndexError): - v[2 + int()] - with assertRaises(IndexError): - v[2 + int()] = a - - # slicing - vv = v[:1] - assert len(vv) == 1 - assert vv[0] == a - vv = v[1:] - assert len(vv) == 1 - assert vv[0] == b - - # remove - v = remove(v, a) - assert len(v) == 1 - assert v[0] == b - v = remove(v, b) - assert len(v) == 0 - - # pop last - v = append(v, a) - v = append(v, b) - v, x = pop(v) - assert len(v) == 1 - assert v[0] == a - assert x == b - v, x = pop(v) - assert len(v) == 0 - assert x == a - with assertRaises(IndexError): - pop(v) - - # pop index - v = append(v, a) - v = append(v, b) - v, x = pop(v, 0) - assert len(v) == 1 - assert v[0] == b - assert x == a - - # TODO: box/unbox - -def test_vec_i64_nested_operations() -> None: - a = vec[i64]([11]) - b = vec[i64]() - v = vec[vec[i64]]() - - # append - v = append(v, a) - assert repr(v) == "vec[vec[i64]]([[11]])" - v = append(v, b) - assert len(v) == 2 - - # get item - assert v[0] == a - assert v[1] == b - - # set item - v[0] = b - assert v[0] == b - v[0] = a - with assertRaises(IndexError): - v[2 + int()] - with assertRaises(IndexError): - v[2 + int()] = a - - # slicing - vv = v[:1] - assert len(vv) == 1 - assert vv[0] == a - vv = v[1:] - assert len(vv) == 1 - assert vv[0] == b - - # remove - v = remove(v, a) - assert len(v) == 1 - assert v[0] == b - v = remove(v, b) - assert len(v) == 0 - - # pop last - v = append(v, a) - v = append(v, b) - v, x = pop(v) - assert len(v) == 1 - assert v[0] == a - assert x == b - v, x = pop(v) - assert len(v) == 0 - assert x == a - with assertRaises(IndexError): - pop(v) - - # pop index - v = append(v, a) - v = append(v, b) - v, x = pop(v, 0) - assert len(v) == 1 - assert v[0] == b - assert x == a - - # TODO: box/unbox - -def test_doubly_nested_vec_i64_operations() -> None: - a = vec[vec[i64]]([vec[i64]([11])]) - b = vec[vec[i64]]([vec[i64]()]) - v = vec[vec[vec[i64]]]() - - # append - v = append(v, a) - assert repr(v) == "vec[vec[vec[i64]]]([[[11]]])" - v = append(v, b) - assert len(v) == 2 - - # get item - assert v[0] == a - assert v[1] == b - - # set item - v[0] = b - assert v[0] == b - v[0] = a - with assertRaises(IndexError): - v[2 + int()] - with assertRaises(IndexError): - v[2 + int()] = a - - # slicing - vv = v[:1] - assert len(vv) == 1 - assert vv[0] == a - vv = v[1:] - assert len(vv) == 1 - assert vv[0] == b - - # remove - v = remove(v, a) - assert len(v) == 1 - assert v[0] == b - v = remove(v, b) - assert len(v) == 0 - - # pop last - v = append(v, a) - v = append(v, b) - v, x = pop(v) - assert len(v) == 1 - assert v[0] == a - assert x == b - v, x = pop(v) - assert len(v) == 0 - assert x == a - with assertRaises(IndexError): - pop(v) - - # pop index - v = append(v, a) - v = append(v, b) - v, x = pop(v, 0) - assert len(v) == 1 - assert v[0] == b - assert x == a - - # TODO: box/unbox - -def f(x: vec[vec[str]]) -> None: - pass - -def test_wrapper_arg_check() -> None: - f_any: Any = f - with assertRaises(TypeError): - f_any(vec[str]()) - with assertRaises(TypeError): - f_any(None) - with assertRaises(TypeError): - f_any([]) - -v: Final = vec[vec[str]]([vec[str](["x"])]) - -def test_final() -> None: - assert v == vec[vec[str]]([vec[str](["x"])]) - -def fdefault(v: vec[vec[str]] = vec[vec[str]]()) -> vec[vec[str]]: - return v - -def test_default_arg() -> None: - assert fdefault() == vec[vec[str]]() - v = vec[vec[str]]([vec[str](["x"])]) - assert fdefault(v) == v - f_any: Any = fdefault - assert f_any() == vec[vec[str]]() - assert f_any(v) == v - -def ftuple(b: bool) -> Tuple[vec[vec[str]], vec[vec[str]]]: - if b: - raise RuntimeError() - return vec[vec[str]]([vec[str]()]), vec[vec[str]]() - -def test_tuple() -> None: - t = ftuple(False) - assert len(t) == 2 - assert t[0] == vec[vec[str]]([vec[str]()]) - assert t[1] == vec[vec[str]]() - a, b = t - assert a == t[0] - assert b == t[1] - with assertRaises(RuntimeError): - ftuple(True) - f_any: Any = ftuple - t2 = f_any(False) - assert t == t2 - with assertRaises(RuntimeError): - f_any(True) diff --git a/mypyc/test-data/run-vecs-t.test b/mypyc/test-data/run-vecs-t.test deleted file mode 100644 index b9c6a6b0710c1..0000000000000 --- a/mypyc/test-data/run-vecs-t.test +++ /dev/null @@ -1,406 +0,0 @@ --- Test cases for vec[t] where t is a boxed, non-vec type (PyObject *). --- Also tests for vec[t | None], which uses the same representation. - -[case testVecTBasicOps_librt_experimental] -from typing import Final, Any, Iterable, Optional, Tuple - -from mypy_extensions import i64 -from librt.vecs import vec, append, remove, pop - -from testutil import assertRaises - -def test_create_empty() -> None: - v = vec[str]() - assert len(v) == 0 - v2 = vec[Optional[str]]() - assert len(v2) == 0 - -def test_create_from_list_and_get_item() -> None: - v = vec[str]([]) - assert len(v) == 0 - - v = vec[str](["xyz", "0" + str(), str(), "foo"]) - assert len(v) == 4 - assert v[0] == "xyz" - assert v[1] == "0" - assert v[2] == "" - assert v[3] == "foo" - - v = vec[str](["1", "2", "3", "4", "5", "6", "7", "8", "9"]) - assert len(v) == 9 - for i in range(i64(9)): - assert v[i] == str(i + 1) - -def test_create_optional_and_get_item() -> None: - v = vec[Optional[str]](["xyz", None]) - assert len(v) == 2 - assert v[0] == "xyz" - assert v[1] is None - -def test_append() -> None: - v = vec[str]() - v = append(v, str() + "3") - assert len(v) == 1 - x = "xyz" + str() - v = append(v, x) - assert len(v) == 2 - assert v[0] == "3" - assert v[1] == "xyz" - v = vec[str]() - for i in range(1024): - v = append(v, str(i * 3)) - assert len(v) == 1024 - for i in range(1024): - assert v[i] == str(i * 3) - -def test_get_item() -> None: - v = vec[str](["3", "4"]) - x: i64 = int() - assert v[x] == "3" - assert v[int()] == "3" - x += 1 - assert v[x] == "4" - assert v[int() + 1] == "4" - with assertRaises(IndexError): - v[x + 1] - with assertRaises(IndexError): - v[2] - with assertRaises(IndexError): - v[2**63 - 1] - -def test_set_item() -> None: - v = vec[str](["3", "4"]) - v[0] = "0" - assert v[0] == "0" - - x: i64 = int() - - v[x] = "4" - assert v[x] == "4" - - v[int()] = "5" - assert v[int()] == "5" - - x += 1 - - v[x] = "-5" - assert v[x] == "-5" - - v[1 + int()] = "xyz" - assert v[1] == "xyz" - - with assertRaises(IndexError): - v[1 + x] = "6" - with assertRaises(IndexError): - v[2 + int()] = "6" - -def test_set_item_optional() -> None: - v = vec[Optional[str]](["3", None]) - v[1] = str() + "a" - assert v[1] == "a" - v[0] = None - assert v[0] is None - -def test_box_and_unbox() -> None: - v = vec[str](["3", "5"]) - o: Any = v - assert len(o) == 2 - assert o[0] == "3" - assert o[1] == "5" - o[1] = "6" - v2: vec[str] = o - assert len(v2) == 2 - assert v2[0] == "3" - assert v2[1] == "6" - o2: Any = None - with assertRaises(TypeError, "vec[str] object expected; got None"): - v2 = o2 - o3: Any = vec[bytes]([]) - with assertRaises(TypeError, "vec[str] object expected; got vec"): - v2 = o3 - -def test_box_and_unbox_optional() -> None: - v = vec[Optional[str]](["3", "4"]) - o: Any = v - o[0] = None - v = o - assert v == vec[Optional[str]]([None, "4"]) - - v2 = vec[str](["4"]) - o2: Any = v2 - o2[0] = "5" - v2 = o2 - assert v2 == vec[str](["5"]) - - o3: Any = vec[str]() - # TODO: Use str | None instead union - with assertRaises(TypeError, "vec[str | None] object expected; got vec"): - v = o3 - - o4: Any = vec[Optional[str]](["3"]) - with assertRaises(TypeError, "vec[str] object expected; got vec"): - v2 = o4 - -def test_construct_from_list_multiply() -> None: - for i in range(50): - v = vec[str]([str(i + 1)] * i) - assert len(v) == i - for j in range(i): - assert v[j] == str(i + 1) - for i in range(50): - v = vec[str](i * [str(i - 1)]) - assert len(v) == i - for j in range(i): - assert v[j] == str(i - 1) - -def test_construct_from_list_comprehension() -> None: - for i in range(50): - l = [i * i for i in range(i)] - v = vec[str]([str(n + 5) for n in l]) - assert len(v) == i - for j in range(i): - assert v[j] == str(j * j + 5) - -def test_construct_from_iterable() -> None: - for i in range(50): - it: Iterable[str] = iter([str(i * i) for i in range(i)]) - v = vec[str](it) - assert len(v) == i - for j in range(i): - assert v[j] == str(j * j) - -def test_equality() -> None: - v0 = vec[str]() - v0b = vec[str]() - v1 = vec[str](["1"]) - v1b = vec[str](["3"]) - assert v0 == v0 - assert v1 == v1 - assert v0 == v0b - assert v0 != v1 - assert v1 != v1b - assert v1 != v0 - -def test_equality_optional() -> None: - v = vec[str]() - vo = vec[Optional[str]]() - assert v != vo - assert vo == vo - vo2 = vec[Optional[str]](["x", None]) - assert vo2 == vec[Optional[str]](["x", None]) - assert vo2 != vec[Optional[str]](["x", "y"]) - assert vo2 != vec[Optional[str]]([None, None]) - assert vo2 != vec[Optional[str]](["x", None, None]) - -def test_str_conversion() -> None: - v = vec[str]() - assert str(v) == "vec[str]([])" - assert repr(v) == "vec[str]([])" - v = vec[str](["126"]) - assert str(v) == "vec[str](['126'])" - v = append(v, "5") - assert str(v) == "vec[str](['126', '5'])" - v = append(v, "xyz") - assert str(v) == "vec[str](['126', '5', 'xyz'])" - v2 = vec[Optional[str]]() - assert str(v2) == "vec[str | None]([])" - v2 = vec[Optional[str]](["x", None]) - assert str(v2) == "vec[str | None](['x', None])" - -def test_for_loop() -> None: - for n in vec[str](): - assert False - a = [] - for n in vec[str](["5"]): - a.append(n) - assert a == ["5"] - v = vec[str](["xyz", "9", "8"]) - a = [] - for n in v: - a.append(n) - assert a == ["xyz", "9", "8"] - assert len(v) == 3 - -def test_contains() -> None: - v = vec[str]() - x = str() - assert x not in v - v = vec[str](["x"]) - assert x + "x" in v - assert x not in v - v2 = vec[str](["xyz", "7", "9"]) - assert x + "xyz" in v2 - assert x + "7" in v2 - assert x + "9" in v2 - assert x not in v2 - assert x + "x" not in v2 - -def test_slicing() -> None: - v = vec[str]() - assert v[:] == vec[str]([]) - assert v[1:] == vec[str]([]) - assert v[:-5] == vec[str]([]) - v = vec[str](["0", "1", "2", "3", "4"]) - assert v[1:4] == vec[str](["1", "2", "3"]) - assert v[2:-1] == vec[str](["2", "3"]) - assert v[-2:-1] == vec[str](["3"]) - assert v[1:] == vec[str](["1", "2", "3", "4"]) - assert v[:-1] == vec[str](["0", "1", "2", "3"]) - assert v[:] == v - assert v[:] is not v - assert v[5:] == vec[str]() - assert v[100:] == vec[str]() - assert v[0:5] ==v - assert v[0:5] is not v - assert v[2:100] == vec[str](["2", "3", "4"]) - assert v[-100:2] == vec[str](["0", "1"]) - assert v[5:100] == vec[str]([]) - assert v[50:100] == vec[str]([]) - assert v[-100:-50] == vec[str]([]) - -def test_slicing_with_step() -> None: - v = vec[str](["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]) - assert v[1:5:2] == vec[str](["1", "3"]) - assert v[1:6:2] == vec[str](["1", "3", "5"]) - assert v[5:1:-2] == vec[str](["5", "3"]) - assert v[6:1:-2] == vec[str](["6", "4", "2"]) - v = vec[str](["0", "1", "2", "3", "4"]) - assert v[::-1] == vec[str](["4", "3", "2", "1", "0"]) - with assertRaises(ValueError): - v[1:3:0] - -def test_remove() -> None: - a = ["4", "7", "xyz"] - for i in a: - v = vec[str](a) - v = remove(v, i) - assert v == vec[str]([j for j in a if j != i]) - v = vec[str](a) - v = remove(v, "4") - v = remove(v, "7") - v = remove(v, "xyz") - assert v == vec[str]() - with assertRaises(ValueError): - remove(v, "4") - v = append(v, "5") - with assertRaises(ValueError): - remove(v, "7") - v = remove(v, "5") - assert len(v) == 0 - v = vec[str](["1", "1", "1"]) - v = remove(v, "1") - assert v == vec[str](["1", "1"]) - v = remove(v, "1") - assert v == vec[str](["1"]) - v = remove(v, "1") - assert v == vec[str]() - f: Any = 1.1 - with assertRaises(TypeError): - remove(v, f) - s: Any = None - with assertRaises(TypeError): - remove(v, s) - -def test_pop_last() -> None: - v = vec[str](["4", "7", "xyz"]) - v, n = pop(v) - assert n == "xyz" - assert v == vec[str](["4", "7"]) - v, n = pop(v) - assert n == "7" - assert v == vec[str](["4"]) - v, n = pop(v) - assert n == "4" - assert v == vec[str]() - with assertRaises(IndexError): - pop(v) - -def test_pop_index() -> None: - v = vec[str](["4", "7", "9", "15", "22"]) - v, n = pop(v, 0) - assert n == "4" - assert v == vec[str](["7", "9", "15", "22"]) - v, n = pop(v, -1) - assert n == "22" - assert v == vec[str](["7", "9", "15"]) - v, n = pop(v, 1) - assert n == "9" - assert v == vec[str](["7", "15"]) - - with assertRaises(IndexError): - pop(v, 2) - - with assertRaises(IndexError): - pop(v, -3) - - v, n = pop(v, -2) - assert n == "7" - assert v == vec[str](["15"]) - v, n = pop(v, 0) - assert n == "15" - assert v == vec[str]() - -def test_pop_optional() -> None: - v = vec[Optional[str]](["x" + str(), None]) - v, s = pop(v) - assert s is None - v, s = pop(v) - assert s == "x" - assert len(v) == 0 - v = vec[Optional[str]](["x" + str(), None]) - v, s = pop(v, 0) - assert s == "x" - v, s = pop(v, -1) - assert s is None - -def f(x: vec[str]) -> None: - pass - -def test_wrapper_arg_check() -> None: - f_any: Any = f - with assertRaises(TypeError): - f_any([]) - -v: Final = vec[str](["x", "y"]) - -def test_final() -> None: - assert v == vec[str](["x", "y"]) - -def fdefault(v: vec[str] = vec[str](["x"])) -> vec[str]: - return v - -def test_default_arg() -> None: - assert fdefault() == vec[str](["x"]) - assert fdefault(vec[str](["y"])) == vec[str](["y"]) - f_any: Any = fdefault - assert f_any() == vec[str](["x"]) - assert f_any(vec[str](["y"])) == vec[str](["y"]) - -def ftuple(b: bool) -> Tuple[vec[str], vec[str]]: - if b: - raise RuntimeError() - return vec[str](["x"]), vec[str]() - -def test_tuple() -> None: - t = ftuple(False) - assert len(t) == 2 - assert t[0] == vec[str](["x"]) - assert t[1] == vec[str]() - a, b = t - assert a == t[0] - assert b == t[1] - with assertRaises(RuntimeError): - ftuple(True) - f_any: Any = ftuple - t2 = f_any(False) - assert t == t2 - with assertRaises(RuntimeError): - f_any(True) - -def test_optional_item_new_syntax() -> None: - v = vec[str | None]() - assert len(v) == 0 - assert str(v) == "vec[str | None]([])" - v = append(v, 'x') - v = append(v, None) - assert str(v) == "vec[str | None](['x', None])" From d20ca79dc2c01ce57481f0bcd6a6ea30965b534e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 2 Feb 2026 16:39:37 +0000 Subject: [PATCH 169/180] WIP temporarily delete various irbuild vec tests --- mypyc/test-data/irbuild-vec-misc.test | 346 ----------------------- mypyc/test-data/irbuild-vec-nested.test | 311 -------------------- mypyc/test-data/irbuild-vec-t.test | 361 ------------------------ 3 files changed, 1018 deletions(-) delete mode 100644 mypyc/test-data/irbuild-vec-misc.test delete mode 100644 mypyc/test-data/irbuild-vec-nested.test delete mode 100644 mypyc/test-data/irbuild-vec-t.test diff --git a/mypyc/test-data/irbuild-vec-misc.test b/mypyc/test-data/irbuild-vec-misc.test deleted file mode 100644 index d48162e1e8300..0000000000000 --- a/mypyc/test-data/irbuild-vec-misc.test +++ /dev/null @@ -1,346 +0,0 @@ --- Test cases for packed/specialized vec item types other than i64, such as --- vec[i32] and vec[float]. Since many of the code paths are the same as for --- vec[i64], only test a subset of functionality. --- --- vec[i64] test cases are in irbuild-vec-i64.test. - -[case testVecMiscCreateEmpty] -from librt.vecs import vec - -from mypy_extensions import i32, i16, u8 - -def create_float() -> vec[float]: - return vec[float]() - -def create_i32() -> vec[i32]: - return vec[i32]() - -def create_i16() -> vec[i16]: - return vec[i16]() - -def create_u8() -> vec[u8]: - return vec[u8]() - -def create_bool() -> vec[bool]: - return vec[bool]() -[out] -def create_float(): - r0 :: vec[float] -L0: - r0 = VecFloatApi.alloc(0, 0) - return r0 -def create_i32(): - r0 :: vec[i32] -L0: - r0 = VecI32Api.alloc(0, 0) - return r0 -def create_i16(): - r0 :: vec[i16] -L0: - r0 = VecI16Api.alloc(0, 0) - return r0 -def create_u8(): - r0 :: vec[u8] -L0: - r0 = VecU8Api.alloc(0, 0) - return r0 -def create_bool(): - r0 :: vec[bool] -L0: - r0 = VecBoolApi.alloc(0, 0) - return r0 - -[case testVecMiscCreateFromList] -from librt.vecs import vec - -from mypy_extensions import i64, i32 - -def create_i32() -> vec[i32]: - return vec[i32]([1, -5]) -[out] -def create_i32(): - r0 :: vec[i32] - r1 :: object - r2, r3, r4 :: ptr -L0: - r0 = VecI32Api.alloc(2, 2) - r1 = r0.buf - r2 = get_element_ptr r1 items :: VecI32BufObject - set_mem r2, 1 :: i32* - r3 = r2 + 4 - set_mem r3, -5 :: i32* - r4 = r3 + 4 - keep_alive r0 - return r0 - -[case testVecMiscLen_64bit] -from librt.vecs import vec - -from mypy_extensions import i64, i32 - -def len_i32(v: vec[i32]) -> i64: - return len(v) -[out] -def len_i32(v): - v :: vec[i32] - r0 :: native_int -L0: - r0 = v.len - return r0 - -[case testVecMiscAppend] -from librt.vecs import vec, append - -from mypy_extensions import i32 - -def append_float(v: vec[float]) -> vec[float]: - return append(v, 1.5) - -def append_i32(v: vec[i32]) -> vec[i32]: - return append(v, 123) -[out] -def append_float(v): - v, r0 :: vec[float] -L0: - r0 = VecFloatApi.append(v, 1.5) - return r0 -def append_i32(v): - v, r0 :: vec[i32] -L0: - r0 = VecI32Api.append(v, 123) - return r0 - -[case testVecMiscGetItem_64bit] -from librt.vecs import vec - -from mypy_extensions import i64 - -def get_item_bool(v: vec[bool], i: i64) -> bool: - return v[i] -[out] -def get_item_bool(v, i): - v :: vec[bool] - i :: i64 - r0 :: native_int - r1 :: bit - r2 :: i64 - r3 :: bit - r4 :: bool - r5 :: i64 - r6 :: object - r7 :: ptr - r8 :: i64 - r9 :: ptr - r10 :: bool -L0: - r0 = v.len - r1 = i < r0 :: unsigned - if r1 goto L4 else goto L1 :: bool -L1: - r2 = i + r0 - r3 = r2 < r0 :: unsigned - if r3 goto L3 else goto L2 :: bool -L2: - r4 = raise IndexError - unreachable -L3: - r5 = r2 - goto L5 -L4: - r5 = i -L5: - r6 = v.buf - r7 = get_element_ptr r6 items :: VecBoolBufObject - r8 = r5 * 1 - r9 = r7 + r8 - r10 = load_mem r9 :: builtins.bool* - keep_alive v - return r10 - -[case testVecMiscPop] -from librt.vecs import vec, pop - -from mypy_extensions import i64 - -def pop_float(v: vec[float], i: i64) -> float: - v, x = pop(v) - return x -[out] -def pop_float(v, i): - v :: vec[float] - i :: i64 - r0 :: tuple[vec[float], float] - r1 :: vec[float] - r2 :: float - r3 :: vec[float] - r4, x :: float -L0: - r0 = VecFloatApi.pop(v, -1) - r1 = borrow r0[0] - r2 = borrow r0[1] - keep_alive steal r0 - r3 = unborrow r1 - v = r3 - r4 = unborrow r2 - x = r4 - return x - -[case testVecMiscRemove] -from librt.vecs import vec, remove - -def remove_float(v: vec[float]) -> vec[float]: - return remove(v, 1.5) -[out] -def remove_float(v): - v, r0 :: vec[float] -L0: - r0 = VecFloatApi.remove(v, 1.5) - return r0 - -[case testVecMiscSlice] -from librt.vecs import vec, remove - -from mypy_extensions import i64 - -def remove_float(v: vec[float], x: i64, y: i64) -> vec[float]: - return v[x:y] -[out] -def remove_float(v, x, y): - v :: vec[float] - x, y :: i64 - r0 :: vec[float] -L0: - r0 = VecFloatApi.slice(v, x, y) - return r0 - -[case testVecMiscForLoop] -from librt.vecs import vec, remove - -from mypy_extensions import i64, i16 - -def for_bool(v: vec[i16]) -> i16: - s: i16 = 0 - for x in v: - s += x - return s -[out] -def for_bool(v): - v :: vec[i16] - s :: i16 - r0, r1 :: native_int - r2 :: bit - r3 :: object - r4 :: ptr - r5 :: native_int - r6 :: ptr - r7, x, r8 :: i16 - r9 :: native_int -L0: - s = 0 - r0 = 0 -L1: - r1 = v.len - r2 = r0 < r1 :: signed - if r2 goto L2 else goto L4 :: bool -L2: - r3 = v.buf - r4 = get_element_ptr r3 items :: VecI16BufObject - r5 = r0 * 2 - r6 = r4 + r5 - r7 = load_mem r6 :: i16* - keep_alive v - x = r7 - r8 = s + x - s = r8 -L3: - r9 = r0 + 1 - r0 = r9 - goto L1 -L4: - return s - -[case testVecMiscNestedGetItem_64bit] -from librt.vecs import vec - -from mypy_extensions import i64, i32 - -def get_item_nested(v: vec[vec[i32]], i: i64) -> vec[i32]: - return v[i] -[out] -def get_item_nested(v, i): - v :: vec[vec[i32]] - i :: i64 - r0 :: native_int - r1 :: bit - r2 :: i64 - r3 :: bit - r4 :: bool - r5 :: i64 - r6 :: object - r7 :: ptr - r8 :: i64 - r9 :: ptr - r10 :: vec[i32] -L0: - r0 = v.len - r1 = i < r0 :: unsigned - if r1 goto L4 else goto L1 :: bool -L1: - r2 = i + r0 - r3 = r2 < r0 :: unsigned - if r3 goto L3 else goto L2 :: bool -L2: - r4 = raise IndexError - unreachable -L3: - r5 = r2 - goto L5 -L4: - r5 = i -L5: - r6 = v.buf - r7 = get_element_ptr r6 items :: VecNestedBufObject - r8 = r5 * 16 - r9 = r7 + r8 - r10 = load_mem r9 :: vec[i32]* - keep_alive v - return r10 - -[case testVecMiscNestedPop_64bit] -from librt.vecs import vec, pop - -from mypy_extensions import i64, i32 - -def get_item_nested(v: vec[vec[i32]], i: i64) -> vec[i32]: - v, x = pop(v, i) - return x -[out] -def get_item_nested(v, i): - v :: vec[vec[i32]] - i :: i64 - r0 :: tuple[vec[vec[i32]], VecNestedBufItem{len:native_int, buf:object_nrc}] - r1, r2 :: vec[vec[i32]] - r3, r4 :: VecNestedBufItem{len:native_int, buf:object_nrc} - r5 :: vec[i32] - r6 :: tuple[vec[vec[i32]], vec[i32]] - r7 :: vec[vec[i32]] - r8 :: vec[i32] - r9 :: vec[vec[i32]] - r10, x :: vec[i32] -L0: - r0 = VecNestedApi.pop(v, i) - r1 = borrow r0[0] - r2 = unborrow r1 - r3 = borrow r0[1] - r4 = unborrow r3 - r5 = VecI32Api.convert_from_nested(r4) - r6 = (r2, r5) - keep_alive steal r0 - r7 = borrow r6[0] - r8 = borrow r6[1] - keep_alive steal r6 - r9 = unborrow r7 - v = r9 - r10 = unborrow r8 - x = r10 - return x diff --git a/mypyc/test-data/irbuild-vec-nested.test b/mypyc/test-data/irbuild-vec-nested.test deleted file mode 100644 index 6b3389394d93d..0000000000000 --- a/mypyc/test-data/irbuild-vec-nested.test +++ /dev/null @@ -1,311 +0,0 @@ --- Test cases for nested vecs - -[case testVecNestedCreateEmpty] -from librt.vecs import vec, append -from mypy_extensions import i64 - -def f() -> vec[vec[str]]: - return vec[vec[str]]() - -def g() -> vec[vec[i64]]: - return vec[vec[i64]]() -[out] -def f(): - r0 :: object - r1 :: ptr - r2 :: vec[vec[str]] -L0: - r0 = load_address PyUnicode_Type - r1 = r0 - r2 = VecNestedApi.alloc(0, 0, r1, 1) - return r2 -def g(): - r0 :: vec[vec[i64]] -L0: - r0 = VecNestedApi.alloc(0, 0, 2, 1) - return r0 - -[case testVecNestedAppend] -from librt.vecs import vec, append - -def f(v: vec[vec[str]], vv: vec[str]) -> vec[vec[str]]: - return append(v, vv) -[out] -def f(v, vv): - v :: vec[vec[str]] - vv :: vec[str] - r0 :: native_int - r1 :: object - r2, r3 :: VecNestedBufItem{len:native_int, buf:object_nrc} - r4 :: vec[vec[str]] -L0: - r0 = vv.len - r1 = vv.buf - r2 = set_element undef VecNestedBufItem, len, r0 - r3 = set_element r2, buf, r1 - r4 = VecNestedApi.append(v, r3) - keep_alive vv - return r4 - -[case testVecNestedVecI64Append] -from librt.vecs import vec, append -from mypy_extensions import i64 - -def f(v: vec[vec[i64]], vv: vec[i64]) -> vec[vec[i64]]: - return append(v, vv) -[out] -def f(v, vv): - v :: vec[vec[i64]] - vv :: vec[i64] - r0 :: native_int - r1 :: object - r2, r3 :: VecNestedBufItem{len:native_int, buf:object_nrc} - r4 :: vec[vec[i64]] -L0: - r0 = vv.len - r1 = vv.buf - r2 = set_element undef VecNestedBufItem, len, r0 - r3 = set_element r2, buf, r1 - r4 = VecNestedApi.append(v, r3) - keep_alive vv - return r4 - -[case testVecNestedLen_64bit] -from librt.vecs import vec -from mypy_extensions import i64 -from typing import Optional - -def f(v: vec[vec[str]]) -> i64: - return len(v) - -def g(v: vec[vec[i64]]) -> i64: - return len(v) -[out] -def f(v): - v :: vec[vec[str]] - r0 :: native_int -L0: - r0 = v.len - return r0 -def g(v): - v :: vec[vec[i64]] - r0 :: native_int -L0: - r0 = v.len - return r0 - -[case testVecNestedGetItem_64bit] -from librt.vecs import vec -from mypy_extensions import i64 - -def f(v: vec[vec[str]], n: i64) -> vec[str]: - return v[n] -[out] -def f(v, n): - v :: vec[vec[str]] - n :: i64 - r0 :: native_int - r1 :: bit - r2 :: i64 - r3 :: bit - r4 :: bool - r5 :: i64 - r6 :: object - r7 :: ptr - r8 :: i64 - r9 :: ptr - r10 :: vec[str] -L0: - r0 = v.len - r1 = n < r0 :: unsigned - if r1 goto L4 else goto L1 :: bool -L1: - r2 = n + r0 - r3 = r2 < r0 :: unsigned - if r3 goto L3 else goto L2 :: bool -L2: - r4 = raise IndexError - unreachable -L3: - r5 = r2 - goto L5 -L4: - r5 = n -L5: - r6 = v.buf - r7 = get_element_ptr r6 items :: VecNestedBufObject - r8 = r5 * 16 - r9 = r7 + r8 - r10 = load_mem r9 :: vec[str]* - keep_alive v - return r10 - -[case testVecNestedI64GetItem_64bit] -from librt.vecs import vec -from mypy_extensions import i64 - -def f(v: vec[vec[i64]], n: i64) -> vec[i64]: - return v[n] -[out] -def f(v, n): - v :: vec[vec[i64]] - n :: i64 - r0 :: native_int - r1 :: bit - r2 :: i64 - r3 :: bit - r4 :: bool - r5 :: i64 - r6 :: object - r7 :: ptr - r8 :: i64 - r9 :: ptr - r10 :: vec[i64] -L0: - r0 = v.len - r1 = n < r0 :: unsigned - if r1 goto L4 else goto L1 :: bool -L1: - r2 = n + r0 - r3 = r2 < r0 :: unsigned - if r3 goto L3 else goto L2 :: bool -L2: - r4 = raise IndexError - unreachable -L3: - r5 = r2 - goto L5 -L4: - r5 = n -L5: - r6 = v.buf - r7 = get_element_ptr r6 items :: VecNestedBufObject - r8 = r5 * 16 - r9 = r7 + r8 - r10 = load_mem r9 :: vec[i64]* - keep_alive v - return r10 - -[case testVecNestedI64GetItemWithBorrow_64bit] -from librt.vecs import vec -from mypy_extensions import i64 - -def f(v: vec[vec[i64]], n: i64) -> i64: - return v[n][n] -[out] -def f(v, n): - v :: vec[vec[i64]] - n :: i64 - r0 :: native_int - r1 :: bit - r2 :: i64 - r3 :: bit - r4 :: bool - r5 :: i64 - r6 :: object - r7 :: ptr - r8 :: i64 - r9 :: ptr - r10 :: vec[i64] - r11 :: native_int - r12 :: bit - r13 :: i64 - r14 :: bit - r15 :: bool - r16 :: i64 - r17 :: object - r18 :: ptr - r19 :: i64 - r20 :: ptr - r21 :: i64 -L0: - r0 = v.len - r1 = n < r0 :: unsigned - if r1 goto L4 else goto L1 :: bool -L1: - r2 = n + r0 - r3 = r2 < r0 :: unsigned - if r3 goto L3 else goto L2 :: bool -L2: - r4 = raise IndexError - unreachable -L3: - r5 = r2 - goto L5 -L4: - r5 = n -L5: - r6 = v.buf - r7 = get_element_ptr r6 items :: VecNestedBufObject - r8 = r5 * 16 - r9 = r7 + r8 - r10 = borrow load_mem r9 :: vec[i64]* - r11 = r10.len - r12 = n < r11 :: unsigned - if r12 goto L9 else goto L6 :: bool -L6: - r13 = n + r11 - r14 = r13 < r11 :: unsigned - if r14 goto L8 else goto L7 :: bool -L7: - r15 = raise IndexError - unreachable -L8: - r16 = r13 - goto L10 -L9: - r16 = n -L10: - r17 = r10.buf - r18 = get_element_ptr r17 items :: VecI64BufObject - r19 = r16 * 8 - r20 = r18 + r19 - r21 = load_mem r20 :: i64* - keep_alive v, r10 - return r21 - -[case testVecDoublyNestedGetItem_64bit] -from librt.vecs import vec -from mypy_extensions import i64 - -def f(v: vec[vec[vec[str]]], n: i64) -> vec[vec[str]]: - return v[n] -[out] -def f(v, n): - v :: vec[vec[vec[str]]] - n :: i64 - r0 :: native_int - r1 :: bit - r2 :: i64 - r3 :: bit - r4 :: bool - r5 :: i64 - r6 :: object - r7 :: ptr - r8 :: i64 - r9 :: ptr - r10 :: vec[vec[str]] -L0: - r0 = v.len - r1 = n < r0 :: unsigned - if r1 goto L4 else goto L1 :: bool -L1: - r2 = n + r0 - r3 = r2 < r0 :: unsigned - if r3 goto L3 else goto L2 :: bool -L2: - r4 = raise IndexError - unreachable -L3: - r5 = r2 - goto L5 -L4: - r5 = n -L5: - r6 = v.buf - r7 = get_element_ptr r6 items :: VecNestedBufObject - r8 = r5 * 16 - r9 = r7 + r8 - r10 = load_mem r9 :: vec[vec[str]]* - keep_alive v - return r10 diff --git a/mypyc/test-data/irbuild-vec-t.test b/mypyc/test-data/irbuild-vec-t.test deleted file mode 100644 index 36132a5cac4cb..0000000000000 --- a/mypyc/test-data/irbuild-vec-t.test +++ /dev/null @@ -1,361 +0,0 @@ --- Test cases for vec[t] where t is a boxed, non-vec type (PyObject *). --- Also tests for vec[t | None], which uses the same representation. - -[case testVecTCreateEmpty] -from librt.vecs import vec, append - -class C: pass - -def primitive() -> vec[str]: - return vec[str]() - -def native_class() -> vec[C]: - return vec[C]() -[out] -def primitive(): - r0 :: object - r1 :: ptr - r2 :: vec[str] -L0: - r0 = load_address PyUnicode_Type - r1 = r0 - r2 = VecTApi.alloc(0, 0, r1) - return r2 -def native_class(): - r0 :: object - r1 :: ptr - r2 :: vec[__main__.C] -L0: - r0 = __main__.C :: type - r1 = r0 - r2 = VecTApi.alloc(0, 0, r1) - return r2 - -[case testVecTAppend] -from librt.vecs import vec, append - -def f(v: vec[str]) -> vec[str]: - return append(v, 'x') -[out] -def f(v): - v :: vec[str] - r0 :: str - r1 :: object - r2 :: ptr - r3 :: vec[str] -L0: - r0 = 'x' - r1 = load_address PyUnicode_Type - r2 = r1 - r3 = VecTApi.append(v, r0, r2) - return r3 - -[case testVecTOptionalCreateEmpty] -from librt.vecs import vec, append -from typing import Optional - -class C: pass - -def primitive() -> vec[Optional[str]]: - return vec[Optional[str]]() - -def native_class() -> vec[Optional[C]]: - return vec[Optional[C]]() -[out] -def primitive(): - r0 :: object - r1, r2 :: ptr - r3 :: vec[str | None] -L0: - r0 = load_address PyUnicode_Type - r1 = r0 - r2 = r1 | 1 - r3 = VecTApi.alloc(0, 0, r2) - return r3 -def native_class(): - r0 :: object - r1, r2 :: ptr - r3 :: vec[__main__.C | None] -L0: - r0 = __main__.C :: type - r1 = r0 - r2 = r1 | 1 - r3 = VecTApi.alloc(0, 0, r2) - return r3 - -[case testVecTOptionalAppend] -from librt.vecs import vec, append -from typing import Optional - -def f(v: vec[Optional[str]]) -> vec[Optional[str]]: - v = append(v, 'x') - return append(v, None) -[out] -def f(v): - v :: vec[str | None] - r0 :: str - r1 :: object - r2, r3 :: ptr - r4 :: vec[str | None] - r5, r6 :: object - r7, r8 :: ptr - r9 :: vec[str | None] -L0: - r0 = 'x' - r1 = load_address PyUnicode_Type - r2 = r1 - r3 = r2 | 1 - r4 = VecTApi.append(v, r0, r3) - v = r4 - r5 = box(None, 1) - r6 = load_address PyUnicode_Type - r7 = r6 - r8 = r7 | 1 - r9 = VecTApi.append(v, r5, r8) - return r9 - -[case testVecTLen_64bit] -from librt.vecs import vec -from mypy_extensions import i64 -from typing import Optional - -def f(v: vec[str]) -> i64: - return len(v) - -def g(v: vec[Optional[str]]) -> i64: - return len(v) -[out] -def f(v): - v :: vec[str] - r0 :: native_int -L0: - r0 = v.len - return r0 -def g(v): - v :: vec[str | None] - r0 :: native_int -L0: - r0 = v.len - return r0 - -[case testVecTGetItem_64bit] -from librt.vecs import vec -from mypy_extensions import i64 - -def f(v: vec[str], n: i64) -> str: - return v[n] -[out] -def f(v, n): - v :: vec[str] - n :: i64 - r0 :: native_int - r1 :: bit - r2 :: i64 - r3 :: bit - r4 :: bool - r5 :: i64 - r6 :: object - r7 :: ptr - r8 :: i64 - r9 :: ptr - r10 :: str -L0: - r0 = v.len - r1 = n < r0 :: unsigned - if r1 goto L4 else goto L1 :: bool -L1: - r2 = n + r0 - r3 = r2 < r0 :: unsigned - if r3 goto L3 else goto L2 :: bool -L2: - r4 = raise IndexError - unreachable -L3: - r5 = r2 - goto L5 -L4: - r5 = n -L5: - r6 = v.buf - r7 = get_element_ptr r6 items :: VecTBufObject - r8 = r5 * 8 - r9 = r7 + r8 - r10 = load_mem r9 :: builtins.str* - keep_alive v - return r10 - -[case testVecTOptionalGetItem_64bit] -from librt.vecs import vec -from mypy_extensions import i64 -from typing import Optional - -def f(v: vec[Optional[str]], n: i64) -> Optional[str]: - return v[n] -[out] -def f(v, n): - v :: vec[str | None] - n :: i64 - r0 :: native_int - r1 :: bit - r2 :: i64 - r3 :: bit - r4 :: bool - r5 :: i64 - r6 :: object - r7 :: ptr - r8 :: i64 - r9 :: ptr - r10 :: union[str, None] -L0: - r0 = v.len - r1 = n < r0 :: unsigned - if r1 goto L4 else goto L1 :: bool -L1: - r2 = n + r0 - r3 = r2 < r0 :: unsigned - if r3 goto L3 else goto L2 :: bool -L2: - r4 = raise IndexError - unreachable -L3: - r5 = r2 - goto L5 -L4: - r5 = n -L5: - r6 = v.buf - r7 = get_element_ptr r6 items :: VecTBufObject - r8 = r5 * 8 - r9 = r7 + r8 - r10 = load_mem r9 :: union* - keep_alive v - return r10 - -[case testNewTPopLast] -from typing import Tuple -from librt.vecs import vec, pop - -def pop_last(v: vec[str]) -> Tuple[vec[str], str]: - return pop(v) -[out] -def pop_last(v): - v :: vec[str] - r0 :: tuple[vec[str], str] -L0: - r0 = VecTApi.pop(v, -1) - return r0 - -[case testVecTConstructFromListComprehension] -from librt.vecs import vec -from mypy_extensions import i64 - -def f(n: i64) -> vec[str]: - return vec[str](['x' for x in range(i64(5))]) -[out] -def f(n): - n :: i64 - r0 :: object - r1 :: ptr - r2, r3 :: vec[str] - r4, x :: i64 - r5 :: bit - r6 :: str - r7 :: object - r8 :: ptr - r9 :: vec[str] - r10 :: i64 -L0: - r0 = load_address PyUnicode_Type - r1 = r0 - r2 = VecTApi.alloc(0, 0, r1) - r3 = r2 - r4 = 0 - x = r4 -L1: - r5 = r4 < 5 :: signed - if r5 goto L2 else goto L4 :: bool -L2: - r6 = 'x' - r7 = load_address PyUnicode_Type - r8 = r7 - r9 = VecTApi.append(r3, r6, r8) - r3 = r9 -L3: - r10 = r4 + 1 - r4 = r10 - x = r10 - goto L1 -L4: - return r3 - -[case testVecTCheckItemType] -from librt.vecs import vec -from typing import Tuple, Any, List, TypeVar - -def bad1(v: vec[Tuple[str, str]]) -> None: pass -def bad2(v: vec[int]) -> None: pass -def bad3(v: vec) -> None: pass -def bad4(v: vec[Any]) -> None: pass -T = TypeVar("T") -def bad5(v: vec[T]) -> None: pass - -def ok1(v: vec[str], v2: vec[bytes]) -> None: pass -def ok2(v: vec[Tuple[str, ...]]) -> None: pass -def ok3(v: vec[object]) -> None: pass -def ok4(v: vec[List[Any]]) -> None: pass -def ok5(v: vec[vec[str]]) -> None: pass -[out] -main:4: error: Invalid item type for "vec" -main:5: error: Invalid item type for "vec" -main:6: error: Invalid item type for "vec" -main:7: error: Invalid item type for "vec" -main:9: error: Invalid item type for "vec" - -[case testVecTCheckItemTypeUnion] -from librt.vecs import vec -from typing import Tuple, Union, Optional, Any, TypeVar -from mypy_extensions import i64 - -def bad1(v: vec[Union[str, bytes]]) -> None: pass -def bad2(v: vec[Union[str, bytes, None]]) -> None: pass -def bad3(v: vec[Union[int, None]]) -> None: pass -def bad4(v: vec[Union[None, int]]) -> None: pass -def bad5(v: vec[Union[i64, None]]) -> None: pass -def bad6(v: vec[Union[None, i64]]) -> None: pass -def bad7(v: vec[Union[bool, None]]) -> None: pass -def bad9(v: vec[Union[float, None]]) -> None: pass -def bad10(v: vec[Union[vec[str], None]]) -> None: pass -def bad11(v: vec[Optional[vec[str]]]) -> None: pass -def bad12(v: vec[Union[str, Any]]) -> None: pass -T = TypeVar("T") -def bad13(v: vec[Union[str, T]]) -> None: pass - -def ok1(v: vec[Union[str, None]]) -> None: pass -def ok2(v: vec[Union[None, str]]) -> None: pass -[out] -main:5: error: Invalid item type for "vec" -main:6: error: Invalid item type for "vec" -main:7: error: Invalid item type for "vec" -main:8: error: Invalid item type for "vec" -main:9: error: Invalid item type for "vec" -main:10: error: Invalid item type for "vec" -main:11: error: Invalid item type for "vec" -main:12: error: Invalid item type for "vec" -main:13: error: Invalid item type for "vec" -main:14: error: Invalid item type for "vec" -main:15: error: Invalid item type for "vec" -main:17: error: Invalid item type for "vec" - -[case testVecTCheckItemTypeDuringConstruction] -from librt.vecs import vec -from typing import Optional, Union - -def f() -> None: - vec[Optional[int]]() - vec[Union[str, bytes]]() - vec[Optional[vec[str]]]() - -[out] -main:5: error: Invalid item type for "vec" -main:6: error: Invalid item type for "vec" -main:7: error: Invalid item type for "vec" From ff1e2d9d6c4f1ef34d00d57e4e3ee0371d57e4f6 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 2 Feb 2026 16:40:29 +0000 Subject: [PATCH 170/180] WIP delete tests from test runners --- mypyc/test/test_irbuild.py | 3 --- mypyc/test/test_run.py | 4 ---- 2 files changed, 7 deletions(-) diff --git a/mypyc/test/test_irbuild.py b/mypyc/test/test_irbuild.py index 92048277565d3..4874a6205103a 100644 --- a/mypyc/test/test_irbuild.py +++ b/mypyc/test/test_irbuild.py @@ -45,9 +45,6 @@ "irbuild-i16.test", "irbuild-u8.test", "irbuild-vec-i64.test", - "irbuild-vec-misc.test", - "irbuild-vec-t.test", - "irbuild-vec-nested.test", "irbuild-vectorcall.test", "irbuild-unreachable.test", "irbuild-isinstance.test", diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index b33efed16d187..7b855841e220f 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -82,10 +82,6 @@ "run-vecs-misc-interp.test", "run-vecs-t-interp.test", "run-vecs-nested-interp.test", - "run-vecs-i64.test", - "run-vecs-misc.test", - "run-vecs-t.test", - "run-vecs-nested.test", ] if sys.version_info >= (3, 12): From bf0866f726aeab0e705095d5bd529d8cca098f92 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 2 Feb 2026 16:42:58 +0000 Subject: [PATCH 171/180] WIP delete codegen related changes temporarily --- mypyc/codegen/emit.py | 123 +----------------------------------- mypyc/codegen/emitclass.py | 4 +- mypyc/codegen/emitfunc.py | 55 ++++++++-------- mypyc/codegen/emitmodule.py | 16 +---- mypyc/test/test_emitfunc.py | 97 ++-------------------------- 5 files changed, 37 insertions(+), 258 deletions(-) diff --git a/mypyc/codegen/emit.py b/mypyc/codegen/emit.py index f75f8d8a6bde6..409018d284108 100644 --- a/mypyc/codegen/emit.py +++ b/mypyc/codegen/emit.py @@ -21,25 +21,16 @@ REG_PREFIX, STATIC_PREFIX, TYPE_PREFIX, - TYPE_VAR_PREFIX, ) from mypyc.ir.class_ir import ClassIR, all_concrete_classes from mypyc.ir.func_ir import FUNC_STATICMETHOD, FuncDecl, FuncIR, get_text_signature -from mypyc.ir.ops import ( - NAMESPACE_MODULE, - NAMESPACE_STATIC, - NAMESPACE_TYPE, - NAMESPACE_TYPE_VAR, - BasicBlock, - Value, -) +from mypyc.ir.ops import BasicBlock, Value from mypyc.ir.rtypes import ( RInstance, RPrimitive, RTuple, RType, RUnion, - RVec, int_rprimitive, is_bool_or_bit_rprimitive, is_bytearray_rprimitive, @@ -65,24 +56,14 @@ is_uint8_rprimitive, object_rprimitive, optional_value_type, - vec_api_by_item_type, - vec_item_type_tags, ) from mypyc.namegen import NameGenerator, exported_name -from mypyc.primitives.registry import builtin_names from mypyc.sametype import is_same_type # Whether to insert debug asserts for all error handling, to quickly # catch errors propagating without exceptions set. DEBUG_ERRORS: Final = False -PREFIX_MAP: Final = { - NAMESPACE_STATIC: STATIC_PREFIX, - NAMESPACE_TYPE: TYPE_PREFIX, - NAMESPACE_MODULE: MODULE_PREFIX, - NAMESPACE_TYPE_VAR: TYPE_VAR_PREFIX, -} - class HeaderDeclaration: """A representation of a declaration in C. @@ -345,13 +326,6 @@ def ctype_spaced(self, rtype: RType) -> str: else: return ctype + " " - def set_undefined_value(self, target: str, rtype: RType) -> None: - if isinstance(rtype, RVec): - self.emit_line(f"{target}.len = -1;") - self.emit_line(f"{target}.buf = NULL;") - else: - self.emit_line(f"{target} = {self.c_undefined_value(rtype)};") - def c_undefined_value(self, rtype: RType) -> str: if not rtype.is_unboxed: return "NULL" @@ -359,8 +333,6 @@ def c_undefined_value(self, rtype: RType) -> str: return rtype.c_undefined elif isinstance(rtype, RTuple): return self.tuple_undefined_value(rtype) - elif isinstance(rtype, RVec): - return f"({self.ctype(rtype)}) {{ -1, NULL }}" assert False, rtype def c_error_value(self, rtype: RType) -> str: @@ -463,12 +435,6 @@ def error_value_check(self, rtype: RType, value: str, compare: str) -> str: return self.tuple_undefined_check_cond( rtype, value, self.c_error_value, compare, check_exception=False ) - elif isinstance(rtype, RVec): - if compare == "==": - return f"{value}.len < 0" - elif compare == "!=": - return f"{value}.len >= 0" - assert False, compare else: return f"{value} {compare} {self.c_error_value(rtype)}" @@ -500,8 +466,6 @@ def tuple_undefined_check_cond( return self.tuple_undefined_check_cond( item_type, tuple_expr_in_c + f".f{i}", c_type_compare_val, compare ) - elif isinstance(item_type, RVec): - return f"{tuple_expr_in_c}.f{i}.len {compare} -1" else: check = f"{tuple_expr_in_c}.f{i} {compare} {c_type_compare_val(item_type)}" if rtuple.error_overlap and check_exception: @@ -521,8 +485,6 @@ def c_initializer_undefined_value(self, rtype: RType) -> str: return f"{{ {int_rprimitive.c_undefined} }}" items = ", ".join([self.c_initializer_undefined_value(t) for t in rtype.types]) return f"{{ {items} }}" - elif isinstance(rtype, RVec): - return "{ -1, NULL }" else: return self.c_undefined_value(rtype) @@ -556,9 +518,6 @@ def emit_inc_ref(self, dest: str, rtype: RType, *, rare: bool = False) -> None: elif isinstance(rtype, RTuple): for i, item_type in enumerate(rtype.types): self.emit_inc_ref(f"{dest}.f{i}", item_type) - elif isinstance(rtype, RVec): - # TODO: Only use the X variant if buf can be NULL - self.emit_line(f"Py_XINCREF({dest}.buf);") elif not rtype.is_unboxed: # Always inline, since this is a simple but very hot op if rtype.may_be_immortal or not HAVE_IMMORTAL: @@ -587,12 +546,6 @@ def emit_dec_ref( elif isinstance(rtype, RTuple): for i, item_type in enumerate(rtype.types): self.emit_dec_ref(f"{dest}.f{i}", item_type, is_xdec=is_xdec, rare=rare) - elif isinstance(rtype, RVec): - # TODO: Only use the X variant if buf can be NULL - if rare: - self.emit_line(f"CPy_XDecRef({dest}.buf);") - else: - self.emit_line(f"CPy_XDECREF({dest}.buf);") elif not rtype.is_unboxed: if rare: self.emit_line(f"CPy_{x}DecRef({dest});") @@ -602,8 +555,6 @@ def emit_dec_ref( self.emit_line(f"CPy_{x}DECREF({dest});") else: self.emit_line(f"CPy_{x}DECREF_NO_IMM({dest});") - elif rtype.is_refcounted: - assert False, f"dec_ref not implemented for {rtype}" # Otherwise assume it's an unboxed, pointerless value and do nothing. def pretty_name(self, typ: RType) -> str: @@ -800,14 +751,6 @@ def emit_cast( elif isinstance(typ, RTuple): assert not optional self.emit_tuple_cast(src, dest, typ, declare_dest, error, src_type) - elif isinstance(typ, RVec): - # TODO: Actually perform the type check, this is a no-op - if declare_dest: - self.emit_line("PyObject *{};".format(dest)) - self.emit_arg_check(src, dest, typ, "", optional) - self.emit_line("{} = {};".format(dest, src)) - if optional: - self.emit_line("}") else: assert False, "Cast not implemented: %s" % typ @@ -951,7 +894,6 @@ def emit_unbox( declare_dest: If True, also declare the variable 'dest' error: What happens on error raise_exception: If True, also raise TypeError on failure - optional: If True, NULL src value is allowed and will map to error value borrow: If True, create a borrowed reference """ @@ -1083,53 +1025,10 @@ def emit_unbox( self.emit_line("}") if optional: self.emit_line("}") - elif isinstance(typ, RVec): - if declare_dest: - self.emit_line(f"{self.ctype(typ)} {dest};") - - if optional: - self.emit_line(f"if ({src} == NULL) {{") - self.emit_line(f"{dest} = {self.c_error_value(typ)};") - self.emit_line("} else {") - - specialized_api_name = vec_api_by_item_type.get(typ.item_type) - if specialized_api_name is not None: - self.emit_line(f"{dest} = {specialized_api_name}.unbox({src});") - else: - depth = typ.depth() - unwrapped = typ.unwrap_item_type() - if unwrapped in vec_item_type_tags: - type_value = str(vec_item_type_tags[unwrapped]) - else: - type_value = self.vec_item_type_c(typ) - if depth == 0: - self.emit_line(f"{dest} = VecTApi.unbox({src}, {type_value});") - else: - self.emit_line(f"{dest} = VecNestedApi.unbox({src}, {type_value}, {depth});") - - self.emit_line(f"if (VEC_IS_ERROR({dest})) {{") - self.emit_line(failure) - self.emit_line("}") - if optional: - self.emit_line("}") else: assert False, "Unboxing not implemented: %s" % typ - def vec_item_type_c(self, typ: RVec) -> str: - item_type = typ.unwrap_item_type() - type_value = f"(size_t){self.type_c_ptr(item_type)}" - if typ.is_optional(): - type_value = f"{type_value} | 1" - return type_value - - def type_c_ptr(self, typ: RPrimitive | RInstance) -> str | None: - if isinstance(typ, RPrimitive) and typ.is_refcounted: - return "&" + builtin_names[typ.name][1] - elif isinstance(typ, RInstance): - return self.type_struct_name(typ.class_ir) - return None - def emit_box( self, src: str, dest: str, typ: RType, declare_dest: bool = False, can_borrow: bool = False ) -> None: @@ -1184,20 +1083,6 @@ def emit_box( inner_name = self.temp_name() self.emit_box(f"{src}.f{i}", inner_name, typ.types[i], declare_dest=True) self.emit_line(f"PyTuple_SET_ITEM({dest}, {i}, {inner_name});") - elif isinstance(typ, RVec): - specialized_api_name = vec_api_by_item_type.get(typ.item_type) - if specialized_api_name is not None: - api = specialized_api_name - elif typ.depth() > 0: - api = "VecNestedApi" - else: - api = "VecTApi" - # Empty vecs of this sort don't describe item type, so it needs to be - # passed explicitly. - item_type = self.vec_item_type_c(typ) - self.emit_line(f"{declaration}{dest} = {api}.box({src}, {item_type});") - return - self.emit_line(f"{declaration}{dest} = {api}.box({src});") else: assert not typ.is_unboxed # Type is boxed -- trivially just assign. @@ -1211,8 +1096,6 @@ def emit_error_check(self, value: str, rtype: RType, failure: str) -> None: else: cond = self.tuple_undefined_check_cond(rtype, value, self.c_error_value, "==") self.emit_line(f"if ({cond}) {{") - elif isinstance(rtype, RVec): - self.emit_line(f"if ({value}.len < 0) {{") elif rtype.error_overlap: # The error value is also valid as a normal value, so we need to also check # for a raised exception. @@ -1237,8 +1120,6 @@ def emit_gc_visit(self, target: str, rtype: RType) -> None: elif isinstance(rtype, RTuple): for i, item_type in enumerate(rtype.types): self.emit_gc_visit(f"{target}.f{i}", item_type) - elif isinstance(rtype, RVec): - self.emit_line(f"Py_VISIT({target}.buf);") elif self.ctype(rtype) == "PyObject *": # The simplest case. self.emit_line(f"Py_VISIT({target});") @@ -1263,8 +1144,6 @@ def emit_gc_clear(self, target: str, rtype: RType) -> None: elif isinstance(rtype, RTuple): for i, item_type in enumerate(rtype.types): self.emit_gc_clear(f"{target}.f{i}", item_type) - elif isinstance(rtype, RVec): - self.emit_line(f"Py_CLEAR({target}.buf);") elif self.ctype(rtype) == "PyObject *" and self.c_undefined_value(rtype) == "NULL": # The simplest case. self.emit_line(f"Py_CLEAR({target});") diff --git a/mypyc/codegen/emitclass.py b/mypyc/codegen/emitclass.py index 387065d93bcfe..8c3fa5de98f85 100644 --- a/mypyc/codegen/emitclass.py +++ b/mypyc/codegen/emitclass.py @@ -653,7 +653,7 @@ def generate_setup_for_class( # We don't need to set this field to NULL since tp_alloc() already # zero-initializes `self`. if value != "NULL": - emitter.set_undefined_value(f"self->{emitter.attr(attr)}", rtype) + emitter.emit_line(rf"self->{emitter.attr(attr)} = {value};") # Initialize attributes to default values, if necessary if defaults_fn is not None: @@ -1194,7 +1194,7 @@ def generate_setter(cl: ClassIR, attr: str, rtype: RType, emitter: Emitter) -> N if deletable: emitter.emit_line("} else") - emitter.set_undefined_value(f" self->{attr_field}", rtype) + emitter.emit_line(f" self->{attr_field} = {emitter.c_undefined_value(rtype)};") if rtype.error_overlap: emitter.emit_attr_bitmap_clear("self", rtype, cl, attr) emitter.emit_line("return 0;") diff --git a/mypyc/codegen/emitfunc.py b/mypyc/codegen/emitfunc.py index c1202d1c928ca..3f1bbab58895f 100644 --- a/mypyc/codegen/emitfunc.py +++ b/mypyc/codegen/emitfunc.py @@ -5,19 +5,25 @@ from typing import Final from mypyc.analysis.blockfreq import frequently_executed_blocks -from mypyc.codegen.emit import ( - DEBUG_ERRORS, - PREFIX_MAP, - Emitter, - TracebackAndGotoHandler, - c_array_initializer, +from mypyc.codegen.emit import DEBUG_ERRORS, Emitter, TracebackAndGotoHandler, c_array_initializer +from mypyc.common import ( + GENERATOR_ATTRIBUTE_PREFIX, + HAVE_IMMORTAL, + MODULE_PREFIX, + NATIVE_PREFIX, + REG_PREFIX, + STATIC_PREFIX, + TYPE_PREFIX, + TYPE_VAR_PREFIX, ) -from mypyc.common import GENERATOR_ATTRIBUTE_PREFIX, HAVE_IMMORTAL, NATIVE_PREFIX, REG_PREFIX from mypyc.ir.class_ir import ClassIR from mypyc.ir.func_ir import FUNC_CLASSMETHOD, FUNC_STATICMETHOD, FuncDecl, FuncIR, all_values from mypyc.ir.ops import ( ERR_FALSE, + NAMESPACE_MODULE, + NAMESPACE_STATIC, NAMESPACE_TYPE, + NAMESPACE_TYPE_VAR, Assign, AssignMulti, BasicBlock, @@ -76,7 +82,6 @@ RStruct, RTuple, RType, - RVec, is_bool_or_bit_rprimitive, is_int32_rprimitive, is_int64_rprimitive, @@ -216,10 +221,6 @@ def error_value_check(self, value: Value, compare: str) -> str: return self.emitter.tuple_undefined_check_cond( typ, self.reg(value), self.c_error_value, compare ) - elif isinstance(typ, RVec): - # Error values for vecs are represented by a negative length. - vec_compare = ">=" if compare == "!=" else "<" - return f"{self.reg(value)}.len {vec_compare} 0" else: return f"{self.reg(value)} {compare} {self.c_error_value(typ)}" @@ -288,16 +289,10 @@ def visit_assign(self, op: Assign) -> None: # clang whines about self assignment (which we might generate # for some casts), so don't emit it. if dest != src: - src_type = op.src.type - dest_type = op.dest.type - if src_type.is_unboxed and not dest_type.is_unboxed: - # We sometimes assign from an integer prepresentation of a pointer - # to a real pointer, and C compilers insist on a cast. + # We sometimes assign from an integer prepresentation of a pointer + # to a real pointer, and C compilers insist on a cast. + if op.src.type.is_unboxed and not op.dest.type.is_unboxed: src = f"(void *){src}" - elif not src_type.is_unboxed and dest_type.is_unboxed: - # We sometimes assign a pointer to an integer type (e.g. to create - # tagged pointers), and here we need an explicit cast. - src = f"({self.emitter.ctype(dest_type)}){src}" self.emit_line(f"{dest} = {src};") def visit_assign_multi(self, op: AssignMulti) -> None: @@ -317,14 +312,11 @@ def visit_assign_multi(self, op: AssignMulti) -> None: ) def visit_load_error_value(self, op: LoadErrorValue) -> None: - reg = self.reg(op) if isinstance(op.type, RTuple): values = [self.c_undefined_value(item) for item in op.type.types] tmp = self.temp_name() self.emit_line("{} {} = {{ {} }};".format(self.ctype(op.type), tmp, ", ".join(values))) - self.emit_line(f"{reg} = {tmp};") - elif isinstance(op.type, RVec): - self.emitter.set_undefined_value(reg, op.type) + self.emit_line(f"{self.reg(op)} = {tmp};") else: self.emit_line(f"{self.reg(op)} = {self.c_error_value(op.type)};") @@ -531,9 +523,16 @@ def visit_set_attr(self, op: SetAttr) -> None: if op.error_kind == ERR_FALSE: self.emitter.emit_line(f"{dest} = 1;") + PREFIX_MAP: Final = { + NAMESPACE_STATIC: STATIC_PREFIX, + NAMESPACE_TYPE: TYPE_PREFIX, + NAMESPACE_MODULE: MODULE_PREFIX, + NAMESPACE_TYPE_VAR: TYPE_VAR_PREFIX, + } + def visit_load_static(self, op: LoadStatic) -> None: dest = self.reg(op) - prefix = PREFIX_MAP[op.namespace] + prefix = self.PREFIX_MAP[op.namespace] name = self.emitter.static_name(op.identifier, op.module_name, prefix) if op.namespace == NAMESPACE_TYPE: name = "(PyObject *)%s" % name @@ -541,7 +540,7 @@ def visit_load_static(self, op: LoadStatic) -> None: def visit_init_static(self, op: InitStatic) -> None: value = self.reg(op.value) - prefix = PREFIX_MAP[op.namespace] + prefix = self.PREFIX_MAP[op.namespace] name = self.emitter.static_name(op.identifier, op.module_name, prefix) if op.namespace == NAMESPACE_TYPE: value = "(PyTypeObject *)%s" % value @@ -846,7 +845,7 @@ def visit_load_address(self, op: LoadAddress) -> None: if isinstance(op.src, Register): src = self.reg(op.src) elif isinstance(op.src, LoadStatic): - prefix = PREFIX_MAP[op.src.namespace] + prefix = self.PREFIX_MAP[op.src.namespace] src = self.emitter.static_name(op.src.identifier, op.src.module_name, prefix) else: src = op.src diff --git a/mypyc/codegen/emitmodule.py b/mypyc/codegen/emitmodule.py index 5589fb987aef6..2b0693ea2f2dd 100644 --- a/mypyc/codegen/emitmodule.py +++ b/mypyc/codegen/emitmodule.py @@ -27,7 +27,7 @@ from mypy.options import Options from mypy.plugin import Plugin, ReportConfigContext from mypy.util import hash_digest, json_dumps -from mypyc.analysis.capsule_deps import find_class_dependencies, find_implicit_op_dependencies +from mypyc.analysis.capsule_deps import find_implicit_op_dependencies from mypyc.codegen.cstring import c_string_initializer from mypyc.codegen.emit import ( Emitter, @@ -56,7 +56,7 @@ short_id_from_name, ) from mypyc.errors import Errors -from mypyc.ir.deps import LIBRT_BASE64, LIBRT_STRINGS, LIBRT_VECS, SourceDep +from mypyc.ir.deps import LIBRT_BASE64, LIBRT_STRINGS, SourceDep from mypyc.ir.func_ir import FuncIR from mypyc.ir.module_ir import ModuleIR, ModuleIRs, deserialize_modules from mypyc.ir.ops import DeserMaps, LoadLiteral @@ -271,12 +271,6 @@ def compile_scc_to_ir( do_copy_propagation(fn, compiler_options) do_flag_elimination(fn, compiler_options) - # Calculate implicit dependencies from class attribute types - for cl in module.classes: - deps = find_class_dependencies(cl) - if deps is not None: - module.dependencies.update(deps) - return modules @@ -638,8 +632,6 @@ def generate_c_for_modules(self) -> list[tuple[str, str]]: ext_declarations.emit_line("#include ") if any(LIBRT_STRINGS in mod.dependencies for mod in self.modules.values()): ext_declarations.emit_line("#include ") - if any(LIBRT_VECS in mod.dependencies for mod in self.modules.values()): - ext_declarations.emit_line('#include "vecs/librt_vecs.h"') # Include headers for conditional source files source_deps = collect_source_dependencies(self.modules) for source_dep in sorted(source_deps, key=lambda d: d.path): @@ -1110,10 +1102,6 @@ def emit_module_exec_func( emitter.emit_line("if (import_librt_strings() < 0) {") emitter.emit_line("return -1;") emitter.emit_line("}") - if LIBRT_VECS in module.dependencies: - emitter.emit_line("if (import_librt_vecs() < 0) {") - emitter.emit_line("return -1;") - emitter.emit_line("}") emitter.emit_line("PyObject* modname = NULL;") if self.multi_phase_init: emitter.emit_line(f"{module_static} = module;") diff --git a/mypyc/test/test_emitfunc.py b/mypyc/test/test_emitfunc.py index 59ab4cb8a0e5d..642e1cd5c3cc3 100644 --- a/mypyc/test/test_emitfunc.py +++ b/mypyc/test/test_emitfunc.py @@ -50,8 +50,6 @@ RStruct, RTuple, RType, - RUnion, - RVec, bool_rprimitive, c_int_rprimitive, cstring_rprimitive, @@ -64,7 +62,6 @@ object_rprimitive, pointer_rprimitive, short_int_rprimitive, - str_rprimitive, ) from mypyc.irbuild.vtable import compute_vtable from mypyc.namegen import NameGenerator @@ -112,11 +109,6 @@ def add_local(name: str, rtype: RType) -> Register: self.tt = add_local( "tt", RTuple([RTuple([int_rprimitive, bool_rprimitive]), bool_rprimitive]) ) - self.vi64 = add_local("vi64", RVec(int64_rprimitive)) - self.vi32 = add_local("vi32", RVec(int32_rprimitive)) - self.vs = add_local("vs", RVec(str_rprimitive)) - self.vs_opt = add_local("vs", RVec(RUnion([str_rprimitive, none_rprimitive]))) - self.vvs = add_local("vvs", RVec(RVec(str_rprimitive))) ir = ClassIR("A", "mod") ir.attributes = { "x": bool_rprimitive, @@ -356,11 +348,11 @@ def test_unbox_int(self) -> None: self.assert_emit( Unbox(self.m, int_rprimitive, 55), """if (likely(PyLong_Check(cpy_r_m))) - cpy_r_r0 = CPyTagged_FromObject(cpy_r_m); - else { - CPy_TypeError("int", cpy_r_m); cpy_r_r0 = CPY_INT_TAG; - } - """, + cpy_r_r0 = CPyTagged_FromObject(cpy_r_m); + else { + CPy_TypeError("int", cpy_r_m); cpy_r_r0 = CPY_INT_TAG; + } + """, ) def test_box_i64(self) -> None: @@ -371,85 +363,6 @@ def test_unbox_i64(self) -> None: Unbox(self.o, int64_rprimitive, 55), """cpy_r_r0 = CPyLong_AsInt64(cpy_r_o);""" ) - def test_box_vec(self) -> None: - self.assert_emit(Box(self.vi64), """cpy_r_r0 = VecI64Api.box(cpy_r_vi64);""") - self.assert_emit(Box(self.vi32), """cpy_r_r0 = VecI32Api.box(cpy_r_vi32);""") - self.assert_emit( - Box(self.vs), """cpy_r_r0 = VecTApi.box(cpy_r_vs, (size_t)&PyUnicode_Type);""" - ) - self.assert_emit( - Box(self.vs_opt), """cpy_r_r0 = VecTApi.box(cpy_r_vs, (size_t)&PyUnicode_Type | 1);""" - ) - self.assert_emit(Box(self.vvs), """cpy_r_r0 = VecNestedApi.box(cpy_r_vvs);""") - - def test_unbox_vec(self) -> None: - self.assert_emit( - Unbox(self.o, RVec(int64_rprimitive), 55), - """cpy_r_r0 = VecI64Api.unbox(cpy_r_o); - if (VEC_IS_ERROR(cpy_r_r0)) { - CPy_TypeError("vec[i64]", cpy_r_o); cpy_r_r0 = (VecI64) { -1, NULL }; - } - """, - ) - self.assert_emit( - Unbox(self.o, RVec(int32_rprimitive), 55), - """cpy_r_r0 = VecI32Api.unbox(cpy_r_o); - if (VEC_IS_ERROR(cpy_r_r0)) { - CPy_TypeError("vec[i32]", cpy_r_o); cpy_r_r0 = (VecI32) { -1, NULL }; - } - """, - ) - self.assert_emit( - Unbox(self.o, RVec(str_rprimitive), 55), - """cpy_r_r0 = VecTApi.unbox(cpy_r_o, (size_t)&PyUnicode_Type); - if (VEC_IS_ERROR(cpy_r_r0)) { - CPy_TypeError("vec[str]", cpy_r_o); cpy_r_r0 = (VecT) { -1, NULL }; - } - """, - ) - self.assert_emit( - Unbox(self.o, RVec(RUnion([str_rprimitive, none_rprimitive])), 55), - """cpy_r_r0 = VecTApi.unbox(cpy_r_o, (size_t)&PyUnicode_Type | 1); - if (VEC_IS_ERROR(cpy_r_r0)) { - CPy_TypeError("vec[str | None]", cpy_r_o); cpy_r_r0 = (VecT) { -1, NULL }; - } - """, - ) - self.assert_emit( - Unbox(self.o, RVec(self.r.type), 55), - """cpy_r_r0 = VecTApi.unbox(cpy_r_o, (size_t)CPyType_A); - if (VEC_IS_ERROR(cpy_r_r0)) { - CPy_TypeError("vec[mod.A]", cpy_r_o); cpy_r_r0 = (VecT) { -1, NULL }; - } - """, - ) - - def test_unbox_vec_nested(self) -> None: - self.assert_emit( - Unbox(self.o, RVec(RVec(str_rprimitive)), 55), - """cpy_r_r0 = VecNestedApi.unbox(cpy_r_o, (size_t)&PyUnicode_Type, 1); - if (VEC_IS_ERROR(cpy_r_r0)) { - CPy_TypeError("vec[vec[str]]", cpy_r_o); cpy_r_r0 = (VecNested) { -1, NULL }; - } - """, - ) - self.assert_emit( - Unbox(self.o, RVec(RVec(RUnion([str_rprimitive, none_rprimitive]))), 55), - """cpy_r_r0 = VecNestedApi.unbox(cpy_r_o, (size_t)&PyUnicode_Type | 1, 1); - if (VEC_IS_ERROR(cpy_r_r0)) { - CPy_TypeError("vec[vec[str | None]]", cpy_r_o); cpy_r_r0 = (VecNested) { -1, NULL }; - } - """, - ) - self.assert_emit( - Unbox(self.o, RVec(RVec(int64_rprimitive)), 55), - """cpy_r_r0 = VecNestedApi.unbox(cpy_r_o, 2, 1); - if (VEC_IS_ERROR(cpy_r_r0)) { - CPy_TypeError("vec[vec[i64]]", cpy_r_o); cpy_r_r0 = (VecNested) { -1, NULL }; - } - """, - ) - def test_list_append(self) -> None: self.assert_emit( CallC( From d9bbaf00f64f6b126559ff40a8b384946d3590b2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 2 Feb 2026 16:44:13 +0000 Subject: [PATCH 172/180] WIP temporarily revert changes to refcount.test --- mypyc/test-data/refcount.test | 432 ---------------------------------- 1 file changed, 432 deletions(-) diff --git a/mypyc/test-data/refcount.test b/mypyc/test-data/refcount.test index 230b9b2d7282b..a71c53041cf7e 100644 --- a/mypyc/test-data/refcount.test +++ b/mypyc/test-data/refcount.test @@ -1499,435 +1499,3 @@ L2: dec_ref r0 :: int L3: return r4 - -[case testVecTSetItem_64bit] -from librt.vecs import vec -from mypy_extensions import i64 - -def f(v: vec[str], i: i64, x: str) -> None: - v[i] = x -[out] -def f(v, i, x): - v :: vec[str] - i :: i64 - x :: str - r0 :: native_int - r1 :: bit - r2 :: i64 - r3 :: bit - r4 :: bool - r5 :: i64 - r6 :: object - r7 :: ptr - r8 :: i64 - r9 :: ptr - r10 :: str -L0: - r0 = v.len - r1 = i < r0 :: unsigned - if r1 goto L4 else goto L1 :: bool -L1: - r2 = i + r0 - r3 = r2 < r0 :: unsigned - if r3 goto L3 else goto L2 :: bool -L2: - r4 = raise IndexError - unreachable -L3: - r5 = r2 - goto L5 -L4: - r5 = i -L5: - r6 = v.buf - r7 = get_element_ptr r6 items :: VecTBufObject - r8 = r5 * 8 - r9 = r7 + r8 - r10 = borrow load_mem r9 :: builtins.str* - dec_ref r10 - inc_ref x - set_mem r9, x :: builtins.str* - return 1 - -[case testVecTConstructFromListMultiply_64bit] -from typing import Optional -from librt.vecs import vec -from mypy_extensions import i64 - -def f(n: i64) -> vec[Optional[str]]: - return vec[Optional[str]]([None] * n) -[out] -def f(n): - n :: i64 - r0, r1 :: object - r2, r3 :: ptr - r4 :: vec[str | None] - r5 :: object - r6 :: ptr - r7 :: i64 - r8, r9 :: ptr - r10 :: bit - r11 :: ptr -L0: - r0 = box(None, 1) - r1 = load_address PyUnicode_Type - r2 = r1 - r3 = r2 | 1 - r4 = VecTApi.alloc(n, n, r3) - r5 = r4.buf - r6 = get_element_ptr r5 items :: VecTBufObject - r7 = n * 8 - r8 = r6 + r7 - r9 = r6 -L1: - r10 = r9 < r8 :: unsigned - if r10 goto L2 else goto L3 :: bool -L2: - inc_ref r0 - set_mem r9, r0 :: union* - r11 = r9 + 8 - r9 = r11 - goto L1 -L3: - return r4 - -[case testVecTForLoop_64bit] -from librt.vecs import vec -from mypy_extensions import i64 - -def f(v: vec[str]) -> i64: - t: i64 = 0 - for s in v: - g(s) - return t - -def g(s: str) -> None: pass -[out] -def f(v): - v :: vec[str] - t :: i64 - r0, r1 :: native_int - r2 :: bit - r3 :: object - r4 :: ptr - r5 :: native_int - r6 :: ptr - r7, s :: str - r8 :: None - r9 :: native_int -L0: - t = 0 - r0 = 0 -L1: - r1 = v.len - r2 = r0 < r1 :: signed - if r2 goto L2 else goto L4 :: bool -L2: - r3 = v.buf - r4 = get_element_ptr r3 items :: VecTBufObject - r5 = r0 * 8 - r6 = r4 + r5 - r7 = load_mem r6 :: builtins.str* - s = r7 - r8 = g(s) - dec_ref s -L3: - r9 = r0 + 1 - r0 = r9 - goto L1 -L4: - return t -def g(s): - s :: str -L0: - return 1 - -[case testVecTConstructFromListExpr_64bit] -from librt.vecs import vec - -def f() -> vec[str]: - return vec[str](['x', 'y']) -[out] -def f(): - r0, r1 :: str - r2 :: object - r3 :: ptr - r4 :: vec[str] - r5 :: object - r6, r7, r8 :: ptr -L0: - r0 = 'x' - r1 = 'y' - r2 = load_address PyUnicode_Type - r3 = r2 - r4 = VecTApi.alloc(2, 2, r3) - r5 = r4.buf - r6 = get_element_ptr r5 items :: VecTBufObject - inc_ref r0 - set_mem r6, r0 :: builtins.str* - r7 = r6 + 8 - inc_ref r1 - set_mem r7, r1 :: builtins.str* - r8 = r7 + 8 - return r4 - -[case testVecI64GetItemBorrowVec_64bit] -from librt.vecs import vec -from mypy_extensions import i64 - -class C: - v: vec[i64] - - def f(self, x: i64) -> i64: - return self.v[x] -[out] -def C.f(self, x): - self :: __main__.C - x :: i64 - r0 :: vec[i64] - r1 :: native_int - r2 :: bit - r3 :: i64 - r4 :: bit - r5 :: bool - r6 :: i64 - r7 :: object - r8 :: ptr - r9 :: i64 - r10 :: ptr - r11 :: i64 -L0: - r0 = borrow self.v - r1 = r0.len - r2 = x < r1 :: unsigned - if r2 goto L4 else goto L1 :: bool -L1: - r3 = x + r1 - r4 = r3 < r1 :: unsigned - if r4 goto L3 else goto L2 :: bool -L2: - r5 = raise IndexError - unreachable -L3: - r6 = r3 - goto L5 -L4: - r6 = x -L5: - r7 = r0.buf - r8 = get_element_ptr r7 items :: VecI64BufObject - r9 = r6 * 8 - r10 = r8 + r9 - r11 = load_mem r10 :: i64* - return r11 - -[case testVecI64LenBorrowVec_64bit] -from librt.vecs import vec -from mypy_extensions import i64 - -class C: - v: vec[i64] - - def f(self, x: i64) -> i64: - return len(self.v) -[out] -def C.f(self, x): - self :: __main__.C - x :: i64 - r0 :: vec[i64] - r1 :: native_int -L0: - r0 = borrow self.v - r1 = r0.len - return r1 - -[case testVecNestedAppend] -from librt.vecs import vec, append - -def f(vv: vec[vec[str]], v: vec[str]) -> vec[vec[str]]: - return append(vv, v) -[out] -def f(vv, v): - vv :: vec[vec[str]] - v :: vec[str] - r0 :: native_int - r1 :: object - r2, r3 :: VecNestedBufItem{len:native_int, buf:object_nrc} - r4 :: vec[vec[str]] -L0: - r0 = v.len - r1 = v.buf - r2 = set_element undef VecNestedBufItem, len, r0 - r3 = set_element r2, buf, r1 - inc_ref vv - r4 = VecNestedApi.append(vv, r3) - return r4 - -[case testVecTGetItem_64bit] -from librt.vecs import vec -from mypy_extensions import i64 - -def f(v: vec[str], n: i64) -> str: - return v[n] -[out] -def f(v, n): - v :: vec[str] - n :: i64 - r0 :: native_int - r1 :: bit - r2 :: i64 - r3 :: bit - r4 :: bool - r5 :: i64 - r6 :: object - r7 :: ptr - r8 :: i64 - r9 :: ptr - r10 :: str -L0: - r0 = v.len - r1 = n < r0 :: unsigned - if r1 goto L4 else goto L1 :: bool -L1: - r2 = n + r0 - r3 = r2 < r0 :: unsigned - if r3 goto L3 else goto L2 :: bool -L2: - r4 = raise IndexError - unreachable -L3: - r5 = r2 - goto L5 -L4: - r5 = n -L5: - r6 = v.buf - r7 = get_element_ptr r6 items :: VecTBufObject - r8 = r5 * 8 - r9 = r7 + r8 - r10 = load_mem r9 :: builtins.str* - return r10 - -[case testVecNestedGetItem_64bit] -from librt.vecs import vec -from mypy_extensions import i64 - -def f(v: vec[vec[str]], n: i64) -> None: - vv = vec[vec[str]]()[n] -[out] -def f(v, n): - v :: vec[vec[str]] - n :: i64 - r0 :: object - r1 :: ptr - r2 :: vec[vec[str]] - r3 :: native_int - r4 :: bit - r5 :: i64 - r6 :: bit - r7 :: bool - r8 :: i64 - r9 :: object - r10 :: ptr - r11 :: i64 - r12 :: ptr - r13, vv :: vec[str] -L0: - r0 = load_address PyUnicode_Type - r1 = r0 - r2 = VecNestedApi.alloc(0, 0, r1, 1) - r3 = r2.len - r4 = n < r3 :: unsigned - if r4 goto L4 else goto L1 :: bool -L1: - r5 = n + r3 - r6 = r5 < r3 :: unsigned - if r6 goto L3 else goto L6 :: bool -L2: - r7 = raise IndexError - unreachable -L3: - r8 = r5 - goto L5 -L4: - r8 = n -L5: - r9 = r2.buf - r10 = get_element_ptr r9 items :: VecNestedBufObject - r11 = r8 * 16 - r12 = r10 + r11 - r13 = load_mem r12 :: vec[str]* - dec_ref r2 - vv = r13 - dec_ref vv - return 1 -L6: - dec_ref r2 - goto L2 - -[case testVecPop] -from librt.vecs import vec, pop, append -from mypy_extensions import i64 - -def f() -> vec[i64]: - v = vec[i64]() - v = append(v, 7) - v, x = pop(v) - return v -[out] -def f(): - r0, v, r1 :: vec[i64] - r2 :: tuple[vec[i64], i64] - r3 :: vec[i64] - r4 :: i64 - r5 :: vec[i64] - r6, x :: i64 -L0: - r0 = VecI64Api.alloc(0, 0) - v = r0 - r1 = VecI64Api.append(v, 7) - v = r1 - r2 = VecI64Api.pop(v, -1) - r3 = borrow r2[0] - r4 = borrow r2[1] - r5 = unborrow r3 - v = r5 - r6 = unborrow r4 - x = r6 - return v - -[case testVecNestedPop] -from librt.vecs import vec, pop - -def f(v: vec[vec[str]]) -> vec[str]: - vv, x = pop(v) - return x -[out] -def f(v): - v :: vec[vec[str]] - r0 :: tuple[vec[vec[str]], VecNestedBufItem{len:native_int, buf:object_nrc}] - r1, r2 :: vec[vec[str]] - r3, r4 :: VecNestedBufItem{len:native_int, buf:object_nrc} - r5 :: vec[str] - r6 :: tuple[vec[vec[str]], vec[str]] - r7 :: vec[vec[str]] - r8 :: vec[str] - r9, vv :: vec[vec[str]] - r10, x :: vec[str] -L0: - inc_ref v - r0 = VecNestedApi.pop(v, -1) - r1 = borrow r0[0] - r2 = unborrow r1 - r3 = borrow r0[1] - r4 = unborrow r3 - r5 = VecTApi.convert_from_nested(r4) - r6 = (r2, r5) - r7 = borrow r6[0] - r8 = borrow r6[1] - r9 = unborrow r7 - vv = r9 - dec_ref vv - r10 = unborrow r8 - x = r10 - return x From 5936862be6ce8c6883d7d7532893c5321246db63 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 2 Feb 2026 16:52:20 +0000 Subject: [PATCH 173/180] Minor tweaks --- mypyc/lib-rt/vecs/vec_nested.c | 3 --- mypyc/lib-rt/vecs/vec_t.c | 3 --- mypyc/lib-rt/vecs/vec_template.c | 3 --- mypyc/test-data/exceptions.test | 2 +- mypyc/test-data/irbuild-dict.test | 4 ++-- 5 files changed, 3 insertions(+), 12 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_nested.c b/mypyc/lib-rt/vecs/vec_nested.c index 02616ce7a4d9b..0034d9e9bfff5 100644 --- a/mypyc/lib-rt/vecs/vec_nested.c +++ b/mypyc/lib-rt/vecs/vec_nested.c @@ -11,9 +11,6 @@ #include "librt_vecs.h" #include "vecs_internal.h" -// Forward declaration of base vec type (defined in librt_vecs.c) -extern PyTypeObject VecType; - static inline VecNested vec_error() { VecNested v = { .len = -1 }; return v; diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index ac20da14a832e..8a03ea1a2e644 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -13,9 +13,6 @@ #include "librt_vecs.h" #include "vecs_internal.h" -// Forward declaration of base vec type (defined in librt_vecs.c) -extern PyTypeObject VecType; - static inline VecT vec_error() { VecT v = { .len = -1 }; return v; diff --git a/mypyc/lib-rt/vecs/vec_template.c b/mypyc/lib-rt/vecs/vec_template.c index 0091f8c07a757..dc1e9df3609fc 100644 --- a/mypyc/lib-rt/vecs/vec_template.c +++ b/mypyc/lib-rt/vecs/vec_template.c @@ -365,9 +365,6 @@ PyTypeObject BUF_TYPE = { .tp_free = PyObject_Del, }; -// Forward declaration of base vec type (defined in librt_vecs.c) -extern PyTypeObject VecType; - PyTypeObject VEC_TYPE = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "vec[" ITEM_TYPE_STR "]", diff --git a/mypyc/test-data/exceptions.test b/mypyc/test-data/exceptions.test index 92e34074618c7..c271c9ae63d5e 100644 --- a/mypyc/test-data/exceptions.test +++ b/mypyc/test-data/exceptions.test @@ -720,7 +720,7 @@ L3: r5 = :: float return r5 -[case testVecNestedRemove] +[case testVecI64Remove] from librt.vecs import vec, remove from mypy_extensions import i64 diff --git a/mypyc/test-data/irbuild-dict.test b/mypyc/test-data/irbuild-dict.test index 2e3cbeba60519..e7a330951ab0d 100644 --- a/mypyc/test-data/irbuild-dict.test +++ b/mypyc/test-data/irbuild-dict.test @@ -216,8 +216,7 @@ L0: return r2 [case testDictIterationMethods] -from typing import Dict, Union -from typing_extensions import TypedDict +from typing import Dict, TypedDict, Union class Person(TypedDict): name: str @@ -237,6 +236,7 @@ def typeddict(d: Person) -> None: for k, v in d.items(): if k == "name": name = v +[typing fixtures/typing-full.pyi] [out] def print_dict_methods(d1, d2): d1, d2 :: dict From fe43e12403ea047f7a4d684128dde9ce244f8d5a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 2 Feb 2026 17:37:22 +0000 Subject: [PATCH 174/180] Fix typo --- mypyc/ir/rtypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 3a8f046ab9426..1ce57f6f9cecb 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -1442,7 +1442,7 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: bool_rprimitive: "VecBoolApi", } -# These are special type item type contants used in nested vecs to represent +# These are special type item type constants used in nested vecs to represent # item types with specialized representations. These must match definitions # in the vecs module (see VEC_ITEM_TYPE_I64 etc.). vec_item_type_tags: Final[dict[RType, int]] = { From 375624c2ae0ca63fdff6ab10ce84b9eacd7f6247 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 3 Feb 2026 14:31:44 +0000 Subject: [PATCH 175/180] Address feedback --- mypyc/analysis/ircheck.py | 2 +- mypyc/ir/rtypes.py | 16 +--------------- mypyc/irbuild/vec.py | 15 +++++++-------- 3 files changed, 9 insertions(+), 24 deletions(-) diff --git a/mypyc/analysis/ircheck.py b/mypyc/analysis/ircheck.py index a49aab26ce074..384bcdbe5cd80 100644 --- a/mypyc/analysis/ircheck.py +++ b/mypyc/analysis/ircheck.py @@ -225,7 +225,7 @@ def is_valid_ptr_displacement_type(rtype: RType) -> bool: def is_pointer_arithmetic(op: IntOp) -> bool: - """Check if op is add/subtract targeting pointer_rprimtive and integer of the same size.""" + """Check if op is add/subtract targeting pointer_rprimitive and integer of the same size.""" if op.op not in (IntOp.ADD, IntOp.SUB): return False if not is_pointer_rprimitive(op.type): diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 1ce57f6f9cecb..502cd152e8e4d 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -322,7 +322,7 @@ def __hash__(self) -> int: "object_ptr", is_unboxed=False, is_refcounted=False, ctype="PyObject **" ) -# Similar to object_primitive, but doesn not use automatic reference +# Similar to object_primitive, but does not use automatic reference # counting. Useful for temporaries. object_non_refcounted_rprimitive: Final = RPrimitive( "builtins.object_nrc", is_unboxed=False, is_refcounted=False @@ -1107,20 +1107,6 @@ def deserialize(cls, data: JsonDict, ctx: DeserMaps) -> RVec: return RVec(deserialize_type(data["item_type"], ctx)) -def vec_depth(t: RVec) -> int: - """Return nesting depth of a RVec type (0 == no nested vecs).""" - it = t.item_type - if isinstance(it, RUnion): - non_opt = optional_value_type(it) - assert non_opt is not None - it = non_opt - if isinstance(it, (RPrimitive, RInstance)): - return 0 - elif isinstance(it, RVec): - return 1 + vec_depth(it) - assert False, t - - @final class RUnion(RType): """union[x, ..., y]""" diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index fdd77d3374b86..0498b7822cace 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -50,7 +50,6 @@ optional_value_type, pointer_rprimitive, vec_api_by_item_type, - vec_depth, vec_item_type_tags, ) from mypyc.primitives.registry import builtin_names @@ -391,7 +390,7 @@ def vec_append(builder: LowLevelIRBuilder, vec: Value, item: Value, line: int) - api_name = vec_api_by_item_type.get(item_type) if api_name is not None: name = f"{api_name}.append" - elif vec_depth(vec_type) == 0: + elif vec_type.depth() == 0: name = "VecTApi.append" item_type_arg = [vec_item_type(builder, item_type, line)] else: @@ -408,7 +407,7 @@ def vec_append(builder: LowLevelIRBuilder, vec: Value, item: Value, line: int) - line=line, ) ) - if vec_depth(vec_type) > 0: + if vec_type.depth() > 0: builder.keep_alive([item], line) return call @@ -422,7 +421,7 @@ def vec_pop(builder: LowLevelIRBuilder, base: Value, index: Value, line: int) -> api_name = vec_api_by_item_type.get(item_type) if api_name is not None: name = f"{api_name}.pop" - elif vec_depth(vec_type) == 0: + elif vec_type.depth() == 0: name = "VecTApi.pop" else: name = "VecNestedApi.pop" @@ -439,7 +438,7 @@ def vec_pop(builder: LowLevelIRBuilder, base: Value, index: Value, line: int) -> line=line, ) ) - if vec_depth(vec_type) > 0: + if vec_type.depth() > 0: orig = result x = builder.add(TupleGet(result, 0, borrow=True)) x = builder.add(Unborrow(x)) @@ -460,7 +459,7 @@ def vec_remove(builder: LowLevelIRBuilder, vec: Value, item: Value, line: int) - if item_type in vec_api_by_item_type: name = f"{vec_api_by_item_type[item_type]}.remove" - elif vec_depth(vec_type) == 0 and not isinstance(item_type, RUnion): + elif vec_type.depth() == 0 and not isinstance(item_type, RUnion): name = "VecTApi.remove" else: coerced_item = convert_to_t_ext_item(builder, coerced_item) @@ -476,7 +475,7 @@ def vec_remove(builder: LowLevelIRBuilder, vec: Value, item: Value, line: int) - line=line, ) ) - if vec_depth(vec_type) > 0: + if vec_type.depth() > 0: builder.keep_alive([item], line) return call @@ -526,7 +525,7 @@ def vec_slice( api_name = vec_api_by_item_type.get(item_type) if api_name is not None: name = f"{api_name}.slice" - elif vec_depth(vec_type) == 0 and not isinstance(item_type, RUnion): + elif vec_type.depth() == 0 and not isinstance(item_type, RUnion): name = "VecTApi.slice" else: name = "VecNestedApi.slice" From 046dcbc6e552e7cc6e1f3be9daa4223beedcbb94 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 3 Feb 2026 15:07:46 +0000 Subject: [PATCH 176/180] Address more feedback --- mypyc/ir/rtypes.py | 5 ++++- mypyc/test-data/irbuild-vec-i64.test | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/mypyc/ir/rtypes.py b/mypyc/ir/rtypes.py index 502cd152e8e4d..82970b39010d4 100644 --- a/mypyc/ir/rtypes.py +++ b/mypyc/ir/rtypes.py @@ -1305,6 +1305,9 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: # Buffers for vec item types that have a packed representation +# +# Note that the 'items' fields are variable-length arrays, and mypyc IR isn't +# able to represent these, so the field type is omitted for now. VecI64BufObject = RStruct( name="VecI64BufObject", @@ -1428,7 +1431,7 @@ def check_native_int_range(rtype: RPrimitive, n: int) -> bool: bool_rprimitive: "VecBoolApi", } -# These are special type item type constants used in nested vecs to represent +# These are special item type constants used in nested vecs to represent # item types with specialized representations. These must match definitions # in the vecs module (see VEC_ITEM_TYPE_I64 etc.). vec_item_type_tags: Final[dict[RType, int]] = { diff --git a/mypyc/test-data/irbuild-vec-i64.test b/mypyc/test-data/irbuild-vec-i64.test index 2b22ecbdc4915..c3c126e6758d2 100644 --- a/mypyc/test-data/irbuild-vec-i64.test +++ b/mypyc/test-data/irbuild-vec-i64.test @@ -317,6 +317,7 @@ from librt.vecs import vec from mypy_extensions import i64 def f(n: i64) -> vec[i64]: + # TODO: Can we pre-allocate a vec with capacity 5? return vec[i64]([x + 1 for x in range(i64(5))]) [out] def f(n): @@ -455,6 +456,7 @@ from librt.vecs import vec, pop from mypy_extensions import i64 def f() -> vec[i64]: + # TODO: Can we pre-allocate a vec with capacity 7? return vec[i64](range(7)) [out] def f(): From 3e1c90e54562af8d5dcb35a1385cf76e9f2f4d79 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 3 Feb 2026 15:15:32 +0000 Subject: [PATCH 177/180] Fix optional items in remove --- mypyc/irbuild/vec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 0498b7822cace..73518c8e8a07a 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -459,7 +459,7 @@ def vec_remove(builder: LowLevelIRBuilder, vec: Value, item: Value, line: int) - if item_type in vec_api_by_item_type: name = f"{vec_api_by_item_type[item_type]}.remove" - elif vec_type.depth() == 0 and not isinstance(item_type, RUnion): + elif vec_type.depth() == 0: name = "VecTApi.remove" else: coerced_item = convert_to_t_ext_item(builder, coerced_item) From 7b2edbd3cfbd27f65264afe36dce4489ca534a6d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 3 Feb 2026 15:15:41 +0000 Subject: [PATCH 178/180] Refactor get item --- mypyc/irbuild/vec.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 73518c8e8a07a..1ae7fa9c5f732 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -282,26 +282,22 @@ def vec_get_item( We inline this, since it's simple but performance-critical. """ - assert isinstance(base.type, RVec) - vtype = base.type # TODO: Support more item types # TODO: Support more index types len_val = vec_len(builder, base) index = vec_check_and_adjust_index(builder, len_val, index, line) - index = builder.coerce(index, c_pyssize_t_rprimitive, line) - item_addr = vec_item_ptr(builder, base, index) - result = builder.load_mem(item_addr, vtype.item_type, borrow=can_borrow) - builder.keep_alives.append(base) - return result + return vec_get_item_unsafe(builder, base, index, line, can_borrow=can_borrow) -def vec_get_item_unsafe(builder: LowLevelIRBuilder, base: Value, index: Value, line: int) -> Value: +def vec_get_item_unsafe( + builder: LowLevelIRBuilder, base: Value, index: Value, line: int, *, can_borrow: bool = False +) -> Value: """Get vec item, assuming index is non-negative and within bounds.""" assert isinstance(base.type, RVec) index = as_platform_int(builder, index, line) vtype = base.type item_addr = vec_item_ptr(builder, base, index) - result = builder.load_mem(item_addr, vtype.item_type) + result = builder.load_mem(item_addr, vtype.item_type, borrow=can_borrow) builder.keep_alive([base], line) return result From 94a7b858aa0934900327431f882cd48dade18163 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 3 Feb 2026 15:23:26 +0000 Subject: [PATCH 179/180] Update test case --- mypyc/test-data/irbuild-vec-i64.test | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mypyc/test-data/irbuild-vec-i64.test b/mypyc/test-data/irbuild-vec-i64.test index c3c126e6758d2..a8d7eacda7e5b 100644 --- a/mypyc/test-data/irbuild-vec-i64.test +++ b/mypyc/test-data/irbuild-vec-i64.test @@ -462,7 +462,7 @@ def f() -> vec[i64]: def f(): r0, r1 :: vec[i64] r2 :: short_int - r3, ___tmp_5 :: i64 + r3, ___tmp_6 :: i64 r4 :: bit r5 :: vec[i64] r6 :: short_int @@ -472,18 +472,18 @@ L0: r1 = r0 r2 = 0 r3 = r2 >> 1 - ___tmp_5 = r3 + ___tmp_6 = r3 L1: r4 = int_lt r2, 14 if r4 goto L2 else goto L4 :: bool L2: - r5 = VecI64Api.append(r1, ___tmp_5) + r5 = VecI64Api.append(r1, ___tmp_6) r1 = r5 L3: r6 = r2 + 2 r2 = r6 r7 = r6 >> 1 - ___tmp_5 = r7 + ___tmp_6 = r7 goto L1 L4: return r1 From daf31b4928d3bec624a94adbd3c20aec71c13bce Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 3 Feb 2026 15:44:13 +0000 Subject: [PATCH 180/180] Address review --- mypyc/irbuild/vec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/irbuild/vec.py b/mypyc/irbuild/vec.py index 1ae7fa9c5f732..6177b492da9c0 100644 --- a/mypyc/irbuild/vec.py +++ b/mypyc/irbuild/vec.py @@ -521,7 +521,7 @@ def vec_slice( api_name = vec_api_by_item_type.get(item_type) if api_name is not None: name = f"{api_name}.slice" - elif vec_type.depth() == 0 and not isinstance(item_type, RUnion): + elif vec_type.depth() == 0: name = "VecTApi.slice" else: name = "VecNestedApi.slice"