Begin rebase of 42465 onto 43888

Author: Keno

base/array.jl

@@ -147,12 +147,20 @@ function vect(X...)
     return copyto!(Vector{T}(undef, length(X)), X)
 end
 
-size(a::Array, d::Integer) = arraysize(a, convert(Int, d))
-size(a::Vector) = (arraysize(a,1),)
-size(a::Matrix) = (arraysize(a,1), arraysize(a,2))
-size(a::Array{<:Any,N}) where {N} = (@inline; ntuple(M -> size(a, M), Val(N))::Dims)
+const ImmutableArray = Core.ImmutableArray
+const IMArray{T,N} = Union{Array{T, N}, ImmutableArray{T,N}}
+const IMVector{T} = IMArray{T, 1}
+const IMMatrix{T} = IMArray{T, 2}
 
-asize_from(a::Array, n) = n > ndims(a) ? () : (arraysize(a,n), asize_from(a, n+1)...)
+ImmutableArray(a::Array) = Core.arrayfreeze(a)
+Array(a::ImmutableArray) = Core.arraythaw(a)
+
+size(a::IMArray, d::Integer) = arraysize(a, convert(Int, d))
+size(a::IMVector) = (arraysize(a,1),)
+size(a::IMMatrix) = (arraysize(a,1), arraysize(a,2))
+size(a::IMArray{<:Any,N}) where {N} = (@_inline_meta; ntuple(M -> size(a, M), Val(N))::Dims)
+
+asize_from(a::IMArray, n) = n > ndims(a) ? () : (arraysize(a,n), asize_from(a, n+1)...)
 
 allocatedinline(T::Type) = (@_pure_meta; ccall(:jl_stored_inline, Cint, (Any,), T) != Cint(0))
 
@@ -223,6 +231,13 @@ function isassigned(a::Array, i::Int...)
     ccall(:jl_array_isassigned, Cint, (Any, UInt), a, ii) == 1
 end
 
+function isassigned(a::ImmutableArray, i::Int...)
+    @_inline_meta
+    ii = (_sub2ind(size(a), i...) % UInt) - 1
+    @boundscheck ii < length(a) % UInt || return false
+    ccall(:jl_array_isassigned, Cint, (Any, UInt), a, ii) == 1
+end
+
 ## copy ##
 
 """
@@ -921,7 +936,10 @@ function getindex end
 @eval getindex(A::Array, i1::Int) = arrayref($(Expr(:boundscheck)), A, i1)
 @eval getindex(A::Array, i1::Int, i2::Int, I::Int...) = (@inline; arrayref($(Expr(:boundscheck)), A, i1, i2, I...))
 
-# Faster contiguous indexing using copyto! for AbstractUnitRange and Colon
+@eval getindex(A::ImmutableArray, i1::Int) = arrayref($(Expr(:boundscheck)), A, i1)
+@eval getindex(A::ImmutableArray, i1::Int, i2::Int, I::Int...) = (@_inline_meta; arrayref($(Expr(:boundscheck)), A, i1, i2, I...))
+
+# Faster contiguous indexing using copyto! for UnitRange and Colon
 function getindex(A::Array, I::AbstractUnitRange{<:Integer})
     @inline
     @boundscheck checkbounds(A, I)

base/compiler/ssair/ir.jl

@@ -319,6 +319,13 @@ function setindex!(x::IRCode, repl::Instruction, s::SSAValue)
     return x
 end
 
+function ssadominates(ir::IRCode, domtree::DomTree, ssa1::Int, ssa2::Int)
+    bb1 = block_for_inst(ir.cfg, ssa1)
+    bb2 = block_for_inst(ir.cfg, ssa2)
+    bb1 == bb2 && return ssa1 < ssa2
+    return dominates(domtree, bb1, bb2)
+end
+
 # SSA values that need renaming
 struct OldSSAValue
     id::Int

base/compiler/ssair/passes.jl

@@ -1760,3 +1760,77 @@ function cfg_simplify!(ir::IRCode)
     compact.active_result_bb = length(bb_starts)
     return finish(compact)
 end
+
+function is_allocation(stmt)
+    isexpr(stmt, :foreigncall) || return false
+    s = stmt.args[1]
+    isa(s, QuoteNode) && (s = s.value)
+    return s === :jl_alloc_array_1d
+end
+
+function memory_opt!(ir::IRCode)
+    compact = IncrementalCompact(ir, false)
+    uses = IdDict{Int, Vector{Int}}()
+    relevant = IdSet{Int}()
+    revisit = Int[]
+    function mark_val(val)
+        isa(val, SSAValue) || return
+        val.id in relevant && pop!(relevant, val.id)
+    end
+    for ((_, idx), stmt) in compact
+        if isa(stmt, ReturnNode)
+            isdefined(stmt, :val) || continue
+            val = stmt.val
+            if isa(val, SSAValue) && val.id in relevant
+                (haskey(uses, val.id)) || (uses[val.id] = Int[])
+                push!(uses[val.id], idx)
+            end
+            continue
+        end
+        (isexpr(stmt, :call) || isexpr(stmt, :foreigncall)) || continue
+        if is_allocation(stmt)
+            push!(relevant, idx)
+            # TODO: Mark everything else here
+            continue
+        end
+        # TODO: Replace this by interprocedural escape analysis
+        if is_known_call(stmt, arrayset, compact)
+            # The value being set escapes, everything else doesn't
+            mark_val(stmt.args[4])
+            arr = stmt.args[3]
+            if isa(arr, SSAValue) && arr.id in relevant
+                (haskey(uses, arr.id)) || (uses[arr.id] = Int[])
+                push!(uses[arr.id], idx)
+            end
+        elseif is_known_call(stmt, Core.arrayfreeze, compact) && isa(stmt.args[2], SSAValue)
+            push!(revisit, idx)
+        else
+            # For now we assume everything escapes
+            # TODO: We could handle PhiNodes specially and improve this
+            for ur in userefs(stmt)
+                mark_val(ur[])
+            end
+        end
+    end
+    ir = finish(compact)
+    isempty(revisit) && return ir
+    domtree = construct_domtree(ir.cfg.blocks)
+    for idx in revisit
+        # Make sure that the value we reference didn't escape
+        id = ir.stmts[idx][:inst].args[2].id
+        (id in relevant) || continue
+
+        # We're ok to steal the memory if we don't dominate any uses
+        ok = true
+        for use in uses[id]
+            if ssadominates(ir, domtree, idx, use)
+                ok = false
+                break
+            end
+        end
+        ok || continue
+
+        ir.stmts[idx][:inst].args[1] = Core.mutating_arrayfreeze
+    end
+    return ir
+end

base/compiler/tfuncs.jl

@@ -1803,6 +1803,21 @@ function builtin_tfunction(interp::AbstractInterpreter, @nospecialize(f), argtyp
                            sv::Union{InferenceState,Nothing})
     if f === tuple
         return tuple_tfunc(argtypes)
+    elseif f === Core.arrayfreeze || f === Core.arraythaw
+        if length(argtypes) != 1
+            isva && return Any
+            return Bottom
+        end
+        a = widenconst(argtypes[1])
+        at = (f === Core.arrayfreeze ? Array : ImmutableArray)
+        rt = (f === Core.arrayfreeze ? ImmutableArray : Array)
+        if a <: at
+            unw = unwrap_unionall(a)
+            if isa(unw, DataType)
+                return rewrap_unionall(rt{unw.parameters[1], unw.parameters[2]}, a)
+            end
+        end
+        return rt
     end
     if isa(f, IntrinsicFunction)
         if is_pure_intrinsic_infer(f) && _all(@nospecialize(a) -> isa(a, Const), argtypes)

base/dict.jl

@@ -373,7 +373,7 @@ end
 function setindex!(h::Dict{K,V}, v0, key0) where V where K
     key = convert(K, key0)
     if !isequal(key, key0)
-        throw(ArgumentError("$(limitrepr(key0)) is not a valid key for type $K"))
+        throw(KeyTypeError(K, key0))
     end
     setindex!(h, v0, key)
 end

base/experimental.jl

@@ -11,6 +11,8 @@ module Experimental
 
 using Base: Threads, sync_varname
 using Base.Meta
+using Base: ImmutableArray
+
 
 """
     Const(A::Array)

src/builtin_proto.h

@@ -55,6 +55,9 @@ DECLARE_BUILTIN(_typebody);
 DECLARE_BUILTIN(typeof);
 DECLARE_BUILTIN(_typevar);
 DECLARE_BUILTIN(donotdelete);
+DECLARE_BUILTIN(arrayfreeze);
+DECLARE_BUILTIN(arraythaw);
+DECLARE_BUILTIN(mutating_arrayfreeze);
 
 JL_CALLABLE(jl_f_invoke_kwsorter);
 #ifdef DEFINE_BUILTIN_GLOBALS

src/builtins.c

@@ -1373,7 +1373,9 @@ JL_CALLABLE(jl_f__typevar)
 JL_CALLABLE(jl_f_arraysize)
 {
     JL_NARGS(arraysize, 2, 2);
-    JL_TYPECHK(arraysize, array, args[0]);
+    if (!jl_is_arrayish(args[0])) {
+        jl_type_error("arraysize", (jl_value_t*)jl_array_type, args[0]);
+    }
     jl_array_t *a = (jl_array_t*)args[0];
     size_t nd = jl_array_ndims(a);
     JL_TYPECHK(arraysize, long, args[1]);
@@ -1412,7 +1414,9 @@ JL_CALLABLE(jl_f_arrayref)
 {
     JL_NARGSV(arrayref, 3);
     JL_TYPECHK(arrayref, bool, args[0]);
-    JL_TYPECHK(arrayref, array, args[1]);
+    if (!jl_is_arrayish(args[1])) {
+        jl_type_error("arrayref", (jl_value_t*)jl_array_type, args[1]);
+    }
     jl_array_t *a = (jl_array_t*)args[1];
     size_t i = array_nd_index(a, &args[2], nargs - 2, "arrayref");
     return jl_arrayref(a, i);
@@ -1735,6 +1739,54 @@ JL_CALLABLE(jl_f_set_binding_type)
     return jl_nothing;
 }
 
+JL_CALLABLE(jl_f_arrayfreeze)
+{
+    JL_NARGSV(arrayfreeze, 1);
+    JL_TYPECHK(arrayfreeze, array, args[0]);
+    jl_array_t *a = (jl_array_t*)args[0];
+    jl_datatype_t *it = (jl_datatype_t *)jl_apply_type2((jl_value_t*)jl_immutable_array_type,
+        jl_tparam0(jl_typeof(a)), jl_tparam1(jl_typeof(a)));
+    JL_GC_PUSH1(&it);
+    // The idea is to elide this copy if the compiler or runtime can prove that
+    // doing so is safe to do.
+    jl_array_t *na = jl_array_copy(a);
+    jl_set_typeof(na, it);
+    JL_GC_POP();
+    return (jl_value_t*)na;
+}
+
+JL_CALLABLE(jl_f_mutating_arrayfreeze)
+{
+    // N.B.: These error checks pretend to be arrayfreeze since this is a drop
+    // in replacement and we don't want to change the visible error type in the
+    // optimizer
+    JL_NARGSV(arrayfreeze, 1);
+    JL_TYPECHK(arrayfreeze, array, args[0]);
+    jl_array_t *a = (jl_array_t*)args[0];
+    jl_datatype_t *it = (jl_datatype_t *)jl_apply_type2((jl_value_t*)jl_immutable_array_type,
+        jl_tparam0(jl_typeof(a)), jl_tparam1(jl_typeof(a)));
+    jl_set_typeof(a, it);
+    return (jl_value_t*)a;
+}
+
+JL_CALLABLE(jl_f_arraythaw)
+{
+    JL_NARGSV(arraythaw, 1);
+    if (((jl_datatype_t*)jl_typeof(args[0]))->name != jl_immutable_array_typename) {
+        jl_type_error("arraythaw", (jl_value_t*)jl_immutable_array_type, args[0]);
+    }
+    jl_array_t *a = (jl_array_t*)args[0];
+    jl_datatype_t *it = (jl_datatype_t *)jl_apply_type2((jl_value_t*)jl_array_type,
+        jl_tparam0(jl_typeof(a)), jl_tparam1(jl_typeof(a)));
+    JL_GC_PUSH1(&it);
+    // The idea is to elide this copy if the compiler or runtime can prove that
+    // doing so is safe to do.
+    jl_array_t *na = jl_array_copy(a);
+    jl_set_typeof(na, it);
+    JL_GC_POP();
+    return (jl_value_t*)na;
+}
+
 // IntrinsicFunctions ---------------------------------------------------------
 
 static void (*runtime_fp[num_intrinsics])(void);
@@ -1890,6 +1942,10 @@ void jl_init_primitives(void) JL_GC_DISABLED
     jl_builtin_arrayset = add_builtin_func("arrayset", jl_f_arrayset);
     jl_builtin_arraysize = add_builtin_func("arraysize", jl_f_arraysize);
 
+    jl_builtin_arrayfreeze = add_builtin_func("arrayfreeze", jl_f_arrayfreeze);
+    jl_builtin_mutating_arrayfreeze = add_builtin_func("mutating_arrayfreeze", jl_f_mutating_arrayfreeze);
+    jl_builtin_arraythaw = add_builtin_func("arraythaw", jl_f_arraythaw);
+
     // method table utils
     jl_builtin_applicable = add_builtin_func("applicable", jl_f_applicable);
     jl_builtin_invoke = add_builtin_func("invoke", jl_f_invoke);
@@ -1965,6 +2021,7 @@ void jl_init_primitives(void) JL_GC_DISABLED
     add_builtin("AbstractArray", (jl_value_t*)jl_abstractarray_type);
     add_builtin("DenseArray", (jl_value_t*)jl_densearray_type);
     add_builtin("Array", (jl_value_t*)jl_array_type);
+    add_builtin("ImmutableArray", (jl_value_t*)jl_immutable_array_type);
 
     add_builtin("Expr", (jl_value_t*)jl_expr_type);
     add_builtin("LineNumberNode", (jl_value_t*)jl_linenumbernode_type);

src/cgutils.cpp

@@ -491,7 +491,7 @@ static Type *_julia_type_to_llvm(jl_codegen_params_t *ctx, LLVMContext &ctxt, jl
     if (isboxed) *isboxed = false;
     if (jt == (jl_value_t*)jl_bottom_type)
         return getVoidTy(ctxt);
-    if (jl_is_concrete_immutable(jt)) {
+    if (jl_is_concrete_immutable(jt) && !jl_is_arrayish_type(jt)) {
         if (jl_datatype_nbits(jt) == 0)
             return getVoidTy(ctxt);
         Type *t = _julia_struct_to_llvm(ctx, ctxt, jt, isboxed);

src/codegen.cpp

@@ -1054,6 +1054,15 @@ static const auto pointer_from_objref_func = new JuliaFunction{
             Attributes(C, {Attribute::NonNull}),
             None); },
 };
+static const auto mutating_arrayfreeze_func = new JuliaFunction{
+    "julia.mutating_arrayfreeze",
+    [](LLVMContext &C) { return FunctionType::get(T_prjlvalue,
+                {T_prjlvalue, T_prjlvalue}, false); },
+    [](LLVMContext &C) { return AttributeList::get(C,
+            Attributes(C, {Attribute::NoUnwind, Attribute::NoRecurse}),
+            Attributes(C, {Attribute::NonNull}),
+            None); },
+};
 
 static const auto jltuple_func = new JuliaFunction{XSTR(jl_f_tuple), get_func_sig, get_func_attrs};
 static std::map<jl_fptr_args_t, JuliaFunction*> builtin_func_map;
@@ -1125,7 +1134,7 @@ static bool deserves_retbox(jl_value_t* t)
 static bool deserves_sret(jl_value_t *dt, Type *T)
 {
     assert(jl_is_datatype(dt));
-    return (size_t)jl_datatype_size(dt) > sizeof(void*) && !T->isFloatingPointTy() && !T->isVectorTy();
+    return (size_t)jl_datatype_size(dt) > sizeof(void*) && !T->isFloatingPointTy() && !T->isVectorTy() && !jl_is_arrayish_type(dt);
 }
 
 
@@ -2996,6 +3005,21 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f,
         }
     }
 
+    else if (f == jl_builtin_mutating_arrayfreeze && nargs == 1) {
+        const jl_cgval_t &ary = argv[1];
+        jl_value_t *aty_dt = jl_unwrap_unionall(ary.typ);
+        if (jl_is_array_type(aty_dt)) {
+            jl_datatype_t *it = (jl_datatype_t *)jl_apply_type2((jl_value_t*)jl_immutable_array_type,
+                jl_tparam0(aty_dt), jl_tparam1(aty_dt));
+            *ret = mark_julia_type(ctx,
+                ctx.builder.CreateCall(prepare_call(mutating_arrayfreeze_func),
+                    { boxed(ctx, ary),
+                    track_pjlvalue(ctx, literal_pointer_val(ctx, (jl_value_t*)it)) }), true, it);
+            return true;
+        }
+        return false;
+    }
+
     else if (f == jl_builtin_arrayset && nargs >= 4) {
         const jl_cgval_t &ary = argv[2];
         jl_cgval_t val = argv[3];
@@ -8176,7 +8200,10 @@ extern "C" void jl_init_llvm(void)
           { jl_f_arrayset_addr,           new JuliaFunction{XSTR(jl_f_arrayset), get_func_sig, get_func_attrs} },
           { jl_f_arraysize_addr,          new JuliaFunction{XSTR(jl_f_arraysize), get_func_sig, get_func_attrs} },
           { jl_f_apply_type_addr,         new JuliaFunction{XSTR(jl_f_apply_type), get_func_sig, get_func_attrs} },
-          { jl_f_donotdelete_addr,        new JuliaFunction{XSTR(jl_f_donotdelete), get_func_sig, get_donotdelete_func_attrs} }
+          { jl_f_donotdelete_addr,        new JuliaFunction{XSTR(jl_f_donotdelete), get_func_sig, get_donotdelete_func_attrs} },
+          { jl_f_arrayfreeze_addr,             new JuliaFunction{XSTR(jl_f_arrayfreeze), get_func_sig, get_func_attrs} },
+          { jl_f_arraythaw_addr,               new JuliaFunction{XSTR(jl_f_arraythaw), get_func_sig, get_func_attrs} },
+          { jl_f_mutating_arrayfreeze_addr,    new JuliaFunction{XSTR(jl_f_mutating_arrayfreeze), get_func_sig, get_func_attrs} }
         };
 
     jl_default_debug_info_kind = (int) DICompileUnit::DebugEmissionKind::FullDebug;