WIP: Make Vararg not a DataType

Currently `Vararg` is a DataType, but is special cased in a
bunch of places to give it special behavior (e.g. in subtyping
and of course in tuple construction). However, unlike all other
DataTypes, it cannot appear as a type parameter, which caused trouble in
PR #38071. Having it be a DataType is a bit of a pun of convenience
in the first place - it's a lot more similar to a tvar (which can
be considered an implementation detail of UnionAll in the same way
Vararg is an implementation detail of Tuple), which has its own
non-type object. This PR does the same to Vararg, and moves it
from being an abstract DataType with special cased behavior to
its own custom type (called `Core.VarargMarker`). The user facing
behavior should be mostly unchanged, since `boot.jl` now has:
```
const Vararg{T, N} = VarargMarker{T, N}
```
i.e. we have a handly UnionAll wrapper that looks just like it
used to. The biggest difference is probably that VarargMarker
does not have `.parameters`, so code that tries to reach into
that explicitly will need to be adjusted. We could provide
a compatibility `getproperty` method to adapt that, but I'd
prefer to see how many packages need updating first, before
going that route.

This is essentially complete, but a few cleanup items remain.

Author: Keno

base/boot.jl

@@ -242,14 +242,23 @@ ccall(:jl_toplevel_eval_in, Any, (Any, Any),
       (f::typeof(Typeof))(x) = ($(_expr(:meta,:nospecialize,:x)); isa(x,Type) ? Type{x} : typeof(x))
       end)
 
-# let the compiler assume that calling Union{} as a constructor does not need
-# to be considered ever (which comes up often as Type{<:T})
-Union{}(a...) = throw(MethodError(Union{}, a))
 
 macro nospecialize(x)
     _expr(:meta, :nospecialize, x)
 end
 
+TypeVar(n::Symbol) = _typevar(n, Union{}, Any)
+TypeVar(n::Symbol, @nospecialize(ub)) = _typevar(n, Union{}, ub)
+TypeVar(n::Symbol, @nospecialize(lb), @nospecialize(ub)) = _typevar(n, lb, ub)
+
+UnionAll(v::TypeVar, @nospecialize(t)) = ccall(:jl_type_unionall, Any, (Any, Any), v, t)
+
+const Vararg{T, N} = VarargMarker{T, N}
+
+# let the compiler assume that calling Union{} as a constructor does not need
+# to be considered ever (which comes up often as Type{<:T})
+Union{}(a...) = throw(MethodError(Union{}, a))
+
 Expr(@nospecialize args...) = _expr(args...)
 
 abstract type Exception end
@@ -378,12 +387,6 @@ mutable struct WeakRef
                                       (Ptr{Cvoid}, Any), getptls(), v)
 end
 
-TypeVar(n::Symbol) = _typevar(n, Union{}, Any)
-TypeVar(n::Symbol, @nospecialize(ub)) = _typevar(n, Union{}, ub)
-TypeVar(n::Symbol, @nospecialize(lb), @nospecialize(ub)) = _typevar(n, lb, ub)
-
-UnionAll(v::TypeVar, @nospecialize(t)) = ccall(:jl_type_unionall, Any, (Any, Any), v, t)
-
 Tuple{}() = ()
 
 struct VecElement{T}

base/compiler/tfuncs.jl

@@ -449,7 +449,7 @@ function typebound_nothrow(b)
     if isType(b)
         b = unwrap_unionall(b.parameters[1])
         b === Union{} && return true
-        return !isa(b, DataType) || b.name != _va_typename
+        return !isa(b, DataType) || !isa(b, Core.VarargMarker)
     end
     return false
 end
@@ -490,7 +490,7 @@ function typeof_concrete_vararg(t::DataType)
         p = t.parameters[i]
         if i == np && isvarargtype(p)
             pp = unwrap_unionall(p)
-            if isconcretetype(pp.parameters[1]) && pp.parameters[2] isa TypeVar
+            if isconcretetype(pp.T) && pp.N isa TypeVar
                 return rewrap_unionall(Type{Tuple{t.parameters[1:np-1]..., pp}}, p)
             end
         elseif !isconcretetype(p)

base/compiler/typelimits.jl

@@ -46,6 +46,8 @@ function is_derived_type(@nospecialize(t), @nospecialize(c), mindepth::Int)
         # see if it is derived from the body
         # also handle the var here, since this construct bounds the mindepth to the smallest possible value
         return is_derived_type(t, c.var.ub, mindepth) || is_derived_type(t, c.body, mindepth)
+    elseif isa(c, Core.VarargMarker)
+        return is_derived_type(t, c.T, mindepth)
     elseif isa(c, DataType)
         if mindepth > 0
             mindepth -= 1
@@ -85,7 +87,7 @@ function _limit_type_size(@nospecialize(t), @nospecialize(c), sources::SimpleVec
         return t # fast path: unparameterized are always simple
     else
         ut = unwrap_unionall(t)
-        if isa(ut, DataType) && ut.name !== _va_typename && isa(c, Type) && c !== Union{} && c <: t
+        if isa(ut, DataType) && isa(c, Type) && c !== Union{} && c <: t
             # TODO: need to check that the UnionAll bounds on t are limited enough too
             return t # t is already wider than the comparison in the type lattice
         elseif is_derived_type_from_any(ut, sources, depth)
@@ -118,19 +120,20 @@ function _limit_type_size(@nospecialize(t), @nospecialize(c), sources::SimpleVec
             b = _limit_type_size(t.b, c.b, sources, depth, allowed_tuplelen)
             return Union{a, b}
         end
+    elseif isa(t, Core.VarargMarker)
+        isa(c, Core.VarargMarker) || return Vararg
+        VaT = _limit_type_size(t.T, c.T, sources, depth + 1, 0)
+        N = t.N
+        if isa(N, TypeVar) || N === c.N
+            return Vararg{VaT, N}
+        end
+        return Vararg{VaT}
     elseif isa(t, DataType)
         if isa(c, DataType)
             tP = t.parameters
             cP = c.parameters
             if t.name === c.name && !isempty(cP)
-                if isvarargtype(t)
-                    VaT = _limit_type_size(tP[1], cP[1], sources, depth + 1, 0)
-                    N = tP[2]
-                    if isa(N, TypeVar) || N === cP[2]
-                        return Vararg{VaT, N}
-                    end
-                    return Vararg{VaT}
-                elseif t.name === Tuple.name
+                if t.name === Tuple.name
                     # for covariant datatypes (Tuple),
                     # apply type-size limit element-wise
                     ltP = length(tP)
@@ -155,21 +158,17 @@ function _limit_type_size(@nospecialize(t), @nospecialize(c), sources::SimpleVec
                     end
                     return Tuple{Q...}
                 end
-            elseif isvarargtype(c)
-                # Tuple{Vararg{T}} --> Tuple{T} is OK
-                return _limit_type_size(t, cP[1], sources, depth, 0)
             end
+        elseif isa(c, Core.VarargMarker)
+            # Tuple{Vararg{T}} --> Tuple{T} is OK
+            return _limit_type_size(t, c.T, sources, depth, 0)
         end
         if isType(t) # allow taking typeof as Type{...}, but ensure it doesn't start nesting
             tt = unwrap_unionall(t.parameters[1])
             if isa(tt, DataType) && !isType(tt)
                 is_derived_type_from_any(tt, sources, depth) && return t
             end
         end
-        if isvarargtype(t)
-            # never replace Vararg with non-Vararg
-            return Vararg
-        end
         if allowed_tuplelen < 1 && t.name === Tuple.name
             return Any
         end

base/essentials.jl

@@ -271,10 +271,9 @@ function rename_unionall(@nospecialize(u))
     return UnionAll(nv, body{nv})
 end
 
-const _va_typename = Vararg.body.body.name
 function isvarargtype(@nospecialize(t))
     t = unwrap_unionall(t)
-    return isa(t, DataType) && (t::DataType).name === _va_typename
+    return isa(t, Core.VarargMarker)
 end
 
 function isvatuple(@nospecialize(t))
@@ -289,14 +288,14 @@ end
 function unwrapva(@nospecialize(t))
     # NOTE: this returns a related type, but it's NOT a subtype of the original tuple
     t2 = unwrap_unionall(t)
-    return isvarargtype(t2) ? rewrap_unionall(t2.parameters[1], t) : t
+    return isvarargtype(t2) ? rewrap_unionall(t2.T, t) : t
 end
 
 function unconstrain_vararg_length(@nospecialize(va))
     # construct a new Vararg type where its length is unconstrained,
     # but its element type still captures any dependencies the input
     # element type may have had on the input length
-    T = unwrap_unionall(va).parameters[1]
+    T = unwrap_unionall(va).T
     return rewrap_unionall(Vararg{T}, va)
 end
 

base/promotion.jl

@@ -157,7 +157,7 @@ function full_va_len(p)
     isempty(p) && return 0, true
     last = p[end]
     if isvarargtype(last)
-        N = unwrap_unionall(last).parameters[2]
+        N = unwrap_unionall(last).N
         if isa(N, Int)
             return length(p)::Int + N - 1, true
         end

base/show.jl

@@ -2212,6 +2212,14 @@ function show(io::IO, tv::TypeVar)
     nothing
 end
 
+function show(io::IO, vm::Core.VarargMarker)
+    print(io, "Vararg{")
+    show(io, vm.T)
+    print(io, ", ")
+    show(io, vm.N)
+    print(io, "}")
+end
+
 module IRShow
     const Compiler = Core.Compiler
     using Core.IR

src/builtins.c

@@ -1002,6 +1002,10 @@ JL_CALLABLE(jl_f_apply_type)
         // substituting typevars (a valid_type_param check here isn't sufficient).
         return (jl_value_t*)jl_type_union(&args[1], nargs-1);
     }
+    else if (args[0] == (jl_value_t*)jl_vararg_marker_type) {
+        JL_NARGS(apply_type, 3, 3);
+        return jl_wrap_vararg(args[1], args[2]);
+    }
     else if (jl_is_unionall(args[0])) {
         for(i=1; i < nargs; i++) {
             jl_value_t *pi = args[i];
@@ -1257,7 +1261,6 @@ static void jl_set_datatype_super(jl_datatype_t *tt, jl_value_t *super)
     if (!jl_is_datatype(super) || !jl_is_abstracttype(super) ||
         tt->super != NULL ||
         tt->name == ((jl_datatype_t*)super)->name ||
-        jl_subtype(super, (jl_value_t*)jl_vararg_type) ||
         jl_is_tuple_type(super) ||
         jl_is_namedtuple_type(super) ||
         jl_subtype(super, (jl_value_t*)jl_type_type) ||
@@ -1577,7 +1580,7 @@ void jl_init_primitives(void) JL_GC_DISABLED
     add_builtin("Union", (jl_value_t*)jl_uniontype_type);
     add_builtin("TypeofBottom", (jl_value_t*)jl_typeofbottom_type);
     add_builtin("Tuple", (jl_value_t*)jl_anytuple_type);
-    add_builtin("Vararg", (jl_value_t*)jl_vararg_type);
+    add_builtin("VarargMarker", (jl_value_t*)jl_vararg_marker_type);
     add_builtin("SimpleVector", (jl_value_t*)jl_simplevector_type);
 
     add_builtin("Module", (jl_value_t*)jl_module_type);

src/dump.c

@@ -567,6 +567,11 @@ static void jl_serialize_value_(jl_serializer_state *s, jl_value_t *v, int as_li
         jl_serialize_value(s, ((jl_tvar_t*)v)->lb);
         jl_serialize_value(s, ((jl_tvar_t*)v)->ub);
     }
+    else if (jl_is_vararg_marker(v)) {
+        write_uint8(s->s, TAG_VARARG);
+        jl_serialize_value(s, ((jl_vararg_marker_t*)v)->T);
+        jl_serialize_value(s, ((jl_vararg_marker_t*)v)->N);
+    }
     else if (jl_is_method(v)) {
         write_uint8(s->s, TAG_METHOD);
         jl_method_t *m = (jl_method_t*)v;
@@ -1747,6 +1752,15 @@ static jl_value_t *jl_deserialize_value(jl_serializer_state *s, jl_value_t **loc
         tv->ub = jl_deserialize_value(s, &tv->ub);
         jl_gc_wb(tv, tv->ub);
         return (jl_value_t*)tv;
+    case TAG_VARARG:
+        v = jl_gc_alloc(s->ptls, sizeof(jl_vararg_marker_t), jl_vararg_marker_type);
+        jl_vararg_marker_t *vm = (jl_vararg_marker_t*)v;
+        arraylist_push(&backref_list, vm);
+        vm->T = (jl_sym_t*)jl_deserialize_value(s, NULL);
+        jl_gc_wb(vm, vm->T);
+        vm->N = jl_deserialize_value(s, &vm->N);
+        jl_gc_wb(vm, vm->N);
+        return (jl_value_t*)vm;
     case TAG_METHOD:
         return jl_deserialize_value_method(s, loc);
     case TAG_METHOD_INSTANCE:
@@ -2580,7 +2594,8 @@ void jl_init_serializer(void)
 
                      jl_bool_type, jl_linenumbernode_type, jl_pinode_type,
                      jl_upsilonnode_type, jl_type_type, jl_bottom_type, jl_ref_type,
-                     jl_pointer_type, jl_vararg_type, jl_abstractarray_type, jl_nothing_type,
+                     jl_pointer_type, jl_abstractarray_type, jl_nothing_type,
+                     jl_vararg_marker_type,
                      jl_densearray_type, jl_function_type, jl_typename_type,
                      jl_builtin_type, jl_task_type, jl_uniontype_type,
                      jl_array_any_type, jl_intrinsic_type,
@@ -2629,6 +2644,7 @@ void jl_init_serializer(void)
     deser_tag[TAG_GOTOIFNOT] = (jl_value_t*)jl_gotoifnot_type;
     deser_tag[TAG_RETURNNODE] = (jl_value_t*)jl_returnnode_type;
     deser_tag[TAG_ARGUMENT] = (jl_value_t*)jl_argument_type;
+    deser_tag[TAG_VARARG] = (jl_value_t*)jl_vararg_marker_type;
 
     intptr_t i = 0;
     while (vals[i] != NULL) {
@@ -2658,7 +2674,6 @@ void jl_init_serializer(void)
     arraylist_push(&builtin_typenames, ((jl_datatype_t*)jl_unwrap_unionall((jl_value_t*)jl_abstractarray_type))->name);
     arraylist_push(&builtin_typenames, ((jl_datatype_t*)jl_unwrap_unionall((jl_value_t*)jl_densearray_type))->name);
     arraylist_push(&builtin_typenames, jl_tuple_typename);
-    arraylist_push(&builtin_typenames, jl_vararg_typename);
     arraylist_push(&builtin_typenames, jl_namedtuple_typename);
 }