IndexedTables and type-stability of IndexedTables.groupby

@bashonubuntu wrote:

Hi,

I am a bit confused by IndexedTable being referred to as a “type-stable” alternative to DataFrames. If I write a function which does some operations and returns an IndexedTable, the @code_warntype still flags the IndexedTable return variable in red. What am I misunderstanding here?

I am also unsure if common operations on IndexedTables like IndexedTables.groupby etc. are type-stable. Is it possible to assert the type of the output of IndexedTables.groupby if I know the types of each column before-hand. Can someone provide an example?

Sharing an example below of a typical use-case in my setting. Is there any way to make data_1, d and d_1 type-stable? The code warntype result flags them as Any. Type-assertion doesn’t work either and the type IndexedTable is marked as red in @code_warntype.

function test(meta_data::R)::DataFrame where {R<:DataFrame} 
     
     data_1 = table(meta_data)
     d = IndexedTables.groupby(df -> 1 ./ (1 .+ df.eV_observed[1]), data_1, (:std_id, :act_yr, :draws_index, :cmp_id), select = (:V_observed, :eV_observed))
     d_1 = table(IndexedTables.columns(d)..., names = (:std_id, :act_yr, :draws_index, :cmp_id, :Prob), copy = false)
     d_final = DataFrames.DataFrame(d_1)::DataFrame
     return d_final::DataFrame
end

@code_warntype test(meta_data) 

Variables
  #self#::Core.Compiler.Const(prob_stage2_create_groupby_new1, false)
  meta_data::DataFrame
  #3209::getfield(Main, Symbol("##3209#3210"))
  data_1::Any
  d::Any
  d_1::Any
  d_final::DataFrame

Body::DataFrame
1 ─ %1  = Main.DataFrame::Core.Compiler.Const(DataFrame, false)
│         (data_1 = Main.table(meta_data))
│         (#3209 = %new(Main.:(##3209#3210)))
│   %4  = #3209::Core.Compiler.Const(getfield(Main, Symbol("##3209#3210"))(), false)
│   %5  = (:V_observed, :eV_observed)::Core.Compiler.Const((:V_observed, :eV_observed), false)
│   %6  = (:select,)::Core.Compiler.Const((:select,), false)
│   %7  = Core.apply_type(Core.NamedTuple, %6)::Core.Compiler.Const(NamedTuple{(:select,),T} where T<:Tuple, false)
│   %8  = Core.tuple(%5)::Core.Compiler.Const(((:V_observed, :eV_observed),), false)
│   %9  = (%7)(%8)::NamedTuple{(:select,),Tuple{Tuple{Symbol,Symbol}}}
│   %10 = IndexedTables.groupby::Core.Compiler.Const(IndexedTables.groupby, false)
│   %11 = Core.kwfunc(%10)::Core.Compiler.Const(getfield(IndexedTables, Symbol("#kw##groupby"))(), false)
│   %12 = IndexedTables.groupby::Core.Compiler.Const(IndexedTables.groupby, false)
│   %13 = data_1::Any
│   %14 = (:std_id, :act_yr, :draws_index, :cmp_id)::Core.Compiler.Const((:std_id, :act_yr, :draws_index, :cmp_id), false)
│         (d = (%11)(%9, %12, %4, %13, %14))
│   %16 = IndexedTables.columns::Core.Compiler.Const(IndexedTables.columns, false)
│   %17 = (%16)(d)::Any
│   %18 = (:std_id, :act_yr, :draws_index, :cmp_id, :Prob)::Core.Compiler.Const((:std_id, :act_yr, :draws_index, :cmp_id, :Prob), false)
│   %19 = (:names, :copy)::Core.Compiler.Const((:names, :copy), false)
│   %20 = Core.apply_type(Core.NamedTuple, %19)::Core.Compiler.Const(NamedTuple{(:names, :copy),T} where T<:Tuple, false)
│   %21 = Core.tuple(%18, false)::Core.Compiler.Const(((:std_id, :act_yr, :draws_index, :cmp_id, :Prob), false), false)
│   %22 = (%20)(%21)::NamedTuple{(:names, :copy),Tuple{NTuple{5,Symbol},Bool}}
│   %24 = Core.tuple(%22, Main.table)::Core.Compiler.PartialStruct(Tuple{NamedTuple{(:names, :copy),Tuple{NTuple{5,Symbol},Bool}},typeof(table)}, Any[NamedTuple{(:names, :copy),Tuple{NTuple{5,Symbol},Bool}}, Core.Compiler.Const(IndexedTables.table, false)])
│         (d_1 = Core._apply(%23, %24, %17))
│   %26 = DataFrames.DataFrame::Core.Compiler.Const(DataFrame, false)
│   %27 = (%26)(d_1)::Any
│         (d_final = Core.typeassert(%27, Main.DataFrame))
│   %29 = Core.typeassert(d_final, Main.DataFrame)::DataFrame
│   %30 = Base.convert(%1, %29)::DataFrame
│   %31 = Core.typeassert(%30, %1)::DataFrame
└──       return %31

Posts: 9

Participants: 2

Read full topic