import narwhals as nw
import polars as pl
import polars.selectors as cs

from survey_kit.utilities.random import RandomData
from survey_kit.utilities.dataframe import summary

from survey_kit.imputation.variable import Variable
from survey_kit.imputation.parameters import Parameters
from survey_kit.imputation.srmi import SRMI
from survey_kit.imputation.selection import Selection

from survey_kit import logger, config
from survey_kit.utilities.dataframe import summary, columns_from_list
from survey_kit.utilities.formula_builder import FormulaBuilder

# Draw some random data

n_rows = 10_000
impute_share = 0.25


df = (
    RandomData(n_rows=n_rows, seed=32565437)
    .index("index")
    .integer("year", 2016, 2020)
    .integer("month", 1, 12)
    .integer("var2", 0, 10)
    .integer("var3", 0, 50)
    .float("var4", 0, 1)
    .integer("var5", 0, 1)
    .float("unrelated_1", 0, 1)
    .float("unrelated_2", 0, 1)
    .float("unrelated_3", 0, 1)
    .float("unrelated_4", 0, 1)
    .float("unrelated_5", 0, 1)
    .np_distribution("epsilon_reg1", "normal", scale=5)
    .np_distribution("epsilon_reg2", "normal", scale=5)
    .float("missing_reg1", 0, 1)
    .float("missing_reg2", 0, 1)
    .to_df()
)


#   Convenience references to them for creating dependent variables
c_var2 = pl.col("var2")
c_var3 = pl.col("var3")
c_var4 = pl.col("var4")
c_var5 = pl.col("var5")

c_e_reg1 = pl.col("epsilon_reg1")
c_e_reg2 = pl.col("epsilon_reg2")


#   Convenience references to them for creating dependent variables
c_var2 = pl.col("var2")
c_var3 = pl.col("var3")
c_var4 = pl.col("var4")
c_var5 = pl.col("var5")


logger.info("var_reg1 is binary and conditional on other variables")
c_reg1 = ((c_var2 * 2 - c_var3 * 3 * c_var5 + c_e_reg1) > 0).alias("var_reg1")

logger.info("var_reg2 is != 0 only if var_reg1 == True")
c_reg2 = (
    pl.when(pl.col("var_reg1"))
    .then((c_var2 * 1.5 - c_var3 * 1 * c_var4 + c_e_reg2))
    .otherwise(pl.lit(0))
    .alias("var_reg2")
)
#   Create a bunch of variables that are functions of the variables created above
df = (
    df.with_columns(c_reg1)
    .with_columns(c_reg2)
    .drop(columns_from_list(df=df, columns="epsilon*"))
    .with_row_index(name="_row_index_")
)

df_original = df

#   Set variables to missing according to the uniform random variables missing_
clear_missing = []
for prefixi in ["reg"]:
    for i in range(1, 3):
        vari = f"var_{prefixi}{i}"
        missingi = f"missing_{prefixi}{i}"

        clear_missing.append(
            pl.when(pl.col(missingi) < impute_share)
            .then(pl.lit(None))
            .otherwise(pl.col(vari))
            .alias(vari)
        )
df = df.with_columns(clear_missing).drop(cs.starts_with("missing_"))

#   Make a fully collinear var for testing
df = df.with_columns(pl.col("unrelated_1").alias("repeat_1"))


summary(df)


#   Actually do the imputation

var_reg1 is binary and conditional on other variables

var_reg2 is != 0 only if var_reg1 == True

┌─────────────┬────────┬─────────────┬────────────┬─────────────┬────────────┬───────────┐
│    Variable ┆      n ┆ n (missing) ┆       mean ┆         std ┆        min ┆       max │
╞═════════════╪════════╪═════════════╪════════════╪═════════════╪════════════╪═══════════╡
│ _row_index_ ┆ 10,000 ┆           0 ┆    4,999.5 ┆ 2,886.89568 ┆        0.0 ┆   9,999.0 │
│       index ┆ 10,000 ┆           0 ┆    4,999.5 ┆ 2,886.89568 ┆        0.0 ┆   9,999.0 │
│        year ┆ 10,000 ┆           0 ┆ 2,017.9851 ┆    1.415937 ┆    2,016.0 ┆   2,020.0 │
│       month ┆ 10,000 ┆           0 ┆     6.5137 ┆    3.432141 ┆        1.0 ┆      12.0 │
│        var2 ┆ 10,000 ┆           0 ┆     4.9782 ┆    3.154508 ┆        0.0 ┆      10.0 │
│        var3 ┆ 10,000 ┆           0 ┆    25.1084 ┆   14.752302 ┆        0.0 ┆      50.0 │
│        var4 ┆ 10,000 ┆           0 ┆   0.505666 ┆    0.287861 ┆   0.000027 ┆  0.999997 │
│ unrelated_1 ┆ 10,000 ┆           0 ┆   0.502449 ┆    0.288359 ┆   0.000119 ┆  0.999997 │
│ unrelated_2 ┆ 10,000 ┆           0 ┆   0.500105 ┆    0.287638 ┆   0.000049 ┆  0.999539 │
│ unrelated_3 ┆ 10,000 ┆           0 ┆   0.499175 ┆     0.28876 ┆   0.000129 ┆   0.99994 │
│ unrelated_4 ┆ 10,000 ┆           0 ┆   0.500655 ┆    0.288698 ┆   0.000133 ┆  0.999972 │
│ unrelated_5 ┆ 10,000 ┆           0 ┆    0.49876 ┆    0.288979 ┆   0.000071 ┆  0.999867 │
│    var_reg2 ┆ 10,000 ┆       2,476 ┆  -2.353019 ┆   10.232008 ┆ -55.354108 ┆ 26.213084 │
│    repeat_1 ┆ 10,000 ┆           0 ┆   0.502449 ┆    0.288359 ┆   0.000119 ┆  0.999997 │
│        var5 ┆ 10,000 ┆           0 ┆     0.4999 ┆    0.500025 ┆        0.0 ┆       1.0 │
│    var_reg1 ┆ 10,000 ┆       2,569 ┆   0.523079 ┆    0.499501 ┆        0.0 ┆       1.0 │
└─────────────┴────────┴─────────────┴────────────┴─────────────┴────────────┴───────────┘

naive plan: (run LazyFrame.explain(optimized=True) to see the optimized plan)

SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")]

UNION

PLAN 0:

WITH_COLUMNS:

[col("n (missing)").cast(Int16), col("min").strict_cast(Float64), col("max").strict_cast(Float64)]

SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")]

WITH_COLUMNS:

["_row_index_".alias("Variable")]

SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")]

SELECT [col("___index___"), col("_row_index__std").alias("std"), col("_row_index__rawn").alias("n"), col("_row_index__min").alias("min"), col("_row_index__max").alias("max"), col("_row_index__mean").alias("mean"), col("_row_index__rawn_missing").alias("n (missing)")]

SELECT [col("___index___"), col("_row_index__std"), col("_row_index__rawn"), col("_row_index__min"), col("_row_index__max"), col("_row_index__mean"), col("_row_index__rawn_missing")]

DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS

PLAN 1:

WITH_COLUMNS:

[col("n (missing)").cast(Int16), col("min").strict_cast(Float64), col("max").strict_cast(Float64)]

SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")]

WITH_COLUMNS:

["index".alias("Variable")]

SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")]

SELECT [col("___index___"), col("index_std").alias("std"), col("index_rawn").alias("n"), col("index_min").alias("min"), col("index_max").alias("max"), col("index_mean").alias("mean"), col("index_rawn_missing").alias("n (missing)")]

SELECT [col("___index___"), col("index_std"), col("index_rawn"), col("index_min"), col("index_max"), col("index_mean"), col("index_rawn_missing")]

DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS

PLAN 2:

WITH_COLUMNS:

[col("n (missing)").cast(Int16), col("min").strict_cast(Float64), col("max").strict_cast(Float64)]

SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")]

WITH_COLUMNS:

["year".alias("Variable")]

SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")]

SELECT [col("___index___"), col("year_std").alias("std"), col("year_rawn").alias("n"), col("year_min").alias("min"), col("year_max").alias("max"), col("year_mean").alias("mean"), col("year_rawn_missing").alias("n (missing)")]

SELECT [col("___index___"), col("year_std"), col("year_rawn"), col("year_min"), col("year_max"), col("year_mean"), col("year_rawn_missing")]

DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS

PLAN 3:

WITH_COLUMNS:

[col("n (missing)").cast(Int16), col("min").strict_cast(Float64), col("max").strict_cast(Float64)]

SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")]

WITH_COLUMNS:

["month".alias("Variable")]

SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")]

SELECT [col("___index___"), col("month_std").alias("std"), col("month_rawn").alias("n"), col("month_min").alias("min"), col("month_max").alias("max"), col("month_mean").alias("mean"), col("month_rawn_missing").alias("n (missing)")]

SELECT [col("___index___"), col("month_std"), col("month_rawn"), col("month_min"), col("month_max"), col("month_mean"), col("month_rawn_missing")]

DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS

PLAN 4:

WITH_COLUMNS:

[col("n (missing)").cast(Int16), col("min").strict_cast(Float64), col("max").strict_cast(Float64)]

SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")]

WITH_COLUMNS:

["var2".alias("Variable")]

SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")]

SELECT [col("___index___"), col("var2_std").alias("std"), col("var2_rawn").alias("n"), col("var2_min").alias("min"), col("var2_max").alias("max"), col("var2_mean").alias("mean"), col("var2_rawn_missing").alias("n (missing)")]

SELECT [col("___index___"), col("var2_std"), col("var2_rawn"), col("var2_min"), col("var2_max"), col("var2_mean"), col("var2_rawn_missing")]

DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS

PLAN 5:

WITH_COLUMNS:

[col("n (missing)").cast(Int16), col("min").strict_cast(Float64), col("max").strict_cast(Float64)]

SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")]

WITH_COLUMNS:

["var3".alias("Variable")]

SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")]

SELECT [col("___index___"), col("var3_std").alias("std"), col("var3_rawn").alias("n"), col("var3_min").alias("min"), col("var3_max").alias("max"), col("var3_mean").alias("mean"), col("var3_rawn_missing").alias("n (missing)")]

SELECT [col("___index___"), col("var3_std"), col("var3_rawn"), col("var3_min"), col("var3_max"), col("var3_mean"), col("var3_rawn_missing")]

DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS

PLAN 6:

WITH_COLUMNS:

[col("n (missing)").cast(Int16)]

SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")]

WITH_COLUMNS:

["var4".alias("Variable")]

SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")]

SELECT [col("___index___"), col("var4_std").alias("std"), col("var4_rawn").alias("n"), col("var4_min").alias("min"), col("var4_max").alias("max"), col("var4_mean").alias("mean"), col("var4_rawn_missing").alias("n (missing)")]

SELECT [col("___index___"), col("var4_std"), col("var4_rawn"), col("var4_min"), col("var4_max"), col("var4_mean"), col("var4_rawn_missing")]

DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS

PLAN 7:

WITH_COLUMNS:

[col("n (missing)").cast(Int16)]

SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")]

WITH_COLUMNS:

["unrelated_1".alias("Variable")]

SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")]

SELECT [col("___index___"), col("unrelated_1_std").alias("std"), col("unrelated_1_rawn").alias("n"), col("unrelated_1_min").alias("min"), col("unrelated_1_max").alias("max"), col("unrelated_1_mean").alias("mean"), col("unrelated_1_rawn_missing").alias("n (missing)")]

SELECT [col("___index___"), col("unrelated_1_std"), col("unrelated_1_rawn"), col("unrelated_1_min"), col("unrelated_1_max"), col("unrelated_1_mean"), col("unrelated_1_rawn_missing")]

DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS

PLAN 8:

WITH_COLUMNS:

[col("n (missing)").cast(Int16)]

SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")]

WITH_COLUMNS:

["unrelated_2".alias("Variable")]

SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")]

SELECT [col("___index___"), col("unrelated_2_std").alias("std"), col("unrelated_2_rawn").alias("n"), col("unrelated_2_min").alias("min"), col("unrelated_2_max").alias("max"), col("unrelated_2_mean").alias("mean"), col("unrelated_2_rawn_missing").alias("n (missing)")]

SELECT [col("___index___"), col("unrelated_2_std"), col("unrelated_2_rawn"), col("unrelated_2_min"), col("unrelated_2_max"), col("unrelated_2_mean"), col("unrelated_2_rawn_missing")]

DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS

PLAN 9:

WITH_COLUMNS:

[col("n (missing)").cast(Int16)]

SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")]

WITH_COLUMNS:

["unrelated_3".alias("Variable")]

SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")]

SELECT [col("___index___"), col("unrelated_3_std").alias("std"), col("unrelated_3_rawn").alias("n"), col("unrelated_3_min").alias("min"), col("unrelated_3_max").alias("max"), col("unrelated_3_mean").alias("mean"), col("unrelated_3_rawn_missing").alias("n (missing)")]

SELECT [col("___index___"), col("unrelated_3_std"), col("unrelated_3_rawn"), col("unrelated_3_min"), col("unrelated_3_max"), col("unrelated_3_mean"), col("unrelated_3_rawn_missing")]

DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS

PLAN 10:

WITH_COLUMNS:

[col("n (missing)").cast(Int16)]

SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")]

WITH_COLUMNS:

["unrelated_4".alias("Variable")]

SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")]

SELECT [col("___index___"), col("unrelated_4_std").alias("std"), col("unrelated_4_rawn").alias("n"), col("unrelated_4_min").alias("min"), col("unrelated_4_max").alias("max"), col("unrelated_4_mean").alias("mean"), col("unrelated_4_rawn_missing").alias("n (missing)")]

SELECT [col("___index___"), col("unrelated_4_std"), col("unrelated_4_rawn"), col("unrelated_4_min"), col("unrelated_4_max"), col("unrelated_4_mean"), col("unrelated_4_rawn_missing")]

DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS

PLAN 11:

WITH_COLUMNS:

[col("n (missing)").cast(Int16)]

SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")]

WITH_COLUMNS:

["unrelated_5".alias("Variable")]

SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")]

SELECT [col("___index___"), col("unrelated_5_std").alias("std"), col("unrelated_5_rawn").alias("n"), col("unrelated_5_min").alias("min"), col("unrelated_5_max").alias("max"), col("unrelated_5_mean").alias("mean"), col("unrelated_5_rawn_missing").alias("n (missing)")]

SELECT [col("___index___"), col("unrelated_5_std"), col("unrelated_5_rawn"), col("unrelated_5_min"), col("unrelated_5_max"), col("unrelated_5_mean"), col("unrelated_5_rawn_missing")]

DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS

PLAN 12:

SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")]

WITH_COLUMNS:

["var_reg2".alias("Variable")]

SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")]

SELECT [col("___index___"), col("var_reg2_std").alias("std"), col("var_reg2_rawn").alias("n"), col("var_reg2_min").alias("min"), col("var_reg2_max").alias("max"), col("var_reg2_mean").alias("mean"), col("var_reg2_rawn_missing").alias("n (missing)")]

SELECT [col("___index___"), col("var_reg2_std"), col("var_reg2_rawn"), col("var_reg2_min"), col("var_reg2_max"), col("var_reg2_mean"), col("var_reg2_rawn_missing")]

DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS

PLAN 13:

WITH_COLUMNS:

[col("n (missing)").cast(Int16)]

SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")]

WITH_COLUMNS:

["repeat_1".alias("Variable")]

SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")]

SELECT [col("___index___"), col("repeat_1_std").alias("std"), col("repeat_1_rawn").alias("n"), col("repeat_1_min").alias("min"), col("repeat_1_max").alias("max"), col("repeat_1_mean").alias("mean"), col("repeat_1_rawn_missing").alias("n (missing)")]

SELECT [col("___index___"), col("repeat_1_std"), col("repeat_1_rawn"), col("repeat_1_min"), col("repeat_1_max"), col("repeat_1_mean"), col("repeat_1_rawn_missing")]

DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS

PLAN 14:

WITH_COLUMNS:

[col("n (missing)").cast(Int16), col("min").strict_cast(Float64), col("max").strict_cast(Float64)]

SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")]

WITH_COLUMNS:

["var5".alias("Variable")]

SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")]

SELECT [col("___index___"), col("var5_std").alias("std"), col("var5_rawn").alias("n"), col("var5_min").alias("min"), col("var5_max").alias("max"), col("var5_mean").alias("mean"), col("var5_rawn_missing").alias("n (missing)")]

SELECT [col("___index___"), col("var5_std"), col("var5_rawn"), col("var5_min"), col("var5_max"), col("var5_mean"), col("var5_rawn_missing")]

DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS

PLAN 15:

WITH_COLUMNS:

[col("min").strict_cast(Float64), col("max").strict_cast(Float64)]

SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")]

WITH_COLUMNS:

["var_reg1".alias("Variable")]

SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")]

SELECT [col("___index___"), col("var_reg1_std").alias("std"), col("var_reg1_rawn").alias("n"), col("var_reg1_min").alias("min"), col("var_reg1_max").alias("max"), col("var_reg1_mean").alias("mean"), col("var_reg1_rawn_missing").alias("n (missing)")]

SELECT [col("___index___"), col("var_reg1_std"), col("var_reg1_rawn"), col("var_reg1_min"), col("var_reg1_max"), col("var_reg1_mean"), col("var_reg1_rawn_missing")]

DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS

END UNION

logger.info(
    "Define the regression model (intentionally include some extraneous variables"
)

f_model = FormulaBuilder(df=df)
f_model.formula_with_varnames_in_brackets(
    "~1+{var_*}+var2+var4+var4*var3*C(var5)+{unrelated_*}+{repeat_*}"
)
logger.info(f_model.formula)

Define the regression model (intentionally include some extraneous variables

~1+var_reg1+var_reg2+var2+var4+var4*var3*C(var5)+unrelated_1+unrelated_2+unrelated_3+unrelated_4+unrelated_5+repeat_1

# Set up the variable to be imputed
vars_impute = []

logger.info("Impute the boolean variable (var_reg1)")
logger.info("   to the default setup for predicted mean matching")
logger.info("   using logit regression")
v_reg1 = Variable(
    impute_var="var_reg1",
    modeltype=Variable.ModelType.pmm,
    model=f_model.formula,
    parameters=Parameters.Regression(model=Parameters.RegressionModel.Logit),
)
logger.info("Add the variable to the list to be imputed")
vars_impute.append(v_reg1)

logger.info("Impute the continuous variable (var_reg2) ")
logger.info("   conditional on var_reg1, using narwhals (nw.col('var_reg1'))")
logger.info("   by setting the model type")
logger.info("   and the formula")
logger.info("   as well as a post-processing edit to set var_reg2=0 when var_reg1==0")
v_reg2 = Variable(
    impute_var="var_reg2",
    sample=Variable.Sample(Where=nw.col("var_reg1")),
    modeltype=Variable.ModelType.pmm,
    model=f_model.formula,
    #   Default parameters
    parameters=Parameters.Regression(),
    transforms=Variable.Transforms(
        post=(
            nw.when(nw.col("var_reg1"))
            .then(nw.col("var_reg2"))
            .otherwise(nw.lit(0))
            .alias("var_reg2")
        )
    ),
)

vars_impute.append(v_reg2)

Impute the boolean variable (var_reg1)

   to the default setup for predicted mean matching

   using logit regression

Add the variable to the list to be imputed

Impute the continuous variable (var_reg2) 

   conditional on var_reg1, using narwhals (nw.col('var_reg1'))

   by setting the model type

   and the formula

   as well as a post-processing edit to set var_reg2=0 when var_reg1==0

logger.info("Set up the imputation")
logger.info("Add LASSO selection before each imputation")
srmi = SRMI(
    df=df,
    variables=vars_impute,
    replication=SRMI.Replication(n_implicates=2, n_iterations=2),
    parallel=SRMI.Parallel(enabled=False),
    bootstrap=SRMI.Bootstrap(enabled=True),
    defaults=SRMI.Defaults(
        selection=Selection(method=Selection.Method.LASSO),
        modeltype=Variable.ModelType.pmm,
        model=f_model.formula,
    ),
    storage=SRMI.Storage(
        path_model=f"{config.path_temp_files}/py_srmi_test_regression",
        force_start=True,
    ),
)

Set up the imputation

Add LASSO selection before each imputation

Removing existing directory C:\Users\jonro\OneDrive\Documents\Coding\survey_kit\.scratch\temp_files/py_srmi_test_regression.srmi

Dropping var_reg1 from formula

Dropping var_reg2 from formula

logger.info("Run it")
srmi.run()

Run it

Variable selection before SRMI run, if necessary

     var_reg1: Method.No

     var_reg2: Method.No

Hyperparameter tuning before SRMI run, if necessary

Removing existing directory C:\Users\jonro\OneDrive\Documents\Coding\survey_kit\.scratch\temp_files/py_srmi_test_regression.srmi/1.srmi.implicate

Removing existing directory C:\Users\jonro\OneDrive\Documents\Coding\survey_kit\.scratch\temp_files/py_srmi_test_regression.srmi/2.srmi.implicate

Dropping var_reg2 from formula

     Running variable selection: Method.LASSO

         Selected model: ~0+var2+var4+C(var5)+var4:var3:C(var5)+unrelated_1+unrelated_2+unrelated_3+unrelated_4+unrelated_5+repeat_1+var3

     Imputation using pmm

C:\Users\jonro\OneDrive\Documents\Coding\survey_kit\.venv\Lib\site-packages\sklearn\utils\validation.py:1406: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  y = column_or_1d(y, warn=True)
R2 = 0.5507

┌────────────────────────┬───────────┐
│               Variable ┆      Beta │
╞════════════════════════╪═══════════╡
│                   var2 ┆    0.2176 │
│                   var4 ┆   -0.3242 │
│           C(var5)FALSE ┆     2.064 │
│            C(var5)TRUE ┆     -2.39 │
│            unrelated_1 ┆   -0.1082 │
│            unrelated_2 ┆   -0.1339 │
│            unrelated_3 ┆   0.07617 │
│            unrelated_4 ┆   -0.2167 │
│            unrelated_5 ┆   -0.1206 │
│               repeat_1 ┆   -0.1082 │
│                   var3 ┆ -0.004198 │
│ var4:var3:C(var5)FALSE ┆ 0.0009905 │
│  var4:var3:C(var5)TRUE ┆   0.02909 │
│            _Intercept_ ┆   -0.3496 │
└────────────────────────┴───────────┘

     error=pmm: donating observed value(s) ['var_reg1'] from 10-nearest matched donors

     Finding 10 nearest neighbors on ['___prediction']

     Randomly picking one and donating ['var_reg1']

     Most common matches: 

shape: (5, 2)
┌──────────────┬─────────┐
│ ___rownumber ┆ nDonors │
│ ---          ┆ ---     │
│ i16          ┆ i8      │
╞══════════════╪═════════╡
│ 823          ┆ 4       │
│ 2000         ┆ 4       │
│ 4710         ┆ 4       │
│ 5629         ┆ 4       │
│ 8345         ┆ 4       │
└──────────────┴─────────┘

Post-imputation statistics for ['var_reg1']

    Where:          None

    Where (impute): col(___imp_missing_var_reg1_1)

┌──────────┬─────────┬───────┬──────────────┬────────┬────────┬──────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┐
│ Variable ┆ Imputed ┆     n ┆ n (not null) ┆   mean ┆    std ┆ mean (not 0) ┆ std (not 0) ┆ q10 (not 0) ┆ q25 (not 0) ┆ q50 (not 0) ┆ q75 (not 0) ┆ q90 (not 0) ┆ min (not 0) ┆ max (not 0) │
╞══════════╪═════════╪═══════╪══════════════╪════════╪════════╪══════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╡
│ var_reg1 ┆         ┆ 10000 ┆        10000 ┆ 0.5233 ┆ 0.4995 ┆            1 ┆           0 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 │
│ var_reg1 ┆       0 ┆  7431 ┆         7431 ┆ 0.5231 ┆ 0.4995 ┆            1 ┆           0 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 │
│ var_reg1 ┆       1 ┆  2569 ┆         2569 ┆ 0.5239 ┆ 0.4995 ┆            1 ┆           0 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 │
└──────────┴─────────┴───────┴──────────────┴────────┴────────┴──────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┘

     Running variable selection: Method.LASSO

         Selected model: ~0+var_reg1+var2+var4+var3+var4:var3+C(var5)+var4:C(var5)+var3:C(var5)+var4:var3:C(var5)+unrelated_1+unrelated_2+unrelated_3+unrelated_4+unrelated_5+repeat_1

     Imputation using pmm

R2 = 0.8185

┌───────────────────────┬──────────┐
│              Variable ┆     Beta │
╞═══════════════════════╪══════════╡
│         var_reg1FALSE ┆      0.0 │
│          var_reg1TRUE ┆     -0.0 │
│                  var2 ┆    1.383 │
│                  var4 ┆   0.5612 │
│                  var3 ┆  0.01672 │
│           C(var5)TRUE ┆  -0.5865 │
│           unrelated_1 ┆   0.1563 │
│           unrelated_2 ┆   0.5452 │
│           unrelated_3 ┆  -0.1947 │
│           unrelated_4 ┆  0.06568 │
│           unrelated_5 ┆   -0.368 │
│              repeat_1 ┆   0.1563 │
│             var4:var3 ┆  -0.9976 │
│      var4:C(var5)TRUE ┆    3.789 │
│      var3:C(var5)TRUE ┆ -0.06316 │
│ var4:var3:C(var5)TRUE ┆  0.06792 │
│           _Intercept_ ┆  0.06084 │
└───────────────────────┴──────────┘

     error=pmm: donating observed value(s) ['var_reg2'] from 10-nearest matched donors

     Finding 10 nearest neighbors on ['___prediction']

     Randomly picking one and donating ['var_reg2']

     Most common matches: 

shape: (5, 2)
┌──────────────┬─────────┐
│ ___rownumber ┆ nDonors │
│ ---          ┆ ---     │
│ i16          ┆ i8      │
╞══════════════╪═════════╡
│ 110          ┆ 4       │
│ 460          ┆ 3       │
│ 519          ┆ 3       │
│ 1061         ┆ 3       │
│ 2180         ┆ 3       │
└──────────────┴─────────┘

Post-imputation statistics for ['var_reg2']

    Where:          col(var_reg1)

    Where (impute): col(___imp_missing_var_reg2_2)

┌──────────┬─────────┬──────┬──────────────┬────────┬───────┬──────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┐
│ Variable ┆ Imputed ┆    n ┆ n (not null) ┆   mean ┆   std ┆ mean (not 0) ┆ std (not 0) ┆ q10 (not 0) ┆ q25 (not 0) ┆ q50 (not 0) ┆ q75 (not 0) ┆ q90 (not 0) ┆ min (not 0) ┆ max (not 0) │
╞══════════╪═════════╪══════╪══════════════╪════════╪═══════╪══════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╡
│ var_reg2 ┆         ┆ 5233 ┆         5233 ┆ -4.196 ┆ 13.47 ┆       -4.403 ┆       13.76 ┆      -24.64 ┆      -12.77 ┆      -1.985 ┆       6.003 ┆       11.24 ┆      -55.35 ┆       26.21 │
│ var_reg2 ┆       0 ┆ 3923 ┆         3923 ┆ -4.147 ┆ 13.38 ┆       -4.356 ┆       13.68 ┆      -24.04 ┆      -12.75 ┆      -1.984 ┆       5.914 ┆       11.27 ┆      -55.35 ┆       26.21 │
│ var_reg2 ┆       1 ┆ 1310 ┆         1310 ┆ -4.345 ┆ 13.75 ┆       -4.543 ┆       14.02 ┆      -25.48 ┆      -13.04 ┆      -1.995 ┆         6.4 ┆       11.19 ┆      -55.35 ┆       22.69 │
└──────────┴─────────┴──────┴──────────────┴────────┴───────┴──────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┘

Updating data according to narwhals expression: when_then(all_horizontal(col(var_reg1), ignore_nulls=False), col(var_reg2), lit(value=0, dtype=None)).alias(name=var_reg2)

Removing existing directory C:\Users\jonro\OneDrive\Documents\Coding\survey_kit\.scratch\temp_files/py_srmi_test_regression.srmi/1.srmi.implicate

     Running variable selection: Method.LASSO

         Selected model: ~0+var_reg2+var2+var4+var3+var4:var3+C(var5)+var4:var3:C(var5)+unrelated_2+unrelated_4+unrelated_5+repeat_1

     Imputation using pmm

C:\Users\jonro\OneDrive\Documents\Coding\survey_kit\.venv\Lib\site-packages\sklearn\utils\validation.py:1406: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  y = column_or_1d(y, warn=True)
R2 = 0.6249

┌───────────────────────┬───────────┐
│              Variable ┆      Beta │
╞═══════════════════════╪═══════════╡
│              var_reg2 ┆   -0.1319 │
│                  var2 ┆    0.2891 │
│                  var4 ┆    -0.157 │
│                  var3 ┆ -0.003246 │
│          C(var5)FALSE ┆     2.413 │
│           C(var5)TRUE ┆    -2.526 │
│           unrelated_2 ┆   -0.2193 │
│           unrelated_4 ┆   -0.3354 │
│           unrelated_5 ┆   -0.2822 │
│              repeat_1 ┆  0.007163 │
│             var4:var3 ┆  -0.08512 │
│ var4:var3:C(var5)TRUE ┆   0.05801 │
│           _Intercept_ ┆   -0.1207 │
└───────────────────────┴───────────┘

     error=pmm: donating observed value(s) ['var_reg1'] from 10-nearest matched donors

     Finding 10 nearest neighbors on ['___prediction']

     Randomly picking one and donating ['var_reg1']

     Most common matches: 

shape: (5, 2)
┌──────────────┬─────────┐
│ ___rownumber ┆ nDonors │
│ ---          ┆ ---     │
│ i16          ┆ i8      │
╞══════════════╪═════════╡
│ 5477         ┆ 4       │
│ 8656         ┆ 4       │
│ 93           ┆ 3       │
│ 400          ┆ 3       │
│ 597          ┆ 3       │
└──────────────┴─────────┘

Post-imputation statistics for ['var_reg1']

    Where:          None

    Where (impute): col(___imp_missing_var_reg1_1)

┌──────────┬─────────┬───────┬──────────────┬────────┬────────┬──────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┐
│ Variable ┆ Imputed ┆     n ┆ n (not null) ┆   mean ┆    std ┆ mean (not 0) ┆ std (not 0) ┆ q10 (not 0) ┆ q25 (not 0) ┆ q50 (not 0) ┆ q75 (not 0) ┆ q90 (not 0) ┆ min (not 0) ┆ max (not 0) │
╞══════════╪═════════╪═══════╪══════════════╪════════╪════════╪══════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╡
│ var_reg1 ┆         ┆ 12569 ┆        12569 ┆ 0.5209 ┆ 0.4996 ┆            1 ┆           0 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 │
│ var_reg1 ┆       0 ┆ 10000 ┆        10000 ┆ 0.5233 ┆ 0.4995 ┆            1 ┆           0 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 │
│ var_reg1 ┆       1 ┆  2569 ┆         2569 ┆ 0.5115 ┆    0.5 ┆            1 ┆           0 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 │
└──────────┴─────────┴───────┴──────────────┴────────┴────────┴──────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┘

     Running variable selection: Method.LASSO

         Selected model: ~0+var_reg1+var2+var4+var3+var4:var3+C(var5)+var4:C(var5)+var3:C(var5)+var4:var3:C(var5)+unrelated_2+unrelated_3+unrelated_4+unrelated_5

     Imputation using pmm

R2 = 0.8298

┌───────────────────────┬──────────┐
│              Variable ┆     Beta │
╞═══════════════════════╪══════════╡
│         var_reg1FALSE ┆      0.0 │
│          var_reg1TRUE ┆     -0.0 │
│                  var2 ┆    1.364 │
│                  var4 ┆    1.554 │
│                  var3 ┆  0.04128 │
│           C(var5)TRUE ┆  -0.4464 │
│           unrelated_2 ┆ -0.06238 │
│           unrelated_3 ┆   0.2616 │
│           unrelated_4 ┆   -0.592 │
│           unrelated_5 ┆   -0.146 │
│             var4:var3 ┆   -1.046 │
│      var4:C(var5)TRUE ┆    2.439 │
│      var3:C(var5)TRUE ┆ -0.07164 │
│ var4:var3:C(var5)TRUE ┆    0.128 │
│           _Intercept_ ┆   0.1934 │
└───────────────────────┴──────────┘

     error=pmm: donating observed value(s) ['var_reg2'] from 10-nearest matched donors

     Finding 10 nearest neighbors on ['___prediction']

     Randomly picking one and donating ['var_reg2']

     Most common matches: 

shape: (5, 2)
┌──────────────┬─────────┐
│ ___rownumber ┆ nDonors │
│ ---          ┆ ---     │
│ i16          ┆ i8      │
╞══════════════╪═════════╡
│ 515          ┆ 3       │
│ 731          ┆ 3       │
│ 787          ┆ 3       │
│ 1559         ┆ 3       │
│ 1668         ┆ 3       │
└──────────────┴─────────┘

Post-imputation statistics for ['var_reg2']

    Where:          col(var_reg1)

    Where (impute): col(___imp_missing_var_reg2_2)

┌──────────┬─────────┬──────┬──────────────┬────────┬───────┬──────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┐
│ Variable ┆ Imputed ┆    n ┆ n (not null) ┆   mean ┆   std ┆ mean (not 0) ┆ std (not 0) ┆ q10 (not 0) ┆ q25 (not 0) ┆ q50 (not 0) ┆ q75 (not 0) ┆ q90 (not 0) ┆ min (not 0) ┆ max (not 0) │
╞══════════╪═════════╪══════╪══════════════╪════════╪═══════╪══════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╡
│ var_reg2 ┆         ┆ 6500 ┆         6500 ┆ -4.311 ┆ 13.47 ┆       -4.582 ┆       13.84 ┆      -25.03 ┆       -13.3 ┆      -2.209 ┆        5.98 ┆        11.1 ┆      -55.35 ┆       26.21 │
│ var_reg2 ┆       0 ┆ 5201 ┆         5201 ┆ -4.232 ┆ 13.41 ┆         -4.5 ┆       13.79 ┆      -24.74 ┆       -13.0 ┆      -2.152 ┆       5.936 ┆       11.19 ┆      -55.35 ┆       26.21 │
│ var_reg2 ┆       1 ┆ 1299 ┆         1299 ┆ -4.628 ┆ 13.69 ┆       -4.908 ┆       14.05 ┆       -25.9 ┆      -14.59 ┆      -2.416 ┆       6.171 ┆       10.83 ┆      -46.63 ┆       24.89 │
└──────────┴─────────┴──────┴──────────────┴────────┴───────┴──────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┘

Updating data according to narwhals expression: when_then(all_horizontal(col(var_reg1), ignore_nulls=False), col(var_reg2), lit(value=0, dtype=None)).alias(name=var_reg2)

var_reg1

var_reg2

Final Estimates by Iteration

Removing existing directory C:\Users\jonro\OneDrive\Documents\Coding\survey_kit\.scratch\temp_files/py_srmi_test_regression.srmi/1.srmi.implicate

Dropping var_reg2 from formula

     Running variable selection: Method.LASSO

         Selected model: ~0+var2+C(var5)+var4:C(var5)+var4:var3:C(var5)+unrelated_1+unrelated_2+unrelated_3+unrelated_5+repeat_1+var4+var3

     Imputation using pmm

C:\Users\jonro\OneDrive\Documents\Coding\survey_kit\.venv\Lib\site-packages\sklearn\utils\validation.py:1406: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  y = column_or_1d(y, warn=True)
R2 = 0.5509

┌────────────────────────┬───────────┐
│               Variable ┆      Beta │
╞════════════════════════╪═══════════╡
│                   var2 ┆    0.1965 │
│           C(var5)FALSE ┆     2.076 │
│            C(var5)TRUE ┆    -2.213 │
│            unrelated_1 ┆    -0.111 │
│            unrelated_2 ┆  -0.09557 │
│            unrelated_3 ┆   -0.1429 │
│            unrelated_5 ┆    -0.408 │
│               repeat_1 ┆    -0.111 │
│                   var4 ┆  -0.04577 │
│                   var3 ┆ -0.005247 │
│       var4:C(var5)TRUE ┆   -0.7446 │
│ var4:var3:C(var5)FALSE ┆  0.003528 │
│  var4:var3:C(var5)TRUE ┆   0.03546 │
│            _Intercept_ ┆   -0.1482 │
└────────────────────────┴───────────┘

     error=pmm: donating observed value(s) ['var_reg1'] from 10-nearest matched donors

     Finding 10 nearest neighbors on ['___prediction']

     Randomly picking one and donating ['var_reg1']

     Most common matches: 

shape: (5, 2)
┌──────────────┬─────────┐
│ ___rownumber ┆ nDonors │
│ ---          ┆ ---     │
│ i16          ┆ i8      │
╞══════════════╪═════════╡
│ 360          ┆ 4       │
│ 379          ┆ 4       │
│ 1046         ┆ 4       │
│ 3449         ┆ 4       │
│ 180          ┆ 3       │
└──────────────┴─────────┘

Post-imputation statistics for ['var_reg1']

    Where:          None

    Where (impute): col(___imp_missing_var_reg1_1)

┌──────────┬─────────┬───────┬──────────────┬────────┬────────┬──────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┐
│ Variable ┆ Imputed ┆     n ┆ n (not null) ┆   mean ┆    std ┆ mean (not 0) ┆ std (not 0) ┆ q10 (not 0) ┆ q25 (not 0) ┆ q50 (not 0) ┆ q75 (not 0) ┆ q90 (not 0) ┆ min (not 0) ┆ max (not 0) │
╞══════════╪═════════╪═══════╪══════════════╪════════╪════════╪══════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╡
│ var_reg1 ┆         ┆ 10000 ┆        10000 ┆ 0.5236 ┆ 0.4995 ┆            1 ┆           0 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 │
│ var_reg1 ┆       0 ┆  7431 ┆         7431 ┆ 0.5231 ┆ 0.4995 ┆            1 ┆           0 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 │
│ var_reg1 ┆       1 ┆  2569 ┆         2569 ┆ 0.5251 ┆ 0.4995 ┆            1 ┆           0 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 │
└──────────┴─────────┴───────┴──────────────┴────────┴────────┴──────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┘

     Running variable selection: Method.LASSO

         Selected model: ~0+var_reg1+var2+var3+var4:var3+C(var5)+var4:C(var5)+var4:var3:C(var5)+unrelated_2+unrelated_4+unrelated_5+var4

     Imputation using pmm

R2 = 0.8246

┌───────────────────────┬─────────┐
│              Variable ┆    Beta │
╞═══════════════════════╪═════════╡
│         var_reg1FALSE ┆     0.0 │
│          var_reg1TRUE ┆    -0.0 │
│                  var2 ┆   1.417 │
│                  var3 ┆ 0.01411 │
│           C(var5)TRUE ┆  -1.319 │
│           unrelated_2 ┆ 0.04522 │
│           unrelated_4 ┆ -0.9803 │
│           unrelated_5 ┆ -0.7875 │
│                  var4 ┆   1.034 │
│             var4:var3 ┆   -1.02 │
│      var4:C(var5)TRUE ┆   3.231 │
│ var4:var3:C(var5)TRUE ┆ 0.02549 │
│           _Intercept_ ┆  0.8182 │
└───────────────────────┴─────────┘

     error=pmm: donating observed value(s) ['var_reg2'] from 10-nearest matched donors

     Finding 10 nearest neighbors on ['___prediction']

     Randomly picking one and donating ['var_reg2']

     Most common matches: 

shape: (5, 2)
┌──────────────┬─────────┐
│ ___rownumber ┆ nDonors │
│ ---          ┆ ---     │
│ i16          ┆ i8      │
╞══════════════╪═════════╡
│ 5577         ┆ 4       │
│ 6529         ┆ 4       │
│ 2500         ┆ 3       │
│ 3041         ┆ 3       │
│ 4038         ┆ 3       │
└──────────────┴─────────┘

Post-imputation statistics for ['var_reg2']

    Where:          col(var_reg1)

    Where (impute): col(___imp_missing_var_reg2_2)

┌──────────┬─────────┬──────┬──────────────┬────────┬───────┬──────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┐
│ Variable ┆ Imputed ┆    n ┆ n (not null) ┆   mean ┆   std ┆ mean (not 0) ┆ std (not 0) ┆ q10 (not 0) ┆ q25 (not 0) ┆ q50 (not 0) ┆ q75 (not 0) ┆ q90 (not 0) ┆ min (not 0) ┆ max (not 0) │
╞══════════╪═════════╪══════╪══════════════╪════════╪═══════╪══════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╡
│ var_reg2 ┆         ┆ 5236 ┆         5236 ┆ -4.283 ┆ 13.47 ┆       -4.494 ┆       13.77 ┆      -24.54 ┆      -13.03 ┆      -2.107 ┆       5.936 ┆       11.17 ┆      -55.35 ┆       26.21 │
│ var_reg2 ┆       0 ┆ 3915 ┆         3915 ┆ -4.199 ┆ 13.41 ┆       -4.408 ┆        13.7 ┆      -24.18 ┆      -12.82 ┆      -1.985 ┆       5.906 ┆       11.24 ┆      -55.35 ┆       26.21 │
│ var_reg2 ┆       1 ┆ 1321 ┆         1321 ┆ -4.532 ┆ 13.67 ┆       -4.747 ┆       13.95 ┆      -25.13 ┆      -13.48 ┆      -2.727 ┆       6.171 ┆        10.7 ┆      -47.42 ┆       26.21 │
└──────────┴─────────┴──────┴──────────────┴────────┴───────┴──────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┘

Updating data according to narwhals expression: when_then(all_horizontal(col(var_reg1), ignore_nulls=False), col(var_reg2), lit(value=0, dtype=None)).alias(name=var_reg2)

Removing existing directory C:\Users\jonro\OneDrive\Documents\Coding\survey_kit\.scratch\temp_files/py_srmi_test_regression.srmi/2.srmi.implicate

     Running variable selection: Method.LASSO

         Selected model: ~0+var_reg2+var2+var4:var3+C(var5)+var4:C(var5)+var4:var3:C(var5)+unrelated_1+unrelated_2+unrelated_3+unrelated_5+repeat_1+var4+var3

     Imputation using pmm

C:\Users\jonro\OneDrive\Documents\Coding\survey_kit\.venv\Lib\site-packages\sklearn\utils\validation.py:1406: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  y = column_or_1d(y, warn=True)
C:\Users\jonro\OneDrive\Documents\Coding\survey_kit\.venv\Lib\site-packages\sklearn\linear_model\_logistic.py:473: ConvergenceWarning: lbfgs failed to converge after 100 iteration(s) (status=1):
STOP: TOTAL NO. OF ITERATIONS REACHED LIMIT

Increase the number of iterations to improve the convergence (max_iter=100).
You might also want to scale the data as shown in:
    https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
  n_iter_i = _check_optimize_result(
R2 = 0.6195

┌───────────────────────┬──────────┐
│              Variable ┆     Beta │
╞═══════════════════════╪══════════╡
│              var_reg2 ┆  -0.1348 │
│                  var2 ┆   0.3141 │
│          C(var5)FALSE ┆    2.275 │
│           C(var5)TRUE ┆   -2.602 │
│           unrelated_1 ┆  0.07694 │
│           unrelated_2 ┆  -0.3327 │
│           unrelated_3 ┆  -0.2426 │
│           unrelated_5 ┆  -0.6309 │
│              repeat_1 ┆  0.07694 │
│                  var4 ┆   0.3434 │
│                  var3 ┆ 0.003656 │
│             var4:var3 ┆ -0.09581 │
│      var4:C(var5)TRUE ┆  0.05559 │
│ var4:var3:C(var5)TRUE ┆  0.05213 │
│           _Intercept_ ┆  -0.3411 │
└───────────────────────┴──────────┘

     error=pmm: donating observed value(s) ['var_reg1'] from 10-nearest matched donors

     Finding 10 nearest neighbors on ['___prediction']

     Randomly picking one and donating ['var_reg1']

     Most common matches: 

shape: (5, 2)
┌──────────────┬─────────┐
│ ___rownumber ┆ nDonors │
│ ---          ┆ ---     │
│ i16          ┆ i8      │
╞══════════════╪═════════╡
│ 390          ┆ 3       │
│ 1113         ┆ 3       │
│ 1864         ┆ 3       │
│ 1931         ┆ 3       │
│ 2325         ┆ 3       │
└──────────────┴─────────┘

Post-imputation statistics for ['var_reg1']

    Where:          None

    Where (impute): col(___imp_missing_var_reg1_1)

┌──────────┬─────────┬───────┬──────────────┬────────┬────────┬──────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┐
│ Variable ┆ Imputed ┆     n ┆ n (not null) ┆   mean ┆    std ┆ mean (not 0) ┆ std (not 0) ┆ q10 (not 0) ┆ q25 (not 0) ┆ q50 (not 0) ┆ q75 (not 0) ┆ q90 (not 0) ┆ min (not 0) ┆ max (not 0) │
╞══════════╪═════════╪═══════╪══════════════╪════════╪════════╪══════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╡
│ var_reg1 ┆         ┆ 12569 ┆        12569 ┆ 0.5206 ┆ 0.4996 ┆            1 ┆           0 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 │
│ var_reg1 ┆       0 ┆ 10000 ┆        10000 ┆ 0.5236 ┆ 0.4995 ┆            1 ┆           0 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 │
│ var_reg1 ┆       1 ┆  2569 ┆         2569 ┆ 0.5088 ┆    0.5 ┆            1 ┆           0 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 ┆           1 │
└──────────┴─────────┴───────┴──────────────┴────────┴────────┴──────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┘

     Running variable selection: Method.LASSO

         Selected model: ~0+var_reg1+var2+var4:var3+C(var5)+var4:C(var5)+var3:C(var5)+var4:var3:C(var5)+unrelated_1+unrelated_2+unrelated_3+unrelated_4+unrelated_5+repeat_1+var4+var3

     Imputation using pmm

R2 = 0.8258

┌───────────────────────┬──────────┐
│              Variable ┆     Beta │
╞═══════════════════════╪══════════╡
│         var_reg1FALSE ┆      0.0 │
│          var_reg1TRUE ┆     -0.0 │
│                  var2 ┆    1.381 │
│           C(var5)TRUE ┆   -1.331 │
│           unrelated_1 ┆ -0.06996 │
│           unrelated_2 ┆  -0.0357 │
│           unrelated_3 ┆  -0.5286 │
│           unrelated_4 ┆  -0.4791 │
│           unrelated_5 ┆  -0.7511 │
│              repeat_1 ┆ -0.06996 │
│                  var4 ┆   0.7273 │
│                  var3 ┆  0.01873 │
│             var4:var3 ┆  -0.9999 │
│      var4:C(var5)TRUE ┆    4.685 │
│      var3:C(var5)TRUE ┆ -0.01287 │
│ var4:var3:C(var5)TRUE ┆ 0.004049 │
│           _Intercept_ ┆   0.9459 │
└───────────────────────┴──────────┘

     error=pmm: donating observed value(s) ['var_reg2'] from 10-nearest matched donors

     Finding 10 nearest neighbors on ['___prediction']

     Randomly picking one and donating ['var_reg2']

     Most common matches: 

shape: (5, 2)
┌──────────────┬─────────┐
│ ___rownumber ┆ nDonors │
│ ---          ┆ ---     │
│ i16          ┆ i8      │
╞══════════════╪═════════╡
│ 3625         ┆ 4       │
│ 1299         ┆ 3       │
│ 1702         ┆ 3       │
│ 2790         ┆ 3       │
│ 2949         ┆ 3       │
└──────────────┴─────────┘

Post-imputation statistics for ['var_reg2']

    Where:          col(var_reg1)

    Where (impute): col(___imp_missing_var_reg2_2)

┌──────────┬─────────┬──────┬──────────────┬────────┬───────┬──────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┐
│ Variable ┆ Imputed ┆    n ┆ n (not null) ┆   mean ┆   std ┆ mean (not 0) ┆ std (not 0) ┆ q10 (not 0) ┆ q25 (not 0) ┆ q50 (not 0) ┆ q75 (not 0) ┆ q90 (not 0) ┆ min (not 0) ┆ max (not 0) │
╞══════════╪═════════╪══════╪══════════════╪════════╪═══════╪══════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╡
│ var_reg2 ┆         ┆ 6494 ┆         6494 ┆ -4.419 ┆ 13.45 ┆       -4.693 ┆       13.81 ┆      -24.74 ┆      -13.28 ┆      -2.505 ┆       5.791 ┆       10.99 ┆      -55.35 ┆       26.21 │
│ var_reg2 ┆       0 ┆ 5194 ┆         5194 ┆ -4.356 ┆ 13.42 ┆       -4.634 ┆       13.79 ┆      -24.68 ┆      -13.26 ┆      -2.263 ┆       5.852 ┆       10.99 ┆      -55.35 ┆       26.21 │
│ var_reg2 ┆       1 ┆ 1300 ┆         1300 ┆ -4.666 ┆ 13.56 ┆       -4.928 ┆       13.88 ┆      -25.06 ┆      -13.49 ┆      -2.894 ┆       5.505 ┆       11.17 ┆      -47.42 ┆       22.69 │
└──────────┴─────────┴──────┴──────────────┴────────┴───────┴──────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┘

Updating data according to narwhals expression: when_then(all_horizontal(col(var_reg1), ignore_nulls=False), col(var_reg2), lit(value=0, dtype=None)).alias(name=var_reg2)

var_reg1

var_reg2

Final Estimates by Iteration

Removing existing directory C:\Users\jonro\OneDrive\Documents\Coding\survey_kit\.scratch\temp_files/py_srmi_test_regression.srmi/2.srmi.implicate

logger.info("Get the results")
_ = df_list = srmi.df_implicates

Get the results

logger.info("\n\nLook at the original")
_ = summary(df_original)

logger.info("\n\nLook at the imputes")
_ = df_list.pipe(summary)

logger.info("\n\nLook at the imputes | var_reg1 == 0")
_ = df_list.filter(~nw.col("var_reg1")).pipe(summary)

logger.info("\n\nLook at the imputes | var_reg1 == 1")
_ = df_list.filter(nw.col("var_reg1")).pipe(summary)

Look at the original

┌──────────────┬────────┬─────────────┬────────────┬─────────────┬────────────┬───────────┐
│     Variable ┆      n ┆ n (missing) ┆       mean ┆         std ┆        min ┆       max │
╞══════════════╪════════╪═════════════╪════════════╪═════════════╪════════════╪═══════════╡
│  _row_index_ ┆ 10,000 ┆           0 ┆    4,999.5 ┆ 2,886.89568 ┆        0.0 ┆   9,999.0 │
│        index ┆ 10,000 ┆           0 ┆    4,999.5 ┆ 2,886.89568 ┆        0.0 ┆   9,999.0 │
│         year ┆ 10,000 ┆           0 ┆ 2,017.9851 ┆    1.415937 ┆    2,016.0 ┆   2,020.0 │
│        month ┆ 10,000 ┆           0 ┆     6.5137 ┆    3.432141 ┆        1.0 ┆      12.0 │
│         var2 ┆ 10,000 ┆           0 ┆     4.9782 ┆    3.154508 ┆        0.0 ┆      10.0 │
│         var3 ┆ 10,000 ┆           0 ┆    25.1084 ┆   14.752302 ┆        0.0 ┆      50.0 │
│         var4 ┆ 10,000 ┆           0 ┆   0.505666 ┆    0.287861 ┆   0.000027 ┆  0.999997 │
│  unrelated_1 ┆ 10,000 ┆           0 ┆   0.502449 ┆    0.288359 ┆   0.000119 ┆  0.999997 │
│  unrelated_2 ┆ 10,000 ┆           0 ┆   0.500105 ┆    0.287638 ┆   0.000049 ┆  0.999539 │
│  unrelated_3 ┆ 10,000 ┆           0 ┆   0.499175 ┆     0.28876 ┆   0.000129 ┆   0.99994 │
│  unrelated_4 ┆ 10,000 ┆           0 ┆   0.500655 ┆    0.288698 ┆   0.000133 ┆  0.999972 │
│  unrelated_5 ┆ 10,000 ┆           0 ┆    0.49876 ┆    0.288979 ┆   0.000071 ┆  0.999867 │
│ missing_reg1 ┆ 10,000 ┆           0 ┆   0.495784 ┆    0.289051 ┆   0.000161 ┆   0.99991 │
│ missing_reg2 ┆ 10,000 ┆           0 ┆   0.502597 ┆    0.288468 ┆   0.000006 ┆  0.999963 │
│     var_reg2 ┆ 10,000 ┆           0 ┆  -2.402477 ┆   10.331745 ┆ -55.354108 ┆ 26.213084 │
│         var5 ┆ 10,000 ┆           0 ┆     0.4999 ┆    0.500025 ┆        0.0 ┆       1.0 │
│     var_reg1 ┆ 10,000 ┆           0 ┆     0.5229 ┆      0.4995 ┆        0.0 ┆       1.0 │
└──────────────┴────────┴─────────────┴────────────┴─────────────┴────────────┴───────────┘

Look at the imputes

┌──────────────┬────────┬─────────────┬────────────┬─────────────┬────────────┬───────────┐
│     Variable ┆      n ┆ n (missing) ┆       mean ┆         std ┆        min ┆       max │
╞══════════════╪════════╪═════════════╪════════════╪═════════════╪════════════╪═══════════╡
│ ___rownumber ┆ 10,000 ┆           0 ┆    4,999.5 ┆ 2,886.89568 ┆        0.0 ┆   9,999.0 │
│  _row_index_ ┆ 10,000 ┆           0 ┆    4,999.5 ┆ 2,886.89568 ┆        0.0 ┆   9,999.0 │
│        index ┆ 10,000 ┆           0 ┆    4,999.5 ┆ 2,886.89568 ┆        0.0 ┆   9,999.0 │
│         year ┆ 10,000 ┆           0 ┆ 2,017.9851 ┆    1.415937 ┆    2,016.0 ┆   2,020.0 │
│        month ┆ 10,000 ┆           0 ┆     6.5137 ┆    3.432141 ┆        1.0 ┆      12.0 │
│         var2 ┆ 10,000 ┆           0 ┆     4.9782 ┆    3.154508 ┆        0.0 ┆      10.0 │
│         var3 ┆ 10,000 ┆           0 ┆    25.1084 ┆   14.752302 ┆        0.0 ┆      50.0 │
│         var4 ┆ 10,000 ┆           0 ┆   0.505666 ┆    0.287861 ┆   0.000027 ┆  0.999997 │
│  unrelated_1 ┆ 10,000 ┆           0 ┆   0.502449 ┆    0.288359 ┆   0.000119 ┆  0.999997 │
│  unrelated_2 ┆ 10,000 ┆           0 ┆   0.500105 ┆    0.287638 ┆   0.000049 ┆  0.999539 │
│  unrelated_3 ┆ 10,000 ┆           0 ┆   0.499175 ┆     0.28876 ┆   0.000129 ┆   0.99994 │
│  unrelated_4 ┆ 10,000 ┆           0 ┆   0.500655 ┆    0.288698 ┆   0.000133 ┆  0.999972 │
│  unrelated_5 ┆ 10,000 ┆           0 ┆    0.49876 ┆    0.288979 ┆   0.000071 ┆  0.999867 │
│     repeat_1 ┆ 10,000 ┆           0 ┆   0.502449 ┆    0.288359 ┆   0.000119 ┆  0.999997 │
│     var_reg2 ┆ 10,000 ┆           0 ┆  -2.246963 ┆    9.922099 ┆ -55.354108 ┆ 26.213084 │
│         var5 ┆ 10,000 ┆           0 ┆     0.4999 ┆    0.500025 ┆        0.0 ┆       1.0 │
│     var_reg1 ┆ 10,000 ┆           0 ┆     0.5201 ┆    0.499621 ┆        0.0 ┆       1.0 │
└──────────────┴────────┴─────────────┴────────────┴─────────────┴────────────┴───────────┘

┌──────────────┬────────┬─────────────┬────────────┬─────────────┬────────────┬───────────┐
│     Variable ┆      n ┆ n (missing) ┆       mean ┆         std ┆        min ┆       max │
╞══════════════╪════════╪═════════════╪════════════╪═════════════╪════════════╪═══════════╡
│ ___rownumber ┆ 10,000 ┆           0 ┆    4,999.5 ┆ 2,886.89568 ┆        0.0 ┆   9,999.0 │
│  _row_index_ ┆ 10,000 ┆           0 ┆    4,999.5 ┆ 2,886.89568 ┆        0.0 ┆   9,999.0 │
│        index ┆ 10,000 ┆           0 ┆    4,999.5 ┆ 2,886.89568 ┆        0.0 ┆   9,999.0 │
│         year ┆ 10,000 ┆           0 ┆ 2,017.9851 ┆    1.415937 ┆    2,016.0 ┆   2,020.0 │
│        month ┆ 10,000 ┆           0 ┆     6.5137 ┆    3.432141 ┆        1.0 ┆      12.0 │
│         var2 ┆ 10,000 ┆           0 ┆     4.9782 ┆    3.154508 ┆        0.0 ┆      10.0 │
│         var3 ┆ 10,000 ┆           0 ┆    25.1084 ┆   14.752302 ┆        0.0 ┆      50.0 │
│         var4 ┆ 10,000 ┆           0 ┆   0.505666 ┆    0.287861 ┆   0.000027 ┆  0.999997 │
│  unrelated_1 ┆ 10,000 ┆           0 ┆   0.502449 ┆    0.288359 ┆   0.000119 ┆  0.999997 │
│  unrelated_2 ┆ 10,000 ┆           0 ┆   0.500105 ┆    0.287638 ┆   0.000049 ┆  0.999539 │
│  unrelated_3 ┆ 10,000 ┆           0 ┆   0.499175 ┆     0.28876 ┆   0.000129 ┆   0.99994 │
│  unrelated_4 ┆ 10,000 ┆           0 ┆   0.500655 ┆    0.288698 ┆   0.000133 ┆  0.999972 │
│  unrelated_5 ┆ 10,000 ┆           0 ┆    0.49876 ┆    0.288979 ┆   0.000071 ┆  0.999867 │
│     repeat_1 ┆ 10,000 ┆           0 ┆   0.502449 ┆    0.288359 ┆   0.000119 ┆  0.999997 │
│     var_reg2 ┆ 10,000 ┆           0 ┆  -2.269353 ┆    9.916197 ┆ -55.354108 ┆ 26.213084 │
│         var5 ┆ 10,000 ┆           0 ┆     0.4999 ┆    0.500025 ┆        0.0 ┆       1.0 │
│     var_reg1 ┆ 10,000 ┆           0 ┆     0.5194 ┆    0.499648 ┆        0.0 ┆       1.0 │
└──────────────┴────────┴─────────────┴────────────┴─────────────┴────────────┴───────────┘

Look at the imputes | var_reg1 == 0

┌──────────────┬───────┬─────────────┬──────────────┬──────────────┬──────────┬──────────┐
│     Variable ┆     n ┆ n (missing) ┆         mean ┆          std ┆      min ┆      max │
╞══════════════╪═══════╪═════════════╪══════════════╪══════════════╪══════════╪══════════╡
│ ___rownumber ┆ 4,799 ┆           0 ┆ 5,038.299229 ┆ 2,889.298096 ┆      1.0 ┆  9,997.0 │
│  _row_index_ ┆ 4,799 ┆           0 ┆ 5,038.299229 ┆ 2,889.298096 ┆      1.0 ┆  9,997.0 │
│        index ┆ 4,799 ┆           0 ┆ 5,038.299229 ┆ 2,889.298096 ┆      1.0 ┆  9,997.0 │
│         year ┆ 4,799 ┆           0 ┆ 2,017.998333 ┆     1.418921 ┆  2,016.0 ┆  2,020.0 │
│        month ┆ 4,799 ┆           0 ┆     6.542405 ┆     3.413955 ┆      1.0 ┆     12.0 │
│         var2 ┆ 4,799 ┆           0 ┆     4.562617 ┆     3.237666 ┆      0.0 ┆     10.0 │
│         var3 ┆ 4,799 ┆           0 ┆    24.898312 ┆    13.384091 ┆      0.0 ┆     50.0 │
│         var4 ┆ 4,799 ┆           0 ┆     0.508898 ┆     0.288576 ┆ 0.000027 ┆ 0.999997 │
│  unrelated_1 ┆ 4,799 ┆           0 ┆     0.501877 ┆     0.286643 ┆ 0.000119 ┆ 0.999921 │
│  unrelated_2 ┆ 4,799 ┆           0 ┆     0.499477 ┆     0.285419 ┆ 0.000079 ┆ 0.999528 │
│  unrelated_3 ┆ 4,799 ┆           0 ┆     0.501495 ┆     0.289221 ┆ 0.000139 ┆  0.99994 │
│  unrelated_4 ┆ 4,799 ┆           0 ┆     0.503626 ┆     0.289203 ┆ 0.000348 ┆ 0.999906 │
│  unrelated_5 ┆ 4,799 ┆           0 ┆     0.500986 ┆     0.291456 ┆ 0.000071 ┆  0.99984 │
│     repeat_1 ┆ 4,799 ┆           0 ┆     0.501877 ┆     0.286643 ┆ 0.000119 ┆ 0.999921 │
│     var_reg2 ┆ 4,799 ┆           0 ┆          0.0 ┆          0.0 ┆      0.0 ┆      0.0 │
│         var5 ┆ 4,799 ┆           0 ┆     0.887477 ┆     0.316042 ┆      0.0 ┆      1.0 │
│     var_reg1 ┆ 4,799 ┆           0 ┆          0.0 ┆          0.0 ┆      0.0 ┆      0.0 │
└──────────────┴───────┴─────────────┴──────────────┴──────────────┴──────────┴──────────┘

┌──────────────┬───────┬─────────────┬──────────────┬──────────────┬──────────┬──────────┐
│     Variable ┆     n ┆ n (missing) ┆         mean ┆          std ┆      min ┆      max │
╞══════════════╪═══════╪═════════════╪══════════════╪══════════════╪══════════╪══════════╡
│ ___rownumber ┆ 4,806 ┆           0 ┆  5,035.05722 ┆ 2,887.183498 ┆      1.0 ┆  9,997.0 │
│  _row_index_ ┆ 4,806 ┆           0 ┆  5,035.05722 ┆ 2,887.183498 ┆      1.0 ┆  9,997.0 │
│        index ┆ 4,806 ┆           0 ┆  5,035.05722 ┆ 2,887.183498 ┆      1.0 ┆  9,997.0 │
│         year ┆ 4,806 ┆           0 ┆ 2,017.990221 ┆       1.4169 ┆  2,016.0 ┆  2,020.0 │
│        month ┆ 4,806 ┆           0 ┆      6.55077 ┆     3.414455 ┆      1.0 ┆     12.0 │
│         var2 ┆ 4,806 ┆           0 ┆     4.552434 ┆     3.245113 ┆      0.0 ┆     10.0 │
│         var3 ┆ 4,806 ┆           0 ┆    24.904078 ┆    13.393172 ┆      0.0 ┆     50.0 │
│         var4 ┆ 4,806 ┆           0 ┆     0.508162 ┆     0.289413 ┆ 0.000027 ┆ 0.999997 │
│  unrelated_1 ┆ 4,806 ┆           0 ┆     0.504055 ┆     0.287282 ┆ 0.000119 ┆ 0.999921 │
│  unrelated_2 ┆ 4,806 ┆           0 ┆     0.501535 ┆     0.286096 ┆ 0.000079 ┆ 0.999528 │
│  unrelated_3 ┆ 4,806 ┆           0 ┆      0.50177 ┆     0.288888 ┆ 0.000139 ┆  0.99994 │
│  unrelated_4 ┆ 4,806 ┆           0 ┆     0.502307 ┆     0.289378 ┆ 0.000141 ┆ 0.999906 │
│  unrelated_5 ┆ 4,806 ┆           0 ┆     0.502701 ┆     0.291023 ┆ 0.000071 ┆  0.99984 │
│     repeat_1 ┆ 4,806 ┆           0 ┆     0.504055 ┆     0.287282 ┆ 0.000119 ┆ 0.999921 │
│     var_reg2 ┆ 4,806 ┆           0 ┆          0.0 ┆          0.0 ┆      0.0 ┆      0.0 │
│         var5 ┆ 4,806 ┆           0 ┆     0.887016 ┆     0.316606 ┆      0.0 ┆      1.0 │
│     var_reg1 ┆ 4,806 ┆           0 ┆          0.0 ┆          0.0 ┆      0.0 ┆      0.0 │
└──────────────┴───────┴─────────────┴──────────────┴──────────────┴──────────┴──────────┘

Look at the imputes | var_reg1 == 1

┌──────────────┬───────┬─────────────┬──────────────┬──────────────┬────────────┬───────────┐
│     Variable ┆     n ┆ n (missing) ┆         mean ┆          std ┆        min ┆       max │
╞══════════════╪═══════╪═════════════╪══════════════╪══════════════╪════════════╪═══════════╡
│ ___rownumber ┆ 5,201 ┆           0 ┆ 4,963.699673 ┆ 2,884.492009 ┆        0.0 ┆   9,999.0 │
│  _row_index_ ┆ 5,201 ┆           0 ┆ 4,963.699673 ┆ 2,884.492009 ┆        0.0 ┆   9,999.0 │
│        index ┆ 5,201 ┆           0 ┆ 4,963.699673 ┆ 2,884.492009 ┆        0.0 ┆   9,999.0 │
│         year ┆ 5,201 ┆           0 ┆  2,017.97289 ┆     1.413205 ┆    2,016.0 ┆   2,020.0 │
│        month ┆ 5,201 ┆           0 ┆     6.487214 ┆     3.448952 ┆        1.0 ┆      12.0 │
│         var2 ┆ 5,201 ┆           0 ┆     5.361661 ┆     3.025873 ┆        0.0 ┆      10.0 │
│         var3 ┆ 5,201 ┆           0 ┆     25.30225 ┆    15.909569 ┆        0.0 ┆      50.0 │
│         var4 ┆ 5,201 ┆           0 ┆     0.502683 ┆     0.287195 ┆   0.000104 ┆  0.999885 │
│  unrelated_1 ┆ 5,201 ┆           0 ┆     0.502976 ┆      0.28996 ┆   0.000248 ┆  0.999997 │
│  unrelated_2 ┆ 5,201 ┆           0 ┆     0.500685 ┆     0.289698 ┆   0.000049 ┆  0.999539 │
│  unrelated_3 ┆ 5,201 ┆           0 ┆     0.497035 ┆     0.288346 ┆   0.000129 ┆  0.999622 │
│  unrelated_4 ┆ 5,201 ┆           0 ┆     0.497914 ┆     0.288232 ┆   0.000133 ┆  0.999972 │
│  unrelated_5 ┆ 5,201 ┆           0 ┆     0.496707 ┆     0.286688 ┆   0.000181 ┆  0.999867 │
│     repeat_1 ┆ 5,201 ┆           0 ┆     0.502976 ┆      0.28996 ┆   0.000248 ┆  0.999997 │
│     var_reg2 ┆ 5,201 ┆           0 ┆    -4.320253 ┆    13.429273 ┆ -55.354108 ┆ 26.213084 │
│         var5 ┆ 5,201 ┆           0 ┆      0.14228 ┆     0.349371 ┆        0.0 ┆       1.0 │
│     var_reg1 ┆ 5,201 ┆           0 ┆          1.0 ┆          0.0 ┆        1.0 ┆       1.0 │
└──────────────┴───────┴─────────────┴──────────────┴──────────────┴────────────┴───────────┘

┌──────────────┬───────┬─────────────┬──────────────┬──────────────┬────────────┬───────────┐
│     Variable ┆     n ┆ n (missing) ┆         mean ┆          std ┆        min ┆       max │
╞══════════════╪═══════╪═════════════╪══════════════╪══════════════╪════════════╪═══════════╡
│ ___rownumber ┆ 5,194 ┆           0 ┆  4,966.59896 ┆ 2,886.517115 ┆        0.0 ┆   9,999.0 │
│  _row_index_ ┆ 5,194 ┆           0 ┆  4,966.59896 ┆ 2,886.517115 ┆        0.0 ┆   9,999.0 │
│        index ┆ 5,194 ┆           0 ┆  4,966.59896 ┆ 2,886.517115 ┆        0.0 ┆   9,999.0 │
│         year ┆ 5,194 ┆           0 ┆ 2,017.980362 ┆     1.415166 ┆    2,016.0 ┆   2,020.0 │
│        month ┆ 5,194 ┆           0 ┆     6.479399 ┆     3.448398 ┆        1.0 ┆      12.0 │
│         var2 ┆ 5,194 ┆           0 ┆      5.37216 ┆     3.015513 ┆        0.0 ┆      10.0 │
│         var3 ┆ 5,194 ┆           0 ┆    25.297459 ┆    15.905758 ┆        0.0 ┆      50.0 │
│         var4 ┆ 5,194 ┆           0 ┆     0.503356 ┆     0.286426 ┆   0.000104 ┆  0.999885 │
│  unrelated_1 ┆ 5,194 ┆           0 ┆     0.500962 ┆     0.289372 ┆   0.000248 ┆  0.999997 │
│  unrelated_2 ┆ 5,194 ┆           0 ┆     0.498781 ┆      0.28908 ┆   0.000049 ┆  0.999539 │
│  unrelated_3 ┆ 5,194 ┆           0 ┆     0.496775 ┆     0.288649 ┆   0.000129 ┆  0.999622 │
│  unrelated_4 ┆ 5,194 ┆           0 ┆     0.499127 ┆     0.288087 ┆   0.000133 ┆  0.999972 │
│  unrelated_5 ┆ 5,194 ┆           0 ┆     0.495114 ┆     0.287055 ┆   0.000181 ┆  0.999867 │
│     repeat_1 ┆ 5,194 ┆           0 ┆     0.500962 ┆     0.289372 ┆   0.000248 ┆  0.999997 │
│     var_reg2 ┆ 5,194 ┆           0 ┆    -4.369182 ┆    13.422283 ┆ -55.354108 ┆ 26.213084 │
│         var5 ┆ 5,194 ┆           0 ┆     0.141702 ┆     0.348778 ┆        0.0 ┆       1.0 │
│     var_reg1 ┆ 5,194 ┆           0 ┆          1.0 ┆          0.0 ┆        1.0 ┆       1.0 │
└──────────────┴───────┴─────────────┴──────────────┴──────────────┴────────────┴───────────┘