In [1]:
import narwhals as nw
import polars as pl
import polars.selectors as cs
from survey_kit.utilities.random import RandomData
from survey_kit.utilities.dataframe import summary
from survey_kit.imputation.variable import Variable
from survey_kit.imputation.parameters import Parameters
from survey_kit.imputation.srmi import SRMI
from survey_kit.imputation.selection import Selection
from survey_kit import logger, config
from survey_kit.utilities.dataframe import summary, columns_from_list
from survey_kit.utilities.formula_builder import FormulaBuilder
In [2]:
# Draw some random data
n_rows = 10_000
impute_share = 0.25
df = (
RandomData(n_rows=n_rows, seed=32565437)
.index("index")
.integer("year", 2016, 2020)
.integer("month", 1, 12)
.integer("var2", 0, 10)
.integer("var3", 0, 50)
.float("var4", 0, 1)
.integer("var5", 0, 1)
.float("unrelated_1", 0, 1)
.float("unrelated_2", 0, 1)
.float("unrelated_3", 0, 1)
.float("unrelated_4", 0, 1)
.float("unrelated_5", 0, 1)
.np_distribution("epsilon_reg1", "normal", scale=5)
.np_distribution("epsilon_reg2", "normal", scale=5)
.float("missing_reg1", 0, 1)
.float("missing_reg2", 0, 1)
.to_df()
)
# Convenience references to them for creating dependent variables
c_var2 = pl.col("var2")
c_var3 = pl.col("var3")
c_var4 = pl.col("var4")
c_var5 = pl.col("var5")
c_e_reg1 = pl.col("epsilon_reg1")
c_e_reg2 = pl.col("epsilon_reg2")
# Convenience references to them for creating dependent variables
c_var2 = pl.col("var2")
c_var3 = pl.col("var3")
c_var4 = pl.col("var4")
c_var5 = pl.col("var5")
logger.info("var_reg1 is binary and conditional on other variables")
c_reg1 = ((c_var2 * 2 - c_var3 * 3 * c_var5 + c_e_reg1) > 0).alias("var_reg1")
logger.info("var_reg2 is != 0 only if var_reg1 == True")
c_reg2 = (
pl.when(pl.col("var_reg1"))
.then((c_var2 * 1.5 - c_var3 * 1 * c_var4 + c_e_reg2))
.otherwise(pl.lit(0))
.alias("var_reg2")
)
# Create a bunch of variables that are functions of the variables created above
df = (
df.with_columns(c_reg1)
.with_columns(c_reg2)
.drop(columns_from_list(df=df, columns="epsilon*"))
.with_row_index(name="_row_index_")
)
df_original = df
# Set variables to missing according to the uniform random variables missing_
clear_missing = []
for prefixi in ["reg"]:
for i in range(1, 3):
vari = f"var_{prefixi}{i}"
missingi = f"missing_{prefixi}{i}"
clear_missing.append(
pl.when(pl.col(missingi) < impute_share)
.then(pl.lit(None))
.otherwise(pl.col(vari))
.alias(vari)
)
df = df.with_columns(clear_missing).drop(cs.starts_with("missing_"))
# Make a fully collinear var for testing
df = df.with_columns(pl.col("unrelated_1").alias("repeat_1"))
summary(df)
# Actually do the imputation
var_reg1 is binary and conditional on other variables
var_reg2 is != 0 only if var_reg1 == True
┌─────────────┬────────┬─────────────┬────────────┬─────────────┬────────────┬───────────┐ │ Variable ┆ n ┆ n (missing) ┆ mean ┆ std ┆ min ┆ max │ ╞═════════════╪════════╪═════════════╪════════════╪═════════════╪════════════╪═══════════╡ │ _row_index_ ┆ 10,000 ┆ 0 ┆ 4,999.5 ┆ 2,886.89568 ┆ 0.0 ┆ 9,999.0 │ │ index ┆ 10,000 ┆ 0 ┆ 4,999.5 ┆ 2,886.89568 ┆ 0.0 ┆ 9,999.0 │ │ year ┆ 10,000 ┆ 0 ┆ 2,017.9851 ┆ 1.415937 ┆ 2,016.0 ┆ 2,020.0 │ │ month ┆ 10,000 ┆ 0 ┆ 6.5137 ┆ 3.432141 ┆ 1.0 ┆ 12.0 │ │ var2 ┆ 10,000 ┆ 0 ┆ 4.9782 ┆ 3.154508 ┆ 0.0 ┆ 10.0 │ │ var3 ┆ 10,000 ┆ 0 ┆ 25.1084 ┆ 14.752302 ┆ 0.0 ┆ 50.0 │ │ var4 ┆ 10,000 ┆ 0 ┆ 0.505666 ┆ 0.287861 ┆ 0.000027 ┆ 0.999997 │ │ unrelated_1 ┆ 10,000 ┆ 0 ┆ 0.502449 ┆ 0.288359 ┆ 0.000119 ┆ 0.999997 │ │ unrelated_2 ┆ 10,000 ┆ 0 ┆ 0.500105 ┆ 0.287638 ┆ 0.000049 ┆ 0.999539 │ │ unrelated_3 ┆ 10,000 ┆ 0 ┆ 0.499175 ┆ 0.28876 ┆ 0.000129 ┆ 0.99994 │ │ unrelated_4 ┆ 10,000 ┆ 0 ┆ 0.500655 ┆ 0.288698 ┆ 0.000133 ┆ 0.999972 │ │ unrelated_5 ┆ 10,000 ┆ 0 ┆ 0.49876 ┆ 0.288979 ┆ 0.000071 ┆ 0.999867 │ │ var_reg2 ┆ 10,000 ┆ 2,476 ┆ -2.353019 ┆ 10.232008 ┆ -55.354108 ┆ 26.213084 │ │ repeat_1 ┆ 10,000 ┆ 0 ┆ 0.502449 ┆ 0.288359 ┆ 0.000119 ┆ 0.999997 │ │ var5 ┆ 10,000 ┆ 0 ┆ 0.4999 ┆ 0.500025 ┆ 0.0 ┆ 1.0 │ │ var_reg1 ┆ 10,000 ┆ 2,569 ┆ 0.523079 ┆ 0.499501 ┆ 0.0 ┆ 1.0 │ └─────────────┴────────┴─────────────┴────────────┴─────────────┴────────────┴───────────┘
Out[2]:
naive plan: (run LazyFrame.explain(optimized=True) to see the optimized plan)
SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")] UNION PLAN 0: WITH_COLUMNS: [col("n (missing)").cast(Int16), col("min").strict_cast(Float64), col("max").strict_cast(Float64)] SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")] WITH_COLUMNS: ["_row_index_".alias("Variable")] SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")] SELECT [col("___index___"), col("_row_index__std").alias("std"), col("_row_index__rawn").alias("n"), col("_row_index__min").alias("min"), col("_row_index__max").alias("max"), col("_row_index__mean").alias("mean"), col("_row_index__rawn_missing").alias("n (missing)")] SELECT [col("___index___"), col("_row_index__std"), col("_row_index__rawn"), col("_row_index__min"), col("_row_index__max"), col("_row_index__mean"), col("_row_index__rawn_missing")] DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS PLAN 1: WITH_COLUMNS: [col("n (missing)").cast(Int16), col("min").strict_cast(Float64), col("max").strict_cast(Float64)] SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")] WITH_COLUMNS: ["index".alias("Variable")] SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")] SELECT [col("___index___"), col("index_std").alias("std"), col("index_rawn").alias("n"), col("index_min").alias("min"), col("index_max").alias("max"), col("index_mean").alias("mean"), col("index_rawn_missing").alias("n (missing)")] SELECT [col("___index___"), col("index_std"), col("index_rawn"), col("index_min"), col("index_max"), col("index_mean"), col("index_rawn_missing")] DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS PLAN 2: WITH_COLUMNS: [col("n (missing)").cast(Int16), col("min").strict_cast(Float64), col("max").strict_cast(Float64)] SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")] WITH_COLUMNS: ["year".alias("Variable")] SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")] SELECT [col("___index___"), col("year_std").alias("std"), col("year_rawn").alias("n"), col("year_min").alias("min"), col("year_max").alias("max"), col("year_mean").alias("mean"), col("year_rawn_missing").alias("n (missing)")] SELECT [col("___index___"), col("year_std"), col("year_rawn"), col("year_min"), col("year_max"), col("year_mean"), col("year_rawn_missing")] DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS PLAN 3: WITH_COLUMNS: [col("n (missing)").cast(Int16), col("min").strict_cast(Float64), col("max").strict_cast(Float64)] SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")] WITH_COLUMNS: ["month".alias("Variable")] SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")] SELECT [col("___index___"), col("month_std").alias("std"), col("month_rawn").alias("n"), col("month_min").alias("min"), col("month_max").alias("max"), col("month_mean").alias("mean"), col("month_rawn_missing").alias("n (missing)")] SELECT [col("___index___"), col("month_std"), col("month_rawn"), col("month_min"), col("month_max"), col("month_mean"), col("month_rawn_missing")] DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS PLAN 4: WITH_COLUMNS: [col("n (missing)").cast(Int16), col("min").strict_cast(Float64), col("max").strict_cast(Float64)] SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")] WITH_COLUMNS: ["var2".alias("Variable")] SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")] SELECT [col("___index___"), col("var2_std").alias("std"), col("var2_rawn").alias("n"), col("var2_min").alias("min"), col("var2_max").alias("max"), col("var2_mean").alias("mean"), col("var2_rawn_missing").alias("n (missing)")] SELECT [col("___index___"), col("var2_std"), col("var2_rawn"), col("var2_min"), col("var2_max"), col("var2_mean"), col("var2_rawn_missing")] DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS PLAN 5: WITH_COLUMNS: [col("n (missing)").cast(Int16), col("min").strict_cast(Float64), col("max").strict_cast(Float64)] SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")] WITH_COLUMNS: ["var3".alias("Variable")] SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")] SELECT [col("___index___"), col("var3_std").alias("std"), col("var3_rawn").alias("n"), col("var3_min").alias("min"), col("var3_max").alias("max"), col("var3_mean").alias("mean"), col("var3_rawn_missing").alias("n (missing)")] SELECT [col("___index___"), col("var3_std"), col("var3_rawn"), col("var3_min"), col("var3_max"), col("var3_mean"), col("var3_rawn_missing")] DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS PLAN 6: WITH_COLUMNS: [col("n (missing)").cast(Int16)] SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")] WITH_COLUMNS: ["var4".alias("Variable")] SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")] SELECT [col("___index___"), col("var4_std").alias("std"), col("var4_rawn").alias("n"), col("var4_min").alias("min"), col("var4_max").alias("max"), col("var4_mean").alias("mean"), col("var4_rawn_missing").alias("n (missing)")] SELECT [col("___index___"), col("var4_std"), col("var4_rawn"), col("var4_min"), col("var4_max"), col("var4_mean"), col("var4_rawn_missing")] DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS PLAN 7: WITH_COLUMNS: [col("n (missing)").cast(Int16)] SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")] WITH_COLUMNS: ["unrelated_1".alias("Variable")] SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")] SELECT [col("___index___"), col("unrelated_1_std").alias("std"), col("unrelated_1_rawn").alias("n"), col("unrelated_1_min").alias("min"), col("unrelated_1_max").alias("max"), col("unrelated_1_mean").alias("mean"), col("unrelated_1_rawn_missing").alias("n (missing)")] SELECT [col("___index___"), col("unrelated_1_std"), col("unrelated_1_rawn"), col("unrelated_1_min"), col("unrelated_1_max"), col("unrelated_1_mean"), col("unrelated_1_rawn_missing")] DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS PLAN 8: WITH_COLUMNS: [col("n (missing)").cast(Int16)] SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")] WITH_COLUMNS: ["unrelated_2".alias("Variable")] SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")] SELECT [col("___index___"), col("unrelated_2_std").alias("std"), col("unrelated_2_rawn").alias("n"), col("unrelated_2_min").alias("min"), col("unrelated_2_max").alias("max"), col("unrelated_2_mean").alias("mean"), col("unrelated_2_rawn_missing").alias("n (missing)")] SELECT [col("___index___"), col("unrelated_2_std"), col("unrelated_2_rawn"), col("unrelated_2_min"), col("unrelated_2_max"), col("unrelated_2_mean"), col("unrelated_2_rawn_missing")] DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS PLAN 9: WITH_COLUMNS: [col("n (missing)").cast(Int16)] SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")] WITH_COLUMNS: ["unrelated_3".alias("Variable")] SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")] SELECT [col("___index___"), col("unrelated_3_std").alias("std"), col("unrelated_3_rawn").alias("n"), col("unrelated_3_min").alias("min"), col("unrelated_3_max").alias("max"), col("unrelated_3_mean").alias("mean"), col("unrelated_3_rawn_missing").alias("n (missing)")] SELECT [col("___index___"), col("unrelated_3_std"), col("unrelated_3_rawn"), col("unrelated_3_min"), col("unrelated_3_max"), col("unrelated_3_mean"), col("unrelated_3_rawn_missing")] DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS PLAN 10: WITH_COLUMNS: [col("n (missing)").cast(Int16)] SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")] WITH_COLUMNS: ["unrelated_4".alias("Variable")] SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")] SELECT [col("___index___"), col("unrelated_4_std").alias("std"), col("unrelated_4_rawn").alias("n"), col("unrelated_4_min").alias("min"), col("unrelated_4_max").alias("max"), col("unrelated_4_mean").alias("mean"), col("unrelated_4_rawn_missing").alias("n (missing)")] SELECT [col("___index___"), col("unrelated_4_std"), col("unrelated_4_rawn"), col("unrelated_4_min"), col("unrelated_4_max"), col("unrelated_4_mean"), col("unrelated_4_rawn_missing")] DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS PLAN 11: WITH_COLUMNS: [col("n (missing)").cast(Int16)] SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")] WITH_COLUMNS: ["unrelated_5".alias("Variable")] SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")] SELECT [col("___index___"), col("unrelated_5_std").alias("std"), col("unrelated_5_rawn").alias("n"), col("unrelated_5_min").alias("min"), col("unrelated_5_max").alias("max"), col("unrelated_5_mean").alias("mean"), col("unrelated_5_rawn_missing").alias("n (missing)")] SELECT [col("___index___"), col("unrelated_5_std"), col("unrelated_5_rawn"), col("unrelated_5_min"), col("unrelated_5_max"), col("unrelated_5_mean"), col("unrelated_5_rawn_missing")] DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS PLAN 12: SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")] WITH_COLUMNS: ["var_reg2".alias("Variable")] SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")] SELECT [col("___index___"), col("var_reg2_std").alias("std"), col("var_reg2_rawn").alias("n"), col("var_reg2_min").alias("min"), col("var_reg2_max").alias("max"), col("var_reg2_mean").alias("mean"), col("var_reg2_rawn_missing").alias("n (missing)")] SELECT [col("___index___"), col("var_reg2_std"), col("var_reg2_rawn"), col("var_reg2_min"), col("var_reg2_max"), col("var_reg2_mean"), col("var_reg2_rawn_missing")] DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS PLAN 13: WITH_COLUMNS: [col("n (missing)").cast(Int16)] SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")] WITH_COLUMNS: ["repeat_1".alias("Variable")] SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")] SELECT [col("___index___"), col("repeat_1_std").alias("std"), col("repeat_1_rawn").alias("n"), col("repeat_1_min").alias("min"), col("repeat_1_max").alias("max"), col("repeat_1_mean").alias("mean"), col("repeat_1_rawn_missing").alias("n (missing)")] SELECT [col("___index___"), col("repeat_1_std"), col("repeat_1_rawn"), col("repeat_1_min"), col("repeat_1_max"), col("repeat_1_mean"), col("repeat_1_rawn_missing")] DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS PLAN 14: WITH_COLUMNS: [col("n (missing)").cast(Int16), col("min").strict_cast(Float64), col("max").strict_cast(Float64)] SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")] WITH_COLUMNS: ["var5".alias("Variable")] SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")] SELECT [col("___index___"), col("var5_std").alias("std"), col("var5_rawn").alias("n"), col("var5_min").alias("min"), col("var5_max").alias("max"), col("var5_mean").alias("mean"), col("var5_rawn_missing").alias("n (missing)")] SELECT [col("___index___"), col("var5_std"), col("var5_rawn"), col("var5_min"), col("var5_max"), col("var5_mean"), col("var5_rawn_missing")] DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS PLAN 15: WITH_COLUMNS: [col("min").strict_cast(Float64), col("max").strict_cast(Float64)] SELECT [col("Variable"), col("n"), col("n (missing)"), col("mean"), col("std"), col("min"), col("max")] WITH_COLUMNS: ["var_reg1".alias("Variable")] SELECT [col("std"), col("n"), col("min"), col("max"), col("mean"), col("n (missing)")] SELECT [col("___index___"), col("var_reg1_std").alias("std"), col("var_reg1_rawn").alias("n"), col("var_reg1_min").alias("min"), col("var_reg1_max").alias("max"), col("var_reg1_mean").alias("mean"), col("var_reg1_rawn_missing").alias("n (missing)")] SELECT [col("___index___"), col("var_reg1_std"), col("var_reg1_rawn"), col("var_reg1_min"), col("var_reg1_max"), col("var_reg1_mean"), col("var_reg1_rawn_missing")] DF ["___index___", "_row_index__rawn", "_row_index__mean", "_row_index__std", ...]; PROJECT */97 COLUMNS END UNION
In [3]:
logger.info(
"Define the regression model (intentionally include some extraneous variables"
)
f_model = FormulaBuilder(df=df)
f_model.formula_with_varnames_in_brackets(
"~1+{var_*}+var2+var4+var4*var3*C(var5)+{unrelated_*}+{repeat_*}"
)
logger.info(f_model.formula)
Define the regression model (intentionally include some extraneous variables
~1+var_reg1+var_reg2+var2+var4+var4*var3*C(var5)+unrelated_1+unrelated_2+unrelated_3+unrelated_4+unrelated_5+repeat_1
In [4]:
# Set up the variable to be imputed
vars_impute = []
logger.info("Impute the boolean variable (var_reg1)")
logger.info(" to the default setup for predicted mean matching")
logger.info(" using logit regression")
v_reg1 = Variable(
impute_var="var_reg1",
modeltype=Variable.ModelType.pmm,
model=f_model.formula,
parameters=Parameters.Regression(model=Parameters.RegressionModel.Logit),
)
logger.info("Add the variable to the list to be imputed")
vars_impute.append(v_reg1)
logger.info("Impute the continuous variable (var_reg2) ")
logger.info(" conditional on var_reg1, using narwhals (nw.col('var_reg1'))")
logger.info(" by setting the model type")
logger.info(" and the formula")
logger.info(" as well as a post-processing edit to set var_reg2=0 when var_reg1==0")
v_reg2 = Variable(
impute_var="var_reg2",
sample=Variable.Sample(Where=nw.col("var_reg1")),
modeltype=Variable.ModelType.pmm,
model=f_model.formula,
# Default parameters
parameters=Parameters.Regression(),
transforms=Variable.Transforms(
post=(
nw.when(nw.col("var_reg1"))
.then(nw.col("var_reg2"))
.otherwise(nw.lit(0))
.alias("var_reg2")
)
),
)
vars_impute.append(v_reg2)
Impute the boolean variable (var_reg1)
to the default setup for predicted mean matching
using logit regression
Add the variable to the list to be imputed
Impute the continuous variable (var_reg2)
conditional on var_reg1, using narwhals (nw.col('var_reg1'))
by setting the model type
and the formula
as well as a post-processing edit to set var_reg2=0 when var_reg1==0
In [5]:
logger.info("Set up the imputation")
logger.info("Add LASSO selection before each imputation")
srmi = SRMI(
df=df,
variables=vars_impute,
replication=SRMI.Replication(n_implicates=2, n_iterations=2),
parallel=SRMI.Parallel(enabled=False),
bootstrap=SRMI.Bootstrap(enabled=True),
defaults=SRMI.Defaults(
selection=Selection(method=Selection.Method.LASSO),
modeltype=Variable.ModelType.pmm,
model=f_model.formula,
),
storage=SRMI.Storage(
path_model=f"{config.path_temp_files}/py_srmi_test_regression",
force_start=True,
),
)
Set up the imputation
Add LASSO selection before each imputation
Removing existing directory C:\Users\jonro\OneDrive\Documents\Coding\survey_kit\.scratch\temp_files/py_srmi_test_regression.srmi
Dropping var_reg1 from formula
Dropping var_reg2 from formula
In [6]:
logger.info("Run it")
srmi.run()
Run it
Variable selection before SRMI run, if necessary
var_reg1: Method.No
var_reg2: Method.No
Hyperparameter tuning before SRMI run, if necessary
Removing existing directory C:\Users\jonro\OneDrive\Documents\Coding\survey_kit\.scratch\temp_files/py_srmi_test_regression.srmi/1.srmi.implicate
Removing existing directory C:\Users\jonro\OneDrive\Documents\Coding\survey_kit\.scratch\temp_files/py_srmi_test_regression.srmi/2.srmi.implicate
Dropping var_reg2 from formula
Running variable selection: Method.LASSO
Selected model: ~0+var2+var4+C(var5)+var4:var3:C(var5)+unrelated_1+unrelated_2+unrelated_3+unrelated_4+unrelated_5+repeat_1+var3
Imputation using pmm
C:\Users\jonro\OneDrive\Documents\Coding\survey_kit\.venv\Lib\site-packages\sklearn\utils\validation.py:1406: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel(). y = column_or_1d(y, warn=True) R2 = 0.5507
┌────────────────────────┬───────────┐ │ Variable ┆ Beta │ ╞════════════════════════╪═══════════╡ │ var2 ┆ 0.2176 │ │ var4 ┆ -0.3242 │ │ C(var5)FALSE ┆ 2.064 │ │ C(var5)TRUE ┆ -2.39 │ │ unrelated_1 ┆ -0.1082 │ │ unrelated_2 ┆ -0.1339 │ │ unrelated_3 ┆ 0.07617 │ │ unrelated_4 ┆ -0.2167 │ │ unrelated_5 ┆ -0.1206 │ │ repeat_1 ┆ -0.1082 │ │ var3 ┆ -0.004198 │ │ var4:var3:C(var5)FALSE ┆ 0.0009905 │ │ var4:var3:C(var5)TRUE ┆ 0.02909 │ │ _Intercept_ ┆ -0.3496 │ └────────────────────────┴───────────┘
error=pmm: donating observed value(s) ['var_reg1'] from 10-nearest matched donors
Finding 10 nearest neighbors on ['___prediction']
Randomly picking one and donating ['var_reg1']
Most common matches:
shape: (5, 2) ┌──────────────┬─────────┐ │ ___rownumber ┆ nDonors │ │ --- ┆ --- │ │ i16 ┆ i8 │ ╞══════════════╪═════════╡ │ 823 ┆ 4 │ │ 2000 ┆ 4 │ │ 4710 ┆ 4 │ │ 5629 ┆ 4 │ │ 8345 ┆ 4 │ └──────────────┴─────────┘
Post-imputation statistics for ['var_reg1']
Where: None
Where (impute): col(___imp_missing_var_reg1_1)
┌──────────┬─────────┬───────┬──────────────┬────────┬────────┬──────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┐ │ Variable ┆ Imputed ┆ n ┆ n (not null) ┆ mean ┆ std ┆ mean (not 0) ┆ std (not 0) ┆ q10 (not 0) ┆ q25 (not 0) ┆ q50 (not 0) ┆ q75 (not 0) ┆ q90 (not 0) ┆ min (not 0) ┆ max (not 0) │ ╞══════════╪═════════╪═══════╪══════════════╪════════╪════════╪══════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╡ │ var_reg1 ┆ ┆ 10000 ┆ 10000 ┆ 0.5233 ┆ 0.4995 ┆ 1 ┆ 0 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 │ │ var_reg1 ┆ 0 ┆ 7431 ┆ 7431 ┆ 0.5231 ┆ 0.4995 ┆ 1 ┆ 0 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 │ │ var_reg1 ┆ 1 ┆ 2569 ┆ 2569 ┆ 0.5239 ┆ 0.4995 ┆ 1 ┆ 0 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 │ └──────────┴─────────┴───────┴──────────────┴────────┴────────┴──────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┘
Running variable selection: Method.LASSO
Selected model: ~0+var_reg1+var2+var4+var3+var4:var3+C(var5)+var4:C(var5)+var3:C(var5)+var4:var3:C(var5)+unrelated_1+unrelated_2+unrelated_3+unrelated_4+unrelated_5+repeat_1
Imputation using pmm
R2 = 0.8185
┌───────────────────────┬──────────┐ │ Variable ┆ Beta │ ╞═══════════════════════╪══════════╡ │ var_reg1FALSE ┆ 0.0 │ │ var_reg1TRUE ┆ -0.0 │ │ var2 ┆ 1.383 │ │ var4 ┆ 0.5612 │ │ var3 ┆ 0.01672 │ │ C(var5)TRUE ┆ -0.5865 │ │ unrelated_1 ┆ 0.1563 │ │ unrelated_2 ┆ 0.5452 │ │ unrelated_3 ┆ -0.1947 │ │ unrelated_4 ┆ 0.06568 │ │ unrelated_5 ┆ -0.368 │ │ repeat_1 ┆ 0.1563 │ │ var4:var3 ┆ -0.9976 │ │ var4:C(var5)TRUE ┆ 3.789 │ │ var3:C(var5)TRUE ┆ -0.06316 │ │ var4:var3:C(var5)TRUE ┆ 0.06792 │ │ _Intercept_ ┆ 0.06084 │ └───────────────────────┴──────────┘
error=pmm: donating observed value(s) ['var_reg2'] from 10-nearest matched donors
Finding 10 nearest neighbors on ['___prediction']
Randomly picking one and donating ['var_reg2']
Most common matches:
shape: (5, 2) ┌──────────────┬─────────┐ │ ___rownumber ┆ nDonors │ │ --- ┆ --- │ │ i16 ┆ i8 │ ╞══════════════╪═════════╡ │ 110 ┆ 4 │ │ 460 ┆ 3 │ │ 519 ┆ 3 │ │ 1061 ┆ 3 │ │ 2180 ┆ 3 │ └──────────────┴─────────┘
Post-imputation statistics for ['var_reg2']
Where: col(var_reg1)
Where (impute): col(___imp_missing_var_reg2_2)
┌──────────┬─────────┬──────┬──────────────┬────────┬───────┬──────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┐ │ Variable ┆ Imputed ┆ n ┆ n (not null) ┆ mean ┆ std ┆ mean (not 0) ┆ std (not 0) ┆ q10 (not 0) ┆ q25 (not 0) ┆ q50 (not 0) ┆ q75 (not 0) ┆ q90 (not 0) ┆ min (not 0) ┆ max (not 0) │ ╞══════════╪═════════╪══════╪══════════════╪════════╪═══════╪══════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╡ │ var_reg2 ┆ ┆ 5233 ┆ 5233 ┆ -4.196 ┆ 13.47 ┆ -4.403 ┆ 13.76 ┆ -24.64 ┆ -12.77 ┆ -1.985 ┆ 6.003 ┆ 11.24 ┆ -55.35 ┆ 26.21 │ │ var_reg2 ┆ 0 ┆ 3923 ┆ 3923 ┆ -4.147 ┆ 13.38 ┆ -4.356 ┆ 13.68 ┆ -24.04 ┆ -12.75 ┆ -1.984 ┆ 5.914 ┆ 11.27 ┆ -55.35 ┆ 26.21 │ │ var_reg2 ┆ 1 ┆ 1310 ┆ 1310 ┆ -4.345 ┆ 13.75 ┆ -4.543 ┆ 14.02 ┆ -25.48 ┆ -13.04 ┆ -1.995 ┆ 6.4 ┆ 11.19 ┆ -55.35 ┆ 22.69 │ └──────────┴─────────┴──────┴──────────────┴────────┴───────┴──────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┘
Updating data according to narwhals expression: when_then(all_horizontal(col(var_reg1), ignore_nulls=False), col(var_reg2), lit(value=0, dtype=None)).alias(name=var_reg2)
Removing existing directory C:\Users\jonro\OneDrive\Documents\Coding\survey_kit\.scratch\temp_files/py_srmi_test_regression.srmi/1.srmi.implicate
Running variable selection: Method.LASSO
Selected model: ~0+var_reg2+var2+var4+var3+var4:var3+C(var5)+var4:var3:C(var5)+unrelated_2+unrelated_4+unrelated_5+repeat_1
Imputation using pmm
C:\Users\jonro\OneDrive\Documents\Coding\survey_kit\.venv\Lib\site-packages\sklearn\utils\validation.py:1406: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel(). y = column_or_1d(y, warn=True) R2 = 0.6249
┌───────────────────────┬───────────┐ │ Variable ┆ Beta │ ╞═══════════════════════╪═══════════╡ │ var_reg2 ┆ -0.1319 │ │ var2 ┆ 0.2891 │ │ var4 ┆ -0.157 │ │ var3 ┆ -0.003246 │ │ C(var5)FALSE ┆ 2.413 │ │ C(var5)TRUE ┆ -2.526 │ │ unrelated_2 ┆ -0.2193 │ │ unrelated_4 ┆ -0.3354 │ │ unrelated_5 ┆ -0.2822 │ │ repeat_1 ┆ 0.007163 │ │ var4:var3 ┆ -0.08512 │ │ var4:var3:C(var5)TRUE ┆ 0.05801 │ │ _Intercept_ ┆ -0.1207 │ └───────────────────────┴───────────┘
error=pmm: donating observed value(s) ['var_reg1'] from 10-nearest matched donors
Finding 10 nearest neighbors on ['___prediction']
Randomly picking one and donating ['var_reg1']
Most common matches:
shape: (5, 2) ┌──────────────┬─────────┐ │ ___rownumber ┆ nDonors │ │ --- ┆ --- │ │ i16 ┆ i8 │ ╞══════════════╪═════════╡ │ 5477 ┆ 4 │ │ 8656 ┆ 4 │ │ 93 ┆ 3 │ │ 400 ┆ 3 │ │ 597 ┆ 3 │ └──────────────┴─────────┘
Post-imputation statistics for ['var_reg1']
Where: None
Where (impute): col(___imp_missing_var_reg1_1)
┌──────────┬─────────┬───────┬──────────────┬────────┬────────┬──────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┐ │ Variable ┆ Imputed ┆ n ┆ n (not null) ┆ mean ┆ std ┆ mean (not 0) ┆ std (not 0) ┆ q10 (not 0) ┆ q25 (not 0) ┆ q50 (not 0) ┆ q75 (not 0) ┆ q90 (not 0) ┆ min (not 0) ┆ max (not 0) │ ╞══════════╪═════════╪═══════╪══════════════╪════════╪════════╪══════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╡ │ var_reg1 ┆ ┆ 12569 ┆ 12569 ┆ 0.5209 ┆ 0.4996 ┆ 1 ┆ 0 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 │ │ var_reg1 ┆ 0 ┆ 10000 ┆ 10000 ┆ 0.5233 ┆ 0.4995 ┆ 1 ┆ 0 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 │ │ var_reg1 ┆ 1 ┆ 2569 ┆ 2569 ┆ 0.5115 ┆ 0.5 ┆ 1 ┆ 0 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 │ └──────────┴─────────┴───────┴──────────────┴────────┴────────┴──────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┘
Running variable selection: Method.LASSO
Selected model: ~0+var_reg1+var2+var4+var3+var4:var3+C(var5)+var4:C(var5)+var3:C(var5)+var4:var3:C(var5)+unrelated_2+unrelated_3+unrelated_4+unrelated_5
Imputation using pmm
R2 = 0.8298
┌───────────────────────┬──────────┐ │ Variable ┆ Beta │ ╞═══════════════════════╪══════════╡ │ var_reg1FALSE ┆ 0.0 │ │ var_reg1TRUE ┆ -0.0 │ │ var2 ┆ 1.364 │ │ var4 ┆ 1.554 │ │ var3 ┆ 0.04128 │ │ C(var5)TRUE ┆ -0.4464 │ │ unrelated_2 ┆ -0.06238 │ │ unrelated_3 ┆ 0.2616 │ │ unrelated_4 ┆ -0.592 │ │ unrelated_5 ┆ -0.146 │ │ var4:var3 ┆ -1.046 │ │ var4:C(var5)TRUE ┆ 2.439 │ │ var3:C(var5)TRUE ┆ -0.07164 │ │ var4:var3:C(var5)TRUE ┆ 0.128 │ │ _Intercept_ ┆ 0.1934 │ └───────────────────────┴──────────┘
error=pmm: donating observed value(s) ['var_reg2'] from 10-nearest matched donors
Finding 10 nearest neighbors on ['___prediction']
Randomly picking one and donating ['var_reg2']
Most common matches:
shape: (5, 2) ┌──────────────┬─────────┐ │ ___rownumber ┆ nDonors │ │ --- ┆ --- │ │ i16 ┆ i8 │ ╞══════════════╪═════════╡ │ 515 ┆ 3 │ │ 731 ┆ 3 │ │ 787 ┆ 3 │ │ 1559 ┆ 3 │ │ 1668 ┆ 3 │ └──────────────┴─────────┘
Post-imputation statistics for ['var_reg2']
Where: col(var_reg1)
Where (impute): col(___imp_missing_var_reg2_2)
┌──────────┬─────────┬──────┬──────────────┬────────┬───────┬──────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┐ │ Variable ┆ Imputed ┆ n ┆ n (not null) ┆ mean ┆ std ┆ mean (not 0) ┆ std (not 0) ┆ q10 (not 0) ┆ q25 (not 0) ┆ q50 (not 0) ┆ q75 (not 0) ┆ q90 (not 0) ┆ min (not 0) ┆ max (not 0) │ ╞══════════╪═════════╪══════╪══════════════╪════════╪═══════╪══════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╡ │ var_reg2 ┆ ┆ 6500 ┆ 6500 ┆ -4.311 ┆ 13.47 ┆ -4.582 ┆ 13.84 ┆ -25.03 ┆ -13.3 ┆ -2.209 ┆ 5.98 ┆ 11.1 ┆ -55.35 ┆ 26.21 │ │ var_reg2 ┆ 0 ┆ 5201 ┆ 5201 ┆ -4.232 ┆ 13.41 ┆ -4.5 ┆ 13.79 ┆ -24.74 ┆ -13.0 ┆ -2.152 ┆ 5.936 ┆ 11.19 ┆ -55.35 ┆ 26.21 │ │ var_reg2 ┆ 1 ┆ 1299 ┆ 1299 ┆ -4.628 ┆ 13.69 ┆ -4.908 ┆ 14.05 ┆ -25.9 ┆ -14.59 ┆ -2.416 ┆ 6.171 ┆ 10.83 ┆ -46.63 ┆ 24.89 │ └──────────┴─────────┴──────┴──────────────┴────────┴───────┴──────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┘
Updating data according to narwhals expression: when_then(all_horizontal(col(var_reg1), ignore_nulls=False), col(var_reg2), lit(value=0, dtype=None)).alias(name=var_reg2)
var_reg1
var_reg2
Final Estimates by Iteration
Removing existing directory C:\Users\jonro\OneDrive\Documents\Coding\survey_kit\.scratch\temp_files/py_srmi_test_regression.srmi/1.srmi.implicate
Dropping var_reg2 from formula
Running variable selection: Method.LASSO
Selected model: ~0+var2+C(var5)+var4:C(var5)+var4:var3:C(var5)+unrelated_1+unrelated_2+unrelated_3+unrelated_5+repeat_1+var4+var3
Imputation using pmm
C:\Users\jonro\OneDrive\Documents\Coding\survey_kit\.venv\Lib\site-packages\sklearn\utils\validation.py:1406: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel(). y = column_or_1d(y, warn=True) R2 = 0.5509
┌────────────────────────┬───────────┐ │ Variable ┆ Beta │ ╞════════════════════════╪═══════════╡ │ var2 ┆ 0.1965 │ │ C(var5)FALSE ┆ 2.076 │ │ C(var5)TRUE ┆ -2.213 │ │ unrelated_1 ┆ -0.111 │ │ unrelated_2 ┆ -0.09557 │ │ unrelated_3 ┆ -0.1429 │ │ unrelated_5 ┆ -0.408 │ │ repeat_1 ┆ -0.111 │ │ var4 ┆ -0.04577 │ │ var3 ┆ -0.005247 │ │ var4:C(var5)TRUE ┆ -0.7446 │ │ var4:var3:C(var5)FALSE ┆ 0.003528 │ │ var4:var3:C(var5)TRUE ┆ 0.03546 │ │ _Intercept_ ┆ -0.1482 │ └────────────────────────┴───────────┘
error=pmm: donating observed value(s) ['var_reg1'] from 10-nearest matched donors
Finding 10 nearest neighbors on ['___prediction']
Randomly picking one and donating ['var_reg1']
Most common matches:
shape: (5, 2) ┌──────────────┬─────────┐ │ ___rownumber ┆ nDonors │ │ --- ┆ --- │ │ i16 ┆ i8 │ ╞══════════════╪═════════╡ │ 360 ┆ 4 │ │ 379 ┆ 4 │ │ 1046 ┆ 4 │ │ 3449 ┆ 4 │ │ 180 ┆ 3 │ └──────────────┴─────────┘
Post-imputation statistics for ['var_reg1']
Where: None
Where (impute): col(___imp_missing_var_reg1_1)
┌──────────┬─────────┬───────┬──────────────┬────────┬────────┬──────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┐ │ Variable ┆ Imputed ┆ n ┆ n (not null) ┆ mean ┆ std ┆ mean (not 0) ┆ std (not 0) ┆ q10 (not 0) ┆ q25 (not 0) ┆ q50 (not 0) ┆ q75 (not 0) ┆ q90 (not 0) ┆ min (not 0) ┆ max (not 0) │ ╞══════════╪═════════╪═══════╪══════════════╪════════╪════════╪══════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╡ │ var_reg1 ┆ ┆ 10000 ┆ 10000 ┆ 0.5236 ┆ 0.4995 ┆ 1 ┆ 0 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 │ │ var_reg1 ┆ 0 ┆ 7431 ┆ 7431 ┆ 0.5231 ┆ 0.4995 ┆ 1 ┆ 0 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 │ │ var_reg1 ┆ 1 ┆ 2569 ┆ 2569 ┆ 0.5251 ┆ 0.4995 ┆ 1 ┆ 0 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 │ └──────────┴─────────┴───────┴──────────────┴────────┴────────┴──────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┘
Running variable selection: Method.LASSO
Selected model: ~0+var_reg1+var2+var3+var4:var3+C(var5)+var4:C(var5)+var4:var3:C(var5)+unrelated_2+unrelated_4+unrelated_5+var4
Imputation using pmm
R2 = 0.8246
┌───────────────────────┬─────────┐ │ Variable ┆ Beta │ ╞═══════════════════════╪═════════╡ │ var_reg1FALSE ┆ 0.0 │ │ var_reg1TRUE ┆ -0.0 │ │ var2 ┆ 1.417 │ │ var3 ┆ 0.01411 │ │ C(var5)TRUE ┆ -1.319 │ │ unrelated_2 ┆ 0.04522 │ │ unrelated_4 ┆ -0.9803 │ │ unrelated_5 ┆ -0.7875 │ │ var4 ┆ 1.034 │ │ var4:var3 ┆ -1.02 │ │ var4:C(var5)TRUE ┆ 3.231 │ │ var4:var3:C(var5)TRUE ┆ 0.02549 │ │ _Intercept_ ┆ 0.8182 │ └───────────────────────┴─────────┘
error=pmm: donating observed value(s) ['var_reg2'] from 10-nearest matched donors
Finding 10 nearest neighbors on ['___prediction']
Randomly picking one and donating ['var_reg2']
Most common matches:
shape: (5, 2) ┌──────────────┬─────────┐ │ ___rownumber ┆ nDonors │ │ --- ┆ --- │ │ i16 ┆ i8 │ ╞══════════════╪═════════╡ │ 5577 ┆ 4 │ │ 6529 ┆ 4 │ │ 2500 ┆ 3 │ │ 3041 ┆ 3 │ │ 4038 ┆ 3 │ └──────────────┴─────────┘
Post-imputation statistics for ['var_reg2']
Where: col(var_reg1)
Where (impute): col(___imp_missing_var_reg2_2)
┌──────────┬─────────┬──────┬──────────────┬────────┬───────┬──────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┐ │ Variable ┆ Imputed ┆ n ┆ n (not null) ┆ mean ┆ std ┆ mean (not 0) ┆ std (not 0) ┆ q10 (not 0) ┆ q25 (not 0) ┆ q50 (not 0) ┆ q75 (not 0) ┆ q90 (not 0) ┆ min (not 0) ┆ max (not 0) │ ╞══════════╪═════════╪══════╪══════════════╪════════╪═══════╪══════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╡ │ var_reg2 ┆ ┆ 5236 ┆ 5236 ┆ -4.283 ┆ 13.47 ┆ -4.494 ┆ 13.77 ┆ -24.54 ┆ -13.03 ┆ -2.107 ┆ 5.936 ┆ 11.17 ┆ -55.35 ┆ 26.21 │ │ var_reg2 ┆ 0 ┆ 3915 ┆ 3915 ┆ -4.199 ┆ 13.41 ┆ -4.408 ┆ 13.7 ┆ -24.18 ┆ -12.82 ┆ -1.985 ┆ 5.906 ┆ 11.24 ┆ -55.35 ┆ 26.21 │ │ var_reg2 ┆ 1 ┆ 1321 ┆ 1321 ┆ -4.532 ┆ 13.67 ┆ -4.747 ┆ 13.95 ┆ -25.13 ┆ -13.48 ┆ -2.727 ┆ 6.171 ┆ 10.7 ┆ -47.42 ┆ 26.21 │ └──────────┴─────────┴──────┴──────────────┴────────┴───────┴──────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┘
Updating data according to narwhals expression: when_then(all_horizontal(col(var_reg1), ignore_nulls=False), col(var_reg2), lit(value=0, dtype=None)).alias(name=var_reg2)
Removing existing directory C:\Users\jonro\OneDrive\Documents\Coding\survey_kit\.scratch\temp_files/py_srmi_test_regression.srmi/2.srmi.implicate
Running variable selection: Method.LASSO
Selected model: ~0+var_reg2+var2+var4:var3+C(var5)+var4:C(var5)+var4:var3:C(var5)+unrelated_1+unrelated_2+unrelated_3+unrelated_5+repeat_1+var4+var3
Imputation using pmm
C:\Users\jonro\OneDrive\Documents\Coding\survey_kit\.venv\Lib\site-packages\sklearn\utils\validation.py:1406: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
y = column_or_1d(y, warn=True)
C:\Users\jonro\OneDrive\Documents\Coding\survey_kit\.venv\Lib\site-packages\sklearn\linear_model\_logistic.py:473: ConvergenceWarning: lbfgs failed to converge after 100 iteration(s) (status=1):
STOP: TOTAL NO. OF ITERATIONS REACHED LIMIT
Increase the number of iterations to improve the convergence (max_iter=100).
You might also want to scale the data as shown in:
https://scikit-learn.org/stable/modules/preprocessing.html
Please also refer to the documentation for alternative solver options:
https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
n_iter_i = _check_optimize_result(
R2 = 0.6195
┌───────────────────────┬──────────┐ │ Variable ┆ Beta │ ╞═══════════════════════╪══════════╡ │ var_reg2 ┆ -0.1348 │ │ var2 ┆ 0.3141 │ │ C(var5)FALSE ┆ 2.275 │ │ C(var5)TRUE ┆ -2.602 │ │ unrelated_1 ┆ 0.07694 │ │ unrelated_2 ┆ -0.3327 │ │ unrelated_3 ┆ -0.2426 │ │ unrelated_5 ┆ -0.6309 │ │ repeat_1 ┆ 0.07694 │ │ var4 ┆ 0.3434 │ │ var3 ┆ 0.003656 │ │ var4:var3 ┆ -0.09581 │ │ var4:C(var5)TRUE ┆ 0.05559 │ │ var4:var3:C(var5)TRUE ┆ 0.05213 │ │ _Intercept_ ┆ -0.3411 │ └───────────────────────┴──────────┘
error=pmm: donating observed value(s) ['var_reg1'] from 10-nearest matched donors
Finding 10 nearest neighbors on ['___prediction']
Randomly picking one and donating ['var_reg1']
Most common matches:
shape: (5, 2) ┌──────────────┬─────────┐ │ ___rownumber ┆ nDonors │ │ --- ┆ --- │ │ i16 ┆ i8 │ ╞══════════════╪═════════╡ │ 390 ┆ 3 │ │ 1113 ┆ 3 │ │ 1864 ┆ 3 │ │ 1931 ┆ 3 │ │ 2325 ┆ 3 │ └──────────────┴─────────┘
Post-imputation statistics for ['var_reg1']
Where: None
Where (impute): col(___imp_missing_var_reg1_1)
┌──────────┬─────────┬───────┬──────────────┬────────┬────────┬──────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┐ │ Variable ┆ Imputed ┆ n ┆ n (not null) ┆ mean ┆ std ┆ mean (not 0) ┆ std (not 0) ┆ q10 (not 0) ┆ q25 (not 0) ┆ q50 (not 0) ┆ q75 (not 0) ┆ q90 (not 0) ┆ min (not 0) ┆ max (not 0) │ ╞══════════╪═════════╪═══════╪══════════════╪════════╪════════╪══════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╡ │ var_reg1 ┆ ┆ 12569 ┆ 12569 ┆ 0.5206 ┆ 0.4996 ┆ 1 ┆ 0 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 │ │ var_reg1 ┆ 0 ┆ 10000 ┆ 10000 ┆ 0.5236 ┆ 0.4995 ┆ 1 ┆ 0 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 │ │ var_reg1 ┆ 1 ┆ 2569 ┆ 2569 ┆ 0.5088 ┆ 0.5 ┆ 1 ┆ 0 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 ┆ 1 │ └──────────┴─────────┴───────┴──────────────┴────────┴────────┴──────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┘
Running variable selection: Method.LASSO
Selected model: ~0+var_reg1+var2+var4:var3+C(var5)+var4:C(var5)+var3:C(var5)+var4:var3:C(var5)+unrelated_1+unrelated_2+unrelated_3+unrelated_4+unrelated_5+repeat_1+var4+var3
Imputation using pmm
R2 = 0.8258
┌───────────────────────┬──────────┐ │ Variable ┆ Beta │ ╞═══════════════════════╪══════════╡ │ var_reg1FALSE ┆ 0.0 │ │ var_reg1TRUE ┆ -0.0 │ │ var2 ┆ 1.381 │ │ C(var5)TRUE ┆ -1.331 │ │ unrelated_1 ┆ -0.06996 │ │ unrelated_2 ┆ -0.0357 │ │ unrelated_3 ┆ -0.5286 │ │ unrelated_4 ┆ -0.4791 │ │ unrelated_5 ┆ -0.7511 │ │ repeat_1 ┆ -0.06996 │ │ var4 ┆ 0.7273 │ │ var3 ┆ 0.01873 │ │ var4:var3 ┆ -0.9999 │ │ var4:C(var5)TRUE ┆ 4.685 │ │ var3:C(var5)TRUE ┆ -0.01287 │ │ var4:var3:C(var5)TRUE ┆ 0.004049 │ │ _Intercept_ ┆ 0.9459 │ └───────────────────────┴──────────┘
error=pmm: donating observed value(s) ['var_reg2'] from 10-nearest matched donors
Finding 10 nearest neighbors on ['___prediction']
Randomly picking one and donating ['var_reg2']
Most common matches:
shape: (5, 2) ┌──────────────┬─────────┐ │ ___rownumber ┆ nDonors │ │ --- ┆ --- │ │ i16 ┆ i8 │ ╞══════════════╪═════════╡ │ 3625 ┆ 4 │ │ 1299 ┆ 3 │ │ 1702 ┆ 3 │ │ 2790 ┆ 3 │ │ 2949 ┆ 3 │ └──────────────┴─────────┘
Post-imputation statistics for ['var_reg2']
Where: col(var_reg1)
Where (impute): col(___imp_missing_var_reg2_2)
┌──────────┬─────────┬──────┬──────────────┬────────┬───────┬──────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┬─────────────┐ │ Variable ┆ Imputed ┆ n ┆ n (not null) ┆ mean ┆ std ┆ mean (not 0) ┆ std (not 0) ┆ q10 (not 0) ┆ q25 (not 0) ┆ q50 (not 0) ┆ q75 (not 0) ┆ q90 (not 0) ┆ min (not 0) ┆ max (not 0) │ ╞══════════╪═════════╪══════╪══════════════╪════════╪═══════╪══════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╪═════════════╡ │ var_reg2 ┆ ┆ 6494 ┆ 6494 ┆ -4.419 ┆ 13.45 ┆ -4.693 ┆ 13.81 ┆ -24.74 ┆ -13.28 ┆ -2.505 ┆ 5.791 ┆ 10.99 ┆ -55.35 ┆ 26.21 │ │ var_reg2 ┆ 0 ┆ 5194 ┆ 5194 ┆ -4.356 ┆ 13.42 ┆ -4.634 ┆ 13.79 ┆ -24.68 ┆ -13.26 ┆ -2.263 ┆ 5.852 ┆ 10.99 ┆ -55.35 ┆ 26.21 │ │ var_reg2 ┆ 1 ┆ 1300 ┆ 1300 ┆ -4.666 ┆ 13.56 ┆ -4.928 ┆ 13.88 ┆ -25.06 ┆ -13.49 ┆ -2.894 ┆ 5.505 ┆ 11.17 ┆ -47.42 ┆ 22.69 │ └──────────┴─────────┴──────┴──────────────┴────────┴───────┴──────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┴─────────────┘
Updating data according to narwhals expression: when_then(all_horizontal(col(var_reg1), ignore_nulls=False), col(var_reg2), lit(value=0, dtype=None)).alias(name=var_reg2)
var_reg1
var_reg2
Final Estimates by Iteration
Removing existing directory C:\Users\jonro\OneDrive\Documents\Coding\survey_kit\.scratch\temp_files/py_srmi_test_regression.srmi/2.srmi.implicate
In [7]:
logger.info("Get the results")
_ = df_list = srmi.df_implicates
Get the results
In [8]:
logger.info("\n\nLook at the original")
_ = summary(df_original)
logger.info("\n\nLook at the imputes")
_ = df_list.pipe(summary)
logger.info("\n\nLook at the imputes | var_reg1 == 0")
_ = df_list.filter(~nw.col("var_reg1")).pipe(summary)
logger.info("\n\nLook at the imputes | var_reg1 == 1")
_ = df_list.filter(nw.col("var_reg1")).pipe(summary)
Look at the original
┌──────────────┬────────┬─────────────┬────────────┬─────────────┬────────────┬───────────┐ │ Variable ┆ n ┆ n (missing) ┆ mean ┆ std ┆ min ┆ max │ ╞══════════════╪════════╪═════════════╪════════════╪═════════════╪════════════╪═══════════╡ │ _row_index_ ┆ 10,000 ┆ 0 ┆ 4,999.5 ┆ 2,886.89568 ┆ 0.0 ┆ 9,999.0 │ │ index ┆ 10,000 ┆ 0 ┆ 4,999.5 ┆ 2,886.89568 ┆ 0.0 ┆ 9,999.0 │ │ year ┆ 10,000 ┆ 0 ┆ 2,017.9851 ┆ 1.415937 ┆ 2,016.0 ┆ 2,020.0 │ │ month ┆ 10,000 ┆ 0 ┆ 6.5137 ┆ 3.432141 ┆ 1.0 ┆ 12.0 │ │ var2 ┆ 10,000 ┆ 0 ┆ 4.9782 ┆ 3.154508 ┆ 0.0 ┆ 10.0 │ │ var3 ┆ 10,000 ┆ 0 ┆ 25.1084 ┆ 14.752302 ┆ 0.0 ┆ 50.0 │ │ var4 ┆ 10,000 ┆ 0 ┆ 0.505666 ┆ 0.287861 ┆ 0.000027 ┆ 0.999997 │ │ unrelated_1 ┆ 10,000 ┆ 0 ┆ 0.502449 ┆ 0.288359 ┆ 0.000119 ┆ 0.999997 │ │ unrelated_2 ┆ 10,000 ┆ 0 ┆ 0.500105 ┆ 0.287638 ┆ 0.000049 ┆ 0.999539 │ │ unrelated_3 ┆ 10,000 ┆ 0 ┆ 0.499175 ┆ 0.28876 ┆ 0.000129 ┆ 0.99994 │ │ unrelated_4 ┆ 10,000 ┆ 0 ┆ 0.500655 ┆ 0.288698 ┆ 0.000133 ┆ 0.999972 │ │ unrelated_5 ┆ 10,000 ┆ 0 ┆ 0.49876 ┆ 0.288979 ┆ 0.000071 ┆ 0.999867 │ │ missing_reg1 ┆ 10,000 ┆ 0 ┆ 0.495784 ┆ 0.289051 ┆ 0.000161 ┆ 0.99991 │ │ missing_reg2 ┆ 10,000 ┆ 0 ┆ 0.502597 ┆ 0.288468 ┆ 0.000006 ┆ 0.999963 │ │ var_reg2 ┆ 10,000 ┆ 0 ┆ -2.402477 ┆ 10.331745 ┆ -55.354108 ┆ 26.213084 │ │ var5 ┆ 10,000 ┆ 0 ┆ 0.4999 ┆ 0.500025 ┆ 0.0 ┆ 1.0 │ │ var_reg1 ┆ 10,000 ┆ 0 ┆ 0.5229 ┆ 0.4995 ┆ 0.0 ┆ 1.0 │ └──────────────┴────────┴─────────────┴────────────┴─────────────┴────────────┴───────────┘
Look at the imputes
┌──────────────┬────────┬─────────────┬────────────┬─────────────┬────────────┬───────────┐ │ Variable ┆ n ┆ n (missing) ┆ mean ┆ std ┆ min ┆ max │ ╞══════════════╪════════╪═════════════╪════════════╪═════════════╪════════════╪═══════════╡ │ ___rownumber ┆ 10,000 ┆ 0 ┆ 4,999.5 ┆ 2,886.89568 ┆ 0.0 ┆ 9,999.0 │ │ _row_index_ ┆ 10,000 ┆ 0 ┆ 4,999.5 ┆ 2,886.89568 ┆ 0.0 ┆ 9,999.0 │ │ index ┆ 10,000 ┆ 0 ┆ 4,999.5 ┆ 2,886.89568 ┆ 0.0 ┆ 9,999.0 │ │ year ┆ 10,000 ┆ 0 ┆ 2,017.9851 ┆ 1.415937 ┆ 2,016.0 ┆ 2,020.0 │ │ month ┆ 10,000 ┆ 0 ┆ 6.5137 ┆ 3.432141 ┆ 1.0 ┆ 12.0 │ │ var2 ┆ 10,000 ┆ 0 ┆ 4.9782 ┆ 3.154508 ┆ 0.0 ┆ 10.0 │ │ var3 ┆ 10,000 ┆ 0 ┆ 25.1084 ┆ 14.752302 ┆ 0.0 ┆ 50.0 │ │ var4 ┆ 10,000 ┆ 0 ┆ 0.505666 ┆ 0.287861 ┆ 0.000027 ┆ 0.999997 │ │ unrelated_1 ┆ 10,000 ┆ 0 ┆ 0.502449 ┆ 0.288359 ┆ 0.000119 ┆ 0.999997 │ │ unrelated_2 ┆ 10,000 ┆ 0 ┆ 0.500105 ┆ 0.287638 ┆ 0.000049 ┆ 0.999539 │ │ unrelated_3 ┆ 10,000 ┆ 0 ┆ 0.499175 ┆ 0.28876 ┆ 0.000129 ┆ 0.99994 │ │ unrelated_4 ┆ 10,000 ┆ 0 ┆ 0.500655 ┆ 0.288698 ┆ 0.000133 ┆ 0.999972 │ │ unrelated_5 ┆ 10,000 ┆ 0 ┆ 0.49876 ┆ 0.288979 ┆ 0.000071 ┆ 0.999867 │ │ repeat_1 ┆ 10,000 ┆ 0 ┆ 0.502449 ┆ 0.288359 ┆ 0.000119 ┆ 0.999997 │ │ var_reg2 ┆ 10,000 ┆ 0 ┆ -2.246963 ┆ 9.922099 ┆ -55.354108 ┆ 26.213084 │ │ var5 ┆ 10,000 ┆ 0 ┆ 0.4999 ┆ 0.500025 ┆ 0.0 ┆ 1.0 │ │ var_reg1 ┆ 10,000 ┆ 0 ┆ 0.5201 ┆ 0.499621 ┆ 0.0 ┆ 1.0 │ └──────────────┴────────┴─────────────┴────────────┴─────────────┴────────────┴───────────┘
┌──────────────┬────────┬─────────────┬────────────┬─────────────┬────────────┬───────────┐ │ Variable ┆ n ┆ n (missing) ┆ mean ┆ std ┆ min ┆ max │ ╞══════════════╪════════╪═════════════╪════════════╪═════════════╪════════════╪═══════════╡ │ ___rownumber ┆ 10,000 ┆ 0 ┆ 4,999.5 ┆ 2,886.89568 ┆ 0.0 ┆ 9,999.0 │ │ _row_index_ ┆ 10,000 ┆ 0 ┆ 4,999.5 ┆ 2,886.89568 ┆ 0.0 ┆ 9,999.0 │ │ index ┆ 10,000 ┆ 0 ┆ 4,999.5 ┆ 2,886.89568 ┆ 0.0 ┆ 9,999.0 │ │ year ┆ 10,000 ┆ 0 ┆ 2,017.9851 ┆ 1.415937 ┆ 2,016.0 ┆ 2,020.0 │ │ month ┆ 10,000 ┆ 0 ┆ 6.5137 ┆ 3.432141 ┆ 1.0 ┆ 12.0 │ │ var2 ┆ 10,000 ┆ 0 ┆ 4.9782 ┆ 3.154508 ┆ 0.0 ┆ 10.0 │ │ var3 ┆ 10,000 ┆ 0 ┆ 25.1084 ┆ 14.752302 ┆ 0.0 ┆ 50.0 │ │ var4 ┆ 10,000 ┆ 0 ┆ 0.505666 ┆ 0.287861 ┆ 0.000027 ┆ 0.999997 │ │ unrelated_1 ┆ 10,000 ┆ 0 ┆ 0.502449 ┆ 0.288359 ┆ 0.000119 ┆ 0.999997 │ │ unrelated_2 ┆ 10,000 ┆ 0 ┆ 0.500105 ┆ 0.287638 ┆ 0.000049 ┆ 0.999539 │ │ unrelated_3 ┆ 10,000 ┆ 0 ┆ 0.499175 ┆ 0.28876 ┆ 0.000129 ┆ 0.99994 │ │ unrelated_4 ┆ 10,000 ┆ 0 ┆ 0.500655 ┆ 0.288698 ┆ 0.000133 ┆ 0.999972 │ │ unrelated_5 ┆ 10,000 ┆ 0 ┆ 0.49876 ┆ 0.288979 ┆ 0.000071 ┆ 0.999867 │ │ repeat_1 ┆ 10,000 ┆ 0 ┆ 0.502449 ┆ 0.288359 ┆ 0.000119 ┆ 0.999997 │ │ var_reg2 ┆ 10,000 ┆ 0 ┆ -2.269353 ┆ 9.916197 ┆ -55.354108 ┆ 26.213084 │ │ var5 ┆ 10,000 ┆ 0 ┆ 0.4999 ┆ 0.500025 ┆ 0.0 ┆ 1.0 │ │ var_reg1 ┆ 10,000 ┆ 0 ┆ 0.5194 ┆ 0.499648 ┆ 0.0 ┆ 1.0 │ └──────────────┴────────┴─────────────┴────────────┴─────────────┴────────────┴───────────┘
Look at the imputes | var_reg1 == 0
┌──────────────┬───────┬─────────────┬──────────────┬──────────────┬──────────┬──────────┐ │ Variable ┆ n ┆ n (missing) ┆ mean ┆ std ┆ min ┆ max │ ╞══════════════╪═══════╪═════════════╪══════════════╪══════════════╪══════════╪══════════╡ │ ___rownumber ┆ 4,799 ┆ 0 ┆ 5,038.299229 ┆ 2,889.298096 ┆ 1.0 ┆ 9,997.0 │ │ _row_index_ ┆ 4,799 ┆ 0 ┆ 5,038.299229 ┆ 2,889.298096 ┆ 1.0 ┆ 9,997.0 │ │ index ┆ 4,799 ┆ 0 ┆ 5,038.299229 ┆ 2,889.298096 ┆ 1.0 ┆ 9,997.0 │ │ year ┆ 4,799 ┆ 0 ┆ 2,017.998333 ┆ 1.418921 ┆ 2,016.0 ┆ 2,020.0 │ │ month ┆ 4,799 ┆ 0 ┆ 6.542405 ┆ 3.413955 ┆ 1.0 ┆ 12.0 │ │ var2 ┆ 4,799 ┆ 0 ┆ 4.562617 ┆ 3.237666 ┆ 0.0 ┆ 10.0 │ │ var3 ┆ 4,799 ┆ 0 ┆ 24.898312 ┆ 13.384091 ┆ 0.0 ┆ 50.0 │ │ var4 ┆ 4,799 ┆ 0 ┆ 0.508898 ┆ 0.288576 ┆ 0.000027 ┆ 0.999997 │ │ unrelated_1 ┆ 4,799 ┆ 0 ┆ 0.501877 ┆ 0.286643 ┆ 0.000119 ┆ 0.999921 │ │ unrelated_2 ┆ 4,799 ┆ 0 ┆ 0.499477 ┆ 0.285419 ┆ 0.000079 ┆ 0.999528 │ │ unrelated_3 ┆ 4,799 ┆ 0 ┆ 0.501495 ┆ 0.289221 ┆ 0.000139 ┆ 0.99994 │ │ unrelated_4 ┆ 4,799 ┆ 0 ┆ 0.503626 ┆ 0.289203 ┆ 0.000348 ┆ 0.999906 │ │ unrelated_5 ┆ 4,799 ┆ 0 ┆ 0.500986 ┆ 0.291456 ┆ 0.000071 ┆ 0.99984 │ │ repeat_1 ┆ 4,799 ┆ 0 ┆ 0.501877 ┆ 0.286643 ┆ 0.000119 ┆ 0.999921 │ │ var_reg2 ┆ 4,799 ┆ 0 ┆ 0.0 ┆ 0.0 ┆ 0.0 ┆ 0.0 │ │ var5 ┆ 4,799 ┆ 0 ┆ 0.887477 ┆ 0.316042 ┆ 0.0 ┆ 1.0 │ │ var_reg1 ┆ 4,799 ┆ 0 ┆ 0.0 ┆ 0.0 ┆ 0.0 ┆ 0.0 │ └──────────────┴───────┴─────────────┴──────────────┴──────────────┴──────────┴──────────┘
┌──────────────┬───────┬─────────────┬──────────────┬──────────────┬──────────┬──────────┐ │ Variable ┆ n ┆ n (missing) ┆ mean ┆ std ┆ min ┆ max │ ╞══════════════╪═══════╪═════════════╪══════════════╪══════════════╪══════════╪══════════╡ │ ___rownumber ┆ 4,806 ┆ 0 ┆ 5,035.05722 ┆ 2,887.183498 ┆ 1.0 ┆ 9,997.0 │ │ _row_index_ ┆ 4,806 ┆ 0 ┆ 5,035.05722 ┆ 2,887.183498 ┆ 1.0 ┆ 9,997.0 │ │ index ┆ 4,806 ┆ 0 ┆ 5,035.05722 ┆ 2,887.183498 ┆ 1.0 ┆ 9,997.0 │ │ year ┆ 4,806 ┆ 0 ┆ 2,017.990221 ┆ 1.4169 ┆ 2,016.0 ┆ 2,020.0 │ │ month ┆ 4,806 ┆ 0 ┆ 6.55077 ┆ 3.414455 ┆ 1.0 ┆ 12.0 │ │ var2 ┆ 4,806 ┆ 0 ┆ 4.552434 ┆ 3.245113 ┆ 0.0 ┆ 10.0 │ │ var3 ┆ 4,806 ┆ 0 ┆ 24.904078 ┆ 13.393172 ┆ 0.0 ┆ 50.0 │ │ var4 ┆ 4,806 ┆ 0 ┆ 0.508162 ┆ 0.289413 ┆ 0.000027 ┆ 0.999997 │ │ unrelated_1 ┆ 4,806 ┆ 0 ┆ 0.504055 ┆ 0.287282 ┆ 0.000119 ┆ 0.999921 │ │ unrelated_2 ┆ 4,806 ┆ 0 ┆ 0.501535 ┆ 0.286096 ┆ 0.000079 ┆ 0.999528 │ │ unrelated_3 ┆ 4,806 ┆ 0 ┆ 0.50177 ┆ 0.288888 ┆ 0.000139 ┆ 0.99994 │ │ unrelated_4 ┆ 4,806 ┆ 0 ┆ 0.502307 ┆ 0.289378 ┆ 0.000141 ┆ 0.999906 │ │ unrelated_5 ┆ 4,806 ┆ 0 ┆ 0.502701 ┆ 0.291023 ┆ 0.000071 ┆ 0.99984 │ │ repeat_1 ┆ 4,806 ┆ 0 ┆ 0.504055 ┆ 0.287282 ┆ 0.000119 ┆ 0.999921 │ │ var_reg2 ┆ 4,806 ┆ 0 ┆ 0.0 ┆ 0.0 ┆ 0.0 ┆ 0.0 │ │ var5 ┆ 4,806 ┆ 0 ┆ 0.887016 ┆ 0.316606 ┆ 0.0 ┆ 1.0 │ │ var_reg1 ┆ 4,806 ┆ 0 ┆ 0.0 ┆ 0.0 ┆ 0.0 ┆ 0.0 │ └──────────────┴───────┴─────────────┴──────────────┴──────────────┴──────────┴──────────┘
Look at the imputes | var_reg1 == 1
┌──────────────┬───────┬─────────────┬──────────────┬──────────────┬────────────┬───────────┐ │ Variable ┆ n ┆ n (missing) ┆ mean ┆ std ┆ min ┆ max │ ╞══════════════╪═══════╪═════════════╪══════════════╪══════════════╪════════════╪═══════════╡ │ ___rownumber ┆ 5,201 ┆ 0 ┆ 4,963.699673 ┆ 2,884.492009 ┆ 0.0 ┆ 9,999.0 │ │ _row_index_ ┆ 5,201 ┆ 0 ┆ 4,963.699673 ┆ 2,884.492009 ┆ 0.0 ┆ 9,999.0 │ │ index ┆ 5,201 ┆ 0 ┆ 4,963.699673 ┆ 2,884.492009 ┆ 0.0 ┆ 9,999.0 │ │ year ┆ 5,201 ┆ 0 ┆ 2,017.97289 ┆ 1.413205 ┆ 2,016.0 ┆ 2,020.0 │ │ month ┆ 5,201 ┆ 0 ┆ 6.487214 ┆ 3.448952 ┆ 1.0 ┆ 12.0 │ │ var2 ┆ 5,201 ┆ 0 ┆ 5.361661 ┆ 3.025873 ┆ 0.0 ┆ 10.0 │ │ var3 ┆ 5,201 ┆ 0 ┆ 25.30225 ┆ 15.909569 ┆ 0.0 ┆ 50.0 │ │ var4 ┆ 5,201 ┆ 0 ┆ 0.502683 ┆ 0.287195 ┆ 0.000104 ┆ 0.999885 │ │ unrelated_1 ┆ 5,201 ┆ 0 ┆ 0.502976 ┆ 0.28996 ┆ 0.000248 ┆ 0.999997 │ │ unrelated_2 ┆ 5,201 ┆ 0 ┆ 0.500685 ┆ 0.289698 ┆ 0.000049 ┆ 0.999539 │ │ unrelated_3 ┆ 5,201 ┆ 0 ┆ 0.497035 ┆ 0.288346 ┆ 0.000129 ┆ 0.999622 │ │ unrelated_4 ┆ 5,201 ┆ 0 ┆ 0.497914 ┆ 0.288232 ┆ 0.000133 ┆ 0.999972 │ │ unrelated_5 ┆ 5,201 ┆ 0 ┆ 0.496707 ┆ 0.286688 ┆ 0.000181 ┆ 0.999867 │ │ repeat_1 ┆ 5,201 ┆ 0 ┆ 0.502976 ┆ 0.28996 ┆ 0.000248 ┆ 0.999997 │ │ var_reg2 ┆ 5,201 ┆ 0 ┆ -4.320253 ┆ 13.429273 ┆ -55.354108 ┆ 26.213084 │ │ var5 ┆ 5,201 ┆ 0 ┆ 0.14228 ┆ 0.349371 ┆ 0.0 ┆ 1.0 │ │ var_reg1 ┆ 5,201 ┆ 0 ┆ 1.0 ┆ 0.0 ┆ 1.0 ┆ 1.0 │ └──────────────┴───────┴─────────────┴──────────────┴──────────────┴────────────┴───────────┘
┌──────────────┬───────┬─────────────┬──────────────┬──────────────┬────────────┬───────────┐ │ Variable ┆ n ┆ n (missing) ┆ mean ┆ std ┆ min ┆ max │ ╞══════════════╪═══════╪═════════════╪══════════════╪══════════════╪════════════╪═══════════╡ │ ___rownumber ┆ 5,194 ┆ 0 ┆ 4,966.59896 ┆ 2,886.517115 ┆ 0.0 ┆ 9,999.0 │ │ _row_index_ ┆ 5,194 ┆ 0 ┆ 4,966.59896 ┆ 2,886.517115 ┆ 0.0 ┆ 9,999.0 │ │ index ┆ 5,194 ┆ 0 ┆ 4,966.59896 ┆ 2,886.517115 ┆ 0.0 ┆ 9,999.0 │ │ year ┆ 5,194 ┆ 0 ┆ 2,017.980362 ┆ 1.415166 ┆ 2,016.0 ┆ 2,020.0 │ │ month ┆ 5,194 ┆ 0 ┆ 6.479399 ┆ 3.448398 ┆ 1.0 ┆ 12.0 │ │ var2 ┆ 5,194 ┆ 0 ┆ 5.37216 ┆ 3.015513 ┆ 0.0 ┆ 10.0 │ │ var3 ┆ 5,194 ┆ 0 ┆ 25.297459 ┆ 15.905758 ┆ 0.0 ┆ 50.0 │ │ var4 ┆ 5,194 ┆ 0 ┆ 0.503356 ┆ 0.286426 ┆ 0.000104 ┆ 0.999885 │ │ unrelated_1 ┆ 5,194 ┆ 0 ┆ 0.500962 ┆ 0.289372 ┆ 0.000248 ┆ 0.999997 │ │ unrelated_2 ┆ 5,194 ┆ 0 ┆ 0.498781 ┆ 0.28908 ┆ 0.000049 ┆ 0.999539 │ │ unrelated_3 ┆ 5,194 ┆ 0 ┆ 0.496775 ┆ 0.288649 ┆ 0.000129 ┆ 0.999622 │ │ unrelated_4 ┆ 5,194 ┆ 0 ┆ 0.499127 ┆ 0.288087 ┆ 0.000133 ┆ 0.999972 │ │ unrelated_5 ┆ 5,194 ┆ 0 ┆ 0.495114 ┆ 0.287055 ┆ 0.000181 ┆ 0.999867 │ │ repeat_1 ┆ 5,194 ┆ 0 ┆ 0.500962 ┆ 0.289372 ┆ 0.000248 ┆ 0.999997 │ │ var_reg2 ┆ 5,194 ┆ 0 ┆ -4.369182 ┆ 13.422283 ┆ -55.354108 ┆ 26.213084 │ │ var5 ┆ 5,194 ┆ 0 ┆ 0.141702 ┆ 0.348778 ┆ 0.0 ┆ 1.0 │ │ var_reg1 ┆ 5,194 ┆ 0 ┆ 1.0 ┆ 0.0 ┆ 1.0 ┆ 1.0 │ └──────────────┴───────┴─────────────┴──────────────┴──────────────┴────────────┴───────────┘