Possible bug in polars-rust, when using fill_null?

when I run the below rust program. rust-polars version 0.35.4

use polars::prelude::*;
use chrono::prelude::*;

    let df = df!(
            "date" => date_range("date",
                NaiveDate::from_ymd_opt(2022, 1, 1).unwrap().and_hms_opt(0, 0, 0).unwrap(), NaiveDate::from_ymd_opt(2022, 1, 5).unwrap().and_hms_opt(0, 0, 0).unwrap(), Duration::parse("1d"),ClosedWindow::Both, TimeUnit::Milliseconds, None)?,
            "sales" => &[1, 2, 3, 4,5],
    )?;

    let period:usize = 3;
    let duration_str = format!("{}{}",period.to_string().as_str(),"i");
    let rolling_options = RollingOptions {
        window_size : Duration::parse(duration_str.as_str()),
        min_periods : period,
        ..Default::default()
    };

    let period = 3;

    let mut lf = df.lazy().with_column(concat_expr([col("sales").slice(0,period).rolling_mean(rolling_options), 
        col("sales").slice(period,lit(Null {}))], false)?.alias("con"));

    println!("{}", lf.clone().collect().unwrap());

    lf = lf.clone().fill_null(0);
    
    println!("{}", lf.clone().collect().unwrap());

I am getting the following output.

shape: (5, 3)
┌─────────────────────┬───────┬──────┐
│ date                ┆ sales ┆ con  │
│ ---                 ┆ ---   ┆ ---  │
│ datetime[ms]        ┆ i32   ┆ f64  │
╞═════════════════════╪═══════╪══════╡
│ 2022-01-01 00:00:00 ┆ 1     ┆ null │
│ 2022-01-02 00:00:00 ┆ 2     ┆ null │
│ 2022-01-03 00:00:00 ┆ 3     ┆ 2.0  │
│ 2022-01-04 00:00:00 ┆ 4     ┆ 4.0  │
│ 2022-01-05 00:00:00 ┆ 5     ┆ 5.0  │
└─────────────────────┴───────┴──────┘
shape: (5, 3)
┌───────────────┬───────┬─────┐
│ date          ┆ sales ┆ con │
│ ---           ┆ ---   ┆ --- │
│ i64           ┆ i32   ┆ f64 │
╞═══════════════╪═══════╪═════╡
│ 1640995200000 ┆ 1     ┆ 0.0 │
│ 1641081600000 ┆ 2     ┆ 0.0 │
│ 1641168000000 ┆ 3     ┆ 2.0 │
│ 1641254400000 ┆ 4     ┆ 4.0 │
│ 1641340800000 ┆ 5     ┆ 5.0 │
└───────────────┴───────┴─────┘

I was expecting the second output to be.

shape: (5, 3)
┌─────────────────────┬───────┬──────┐
│ date                ┆ sales ┆ con  │
│ ---                 ┆ ---   ┆ ---  │
│ datetime[ms]        ┆ i32   ┆ f64  │
╞═════════════════════╪═══════╪══════╡
│ 2022-01-01 00:00:00 ┆ 1     ┆ 0.0  │
│ 2022-01-02 00:00:00 ┆ 2     ┆ 0.0  │
│ 2022-01-03 00:00:00 ┆ 3     ┆ 2.0  │
│ 2022-01-04 00:00:00 ┆ 4     ┆ 4.0  │
│ 2022-01-05 00:00:00 ┆ 5     ┆ 5.0  │
└─────────────────────┴───────┴──────┘

can any rust-polars experts confirm, if I am doing any mistake or it is a possible bug in rust-polars? Any help here would be much appreciated. Thanks in advance.

  • As of now, have a workaround for the same, the int value is nothing but the epoch value. casting it back to DateTime is giving the desired result ` lf = lf.clone().select([col(“date”).cast(DataType::Datetime(TimeUnit::Milliseconds, None)), all().exclude([“date”])]); `

    – 




  • 1

    Hard to say whether it’s a bug; it’s pretty clear that Polars is looking for the common supertype of datetime[ms] and some kind of Int (32, 64, not sure) and deeming it to be Int, not datetime[ms]. But I’m not sure whether that’s intended or not. You could of course simply do fill_null on just the column you care about and not the whole frame.

    – 

  • The above problem is not there with python-polars, only with rust version. But anyway your suggestion of using fill_null on a specific column should solve my issue. Thanks

    – 

Leave a Comment