Skip to content
This repository was archived by the owner on Sep 12, 2024. It is now read-only.

fix(data-loader): used ISO8601 extended strings in queries #919

Merged
merged 2 commits into from
Apr 9, 2022
Merged

fix(data-loader): used ISO8601 extended strings in queries #919

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3fa8076
fix(data-loader): used ISO8601 extended strings in queries
kartikay-bagla Apr 9, 2022
3d9a77c
fix(data-loader): updated tests for iso8601 datetimes
kartikay-bagla Apr 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions chaos_genius/core/utils/data_loader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,14 +105,14 @@ def _convert_date_to_string(self, date: date, offset: str):
# we shouldn't need to take offset as a string, but rather
# take in a pytz timezone and skip using strings.
date = date.strftime("%Y-%m-%d")
date += f" 00:00:00 {offset}"
date += f"T00:00:00{offset}"
if not self.kpi_info.get("timezone_aware"):
date = (
pd.Timestamp(datetime.strptime(date, "%Y-%m-%d %H:%M:%S %z"))
pd.Timestamp(datetime.strptime(date, "%Y-%m-%dT%H:%M:%S%z"))
.tz_convert(self.connection_info["database_timezone"])
.tz_localize(None)
# TODO: We should also use date.isoformat() here
.strftime("%Y-%m-%d %H:%M:%S")
.strftime("%Y-%m-%dT%H:%M:%S")
)
return date

Expand Down
90 changes: 37 additions & 53 deletions tests/test_data_loader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,16 +42,13 @@ def get_data_source(*args, **kwargs):

# tail
tail = 5
dl = data_loader.DataLoader(
kpi_info,
tail=tail
)
dl = data_loader.DataLoader(kpi_info, tail=tail)
output_query = f"""select * from "cloud_cost" limit {tail}"""
assert output_query == dl._build_query().strip()

# count
dl = data_loader.DataLoader(kpi_info)
output_query = 'select count(*) from "cloud_cost\"'
output_query = 'select count(*) from "cloud_cost"'
assert output_query == dl._build_query(count=True).strip()

# query
Expand All @@ -67,7 +64,9 @@ def get_data_source(*args, **kwargs):
}

dl = data_loader.DataLoader(kpi_info)
output_query = r"select \* from \(select \* from cloud_cost\) as \"[a-z]{10}\""
output_query = (
r"select \* from \(select \* from cloud_cost\) as \"[a-z]{10}\""
)
assert re.match(output_query, dl._build_query().strip())

# tests for different date inputs
Expand All @@ -84,145 +83,130 @@ def get_data_source(*args, **kwargs):
}

# no end_date, no start_date, no days_before
dl = data_loader.DataLoader(
kpi_info
)
dl = data_loader.DataLoader(kpi_info)
output_query = """select * from "cloud_cost\""""
assert output_query == dl._build_query().strip()

# end_date, no start_date, no days_before
end_date = date(2020, 1, 1)
dl = data_loader.DataLoader(
kpi_info,
end_date=end_date
)
dl = data_loader.DataLoader(kpi_info, end_date=end_date)
end_date = end_date + timedelta(days=1)
output_query = f"""select * from "cloud_cost" where "date" < '{end_date.strftime("%Y-%m-%d")} 00:00:00'"""
output_query = f"""select * from "cloud_cost" where "date" < '{end_date.strftime("%Y-%m-%d")}T00:00:00'"""
assert output_query == dl._build_query().strip()

# no end_date, start_date, no days_before
start_date = date(2019, 1, 1)
dl = data_loader.DataLoader(
kpi_info,
start_date=start_date
)
output_query = f"""select * from "cloud_cost" where "date" >= '{start_date.strftime("%Y-%m-%d")} 00:00:00'"""
dl = data_loader.DataLoader(kpi_info, start_date=start_date)
output_query = f"""select * from "cloud_cost" where "date" >= '{start_date.strftime("%Y-%m-%d")}T00:00:00'"""
assert output_query == dl._build_query().strip()

# end_date, start_date, no days_before
end_date = date(2020, 1, 1)
start_date = date(2019, 1, 1)
dl = data_loader.DataLoader(
kpi_info,
end_date=end_date,
start_date=start_date
kpi_info, end_date=end_date, start_date=start_date
)
end_date = end_date + timedelta(days=1)
output_query = f"""select * from "cloud_cost" where "date" >= '{start_date.strftime("%Y-%m-%d")} 00:00:00' and "date" < '{end_date.strftime("%Y-%m-%d")} 00:00:00'"""
output_query = f"""select * from "cloud_cost" where "date" >= '{start_date.strftime("%Y-%m-%d")}T00:00:00' and "date" < '{end_date.strftime("%Y-%m-%d")}T00:00:00'"""
assert output_query == dl._build_query().strip()

# no end_date, no start_date, days_before
with pytest.raises(ValueError, match='If days_before is specified, either start_date or end_date must be specified'):
dl = data_loader.DataLoader(
kpi_info,
days_before=1
)
with pytest.raises(
ValueError,
match="If days_before is specified, either start_date or end_date must be specified",
):
dl = data_loader.DataLoader(kpi_info, days_before=1)

# end_date, no start_date, days_before
end_date = date(2020, 1, 1)
days_before = 30
dl = data_loader.DataLoader(
kpi_info,
end_date=end_date,
days_before=days_before
kpi_info, end_date=end_date, days_before=days_before
)
start_date = end_date - timedelta(days=days_before)
end_date = end_date + timedelta(days=1)
output_query = f"""select * from "cloud_cost" where "date" >= '{start_date.strftime("%Y-%m-%d")} 00:00:00' and "date" < '{end_date.strftime("%Y-%m-%d")} 00:00:00'"""
output_query = f"""select * from "cloud_cost" where "date" >= '{start_date.strftime("%Y-%m-%d")}T00:00:00' and "date" < '{end_date.strftime("%Y-%m-%d")}T00:00:00'"""
assert output_query == dl._build_query().strip()

# no end_date, start_date, days_before
start_date = date(2019, 1, 1)
days_before = 30
dl = data_loader.DataLoader(
kpi_info,
start_date=start_date,
days_before=days_before
kpi_info, start_date=start_date, days_before=days_before
)
end_date = start_date + timedelta(days=days_before)
end_date = end_date + timedelta(days=1)
output_query = f"""select * from "cloud_cost" where "date" >= '{start_date.strftime("%Y-%m-%d")} 00:00:00' and "date" < '{end_date.strftime("%Y-%m-%d")} 00:00:00'"""
output_query = f"""select * from "cloud_cost" where "date" >= '{start_date.strftime("%Y-%m-%d")}T00:00:00' and "date" < '{end_date.strftime("%Y-%m-%d")}T00:00:00'"""
assert output_query == dl._build_query().strip()

# end_date, start_date, days_before
end_date = date(2020, 1, 1)
start_date = date(2019, 1, 1)
days_before = 30
with pytest.raises(ValueError, match='end_date, start_date and days_before cannot be specified at the same time'):
with pytest.raises(
ValueError,
match="end_date, start_date and days_before cannot be specified at the same time",
):
dl = data_loader.DataLoader(
kpi_info,
end_date=end_date,
start_date=start_date,
days_before=days_before
days_before=days_before,
)

# tz naive testing
data_source["database_timezone"] = "Asia/Kolkata"
end_date = date(2020, 1, 1)
start_date = date(2019, 1, 1)
dl = data_loader.DataLoader(
kpi_info,
end_date=end_date,
start_date=start_date
kpi_info, end_date=end_date, start_date=start_date
)
end_date = end_date + timedelta(days=1)
output_query = f"""select * from "cloud_cost" where "date" >= '{start_date.strftime("%Y-%m-%d")} 05:30:00' and "date" < '{end_date.strftime("%Y-%m-%d")} 05:30:00'"""
output_query = f"""select * from "cloud_cost" where "date" >= '{start_date.strftime("%Y-%m-%d")}T05:30:00' and "date" < '{end_date.strftime("%Y-%m-%d")}T05:30:00'"""
assert output_query == dl._build_query().strip()

data_source["database_timezone"] = "America/New_York"
start_date = start_date - timedelta(days=1)
end_date = end_date - timedelta(days=1)
output_query = f"""select * from "cloud_cost" where "date" >= '{start_date.strftime("%Y-%m-%d")} 20:00:00' and "date" < '{end_date.strftime("%Y-%m-%d")} 20:00:00'"""
output_query = f"""select * from "cloud_cost" where "date" >= '{start_date.strftime("%Y-%m-%d")}T20:00:00' and "date" < '{end_date.strftime("%Y-%m-%d")}T20:00:00'"""

# tz aware testing
kpi_info["timezone_aware"] = True
data_source["database_timezone"] = "Asia/Kolkata"
end_date = date(2020, 1, 1)
start_date = date(2019, 1, 1)
dl = data_loader.DataLoader(
kpi_info,
end_date=end_date,
start_date=start_date
kpi_info, end_date=end_date, start_date=start_date
)
end_date = end_date + timedelta(days=1)
output_query = f"""select * from "cloud_cost" where "date" >= '{start_date.strftime("%Y-%m-%d")} 00:00:00 +00:00' and "date" < '{end_date.strftime("%Y-%m-%d")} 00:00:00 +00:00'"""
output_query = f"""select * from "cloud_cost" where "date" >= '{start_date.strftime("%Y-%m-%d")}T00:00:00+00:00' and "date" < '{end_date.strftime("%Y-%m-%d")}T00:00:00+00:00'"""
assert output_query == dl._build_query().strip()

data_source["database_timezone"] = "America/New_York"
start_date = start_date - timedelta(days=1)
end_date = end_date - timedelta(days=1)
output_query = f"""select * from "cloud_cost" where "date" >= '{start_date.strftime("%Y-%m-%d")} 00:00:00 +00:00' and "date" < '{end_date.strftime("%Y-%m-%d")} 00:00:00 +00:00'"""
output_query = f"""select * from "cloud_cost" where "date" >= '{start_date.strftime("%Y-%m-%d")}T00:00:00+00:00' and "date" < '{end_date.strftime("%Y-%m-%d")}T00:00:00+00:00'"""

from chaos_genius import settings

settings.TIMEZONE = "Asia/Kolkata"

import importlib

importlib.reload(data_loader)

kpi_info["timezone_aware"] = True
data_source["database_timezone"] = "Asia/Kolkata"
end_date = date(2020, 1, 1)
start_date = date(2019, 1, 1)
dl = data_loader.DataLoader(
kpi_info,
end_date=end_date,
start_date=start_date
kpi_info, end_date=end_date, start_date=start_date
)
end_date = end_date + timedelta(days=1)
output_query = f"""select * from "cloud_cost" where "date" >= '{start_date.strftime("%Y-%m-%d")} 00:00:00 +05:30' and "date" < '{end_date.strftime("%Y-%m-%d")} 00:00:00 +05:30'"""
output_query = f"""select * from "cloud_cost" where "date" >= '{start_date.strftime("%Y-%m-%d")}T00:00:00+05:30' and "date" < '{end_date.strftime("%Y-%m-%d")}T00:00:00+05:30'"""
assert output_query == dl._build_query().strip()

data_source["database_timezone"] = "America/New_York"
start_date = start_date - timedelta(days=1)
end_date = end_date - timedelta(days=1)
output_query = f"""select * from "cloud_cost" where "date" >= '{start_date.strftime("%Y-%m-%d")} 00:00:00 +05:30' and "date" < '{end_date.strftime("%Y-%m-%d")} 00:00:00 +05:30'"""
output_query = f"""select * from "cloud_cost" where "date" >= '{start_date.strftime("%Y-%m-%d")}T00:00:00+05:30' and "date" < '{end_date.strftime("%Y-%m-%d")}T00:00:00+05:30'"""