Skip to content

Commit 76d2e56

Browse files
authored
[airflow] Avoid deprecated values (AIR302) (#14582)
1 parent 30d80d9 commit 76d2e56

13 files changed

+301
-12
lines changed

crates/ruff_linter/resources/test/fixtures/airflow/AIR301.py

+15
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,14 @@
11
from airflow import DAG, dag
2+
from airflow.timetables.simple import NullTimetable
23

34
DAG(dag_id="class_default_schedule")
45

56
DAG(dag_id="class_schedule", schedule="@hourly")
67

8+
DAG(dag_id="class_schedule_interval", schedule_interval="@hourly")
9+
10+
DAG(dag_id="class_timetable", timetable=NullTimetable())
11+
712

813
@dag()
914
def decorator_default_schedule():
@@ -13,3 +18,13 @@ def decorator_default_schedule():
1318
@dag(schedule="0 * * * *")
1419
def decorator_schedule():
1520
pass
21+
22+
23+
@dag(schedule_interval="0 * * * *")
24+
def decorator_schedule_interval():
25+
pass
26+
27+
28+
@dag(timetable=NullTimetable())
29+
def decorator_timetable():
30+
pass
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
from airflow import DAG, dag
2+
from airflow.timetables.simple import NullTimetable
3+
4+
DAG(dag_id="class_schedule", schedule="@hourly")
5+
6+
DAG(dag_id="class_schedule_interval", schedule_interval="@hourly")
7+
8+
DAG(dag_id="class_timetable", timetable=NullTimetable())
9+
10+
11+
@dag(schedule="0 * * * *")
12+
def decorator_schedule():
13+
pass
14+
15+
16+
@dag(schedule_interval="0 * * * *")
17+
def decorator_schedule_interval():
18+
pass
19+
20+
21+
@dag(timetable=NullTimetable())
22+
def decorator_timetable():
23+
pass
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
from airflow.utils import dates
2+
from airflow.utils.dates import date_range, datetime_to_nano, days_ago
3+
4+
date_range
5+
days_ago
6+
7+
dates.date_range
8+
dates.days_ago
9+
10+
# This one was not deprecated.
11+
datetime_to_nano
12+
dates.datetime_to_nano

crates/ruff_linter/src/checkers/ast/analyze/expression.rs

+9
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,9 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
220220
if checker.enabled(Rule::RegexFlagAlias) {
221221
refurb::rules::regex_flag_alias(checker, expr);
222222
}
223+
if checker.enabled(Rule::Airflow3Removal) {
224+
airflow::rules::removed_in_3(checker, expr);
225+
}
223226

224227
// Ex) List[...]
225228
if checker.any_enabled(&[
@@ -380,6 +383,9 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
380383
if checker.enabled(Rule::ByteStringUsage) {
381384
flake8_pyi::rules::bytestring_attribute(checker, expr);
382385
}
386+
if checker.enabled(Rule::Airflow3Removal) {
387+
airflow::rules::removed_in_3(checker, expr);
388+
}
383389
}
384390
Expr::Call(
385391
call @ ast::ExprCall {
@@ -1084,6 +1090,9 @@ pub(crate) fn expression(expr: &Expr, checker: &mut Checker) {
10841090
if checker.enabled(Rule::UnnecessaryRegularExpression) {
10851091
ruff::rules::unnecessary_regular_expression(checker, call);
10861092
}
1093+
if checker.enabled(Rule::Airflow3Removal) {
1094+
airflow::rules::removed_in_3(checker, expr);
1095+
}
10871096
}
10881097
Expr::Dict(dict) => {
10891098
if checker.any_enabled(&[

crates/ruff_linter/src/codes.rs

+1
Original file line numberDiff line numberDiff line change
@@ -1040,6 +1040,7 @@ pub fn code_to_rule(linter: Linter, code: &str) -> Option<(RuleGroup, Rule)> {
10401040
// airflow
10411041
(Airflow, "001") => (RuleGroup::Stable, rules::airflow::rules::AirflowVariableNameTaskIdMismatch),
10421042
(Airflow, "301") => (RuleGroup::Preview, rules::airflow::rules::AirflowDagNoScheduleArgument),
1043+
(Airflow, "302") => (RuleGroup::Preview, rules::airflow::rules::Airflow3Removal),
10431044

10441045
// perflint
10451046
(Perflint, "101") => (RuleGroup::Stable, rules::perflint::rules::UnnecessaryListCast),

crates/ruff_linter/src/rules/airflow/mod.rs

+2
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ mod tests {
1414

1515
#[test_case(Rule::AirflowVariableNameTaskIdMismatch, Path::new("AIR001.py"))]
1616
#[test_case(Rule::AirflowDagNoScheduleArgument, Path::new("AIR301.py"))]
17+
#[test_case(Rule::Airflow3Removal, Path::new("AIR302_args.py"))]
18+
#[test_case(Rule::Airflow3Removal, Path::new("AIR302_names.py"))]
1719
fn rules(rule_code: Rule, path: &Path) -> Result<()> {
1820
let snapshot = format!("{}_{}", rule_code.noqa_code(), path.to_string_lossy());
1921
let diagnostics = test_path(

crates/ruff_linter/src/rules/airflow/rules/dag_schedule_argument.rs

+8-2
Original file line numberDiff line numberDiff line change
@@ -73,8 +73,14 @@ pub(crate) fn dag_no_schedule_argument(checker: &mut Checker, expr: &Expr) {
7373
return;
7474
}
7575

76-
// If there's a `schedule` keyword argument, we are good.
77-
if arguments.find_keyword("schedule").is_some() {
76+
// If there's a schedule keyword argument, we are good.
77+
// This includes the canonical 'schedule', and the deprecated 'timetable'
78+
// and 'schedule_interval'. Usages of deprecated schedule arguments are
79+
// covered by AIR302.
80+
if ["schedule", "schedule_interval", "timetable"]
81+
.iter()
82+
.any(|a| arguments.find_keyword(a).is_some())
83+
{
7884
return;
7985
}
8086

Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
pub(crate) use dag_schedule_argument::*;
2+
pub(crate) use removal_in_3::*;
23
pub(crate) use task_variable_name::*;
34

45
mod dag_schedule_argument;
6+
mod removal_in_3;
57
mod task_variable_name;
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
use ruff_diagnostics::{Diagnostic, Violation};
2+
use ruff_macros::{derive_message_formats, ViolationMetadata};
3+
use ruff_python_ast::{name::QualifiedName, Arguments, Expr, ExprAttribute, ExprCall};
4+
use ruff_python_semantic::Modules;
5+
use ruff_text_size::Ranged;
6+
7+
use crate::checkers::ast::Checker;
8+
9+
#[derive(Debug, Eq, PartialEq)]
10+
enum Replacement {
11+
None,
12+
Name(String),
13+
}
14+
15+
/// ## What it does
16+
/// Checks for uses of deprecated Airflow functions and values.
17+
///
18+
/// ## Why is this bad?
19+
/// Airflow 3.0 removed various deprecated functions, members, and other
20+
/// values. Some have more modern replacements. Others are considered too niche
21+
/// and not worth to be maintained in Airflow.
22+
///
23+
/// ## Example
24+
/// ```python
25+
/// from airflow.utils.dates import days_ago
26+
///
27+
///
28+
/// yesterday = days_ago(today, 1)
29+
/// ```
30+
///
31+
/// Use instead:
32+
/// ```python
33+
/// from datetime import timedelta
34+
///
35+
///
36+
/// yesterday = today - timedelta(days=1)
37+
/// ```
38+
#[derive(ViolationMetadata)]
39+
pub(crate) struct Airflow3Removal {
40+
deprecated: String,
41+
replacement: Replacement,
42+
}
43+
44+
impl Violation for Airflow3Removal {
45+
#[derive_message_formats]
46+
fn message(&self) -> String {
47+
let Airflow3Removal {
48+
deprecated,
49+
replacement,
50+
} = self;
51+
match replacement {
52+
Replacement::None => format!("`{deprecated}` is removed in Airflow 3.0"),
53+
Replacement::Name(name) => {
54+
format!("`{deprecated}` is removed in Airflow 3.0; use {name} instead")
55+
}
56+
}
57+
}
58+
}
59+
60+
fn diagnostic_for_argument(
61+
arguments: &Arguments,
62+
deprecated: &str,
63+
replacement: Option<&str>,
64+
) -> Option<Diagnostic> {
65+
let keyword = arguments.find_keyword(deprecated)?;
66+
Some(Diagnostic::new(
67+
Airflow3Removal {
68+
deprecated: (*deprecated).to_string(),
69+
replacement: match replacement {
70+
Some(name) => Replacement::Name(name.to_owned()),
71+
None => Replacement::None,
72+
},
73+
},
74+
keyword
75+
.arg
76+
.as_ref()
77+
.map_or_else(|| keyword.range(), Ranged::range),
78+
))
79+
}
80+
81+
fn removed_argument(checker: &mut Checker, qualname: &QualifiedName, arguments: &Arguments) {
82+
#[allow(clippy::single_match)]
83+
match qualname.segments() {
84+
["airflow", .., "DAG" | "dag"] => {
85+
checker.diagnostics.extend(diagnostic_for_argument(
86+
arguments,
87+
"schedule_interval",
88+
Some("schedule"),
89+
));
90+
checker.diagnostics.extend(diagnostic_for_argument(
91+
arguments,
92+
"timetable",
93+
Some("schedule"),
94+
));
95+
}
96+
_ => {}
97+
};
98+
}
99+
100+
fn removed_name(checker: &mut Checker, expr: &Expr, ranged: impl Ranged) {
101+
let result =
102+
checker
103+
.semantic()
104+
.resolve_qualified_name(expr)
105+
.and_then(|qualname| match qualname.segments() {
106+
["airflow", "utils", "dates", "date_range"] => {
107+
Some((qualname.to_string(), Replacement::None))
108+
}
109+
["airflow", "utils", "dates", "days_ago"] => Some((
110+
qualname.to_string(),
111+
Replacement::Name("datetime.timedelta()".to_string()),
112+
)),
113+
_ => None,
114+
});
115+
if let Some((deprecated, replacement)) = result {
116+
checker.diagnostics.push(Diagnostic::new(
117+
Airflow3Removal {
118+
deprecated,
119+
replacement,
120+
},
121+
ranged.range(),
122+
));
123+
}
124+
}
125+
126+
/// AIR302
127+
pub(crate) fn removed_in_3(checker: &mut Checker, expr: &Expr) {
128+
if !checker.semantic().seen_module(Modules::AIRFLOW) {
129+
return;
130+
}
131+
132+
match expr {
133+
Expr::Call(ExprCall {
134+
func, arguments, ..
135+
}) => {
136+
if let Some(qualname) = checker.semantic().resolve_qualified_name(func) {
137+
removed_argument(checker, &qualname, arguments);
138+
};
139+
}
140+
Expr::Attribute(ExprAttribute { attr: ranged, .. }) => removed_name(checker, expr, ranged),
141+
ranged @ Expr::Name(_) => removed_name(checker, expr, ranged),
142+
_ => {}
143+
}
144+
}
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,20 @@
11
---
22
source: crates/ruff_linter/src/rules/airflow/mod.rs
33
---
4-
AIR301.py:3:1: AIR301 DAG should have an explicit `schedule` argument
4+
AIR301.py:4:1: AIR301 DAG should have an explicit `schedule` argument
55
|
6-
1 | from airflow import DAG, dag
7-
2 |
8-
3 | DAG(dag_id="class_default_schedule")
6+
2 | from airflow.timetables.simple import NullTimetable
7+
3 |
8+
4 | DAG(dag_id="class_default_schedule")
99
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ AIR301
10-
4 |
11-
5 | DAG(dag_id="class_schedule", schedule="@hourly")
10+
5 |
11+
6 | DAG(dag_id="class_schedule", schedule="@hourly")
1212
|
1313

14-
AIR301.py:8:2: AIR301 DAG should have an explicit `schedule` argument
14+
AIR301.py:13:2: AIR301 DAG should have an explicit `schedule` argument
1515
|
16-
8 | @dag()
16+
13 | @dag()
1717
| ^^^^^ AIR301
18-
9 | def decorator_default_schedule():
19-
10 | pass
18+
14 | def decorator_default_schedule():
19+
15 | pass
2020
|
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
---
2+
source: crates/ruff_linter/src/rules/airflow/mod.rs
3+
---
4+
AIR302_args.py:6:39: AIR302 `schedule_interval` is removed in Airflow 3.0; use schedule instead
5+
|
6+
4 | DAG(dag_id="class_schedule", schedule="@hourly")
7+
5 |
8+
6 | DAG(dag_id="class_schedule_interval", schedule_interval="@hourly")
9+
| ^^^^^^^^^^^^^^^^^ AIR302
10+
7 |
11+
8 | DAG(dag_id="class_timetable", timetable=NullTimetable())
12+
|
13+
14+
AIR302_args.py:8:31: AIR302 `timetable` is removed in Airflow 3.0; use schedule instead
15+
|
16+
6 | DAG(dag_id="class_schedule_interval", schedule_interval="@hourly")
17+
7 |
18+
8 | DAG(dag_id="class_timetable", timetable=NullTimetable())
19+
| ^^^^^^^^^ AIR302
20+
|
21+
22+
AIR302_args.py:16:6: AIR302 `schedule_interval` is removed in Airflow 3.0; use schedule instead
23+
|
24+
16 | @dag(schedule_interval="0 * * * *")
25+
| ^^^^^^^^^^^^^^^^^ AIR302
26+
17 | def decorator_schedule_interval():
27+
18 | pass
28+
|
29+
30+
AIR302_args.py:21:6: AIR302 `timetable` is removed in Airflow 3.0; use schedule instead
31+
|
32+
21 | @dag(timetable=NullTimetable())
33+
| ^^^^^^^^^ AIR302
34+
22 | def decorator_timetable():
35+
23 | pass
36+
|
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
---
2+
source: crates/ruff_linter/src/rules/airflow/mod.rs
3+
---
4+
AIR302_names.py:4:1: AIR302 `airflow.utils.dates.date_range` is removed in Airflow 3.0
5+
|
6+
2 | from airflow.utils.dates import date_range, datetime_to_nano, days_ago
7+
3 |
8+
4 | date_range
9+
| ^^^^^^^^^^ AIR302
10+
5 | days_ago
11+
|
12+
13+
AIR302_names.py:5:1: AIR302 `airflow.utils.dates.days_ago` is removed in Airflow 3.0; use datetime.timedelta() instead
14+
|
15+
4 | date_range
16+
5 | days_ago
17+
| ^^^^^^^^ AIR302
18+
6 |
19+
7 | dates.date_range
20+
|
21+
22+
AIR302_names.py:7:7: AIR302 `airflow.utils.dates.date_range` is removed in Airflow 3.0
23+
|
24+
5 | days_ago
25+
6 |
26+
7 | dates.date_range
27+
| ^^^^^^^^^^ AIR302
28+
8 | dates.days_ago
29+
|
30+
31+
AIR302_names.py:8:7: AIR302 `airflow.utils.dates.days_ago` is removed in Airflow 3.0; use datetime.timedelta() instead
32+
|
33+
7 | dates.date_range
34+
8 | dates.days_ago
35+
| ^^^^^^^^ AIR302
36+
9 |
37+
10 | # This one was not deprecated.
38+
|

ruff.schema.json

+1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)