test: enable setting min oracle version on tests (#184)

brush-shell/Cargo.toml

@@ -69,3 +69,4 @@ regex = "1.11.0"
 serde = { version = "1.0.210", features = ["derive"] }
 serde_yaml = "0.9.34"
 strip-ansi-escapes = "0.2.0"
+version-compare = "0.2.0"

brush-shell/tests/cases/builtins/shopt.yaml

@@ -1,11 +1,13 @@
 name: "Builtins: shopt"
 cases:
   - name: "shopt defaults"
+    min_oracle_version: 5.2
     known_failure: true # TODO: new options from newer version of bash?
     stdin: |
       shopt | sort | grep -v extglob
 
   - name: "shopt interactive defaults"
+    min_oracle_version: 5.2
     known_failure: true # TODO: new options from newer version of bash?
     pty: true
     args: ["-i", "-c", "shopt | sort | grep -v extglob"]

brush-shell/tests/cases/word_expansion.yaml

@@ -721,6 +721,7 @@ cases:
       echo "\${arr2[*]@K}: ${arr2[*]@K}"
 
   - name: "Parameter quote transformations - k"
+    min_oracle_version: 5.2
     stdin: |
       var='""'
       echo "\${var@k}: ${var@k}"

brush-shell/tests/compat_tests.rs

@@ -26,19 +26,24 @@ struct ShellConfig {
 struct TestConfig {
     pub name: String,
     pub oracle_shell: ShellConfig,
+    pub oracle_version_str: Option<String>,
     pub test_shell: ShellConfig,
     pub options: TestOptions,
 }
 
 impl TestConfig {
-    pub fn for_bash_testing(options: &TestOptions) -> Self {
+    pub fn for_bash_testing(options: &TestOptions) -> Result<Self> {
+        // Check for bash version.
+        let bash_version_str = get_bash_version_str(Path::new(&options.bash_path))?;
+
         // Skip rc file and profile for deterministic behavior across systems/distros.
-        Self {
+        Ok(Self {
             name: String::from(BASH_CONFIG_NAME),
             oracle_shell: ShellConfig {
                 which: WhichShell::NamedShell(options.bash_path.clone()),
                 default_args: vec![String::from("--norc"), String::from("--noprofile")],
             },
+            oracle_version_str: Some(bash_version_str),
             test_shell: ShellConfig {
                 which: WhichShell::ShellUnderTest(String::from("brush")),
                 // Disable a few fancy UI options for shells under test.
@@ -49,17 +54,19 @@ impl TestConfig {
                 ],
             },
             options: options.clone(),
-        }
+        })
     }
 
-    pub fn for_sh_testing(options: &TestOptions) -> Self {
+    #[allow(clippy::unnecessary_wraps)]
+    pub fn for_sh_testing(options: &TestOptions) -> Result<Self> {
         // Skip rc file and profile for deterministic behavior across systems/distros.
-        Self {
+        Ok(Self {
             name: String::from(SH_CONFIG_NAME),
             oracle_shell: ShellConfig {
                 which: WhichShell::NamedShell(String::from("sh")),
                 default_args: vec![],
             },
+            oracle_version_str: None,
             test_shell: ShellConfig {
                 which: WhichShell::ShellUnderTest(String::from("brush")),
                 // Disable a few fancy UI options for shells under test.
@@ -71,7 +78,7 @@ impl TestConfig {
                 ],
             },
             options: options.clone(),
-        }
+        })
     }
 }
 
@@ -92,11 +99,11 @@ async fn cli_integration_tests(options: TestOptions) -> Result<()> {
     let mut test_configs = vec![];
 
     if options.should_enable_config(BASH_CONFIG_NAME) {
-        test_configs.push(TestConfig::for_bash_testing(&options));
+        test_configs.push(TestConfig::for_bash_testing(&options)?);
     }
 
     if options.should_enable_config(SH_CONFIG_NAME) {
-        test_configs.push(TestConfig::for_sh_testing(&options));
+        test_configs.push(TestConfig::for_sh_testing(&options)?);
     }
 
     // Spawn each test case set separately.
@@ -307,6 +314,8 @@ struct TestCase {
     #[serde(default)]
     pub incompatible_configs: HashSet<String>,
     #[serde(default)]
+    pub min_oracle_version: Option<String>,
+    #[serde(default)]
     pub timeout_in_seconds: Option<u64>,
 }
 
@@ -405,6 +414,25 @@ impl TestCaseSet {
                 continue;
             }
 
+            // Make sure the oracle meets any min version listed.
+            if let Some(min_oracle_version_str) = &test_case.min_oracle_version {
+                if let Some(actual_oracle_version_str) = &test_config.oracle_version_str {
+                    let actual_oracle_version =
+                        version_compare::Version::from(actual_oracle_version_str.as_str())
+                            .ok_or_else(|| anyhow::anyhow!("failed to parse oracle version"))?;
+
+                    let min_oracle_version = version_compare::Version::from(min_oracle_version_str)
+                        .ok_or_else(|| anyhow::anyhow!("failed to parse min oracle version"))?;
+
+                    if matches!(
+                        actual_oracle_version.compare(min_oracle_version),
+                        version_compare::Cmp::Lt
+                    ) {
+                        continue;
+                    }
+                }
+            }
+
             // Make sure it passes filters.
             if !test_config.options.should_run_test(self, test_case) {
                 continue;
@@ -1336,6 +1364,21 @@ fn write_diff(
     Ok(())
 }
 
+fn get_bash_version_str(bash_path: &Path) -> Result<String> {
+    let output = std::process::Command::new(bash_path)
+        .arg("--norc")
+        .arg("--noprofile")
+        .arg("-c")
+        .arg("echo -n ${BASH_VERSINFO[0]}.${BASH_VERSINFO[1]}.${BASH_VERSINFO[2]}")
+        .output()
+        .context("failed to retrieve bash version")?
+        .stdout;
+
+    let ver_str = String::from_utf8(output)?;
+
+    Ok(ver_str)
+}
+
 fn main() -> Result<()> {
     let unparsed_args: Vec<_> = std::env::args().collect();
     let mut options = TestOptions::parse_from(unparsed_args);