Add prehash cache support

czkawka_cli/src/commands.rs

@@ -22,7 +22,7 @@ pub enum Commands {
         minimal_file_size: u64,
         #[structopt(short = "i", long, parse(try_from_str = parse_maximal_file_size), default_value = "18446744073709551615", help = "Maximum size in bytes", long_help = "Maximum size of checked files in bytes, assigning lower value may speed up searching")]
         maximal_file_size: u64,
-        #[structopt(short = "c", long, parse(try_from_str = parse_minimal_file_size), default_value = "524288", help = "Minimum cached file size in bytes", long_help = "Minimum size of cached files in bytes, assigning bigger value may speed up will cause that lower amount of files will be cached, but loading of cache will be faster")]
+        #[structopt(short = "c", long, parse(try_from_str = parse_minimal_file_size), default_value = "257144", help = "Minimum cached file size in bytes", long_help = "Minimum size of cached files in bytes, assigning bigger value may speed up will cause that lower amount of files will be cached, but loading of cache will be faster")]
         minimal_cached_file_size: u64,
         #[structopt(flatten)]
         allowed_extensions: AllowedExtensions,

czkawka_core/src/similar_videos.rs

@@ -457,7 +457,7 @@ impl SimilarVideos {
                 hashmap_with_file_entries.insert(file_entry.vhash.src_path().to_string_lossy().to_string(), file_entry.clone());
                 vector_of_hashes.push(file_entry.vhash.clone());
             } else {
-                self.text_messages.errors.push(file_entry.error.clone());
+                self.text_messages.warnings.push(file_entry.error.clone());
             }
         }
 

czkawka_gui/src/connect_button_search.rs

@@ -92,7 +92,9 @@ pub fn connect_button_search(
     let radio_button_hash_type_xxh3 = gui_data.main_notebook.radio_button_hash_type_xxh3.clone();
     let check_button_settings_hide_hard_links = gui_data.settings.check_button_settings_hide_hard_links.clone();
     let check_button_settings_use_cache = gui_data.settings.check_button_settings_use_cache.clone();
+    let check_button_duplicates_use_prehash_cache = gui_data.settings.check_button_duplicates_use_prehash_cache.clone();
     let entry_settings_cache_file_minimal_size = gui_data.settings.entry_settings_cache_file_minimal_size.clone();
+    let entry_settings_prehash_cache_file_minimal_size = gui_data.settings.entry_settings_prehash_cache_file_minimal_size.clone();
     let radio_button_similar_hash_size_4 = gui_data.main_notebook.radio_button_similar_hash_size_4.clone();
     let radio_button_similar_hash_size_8 = gui_data.main_notebook.radio_button_similar_hash_size_8.clone();
     let radio_button_similar_hash_size_16 = gui_data.main_notebook.radio_button_similar_hash_size_16.clone();
@@ -118,7 +120,7 @@ pub fn connect_button_search(
         let allowed_extensions = entry_allowed_extensions.text().as_str().to_string();
         let hide_hard_links = check_button_settings_hide_hard_links.is_active();
         let use_cache = check_button_settings_use_cache.is_active();
-        let minimal_cache_file_size = entry_settings_cache_file_minimal_size.text().as_str().parse::<u64>().unwrap_or(2 * 1024 * 1024);
+        let minimal_cache_file_size = entry_settings_cache_file_minimal_size.text().as_str().parse::<u64>().unwrap_or(1024 * 1024 / 4);
 
         let show_dialog = Arc::new(AtomicBool::new(true));
 
@@ -170,6 +172,9 @@ pub fn connect_button_search(
                     panic!("No radio button is pressed");
                 }
 
+                let use_prehash_cache = check_button_duplicates_use_prehash_cache.is_active();
+                let minimal_prehash_cache_file_size = entry_settings_prehash_cache_file_minimal_size.text().as_str().parse::<u64>().unwrap_or(0);
+
                 let delete_outdated_cache = check_button_settings_duplicates_delete_outdated_cache.is_active();
 
                 let futures_sender_duplicate_files = futures_sender_duplicate_files.clone();
@@ -184,10 +189,12 @@ pub fn connect_button_search(
                     df.set_minimal_file_size(minimal_file_size);
                     df.set_maximal_file_size(maximal_file_size);
                     df.set_minimal_cache_file_size(minimal_cache_file_size);
+                    df.set_minimal_prehash_cache_file_size(minimal_prehash_cache_file_size);
                     df.set_check_method(check_method);
                     df.set_hash_type(hash_type);
                     df.set_ignore_hard_links(hide_hard_links);
                     df.set_use_cache(use_cache);
+                    df.set_use_prehash_cache(use_prehash_cache);
                     df.set_delete_outdated_cache(delete_outdated_cache);
                     df.find_duplicates(Some(&stop_receiver), Some(&futures_sender_duplicate_files));
                     let _ = glib_stop_sender.send(Message::Duplicates(df));

czkawka_gui/src/connect_button_select.rs

@@ -10,10 +10,10 @@ use crate::notebook_enums::*;
 pub fn connect_button_select(gui_data: &GuiData) {
     let mut hashmap: HashMap<NotebookMainEnum, Vec<PopoverTypes>> = Default::default();
     {
-        hashmap.insert(NotebookMainEnum::SimilarImages, vec![PopoverTypes::All, PopoverTypes::ImageSize, PopoverTypes::Reverse, PopoverTypes::Custom, PopoverTypes::Date]);
-        hashmap.insert(NotebookMainEnum::SimilarVideos, vec![PopoverTypes::All, PopoverTypes::Reverse, PopoverTypes::Custom, PopoverTypes::Date]);
+        hashmap.insert(NotebookMainEnum::SimilarImages, vec![PopoverTypes::All, PopoverTypes::Size, PopoverTypes::Reverse, PopoverTypes::Custom, PopoverTypes::Date]);
+        hashmap.insert(NotebookMainEnum::SimilarVideos, vec![PopoverTypes::All, PopoverTypes::Reverse, PopoverTypes::Custom, PopoverTypes::Date, PopoverTypes::Size]);
         hashmap.insert(NotebookMainEnum::Duplicate, vec![PopoverTypes::All, PopoverTypes::Reverse, PopoverTypes::Custom, PopoverTypes::Date]);
-        hashmap.insert(NotebookMainEnum::SameMusic, vec![PopoverTypes::All, PopoverTypes::Reverse, PopoverTypes::Custom, PopoverTypes::Date]);
+        hashmap.insert(NotebookMainEnum::SameMusic, vec![PopoverTypes::All, PopoverTypes::Reverse, PopoverTypes::Custom, PopoverTypes::Date, PopoverTypes::Size]);
 
         hashmap.insert(NotebookMainEnum::EmptyFiles, vec![PopoverTypes::All, PopoverTypes::Reverse, PopoverTypes::Custom]);
         hashmap.insert(NotebookMainEnum::EmptyDirectories, vec![PopoverTypes::All, PopoverTypes::Reverse, PopoverTypes::Custom]);
@@ -63,7 +63,7 @@ fn show_required_popovers(popovers: &GuiPopovers, current_mode: &NotebookMainEnu
         buttons_popover_unselect_all.hide();
     }
 
-    if vec.contains(&PopoverTypes::ImageSize) {
+    if vec.contains(&PopoverTypes::Size) {
         buttons_popover_select_all_images_except_biggest.show();
         buttons_popover_select_all_images_except_smallest.show();
         separator_select_image_size.show();

czkawka_gui/src/connect_popovers.rs

@@ -345,7 +345,7 @@ fn popover_custom_select_unselect(popover: &gtk::Popover, window_main: &Window,
     }
 }
 
-fn popover_all_except_biggest_smallest(popover: &gtk::Popover, tree_view: &gtk::TreeView, column_color: i32, column_size_as_bytes: i32, column_dimensions: i32, column_button_selection: u32, except_biggest: bool) {
+fn popover_all_except_biggest_smallest(popover: &gtk::Popover, tree_view: &gtk::TreeView, column_color: i32, column_size_as_bytes: i32, column_dimensions: Option<i32>, column_button_selection: u32, except_biggest: bool) {
     let model = get_list_store(tree_view);
 
     if let Some(iter) = model.iter_first() {
@@ -373,22 +373,38 @@ fn popover_all_except_biggest_smallest(popover: &gtk::Popover, tree_view: &gtk::
                 }
                 tree_iter_array.push(iter.clone());
                 let size_as_bytes = model.value(&iter, column_size_as_bytes).get::<u64>().unwrap();
-                let dimensions_string = model.value(&iter, column_dimensions).get::<String>().unwrap();
 
-                let dimensions = change_dimension_to_krotka(dimensions_string);
-                let number_of_pixels = dimensions.0 * dimensions.1;
+                // If dimension exists, then needs to be checked images
+                if let Some(column_dimensions) = column_dimensions {
+                    let dimensions_string = model.value(&iter, column_dimensions).get::<String>().unwrap();
 
-                if except_biggest {
-                    if number_of_pixels > number_of_pixels_min_max || (number_of_pixels == number_of_pixels_min_max && size_as_bytes > size_as_bytes_min_max) {
-                        number_of_pixels_min_max = number_of_pixels;
-                        size_as_bytes_min_max = size_as_bytes;
-                        used_index = Some(current_index);
+                    let dimensions = change_dimension_to_krotka(dimensions_string);
+                    let number_of_pixels = dimensions.0 * dimensions.1;
+
+                    if except_biggest {
+                        if number_of_pixels > number_of_pixels_min_max || (number_of_pixels == number_of_pixels_min_max && size_as_bytes > size_as_bytes_min_max) {
+                            number_of_pixels_min_max = number_of_pixels;
+                            size_as_bytes_min_max = size_as_bytes;
+                            used_index = Some(current_index);
+                        }
+                    } else {
+                        if number_of_pixels < number_of_pixels_min_max || (number_of_pixels == number_of_pixels_min_max && size_as_bytes < size_as_bytes_min_max) {
+                            number_of_pixels_min_max = number_of_pixels;
+                            size_as_bytes_min_max = size_as_bytes;
+                            used_index = Some(current_index);
+                        }
                     }
                 } else {
-                    if number_of_pixels < number_of_pixels_min_max || (number_of_pixels == number_of_pixels_min_max && size_as_bytes < size_as_bytes_min_max) {
-                        number_of_pixels_min_max = number_of_pixels;
-                        size_as_bytes_min_max = size_as_bytes;
-                        used_index = Some(current_index);
+                    if except_biggest {
+                        if size_as_bytes > size_as_bytes_min_max {
+                            size_as_bytes_min_max = size_as_bytes;
+                            used_index = Some(current_index);
+                        }
+                    } else {
+                        if size_as_bytes < size_as_bytes_min_max {
+                            size_as_bytes_min_max = size_as_bytes;
+                            used_index = Some(current_index);
+                        }
                     }
                 }
 
@@ -593,9 +609,9 @@ pub fn connect_popovers(gui_data: &GuiData) {
         popover_all_except_biggest_smallest(
             &popover_select,
             tree_view,
-            nb_object.column_color.expect("AEB can't be used without headers"),
-            nb_object.column_size_as_bytes.expect("AEB needs size as bytes column"),
-            nb_object.column_dimensions.expect("AEB needs dimensions column"),
+            nb_object.column_color.expect("AEBI can't be used without headers"),
+            nb_object.column_size_as_bytes.expect("AEBI needs size as bytes column"),
+            nb_object.column_dimensions,
             nb_object.column_selection as u32,
             true,
         );
@@ -613,9 +629,9 @@ pub fn connect_popovers(gui_data: &GuiData) {
         popover_all_except_biggest_smallest(
             &popover_select,
             tree_view,
-            nb_object.column_color.expect("AES can't be used without headers"),
-            nb_object.column_size_as_bytes.expect("AES needs size as bytes column"),
-            nb_object.column_dimensions.expect("AES needs dimensions column"),
+            nb_object.column_color.expect("AESI can't be used without headers"),
+            nb_object.column_size_as_bytes.expect("AESI needs size as bytes column"),
+            nb_object.column_dimensions,
             nb_object.column_selection as u32,
             false,
         );

czkawka_gui/src/gui_settings.rs

@@ -17,6 +17,8 @@ pub struct GuiSettings {
     // Duplicates
     pub check_button_settings_hide_hard_links: gtk::CheckButton,
     pub entry_settings_cache_file_minimal_size: gtk::Entry,
+    pub entry_settings_prehash_cache_file_minimal_size: gtk::Entry,
+    pub check_button_duplicates_use_prehash_cache: gtk::CheckButton,
     pub check_button_settings_show_preview_duplicates: gtk::CheckButton,
     pub check_button_settings_duplicates_delete_outdated_cache: gtk::CheckButton,
     pub button_settings_duplicates_clear_cache: gtk::Button,
@@ -71,6 +73,8 @@ impl GuiSettings {
         let check_button_settings_show_preview_duplicates: gtk::CheckButton = builder.object("check_button_settings_show_preview_duplicates").unwrap();
         let check_button_settings_duplicates_delete_outdated_cache: gtk::CheckButton = builder.object("check_button_settings_duplicates_delete_outdated_cache").unwrap();
         let button_settings_duplicates_clear_cache: gtk::Button = builder.object("button_settings_duplicates_clear_cache").unwrap();
+        let check_button_duplicates_use_prehash_cache: gtk::CheckButton = builder.object("check_button_duplicates_use_prehash_cache").unwrap();
+        let entry_settings_prehash_cache_file_minimal_size: gtk::Entry = builder.object("entry_settings_prehash_cache_file_minimal_size").unwrap();
 
         check_button_settings_hide_hard_links.set_tooltip_text(Some(
             "Hides all files except one, if are points to same data(are hardlinked).\n\nE.g. in case where on disk there is 7 files which are hardlinked to specific data and one different file with same data but different inode, then in duplicate finder will be visible only one unique file and one file from hardlinked ones.",
@@ -80,7 +84,10 @@ impl GuiSettings {
         ));
         check_button_settings_show_preview_duplicates.set_tooltip_text(Some("Shows preview at right side, when selecting image file."));
         check_button_settings_duplicates_delete_outdated_cache.set_tooltip_text(Some("Allows to delete outdated cache results which points to non-existent files.\n\nWhen enabled, app make sure when loading records, that all points to valid files and ignore broken ones.\n\nDisabling this option, will help to scan files on external drives, so cache entries about them will not be purged in next scan.\n\nIn case of having hundred of thousands records in cache, it is suggested to enable this option, to speedup cache loading and saving at start and end of scan."));
-        button_settings_duplicates_clear_cache.set_tooltip_text(Some("Manually clear cache from outdated entries.\nShould be used only if automatic clearing was disabled."));
+        button_settings_duplicates_clear_cache.set_tooltip_text(Some("Manually clear cache from outdated entries.\n\nShould be used only if automatic clearing was disabled."));
+        check_button_duplicates_use_prehash_cache.set_tooltip_text(Some(
+            "Enables caching of prehash(hash computed from small part of file) which allows to earlier throw out non duplicated results.\n\nIt is disabled by default because can cause in some situations slowdowns.\n\nIt is heavily recommended to use it when scanning hundred of thousands or million files, because it can speedup search multiple times.",
+        ));
 
         // Similar Images
         let check_button_settings_show_preview_similar_images: gtk::CheckButton = builder.object("check_button_settings_show_preview_similar_images").unwrap();
@@ -126,6 +133,8 @@ impl GuiSettings {
             check_button_settings_use_trash,
             check_button_settings_hide_hard_links,
             entry_settings_cache_file_minimal_size,
+            entry_settings_prehash_cache_file_minimal_size,
+            check_button_duplicates_use_prehash_cache,
             check_button_settings_show_preview_duplicates,
             check_button_settings_duplicates_delete_outdated_cache,
             button_settings_duplicates_clear_cache,

czkawka_gui/src/help_functions.rs

@@ -33,7 +33,7 @@ pub const KEY_END: u32 = 110;
 #[derive(Eq, PartialEq)]
 pub enum PopoverTypes {
     All,
-    ImageSize,
+    Size,
     Reverse,
     Custom,
     Date,