Enhancements to support parallel run of visual tests (#16)

* enhancements to support parallel run of visual tests

README.md

@@ -51,6 +51,9 @@ The report should get generated after test have finished. To generate report man
 ```
     reportgen path/to/output directory
 ```
+# Pabot users
+Visual tests can be parallel executed using pabot . However there may be issues with the auto generated report after tests have finished.
+A workaround can be to generate report using **reportgen** to ensure report has no discrepancies.
 
 # Keyword Documentation
 **open eyes** - Arguments: library Eg AppiumLibrary (optional) - (Gets current selenium/appium instance) <br/>

RobotEyes/__init__.py

@@ -24,10 +24,12 @@ def __init__(self, mode, tolerance=0):
         self.sys = platform.system()
         self.tolerance = tolerance
         self.stats = {}
+        self.content = ''
+        self.fail = False
 
     def open_eyes(self, lib='Selenium2Library'):
-        self.output_dir = BuiltIn().replace_variables('${OUTPUT DIR}')
-        self.images_base_folder = self.output_dir + IMAGES_FOLDER
+        self.output_dir = self._output_dir()
+        self.images_base_folder = os.path.join(self.output_dir, IMAGES_FOLDER)
 
         try:
             s2l = BuiltIn().get_library_instance(lib)
@@ -53,16 +55,8 @@ def open_eyes(self, lib='Selenium2Library'):
         # Delete visualReport.html if modified_at is older than a specified time difference.
         # Couldn't think of a better way to do this.
         # New report gets appended to old test run report if this is not done.
-        self._delete_report_if_old(self.output_dir + REPORT_FILE) if os.path.exists(self.output_dir + REPORT_FILE) else ''
-
-        if not os.path.exists(self.output_dir + REPORT_FILE):
-            file = open(self.output_dir + REPORT_FILE, 'w')
-            file.write(HEADER)
-            file.close()
-
-        with open(self.output_dir + REPORT_FILE, 'r') as file:
-            content = file.read()
-            self.content = content.replace(FOOTER, '')
+        self.report_path = os.path.join(self.output_dir, REPORT_FILE)
+        self._delete_report_if_old(self.report_path) if os.path.exists(self.report_path) else ''
 
     # Captures full screen
     def capture_full_screen(self, tolerance=None, blur=[], radius=50):
@@ -90,6 +84,7 @@ def capture_mobile_element(self, selector, tolerance=None):
         top = location['y']
         right = location['x'] + size['width']
         bottom = location['y'] + size['height']
+
         image = Image.open(self.path + '/img' + str(self.count) + '.png')
         image = image.crop((left, top, right, bottom))
         image.save(self.path + '/img' + str(self.count) + '.png')
@@ -136,14 +131,13 @@ def scroll_to_element(self, selector):
     def compare_images(self):
         if self.mode.lower() == MODE_TEST:
             test_name = self.test_name.replace(' ', '_')
-            baseline_path = self.images_base_folder + BASELINE_IMAGE_BASE_FOLDER + test_name
-            actual_path = self.images_base_folder + ACTUAL_IMAGE_BASE_FOLDER + test_name
-            diff_path = self.images_base_folder + DIFF_IMAGE_BASE_FOLDER + test_name
+            baseline_path = os.path.join(self.images_base_folder, BASELINE_IMAGE_BASE_FOLDER, test_name)
+            actual_path = os.path.join(self.images_base_folder, ACTUAL_IMAGE_BASE_FOLDER, test_name)
+            diff_path = os.path.join(self.images_base_folder, DIFF_IMAGE_BASE_FOLDER, test_name)
 
             if not os.path.exists(diff_path):
                 os.makedirs(diff_path)
 
-            fail = False
             self.content += make_parent_row(self.test_name)
 
             # compare actual and baseline images and save the diff image
@@ -153,27 +147,18 @@ def compare_images(self):
                 d_path = ''
 
                 if filename.endswith('.png'):
-                    b_path = baseline_path + '/' + filename
-                    a_path = actual_path + '/' + filename
-                    d_path = diff_path + '/' + filename
+                    b_path = os.path.join(baseline_path, filename)
+                    a_path = os.path.join(actual_path, filename)
+                    d_path = os.path.join(diff_path, filename)
 
                     if os.path.exists(b_path):
                         self._resize(b_path, a_path)
-                        difference = self._compare(b_path, a_path, d_path)
-                        threshold = float(self.stats[filename])
 
-                        if difference > threshold:
-                            color = 'red'
-                            result = '%s<%s' % (threshold, difference)
-                            fail = True
+                        difference = self._compare(b_path, a_path, d_path)
 
-                        elif difference == threshold:
-                            color = 'green'
-                            result = '%s==%s' % (threshold, difference)
+                        threshold = float(self.stats[filename])
 
-                        else:
-                            color = 'green'
-                            result = '%s>%s' % (threshold, difference)
+                        color, result = self._get_result(difference, threshold)
 
                         text = '%s %s' % (result, color)
                         final_result = [color, result]
@@ -189,11 +174,9 @@ def compare_images(self):
             self.content += INNER_TABLE_END
             self.content += FOOTER
 
-            file = open(self.output_dir + REPORT_FILE, 'w')
-            file.write(self.content)
-            file.close()
+            self._write_to_report()
 
-            BuiltIn().run_keyword('Fail', 'Image dissimilarity exceeds threshold') if fail else ''
+            BuiltIn().run_keyword('Fail', 'Image dissimilarity exceeds threshold') if self.fail else ''
 
     def _compare(self, b_path, a_path, d_path):
         compare_cmd = 'compare -metric RMSE -subimage-search -dissimilarity-threshold 1.0 %s %s %s' \
@@ -252,26 +235,29 @@ def _get_coordinates(self, prefix, locator):
         return coord
 
     def _delete_folder_if_exists(self, test_name_folder):
+        actual_image_test_folder = os.path.join(self.images_base_folder, ACTUAL_IMAGE_BASE_FOLDER, test_name_folder)
+        diff_image_test_folder = os.path.join(self.images_base_folder, DIFF_IMAGE_BASE_FOLDER, test_name_folder)
+        baseline_image_test_folder = os.path.join(self.images_base_folder, BASELINE_IMAGE_BASE_FOLDER, test_name_folder)
+
         if self.mode.lower() == MODE_TEST:
-            if os.path.exists(self.images_base_folder + ACTUAL_IMAGE_BASE_FOLDER + test_name_folder):
-                shutil.rmtree(self.images_base_folder + ACTUAL_IMAGE_BASE_FOLDER + test_name_folder)
+            if os.path.exists(actual_image_test_folder):
+                shutil.rmtree(actual_image_test_folder)
 
-            if os.path.exists(self.images_base_folder + DIFF_IMAGE_BASE_FOLDER + test_name_folder):
-                shutil.rmtree(self.images_base_folder + DIFF_IMAGE_BASE_FOLDER + test_name_folder)
+            if os.path.exists(diff_image_test_folder):
+                shutil.rmtree(diff_image_test_folder)
 
         elif self.mode.lower() == MODE_BASELINE:
-            if os.path.exists(self.images_base_folder + BASELINE_IMAGE_BASE_FOLDER + test_name_folder):
-                shutil.rmtree(self.images_base_folder + BASELINE_IMAGE_BASE_FOLDER + test_name_folder)
-
+            if os.path.exists(baseline_image_test_folder):
+                shutil.rmtree(baseline_image_test_folder)
         else:
-            raise AssertionError('Mode should be test or baseline')
+            raise Exception('Mode should be test or baseline')
 
     def _create_empty_folder(self, test_name_folder):
         if self.mode.lower() == MODE_BASELINE:
-            self.path = self.images_base_folder + BASELINE_IMAGE_BASE_FOLDER + test_name_folder
+            self.path = os.path.join(self.images_base_folder, BASELINE_IMAGE_BASE_FOLDER, test_name_folder)
 
         elif self.mode.lower() == MODE_TEST:
-            self.path = self.images_base_folder + ACTUAL_IMAGE_BASE_FOLDER + test_name_folder
+            self.path = os.path.join(self.images_base_folder, ACTUAL_IMAGE_BASE_FOLDER, test_name_folder)
 
         if not os.path.exists(self.path):
             os.makedirs(self.path)
@@ -315,3 +301,40 @@ def _delete_report_if_old(self, path):
         t2 = time.time()
         diff = int(t2 - t1)
         os.remove(path) if diff > REPORT_EXPIRATION_THRESHOLD else ''
+
+    def _output_dir(self):
+        output_dir = BuiltIn().replace_variables('${OUTPUT DIR}')
+
+        if 'pabot_results' in output_dir:
+            index = output_dir.find('/pabot_results')
+            return output_dir[:index]
+        return output_dir
+
+    def _get_result(self, difference, threshold):
+        if difference > threshold:
+            color = 'red'
+            result = '%s<%s' % (threshold, difference)
+            self.fail = True
+
+        elif difference == threshold:
+            color = 'green'
+            result = '%s==%s' % (threshold, difference)
+
+        else:
+            color = 'green'
+            result = '%s>%s' % (threshold, difference)
+
+        return color, result
+
+    def _write_to_report(self):
+        if os.path.exists(self.report_path):
+            with open(self.report_path, 'r') as file:
+                old_content = file.read()
+                old_content = old_content.replace(FOOTER, '')
+                self.content = old_content + self.content
+        else:
+            self.content = HEADER + self.content
+
+        file = open(self.report_path, 'w')
+        file.write(self.content)
+        file.close()

RobotEyes/constants.py

@@ -1,8 +1,8 @@
-IMAGES_FOLDER = '/visual_images'
-ACTUAL_IMAGE_BASE_FOLDER = '/actual/'
-DIFF_IMAGE_BASE_FOLDER = '/diff/'
-BASELINE_IMAGE_BASE_FOLDER = '/baseline/'
+IMAGES_FOLDER = 'visual_images'
+ACTUAL_IMAGE_BASE_FOLDER = 'actual'
+DIFF_IMAGE_BASE_FOLDER = 'diff'
+BASELINE_IMAGE_BASE_FOLDER = 'baseline'
 MODE_TEST = 'test'
 MODE_BASELINE = 'baseline'
-REPORT_FILE = '/visualReport.html'
+REPORT_FILE = 'visualReport.html'
 REPORT_EXPIRATION_THRESHOLD = 5