From 37beb72b6e9ce8cc55233c6062833448b5555473 Mon Sep 17 00:00:00 2001 From: Mohammed Sazid-Al-Rashid Date: Fri, 25 Nov 2022 16:36:36 +0600 Subject: [PATCH 01/16] Testing image text - Change state of the search by providing value with left + width and top + height - Change state of the search by keeping original colour --- .../Desktop/Windows/BuiltInFunctions.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py b/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py index e85490ff3..41a826b47 100644 --- a/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py +++ b/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py @@ -915,6 +915,11 @@ def image_search(step_data_set): confidence = 0.85 parent_dataset = [] image_text = "" + left_width = 1 + top_height = 1 + colour_state = "black_white" + + for left, mid, right in step_data_set: left = left.strip().lower() mid = mid.strip().lower() @@ -927,6 +932,12 @@ def image_search(step_data_set): confidence = float(right.replace("%", "").replace(" ", "").lower()) / 100 elif "text" in left: image_text = right + elif "left+width" in left: + left_width = right + elif "top+height" in left: + top_height = right + elif "colour" in left: + colour_state = right else: file_name = right.strip() if "~" in file_name: @@ -968,11 +979,14 @@ def image_search(step_data_set): pytesseract.tesseract_cmd = os.environ["PROGRAMFILES"] + r"\Tesseract-OCR\tesseract.exe" image_text = image_text.replace(" ", "").lower() - PIL.ImageGrab.grab().crop((left, top, left + width, top + height)).save("sample.jpg") + PIL.ImageGrab.grab().crop((left, top, left + width * left_width, top + height * top_height)).save("sample.jpg") imge = cv2.imread("sample.jpg") gray = cv2.cvtColor(imge, cv2.COLOR_BGR2GRAY) - data = pytesseract.image_to_boxes(gray) + if colour_state == "black_white": + data = pytesseract.image_to_boxes(gray) + else: + data = pytesseract.image_to_boxes(imge) all_letters = data.split("\n") print(all_letters) full_string = "" From ff014df7eb85984cf494fd1d9d7400434ae8b7d1 Mon Sep 17 00:00:00 2001 From: Mohammed Sazid-Al-Rashid Date: Sun, 27 Nov 2022 01:48:33 +0600 Subject: [PATCH 02/16] Missing image text parameter --- .../Built_In_Automation/Desktop/Windows/BuiltInFunctions.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py b/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py index 41a826b47..8302b53bc 100644 --- a/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py +++ b/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py @@ -1122,6 +1122,8 @@ def Get_Element(data_set, wait_time=Shared_Resources.Get_Shared_Variables("eleme elif "path" in left: element_path = right.strip() elif "image" in left: element_image.append((left, mid, right)) + elif "imagetext" in left: + element_image.append((left, mid, right)) elif mid == "parent parameter": parent = True From 697d651c35284c9470a417d2cd26a4419225fc39 Mon Sep 17 00:00:00 2001 From: Mohammed Sazid-Al-Rashid Date: Mon, 28 Nov 2022 02:20:04 +0600 Subject: [PATCH 03/16] Ongoing , not complete --- .../Desktop/Windows/BuiltInFunctions.py | 197 +++++++++++++----- 1 file changed, 146 insertions(+), 51 deletions(-) diff --git a/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py b/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py index 8302b53bc..a9fec8063 100644 --- a/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py +++ b/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py @@ -34,6 +34,7 @@ from PIL import Image, ImageGrab + python_folder = [] for location in subprocess.getoutput("where python").split("\n"): if "Microsoft" not in location: @@ -915,9 +916,12 @@ def image_search(step_data_set): confidence = 0.85 parent_dataset = [] image_text = "" - left_width = 1 - top_height = 1 - colour_state = "black_white" + left_width = "" + top_height = "" + colour_state = "" + method_image = "" + language = "" + for left, mid, right in step_data_set: @@ -932,18 +936,37 @@ def image_search(step_data_set): confidence = float(right.replace("%", "").replace(" ", "").lower()) / 100 elif "text" in left: image_text = right - elif "left+width" in left: - left_width = right - elif "top+height" in left: - top_height = right + elif "left_width" in left: + left_width = float(right) + elif "top_height" in left: + top_height = float(right) elif "colour" in left: colour_state = right + elif 'language' in left: + language = right + elif 'method' in left: + method_image = right + else: - file_name = right.strip() - if "~" in file_name: - file_name = str(Path(os.path.expanduser(file_name))) + if not image_text: + pass + else: + file_name = right.strip() + if "~" in file_name: + file_name = str(Path(os.path.expanduser(file_name))) if mid == "parent parameter": parent_dataset.append((left, "element parameter", right)) + if left_width == '': + left_width = 1 + if top_height == '': + top_height = 1 + if colour_state == '': + colour_state = 'black_white' + if method_image == '': + method = 'method_1' + if language == '': + language = 'en' + if parent_dataset: parent = Get_Element(parent_dataset) @@ -974,53 +997,95 @@ def image_search(step_data_set): # Find element information try: if image_text: - import cv2 - from pytesseract import pytesseract - pytesseract.tesseract_cmd = os.environ["PROGRAMFILES"] + r"\Tesseract-OCR\tesseract.exe" - image_text = image_text.replace(" ", "").lower() - PIL.ImageGrab.grab().crop((left, top, left + width * left_width, top + height * top_height)).save("sample.jpg") - imge = cv2.imread("sample.jpg") - gray = cv2.cvtColor(imge, cv2.COLOR_BGR2GRAY) + if method_image == 'method_1': + import cv2 + from pytesseract import pytesseract + pytesseract.tesseract_cmd = os.environ["PROGRAMFILES"] + r"\Tesseract-OCR\tesseract.exe" + + image_text = image_text.replace(" ", "").lower() + PIL.ImageGrab.grab().crop((left, top, left + width * left_width, top + height * top_height)).save("sample.jpg") + imge = cv2.imread("sample.jpg") + gray = cv2.cvtColor(imge, cv2.COLOR_BGR2GRAY) + + if colour_state == "black_white": + data = pytesseract.image_to_boxes(gray) + else: + data = pytesseract.image_to_boxes(imge) + all_letters = data.split("\n") + print(all_letters) + full_string = "" + for i in all_letters: + full_string += i[0] if len(i) > 0 else "~" + full_string = full_string.lower() + + all_pos = [m.start() for m in re.finditer(image_text, full_string)] + + if -len(all_pos) <= idx < len(all_pos): + CommonUtil.ExecLog(sModuleInfo, "Found %s text elements. Returning element of index %s" % (len(all_pos), idx), 1) + i = all_pos[idx] + elif len(all_pos) != 0: + CommonUtil.ExecLog(sModuleInfo, "Found %s text elements. Index out of range" % len(all_pos), 3) + return "zeuz_failed" + else: + CommonUtil.ExecLog(sModuleInfo, 'Could not find text "%s"' % image_text, 3) + return "zeuz_failed" + + msg = "" + a = all_letters[i:i + len(image_text)] + for i in a: + msg += i + "\n" + left_top = list(map(int, a[0].split(" ")[1:3])) + right_bottom = list(map(int, a[-1].split(" ")[3:5])) + center = left + (right_bottom[0] + left_top[0]) // 2, top + height - (right_bottom[1] + left_top[1]) // 2 + msg += "Center = " + str(center) + "\n" + # pyautogui.moveTo(center) + + element = left_top[0] + left, height - right_bottom[1] + top, right_bottom[0] - left_top[0], right_bottom[1] - left_top[1] + msg += "Coordinates = " + str(element) + "\n" + CommonUtil.ExecLog(sModuleInfo, msg, 5) + + return _Element(element) - if colour_state == "black_white": - data = pytesseract.image_to_boxes(gray) - else: - data = pytesseract.image_to_boxes(imge) - all_letters = data.split("\n") - print(all_letters) - full_string = "" - for i in all_letters: - full_string += i[0] if len(i) > 0 else "~" - full_string = full_string.lower() - - all_pos = [m.start() for m in re.finditer(image_text, full_string)] - - if -len(all_pos) <= idx < len(all_pos): - CommonUtil.ExecLog(sModuleInfo, "Found %s text elements. Returning element of index %s" % (len(all_pos), idx), 1) - i = all_pos[idx] - elif len(all_pos) != 0: - CommonUtil.ExecLog(sModuleInfo, "Found %s text elements. Index out of range" % len(all_pos), 3) - return "zeuz_failed" else: - CommonUtil.ExecLog(sModuleInfo, 'Could not find text "%s"' % image_text, 3) - return "zeuz_failed" + import easyocr + import numpy as np + import cv2 + from pytesseract import pytesseract + import matplotlib.pyplot as plt + + pytesseract.tesseract_cmd = os.environ["PROGRAMFILES"] + r"\Tesseract-OCR\tesseract.exe" + + PIL.ImageGrab.grab().save("sample.png") + reader = easyocr.Reader([language]) + output = reader.readtext("sample.png") + item = [] + for text in output: + if image_text in text: + item.append([text]) + print(text) + cord = np.array(item[idx][0]) + cord1 = cord.tolist() + + x_min, y_min = [min(cord_val) for cord_val in zip(*cord1[0])] + x_max, y_max = [max(cord_val) for cord_val in zip(*cord1[0])] + img = cv2.imread('sample.png') + x = cv2.rectangle(img, (x_min, y_min), (x_max, y_max), (0, 0, 255), 2) + + img = cv2.imread("sample.png") + cv2.rectangle(img, (x_min, y_min), (x_max, y_max), (0, 0, 255), 2) + image1 = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + + plt.imshow(image1) + plt.show() + return _Element(element) + + + + - msg = "" - a = all_letters[i:i + len(image_text)] - for i in a: - msg += i + "\n" - left_top = list(map(int, a[0].split(" ")[1:3])) - right_bottom = list(map(int, a[-1].split(" ")[3:5])) - center = left + (right_bottom[0] + left_top[0]) // 2, top + height - (right_bottom[1] + left_top[1]) // 2 - msg += "Center = " + str(center) + "\n" - # pyautogui.moveTo(center) - element = left_top[0] + left, height - right_bottom[1] + top, right_bottom[0] - left_top[0], right_bottom[1] - left_top[1] - msg += "Coordinates = " + str(element) + "\n" - CommonUtil.ExecLog(sModuleInfo, msg, 5) - return _Element(element) else: # Scale image if required regex = re.compile(r"(\d+)\s*x\s*(\d+)", re.IGNORECASE) # Create regex object with expression @@ -1125,6 +1190,7 @@ def Get_Element(data_set, wait_time=Shared_Resources.Get_Shared_Variables("eleme elif "imagetext" in left: element_image.append((left, mid, right)) + elif mid == "parent parameter": parent = True if "class" in left: parent_class = [right, _count_star(left)] @@ -1170,6 +1236,30 @@ def Get_Element(data_set, wait_time=Shared_Resources.Get_Shared_Variables("eleme s = time.time() while True: if element_image: + + left_width = 1 + top_height = 1 + colour_state = 'black_white' + method = 'method_1' + language = 'en' + for left, mid, right in data_set: + left = left.strip().lower() + mid = mid.strip().lower() + if mid == "element parameter": + if "left_width" in left: + left_width = right + elif "top_height" in left: + top_height = right + elif "colour" in left: + colour_state = right + elif 'language' in left: + language = right + elif 'method_image' in left: + method_image = right + + + + _get_main_window(window_name) for i in (("name", parent_name), ("class", parent_class), ("automationid", parent_automation), ("control", parent_control), ("path", parent_path), ("window", window_name), ("index", parent_index)): if i[1]: @@ -1179,6 +1269,11 @@ def Get_Element(data_set, wait_time=Shared_Resources.Get_Shared_Variables("eleme data = (i[0], "parent parameter", i[1]) element_image.append(data) element_image.append(("index", "element parameter", str(element_index))) + element_image.append(("left_width", "element parameter", str(left_width))) + element_image.append(("top_height", "element parameter", str(top_height))) + element_image.append(("colour_state", "element parameter", str(colour_state))) + element_image.append(("language", "element parameter", str(language))) + element_image.append(("method_image", "element parameter", str(method_image))) result = image_search(element_image) return result if element_path: From 329c563c4d7a5f9cc6f29cf7daab5793cf561566 Mon Sep 17 00:00:00 2001 From: Mohammed Sazid-Al-Rashid Date: Mon, 28 Nov 2022 15:16:00 +0600 Subject: [PATCH 04/16] complemented for now --- .../Desktop/Windows/BuiltInFunctions.py | 37 ++++++++++--------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py b/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py index a9fec8063..705b18fb3 100644 --- a/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py +++ b/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py @@ -1060,32 +1060,35 @@ def image_search(step_data_set): reader = easyocr.Reader([language]) output = reader.readtext("sample.png") item = [] + count = 0 for text in output: - if image_text in text: + if not image_text in text: + continue + else: item.append([text]) print(text) + CommonUtil.ExecLog(sModuleInfo,"Found %s text. Returning element of index %s" % (image_text, count), 1) + count = count + 1 + if item == []: + CommonUtil.ExecLog(sModuleInfo, 'Could not find text "%s"' % image_text, 3) + return "zeuz_failed" cord = np.array(item[idx][0]) cord1 = cord.tolist() x_min, y_min = [min(cord_val) for cord_val in zip(*cord1[0])] x_max, y_max = [max(cord_val) for cord_val in zip(*cord1[0])] - img = cv2.imread('sample.png') - x = cv2.rectangle(img, (x_min, y_min), (x_max, y_max), (0, 0, 255), 2) - - img = cv2.imread("sample.png") - cv2.rectangle(img, (x_min, y_min), (x_max, y_max), (0, 0, 255), 2) - image1 = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) - - plt.imshow(image1) - plt.show() + # img = cv2.imread("sample.png") + # cropping = img[y_min:y_max , x_min:x_max] + # cv2.imwrite(r"C:\Users\Sazid\Desktop\final_csv_result\crop_{0}.png".format(idx), cropping) + # + # cv2.rectangle(img, (x_min, y_min), (x_max, y_max), (0, 0, 255), 2) + # image1 = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + # + # plt.imshow(image1) + # plt.show() + element = x_min,y_min,x_max-x_min,y_max-y_min return _Element(element) - - - - - - else: # Scale image if required regex = re.compile(r"(\d+)\s*x\s*(\d+)", re.IGNORECASE) # Create regex object with expression @@ -1240,7 +1243,7 @@ def Get_Element(data_set, wait_time=Shared_Resources.Get_Shared_Variables("eleme left_width = 1 top_height = 1 colour_state = 'black_white' - method = 'method_1' + method_image = 'method_1' language = 'en' for left, mid, right in data_set: left = left.strip().lower() From b8f9820bcef4b4d124bacba5b81b320855bc7e4e Mon Sep 17 00:00:00 2001 From: Mohammed Sazid-Al-Rashid Date: Mon, 28 Nov 2022 19:36:47 +0600 Subject: [PATCH 05/16] Added sequence matcher for smaller text --- .../Desktop/Windows/BuiltInFunctions.py | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py b/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py index 705b18fb3..2aceed814 100644 --- a/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py +++ b/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py @@ -1053,6 +1053,7 @@ def image_search(step_data_set): import cv2 from pytesseract import pytesseract import matplotlib.pyplot as plt + from difflib import SequenceMatcher pytesseract.tesseract_cmd = os.environ["PROGRAMFILES"] + r"\Tesseract-OCR\tesseract.exe" @@ -1062,13 +1063,24 @@ def image_search(step_data_set): item = [] count = 0 for text in output: - if not image_text in text: - continue - else: + def seq(a, b): + c = SequenceMatcher(a=a, b=b).ratio() + if c > 0.8: + return c + else: + return .00004 + + rslt = seq(image_text, text[1]) + if rslt > 0.8: + # if image_text in text[1]: item.append([text]) print(text) - CommonUtil.ExecLog(sModuleInfo,"Found %s text. Returning element of index %s" % (image_text, count), 1) + CommonUtil.ExecLog(sModuleInfo, "Found %s text. Returning element of index %s" % (image_text, count), 1) count = count + 1 + else: + print(text) + continue + if item == []: CommonUtil.ExecLog(sModuleInfo, 'Could not find text "%s"' % image_text, 3) return "zeuz_failed" From 37202dd437127d1e227627133f234b9c16983a47 Mon Sep 17 00:00:00 2001 From: Mohammed Sazid-Al-Rashid Date: Wed, 7 Dec 2022 21:10:59 +0600 Subject: [PATCH 06/16] put user based confidence --- .../Desktop/Windows/BuiltInFunctions.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py b/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py index 2aceed814..db0606321 100644 --- a/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py +++ b/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py @@ -921,6 +921,7 @@ def image_search(step_data_set): colour_state = "" method_image = "" language = "" + easy_ocr_conf = "" @@ -944,8 +945,10 @@ def image_search(step_data_set): colour_state = right elif 'language' in left: language = right - elif 'method' in left: + elif 'method_image' in left: method_image = right + elif 'easy_ocr_conf' in left: + easy_ocr_conf = float(right) else: if not image_text: @@ -1065,13 +1068,13 @@ def image_search(step_data_set): for text in output: def seq(a, b): c = SequenceMatcher(a=a, b=b).ratio() - if c > 0.8: + if c > easy_ocr_conf: return c else: return .00004 rslt = seq(image_text, text[1]) - if rslt > 0.8: + if rslt > easy_ocr_conf: # if image_text in text[1]: item.append([text]) print(text) @@ -1086,9 +1089,10 @@ def seq(a, b): return "zeuz_failed" cord = np.array(item[idx][0]) cord1 = cord.tolist() - + # x_min, y_min = [min(cord_val) for cord_val in zip(*cord1[0])] x_max, y_max = [max(cord_val) for cord_val in zip(*cord1[0])] + # img = cv2.imread("sample.png") # cropping = img[y_min:y_max , x_min:x_max] # cv2.imwrite(r"C:\Users\Sazid\Desktop\final_csv_result\crop_{0}.png".format(idx), cropping) @@ -1256,6 +1260,7 @@ def Get_Element(data_set, wait_time=Shared_Resources.Get_Shared_Variables("eleme top_height = 1 colour_state = 'black_white' method_image = 'method_1' + easy_ocr_conf = 0.8 language = 'en' for left, mid, right in data_set: left = left.strip().lower() @@ -1271,6 +1276,8 @@ def Get_Element(data_set, wait_time=Shared_Resources.Get_Shared_Variables("eleme language = right elif 'method_image' in left: method_image = right + elif 'easey_ocr_conf' in left: + easy_ocr_conf = right From 91ca8bdd730569d5646505cfab3f090076bc3a90 Mon Sep 17 00:00:00 2001 From: Mohammed Sazid-Al-Rashid Date: Thu, 8 Dec 2022 00:39:03 +0600 Subject: [PATCH 07/16] Implemented option for manually changing the confidence value --- .../Desktop/Windows/BuiltInFunctions.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py b/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py index db0606321..02946cc19 100644 --- a/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py +++ b/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py @@ -921,7 +921,7 @@ def image_search(step_data_set): colour_state = "" method_image = "" language = "" - easy_ocr_conf = "" + image_conf = 0.8 @@ -947,8 +947,8 @@ def image_search(step_data_set): language = right elif 'method_image' in left: method_image = right - elif 'easy_ocr_conf' in left: - easy_ocr_conf = float(right) + elif 'image_conf' in left: + image_conf = float(right) else: if not image_text: @@ -971,6 +971,7 @@ def image_search(step_data_set): language = 'en' + if parent_dataset: parent = Get_Element(parent_dataset) if type(parent) == str and parent == "zeuz_failed": @@ -1068,13 +1069,13 @@ def image_search(step_data_set): for text in output: def seq(a, b): c = SequenceMatcher(a=a, b=b).ratio() - if c > easy_ocr_conf: + if c > 0.8: return c else: return .00004 rslt = seq(image_text, text[1]) - if rslt > easy_ocr_conf: + if rslt >= float(image_conf): # if image_text in text[1]: item.append([text]) print(text) @@ -1255,12 +1256,11 @@ def Get_Element(data_set, wait_time=Shared_Resources.Get_Shared_Variables("eleme s = time.time() while True: if element_image: - left_width = 1 top_height = 1 colour_state = 'black_white' method_image = 'method_1' - easy_ocr_conf = 0.8 + image_conf = 0.8 language = 'en' for left, mid, right in data_set: left = left.strip().lower() @@ -1276,8 +1276,8 @@ def Get_Element(data_set, wait_time=Shared_Resources.Get_Shared_Variables("eleme language = right elif 'method_image' in left: method_image = right - elif 'easey_ocr_conf' in left: - easy_ocr_conf = right + elif 'image_conf' in left: + image_conf = right @@ -1296,6 +1296,8 @@ def Get_Element(data_set, wait_time=Shared_Resources.Get_Shared_Variables("eleme element_image.append(("colour_state", "element parameter", str(colour_state))) element_image.append(("language", "element parameter", str(language))) element_image.append(("method_image", "element parameter", str(method_image))) + element_image.append(("image_conf", "element parameter", str(image_conf))) + result = image_search(element_image) return result if element_path: From de18487c3ac7956d193d69b8115caa22de969913 Mon Sep 17 00:00:00 2001 From: Mohammed Sazid-Al-Rashid Date: Fri, 9 Dec 2022 18:04:11 +0600 Subject: [PATCH 08/16] - Take screenshot of text image location. Taking this screenshot is necessery when texts doesn't show upin full form , rather displays some ambigious elements instead of text. - First it will extract the desired text bounding box. Then will crop image according to the boundin box and put the data in the user defined location. --- .../Desktop/Windows/BuiltInFunctions.py | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py b/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py index 02946cc19..447b8d74b 100644 --- a/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py +++ b/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py @@ -922,6 +922,7 @@ def image_search(step_data_set): method_image = "" language = "" image_conf = 0.8 + text_screenshot = '' @@ -949,6 +950,8 @@ def image_search(step_data_set): method_image = right elif 'image_conf' in left: image_conf = float(right) + elif 't_screenshot' in left: + text_screenshot = right else: if not image_text: @@ -1063,14 +1066,16 @@ def image_search(step_data_set): PIL.ImageGrab.grab().save("sample.png") reader = easyocr.Reader([language]) - output = reader.readtext("sample.png") + output = reader.readtext("sample.png",paragraph=False) item = [] count = 0 for text in output: def seq(a, b): c = SequenceMatcher(a=a, b=b).ratio() + print(c) if c > 0.8: return c + else: return .00004 @@ -1094,9 +1099,12 @@ def seq(a, b): x_min, y_min = [min(cord_val) for cord_val in zip(*cord1[0])] x_max, y_max = [max(cord_val) for cord_val in zip(*cord1[0])] - # img = cv2.imread("sample.png") - # cropping = img[y_min:y_max , x_min:x_max] - # cv2.imwrite(r"C:\Users\Sazid\Desktop\final_csv_result\crop_{0}.png".format(idx), cropping) + if text_screenshot !='': + img = cv2.imread("sample.png") + cropping = img[y_min:y_max , x_min:x_max] + cv2.imwrite(text_screenshot, cropping) + else: + pass # # cv2.rectangle(img, (x_min, y_min), (x_max, y_max), (0, 0, 255), 2) # image1 = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) @@ -1262,6 +1270,7 @@ def Get_Element(data_set, wait_time=Shared_Resources.Get_Shared_Variables("eleme method_image = 'method_1' image_conf = 0.8 language = 'en' + text_screenshot = '' for left, mid, right in data_set: left = left.strip().lower() mid = mid.strip().lower() @@ -1278,6 +1287,8 @@ def Get_Element(data_set, wait_time=Shared_Resources.Get_Shared_Variables("eleme method_image = right elif 'image_conf' in left: image_conf = right + elif 't_screenshot' in left: + text_screenshot = right @@ -1297,6 +1308,8 @@ def Get_Element(data_set, wait_time=Shared_Resources.Get_Shared_Variables("eleme element_image.append(("language", "element parameter", str(language))) element_image.append(("method_image", "element parameter", str(method_image))) element_image.append(("image_conf", "element parameter", str(image_conf))) + element_image.append(("t_screenshot", "element parameter", str(text_screenshot))) + result = image_search(element_image) return result From 0e9231dfb848670dcaa38d4af19c3526676f8cd9 Mon Sep 17 00:00:00 2001 From: Mohammed Sazid-Al-Rashid Date: Fri, 9 Dec 2022 19:05:05 +0600 Subject: [PATCH 09/16] Implemented ability to manipulate paragraph mode manually --- .../Desktop/Windows/BuiltInFunctions.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py b/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py index 447b8d74b..64f5a1f7e 100644 --- a/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py +++ b/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py @@ -923,6 +923,7 @@ def image_search(step_data_set): language = "" image_conf = 0.8 text_screenshot = '' + easyocr_paragraph = '' @@ -952,6 +953,8 @@ def image_search(step_data_set): image_conf = float(right) elif 't_screenshot' in left: text_screenshot = right + elif 'easyocr_paragraph' in left: + easyocr_paragraph = right else: if not image_text: @@ -1066,7 +1069,11 @@ def image_search(step_data_set): PIL.ImageGrab.grab().save("sample.png") reader = easyocr.Reader([language]) - output = reader.readtext("sample.png",paragraph=False) + if easyocr_paragraph == 'true': + output = reader.readtext("sample.png",paragraph=True) + else: + output = reader.readtext("sample.png", paragraph=False) + item = [] count = 0 for text in output: @@ -1271,6 +1278,7 @@ def Get_Element(data_set, wait_time=Shared_Resources.Get_Shared_Variables("eleme image_conf = 0.8 language = 'en' text_screenshot = '' + easyocr_paragraph = '' for left, mid, right in data_set: left = left.strip().lower() mid = mid.strip().lower() @@ -1289,6 +1297,8 @@ def Get_Element(data_set, wait_time=Shared_Resources.Get_Shared_Variables("eleme image_conf = right elif 't_screenshot' in left: text_screenshot = right + elif 'easyocr_paragraph' in left: + easyocr_paragraph = right @@ -1309,6 +1319,8 @@ def Get_Element(data_set, wait_time=Shared_Resources.Get_Shared_Variables("eleme element_image.append(("method_image", "element parameter", str(method_image))) element_image.append(("image_conf", "element parameter", str(image_conf))) element_image.append(("t_screenshot", "element parameter", str(text_screenshot))) + element_image.append(("easyocr_paragraph", "element parameter", str(easyocr_paragraph))) + result = image_search(element_image) From fb2cad0e86828b278532eb20399cb82f9a58ab0a Mon Sep 17 00:00:00 2001 From: Mohammed Sazid-Al-Rashid Date: Sat, 10 Dec 2022 02:46:25 +0600 Subject: [PATCH 10/16] Changed default confidance value for image ocr from 0.8 to 0.9 --- .../Built_In_Automation/Desktop/Windows/BuiltInFunctions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py b/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py index 64f5a1f7e..a48917910 100644 --- a/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py +++ b/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py @@ -921,7 +921,7 @@ def image_search(step_data_set): colour_state = "" method_image = "" language = "" - image_conf = 0.8 + image_conf = 0.9 text_screenshot = '' easyocr_paragraph = '' @@ -1275,7 +1275,7 @@ def Get_Element(data_set, wait_time=Shared_Resources.Get_Shared_Variables("eleme top_height = 1 colour_state = 'black_white' method_image = 'method_1' - image_conf = 0.8 + image_conf = 0.9 language = 'en' text_screenshot = '' easyocr_paragraph = '' From 5c2f59a373d229eff992cec800f1957a03301cd7 Mon Sep 17 00:00:00 2001 From: Mohammed Sazid-Al-Rashid Date: Tue, 13 Dec 2022 19:17:53 +0600 Subject: [PATCH 11/16] the screenshot part is now introduced inside the node itself. user won't have to use them seperately. --- .../Desktop/Windows/BuiltInFunctions.py | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py b/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py index a48917910..d91e7db5a 100644 --- a/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py +++ b/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py @@ -1076,6 +1076,7 @@ def image_search(step_data_set): item = [] count = 0 + crop_counter = 0 for text in output: def seq(a, b): c = SequenceMatcher(a=a, b=b).ratio() @@ -1109,7 +1110,25 @@ def seq(a, b): if text_screenshot !='': img = cv2.imread("sample.png") cropping = img[y_min:y_max , x_min:x_max] - cv2.imwrite(text_screenshot, cropping) + cv2.imwrite('cropped_image.png', cropping) + from PIL import Image + + image = Image.open('cropped_image.png') + gray_image = image.convert('L') + output_2 = pytesseract.image_to_string(gray_image) + print(output_2) + + if output_2 in image_text or image_text in output_2: + element = x_min, y_min, x_max - x_min, y_max - y_min + crop_counter = 1 + return _Element(element) + else: + CommonUtil.ExecLog(sModuleInfo, 'Could not find text "%s"' % image_text, 3) + return "zeuz_failed" + + + + else: pass # @@ -1118,8 +1137,11 @@ def seq(a, b): # # plt.imshow(image1) # plt.show() - element = x_min,y_min,x_max-x_min,y_max-y_min - return _Element(element) + if crop_counter == 0: + element = x_min,y_min,x_max-x_min,y_max-y_min + return _Element(element) + else: + pass else: # Scale image if required From 5985460bce65bc3cb801cdbe1088680940db67bc Mon Sep 17 00:00:00 2001 From: Mohammed Sazid-Al-Rashid Date: Wed, 14 Dec 2022 19:08:55 +0600 Subject: [PATCH 12/16] Added --- .../Desktop/Windows/BuiltInFunctions.py | 123 +++++++++++++++++- 1 file changed, 120 insertions(+), 3 deletions(-) diff --git a/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py b/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py index d91e7db5a..a07ab321c 100644 --- a/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py +++ b/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py @@ -1178,6 +1178,114 @@ def seq(a, b): except: return CommonUtil.Exception_Handler(sys.exc_info()) +@logger +def new_image_text(step_data_set): + sModuleInfo = inspect.currentframe().f_code.co_name + " : " + MODULE_NAME + + import easyocr + import numpy as np + import cv2 + from pytesseract import pytesseract + import matplotlib.pyplot as plt + from difflib import SequenceMatcher + + pytesseract.tesseract_cmd = os.environ["PROGRAMFILES"] + r"\Tesseract-OCR\tesseract.exe" + + + file_name = "" + resolution = "" + idx = 0 + confidence = 0.85 + parent_dataset = [] + image_text = "" + left_width = "" + top_height = "" + colour_state = "" + method_image = "" + language = "" + image_conf = 0.9 + text_screenshot = '' + easyocr_paragraph = '' + + + + for left, mid, right in step_data_set: + left = left.strip().lower() + mid = mid.strip().lower() + if mid == "element parameter": + if "index" in left: + idx = int(right.strip()) + elif "ntext" in left: + image_text = right + elif 'language' in left: + language = right + elif 'image_conf' in left: + image_conf = float(right) + elif 't_screenshot' in left: + text_screenshot = right + elif 'easyocr_paragraph' in left: + easyocr_paragraph = right + + PIL.ImageGrab.grab().save("sample.png") + reader = easyocr.Reader([language]) + if easyocr_paragraph == 'true': + output = reader.readtext("sample.png", paragraph=True) + else: + output = reader.readtext("sample.png", paragraph=False) + + item = [] + count = 0 + crop_counter = 0 + for text in output: + def seq(a, b): + c = SequenceMatcher(a=a, b=b).ratio() + print(c) + if c > 0.8: + return c + + else: + return .00004 + + rslt = seq(image_text, text[1]) + if rslt >= float(image_conf): + # if image_text in text[1]: + item.append([text]) + print(text) + CommonUtil.ExecLog(sModuleInfo, "Found %s text. Returning element of index %s" % (image_text, count), 1) + count = count + 1 + else: + print(text) + continue + + if item == []: + CommonUtil.ExecLog(sModuleInfo, 'Could not find text "%s"' % image_text, 3) + return "zeuz_failed" + cord = np.array(item[idx][0]) + cord1 = cord.tolist() + # + x_min, y_min = [min(cord_val) for cord_val in zip(*cord1[0])] + x_max, y_max = [max(cord_val) for cord_val in zip(*cord1[0])] + + if text_screenshot != '': + img = cv2.imread("sample.png") + cropping = img[y_min:y_max, x_min:x_max] + cv2.imwrite('cropped_image.png', cropping) + from PIL import Image + + image = Image.open('cropped_image.png') + gray_image = image.convert('L') + output_2 = pytesseract.image_to_string(gray_image) + print(output_2) + + if output_2 in image_text or image_text in output_2: + element = x_min, y_min, x_max - x_min, y_max - y_min + crop_counter = 1 + return _Element(element) + else: + CommonUtil.ExecLog(sModuleInfo, 'Could not find text "%s"' % image_text, 3) + return "zeuz_failed" + + def _scale_image(file_name, size_w, size_h): """ This function calculates ratio and scales an image for comparison by _pyautogui() """ @@ -1246,6 +1354,8 @@ def Get_Element(data_set, wait_time=Shared_Resources.Get_Shared_Variables("eleme element_image.append((left, mid, right)) elif "imagetext" in left: element_image.append((left, mid, right)) + elif "ntext" in left: + element_image.append((left, mid, right)) elif mid == "parent parameter": @@ -1301,6 +1411,7 @@ def Get_Element(data_set, wait_time=Shared_Resources.Get_Shared_Variables("eleme language = 'en' text_screenshot = '' easyocr_paragraph = '' + for left, mid, right in data_set: left = left.strip().lower() mid = mid.strip().lower() @@ -1325,6 +1436,7 @@ def Get_Element(data_set, wait_time=Shared_Resources.Get_Shared_Variables("eleme + _get_main_window(window_name) for i in (("name", parent_name), ("class", parent_class), ("automationid", parent_automation), ("control", parent_control), ("path", parent_path), ("window", window_name), ("index", parent_index)): if i[1]: @@ -1344,9 +1456,12 @@ def Get_Element(data_set, wait_time=Shared_Resources.Get_Shared_Variables("eleme element_image.append(("easyocr_paragraph", "element parameter", str(easyocr_paragraph))) - - result = image_search(element_image) - return result + if 'ntext' in element_image[0][0]: + result = new_image_text(element_image) + return result + else: + result = image_search(element_image) + return result if element_path: all_elements = Element_path_search(window_name, element_path) if all_elements == "zeuz_failed" and time.time() < s + wait_time: @@ -2277,3 +2392,5 @@ def save_attribute_values_in_list(data_set): return Shared_Resources.Set_Shared_Variables(variable_name, []) except Exception: return CommonUtil.Exception_Handler(sys.exc_info()) + + From d9d33f5c3f77c3bf46ea84820b87fb00c6cb7472 Mon Sep 17 00:00:00 2001 From: Mohammed Sazid-Al-Rashid Date: Thu, 15 Dec 2022 02:02:19 +0600 Subject: [PATCH 13/16] seperated and created a new image text function , because older one cause problem when I mixed it up with web automation. --- .../Desktop/Windows/BuiltInFunctions.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py b/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py index a07ab321c..bc6fc7a93 100644 --- a/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py +++ b/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py @@ -1125,18 +1125,8 @@ def seq(a, b): else: CommonUtil.ExecLog(sModuleInfo, 'Could not find text "%s"' % image_text, 3) return "zeuz_failed" - - - - else: pass - # - # cv2.rectangle(img, (x_min, y_min), (x_max, y_max), (0, 0, 255), 2) - # image1 = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) - # - # plt.imshow(image1) - # plt.show() if crop_counter == 0: element = x_min,y_min,x_max-x_min,y_max-y_min return _Element(element) @@ -1284,6 +1274,10 @@ def seq(a, b): else: CommonUtil.ExecLog(sModuleInfo, 'Could not find text "%s"' % image_text, 3) return "zeuz_failed" + else: + element = x_min, y_min, x_max - x_min, y_max - y_min + return _Element(element) + From e0b5c33daf5caeb92527e93dd1895a73f175adad Mon Sep 17 00:00:00 2001 From: Nadim Al Rashid Date: Mon, 19 Dec 2022 02:19:09 +0600 Subject: [PATCH 14/16] seperated and created a new image text function , because older one cause problem when I mixed it up with web automation. --- .../Desktop/Windows/BuiltInFunctions.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py b/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py index a07ab321c..bc6fc7a93 100644 --- a/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py +++ b/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py @@ -1125,18 +1125,8 @@ def seq(a, b): else: CommonUtil.ExecLog(sModuleInfo, 'Could not find text "%s"' % image_text, 3) return "zeuz_failed" - - - - else: pass - # - # cv2.rectangle(img, (x_min, y_min), (x_max, y_max), (0, 0, 255), 2) - # image1 = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) - # - # plt.imshow(image1) - # plt.show() if crop_counter == 0: element = x_min,y_min,x_max-x_min,y_max-y_min return _Element(element) @@ -1284,6 +1274,10 @@ def seq(a, b): else: CommonUtil.ExecLog(sModuleInfo, 'Could not find text "%s"' % image_text, 3) return "zeuz_failed" + else: + element = x_min, y_min, x_max - x_min, y_max - y_min + return _Element(element) + From 5492d79029f9bcb830d80c893ed8a6d93f7c0ede Mon Sep 17 00:00:00 2001 From: Nadim Al Rashid Date: Wed, 21 Dec 2022 19:53:04 +0600 Subject: [PATCH 15/16] Using 'image_conf' was executing previous action..so turned it to 't_conf' --- .../Desktop/Windows/BuiltInFunctions.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py b/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py index bc6fc7a93..816dda946 100644 --- a/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py +++ b/Framework/Built_In_Automation/Desktop/Windows/BuiltInFunctions.py @@ -921,7 +921,7 @@ def image_search(step_data_set): colour_state = "" method_image = "" language = "" - image_conf = 0.9 + t_conf = 0.9 text_screenshot = '' easyocr_paragraph = '' @@ -949,8 +949,8 @@ def image_search(step_data_set): language = right elif 'method_image' in left: method_image = right - elif 'image_conf' in left: - image_conf = float(right) + elif 't_conf' in left: + t_conf = float(right) elif 't_screenshot' in left: text_screenshot = right elif 'easyocr_paragraph' in left: @@ -1088,7 +1088,7 @@ def seq(a, b): return .00004 rslt = seq(image_text, text[1]) - if rslt >= float(image_conf): + if rslt >= float(t_conf): # if image_text in text[1]: item.append([text]) print(text) @@ -1193,7 +1193,7 @@ def new_image_text(step_data_set): colour_state = "" method_image = "" language = "" - image_conf = 0.9 + t_conf = 0.9 text_screenshot = '' easyocr_paragraph = '' @@ -1209,8 +1209,8 @@ def new_image_text(step_data_set): image_text = right elif 'language' in left: language = right - elif 'image_conf' in left: - image_conf = float(right) + elif 't_conf' in left: + t_conf = float(right) elif 't_screenshot' in left: text_screenshot = right elif 'easyocr_paragraph' in left: @@ -1237,7 +1237,7 @@ def seq(a, b): return .00004 rslt = seq(image_text, text[1]) - if rslt >= float(image_conf): + if rslt >= float(t_conf): # if image_text in text[1]: item.append([text]) print(text) @@ -1401,7 +1401,7 @@ def Get_Element(data_set, wait_time=Shared_Resources.Get_Shared_Variables("eleme top_height = 1 colour_state = 'black_white' method_image = 'method_1' - image_conf = 0.9 + t_conf = 0.9 language = 'en' text_screenshot = '' easyocr_paragraph = '' @@ -1420,8 +1420,8 @@ def Get_Element(data_set, wait_time=Shared_Resources.Get_Shared_Variables("eleme language = right elif 'method_image' in left: method_image = right - elif 'image_conf' in left: - image_conf = right + elif 't_conf' in left: + t_conf = right elif 't_screenshot' in left: text_screenshot = right elif 'easyocr_paragraph' in left: @@ -1445,7 +1445,7 @@ def Get_Element(data_set, wait_time=Shared_Resources.Get_Shared_Variables("eleme element_image.append(("colour_state", "element parameter", str(colour_state))) element_image.append(("language", "element parameter", str(language))) element_image.append(("method_image", "element parameter", str(method_image))) - element_image.append(("image_conf", "element parameter", str(image_conf))) + element_image.append(("t_conf", "element parameter", str(t_conf))) element_image.append(("t_screenshot", "element parameter", str(text_screenshot))) element_image.append(("easyocr_paragraph", "element parameter", str(easyocr_paragraph))) From 2aaec92f709548c64213f8f567348cb875322ae7 Mon Sep 17 00:00:00 2001 From: Nadim Al Rashid Date: Thu, 26 Jan 2023 02:25:26 +0600 Subject: [PATCH 16/16] Fix on image check for element action --- .../Desktop/CrossPlatform/BuiltInFunctions.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Framework/Built_In_Automation/Desktop/CrossPlatform/BuiltInFunctions.py b/Framework/Built_In_Automation/Desktop/CrossPlatform/BuiltInFunctions.py index 13df5293d..005bc91fb 100644 --- a/Framework/Built_In_Automation/Desktop/CrossPlatform/BuiltInFunctions.py +++ b/Framework/Built_In_Automation/Desktop/CrossPlatform/BuiltInFunctions.py @@ -11,6 +11,8 @@ # Modules # # # ######################### +from Framework.Built_In_Automation.Database.BuiltInFunctions import sr + try: import pyautogui as gui except: @@ -641,10 +643,13 @@ def check_for_element(data_set): # Parse data set try: + variable_name = None file_name = "" for row in data_set: if row[1] == "element parameter": file_name = row[2] + elif "action" in row[1]: + variable_name = row[2].strip() if file_name == "": CommonUtil.ExecLog( @@ -671,9 +676,13 @@ def check_for_element(data_set): if element in failed_tag_list: CommonUtil.ExecLog(sModuleInfo, "Element not found", 3) + x = 'False' + sr.Set_Shared_Variables(variable_name, x) return "zeuz_failed" else: CommonUtil.ExecLog(sModuleInfo, "Found element", 1) + x = 'True' + sr.Set_Shared_Variables(variable_name, x) return "passed" except Exception: