0
0
mirror of https://github.com/ok-oldking/ok-wuthering-waves.git synced 2025-04-24 08:25:16 +00:00

优化声骸拾取

优化声骸拾取

优化声骸拾取

优化声骸拾取

优化声骸拾取
This commit is contained in:
firedcto@gmail.com 2025-04-16 02:17:13 +08:00
parent 55500510a9
commit 35c6b30f5e
14 changed files with 11237 additions and 59 deletions

10791
assets/echo_model/best.xml Normal file

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,21 @@
description: Ultralytics YOLO11s model trained on split_dataset\data.yaml
author: Ultralytics
date: '2025-04-16T09:53:39.504253'
version: 8.3.108
license: AGPL-3.0 License (https://ultralytics.com/license)
docs: https://docs.ultralytics.com
stride: 32
task: detect
batch: 1
imgsz:
- 640
- 640
names:
0: echo
args:
batch: 1
fraction: 1.0
half: false
int8: false
dynamic: false
nms: false

271
src/OpenVinoYolo8Detect.py Normal file
View File

@ -0,0 +1,271 @@
import os
import random
import time
from typing import Tuple
# import onnxruntime as ort # Removed onnxruntime
from openvino import Core # Added OpenVINO Core
import cv2
import numpy as np
from ok import Logger, Box, sort_boxes
logger = Logger.get_logger(__name__)
class OpenVinoYolo8Detect: # Renamed class
def __init__(self, weights='echo.onnx', model_h=640, model_w=640, iou_thres=0.45):
"""
yolov OpenVINO inference
dic_labels: {0: 'person', 1: 'bicycle'}
"""
self.dic_labels = {0: 'echo'}
self.weights = weights
self.model_size = (model_w, model_h)
self.iou_threshold = iou_thres
self.openfile_name_model = weights
# --- OpenVINO Initialization ---
self.core = Core()
# self.core.set_property("CPU", {"INFERENCE_NUM_THREADS": str(1)})
device = "CPU" # Default device, tries GPU then CPU etc.
try:
logger.info(f"Compiling OpenVINO model for {device}...")
# Read and compile the ONNX model directly
model = self.core.read_model(model=self.openfile_name_model)
self.compiled_model = self.core.compile_model(model=model, device_name=device,
config={"PERFORMANCE_HINT": "LATENCY"},)
# Get input/output names (usually one input, one output for YOLOv5)
self.input_layer = self.compiled_model.input(0)
self.output_layer = self.compiled_model.output(0)
self.input_width = self.input_layer.shape[2]
self.input_height = self.input_layer.shape[3]
logger.info(f"OpenVINO model compiled successfully for {self.compiled_model} {self.input_width}x{self.input_height}.")
except Exception as e:
logger.error(f"Error initializing OpenVINO: {e}")
raise RuntimeError("Could not initialize OpenVINO model") from e
# --- End OpenVINO Initialization ---
def letterbox(self, img: np.ndarray, new_shape: Tuple[int, int] = (640, 640)) -> Tuple[np.ndarray, Tuple[int, int]]:
"""
Resize and reshape images while maintaining aspect ratio by adding padding.
Args:
img (np.ndarray): Input image to be resized.
new_shape (Tuple[int, int]): Target shape (height, width) for the image.
Returns:
(np.ndarray): Resized and padded image.
(Tuple[int, int]): Padding values (top, left) applied to the image.
"""
shape = img.shape[:2] # current shape [height, width]
# Scale ratio (new / old)
r = min(new_shape[0] / shape[0], new_shape[1] / shape[1])
# Compute padding
new_unpad = int(round(shape[1] * r)), int(round(shape[0] * r))
dw, dh = (new_shape[1] - new_unpad[0]) / 2, (new_shape[0] - new_unpad[1]) / 2 # wh padding
if shape[::-1] != new_unpad: # resize
img = cv2.resize(img, new_unpad, interpolation=cv2.INTER_LINEAR)
top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1))
left, right = int(round(dw - 0.1)), int(round(dw + 0.1))
img = cv2.copyMakeBorder(img, top, bottom, left, right, cv2.BORDER_CONSTANT, value=(114, 114, 114))
return img, (top, left)
def _preprocess(self, img):
"""图像预处理(保持宽高比的缩放填充) - unchanged"""
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
img, pad = self.letterbox(img, (self.input_width, self.input_height))
# Normalize the image data by dividing it by 255.0
image_data = np.array(img) / 255.0
# Transpose the image to have the channel dimension as the first dimension
image_data = np.transpose(image_data, (2, 0, 1)) # Channel first
# Expand the dimensions of the image data to match the expected input shape
image_data = np.expand_dims(image_data, axis=0).astype(np.float32)
# Return the preprocessed image data
return image_data, pad
def _postprocess(self, outputs, padding, orig_shape, confidence_threshold, label):
"""
Perform post-processing on the model's output to extract and visualize detections.
This method processes the raw model output to extract bounding boxes, scores, and class IDs.
It applies non-maximum suppression to filter overlapping detections and draws the results on the input image.
Args:
input_image (np.ndarray): The input image.
output (List[np.ndarray]): The output arrays from the model.
pad (Tuple[int, int]): Padding values (top, left) used during letterboxing.
Returns:
(np.ndarray): The input image with detections drawn on it.
"""
# Transpose and squeeze the output to match the expected shape
outputs = np.transpose(np.squeeze(outputs[0]))
# Get the number of rows in the outputs array
rows = outputs.shape[0]
# Lists to store the bounding boxes, scores, and class IDs of the detections
boxes = []
scores = []
class_ids = []
# Calculate the scaling factors for the bounding box coordinates
gain = min(orig_shape[0] / orig_shape[0], self.input_width / orig_shape[1])
outputs[:, 0] -= padding[1]
outputs[:, 1] -= padding[0]
# Iterate over each row in the outputs array
for i in range(rows):
# Extract the class scores from the current row
classes_scores = outputs[i][4:]
# Find the maximum score among the class scores
max_score = np.amax(classes_scores)
class_id = np.argmax(classes_scores)
# If the maximum score is above the confidence threshold
if max_score >= confidence_threshold and (label==-1 or label==class_id):
# Get the class ID with the highest score
# Extract the bounding box coordinates from the current row
x, y, w, h = outputs[i][0], outputs[i][1], outputs[i][2], outputs[i][3]
# Calculate the scaled coordinates of the bounding box
left = int((x - w / 2) / gain)
top = int((y - h / 2) / gain)
width = int(w / gain)
height = int(h / gain)
# Add the class ID, score, and box coordinates to the respective lists
class_ids.append(class_id)
scores.append(max_score)
boxes.append([left, top, width, height])
# Apply non-maximum suppression to filter out overlapping bounding boxes
indices = cv2.dnn.NMSBoxes(boxes, scores, confidence_threshold, self.iou_threshold)
# Iterate over the selected indices after non-maximum suppression
results = []
for i in indices:
# Get the box, score, and class ID corresponding to the index
box = boxes[i]
box_obj = Box(box[0], box[1], box[2], box[3]) # Use Box class if available
box_obj.name = self.dic_labels.get(int(class_ids[i]), 'unknown')
box_obj.confidence = scores[i]
results.append(box_obj)
# Draw the detection on the input image
# self.draw_detections(input_image, box, score, class_id)
return results
# 推理
def detect(self, image, threshold=0.5, label=-1):
'''
预测
'''
try:
h, w = image.shape[:2]
img_data, pad = self._preprocess(image)
# input_tensor = np.expand_dims(img_data, axis=0) # Add batch dimension
# --- OpenVINO Inference ---
# Input is a dictionary {input_layer_name: data}
# Output is a dictionary {output_layer_name: data}
results = self.compiled_model({self.input_layer: img_data})
# Extract the output tensor using the output layer obtained during init
outputs = results[self.output_layer]
# --- End OpenVINO Inference ---
boxes = self._postprocess(outputs, pad, (h, w), threshold, label)
return sort_boxes(boxes)
except Exception as e:
logger.error(f'OpenVINO yolo detect error:', e) # Added exc_info
return []
# --- Main execution part needs to be updated to use the new class ---
if __name__ == '__main__':
# Ensure ok module and Box class are available, or provide stubs
class MockLogger:
def get_logger(self, name): return self
def debug(self, msg, *args): print(f"DEBUG: {msg}")
def info(self, msg, *args): print(f"INFO: {msg}")
def warning(self, msg, *args): print(f"WARN: {msg}")
def error(self, msg, *args, **kwargs): print(f"ERROR: {msg}")
class MockOk:
class og: use_dml = False # Simulate ok.og.use_dml if needed
Logger = MockLogger()
if 'ok' not in globals(): # Define stubs if 'ok' module is not present
ok = MockOk()
Logger = ok.Logger
class Box:
def __init__(self, x, y, w, h):
self.x, self.y, self.w, self.h = x, y, w, h
self.name = "unknown"
self.confidence = 0.0
def __repr__(self):
return f"Box(x={self.x}, y={self.y}, w={self.w}, h={self.h}, name='{self.name}', conf={self.confidence:.2f})"
image_path = "tests/images/echo.png"
weights = "assets/echo_model/best.xml" # OpenVINO reads ONNX directly
model_h = 640
model_w = 640
# NOTE: You need a real ONNX model at the 'weights' path for inference to work.
if not os.path.exists(weights):
print(f"ERROR: Model file not found at {weights}. Inference will fail.")
print("Please download or place yolov5s_320.onnx in assets/yolo/")
# Create a dummy file to avoid immediate crash during init, but it won't work
# with open(weights, 'w') as f: f.write('') # This won't be a valid model
# Use the new OpenVINO class
yolov = OpenVinoYolo8Detect(weights=weights, model_w=model_w, model_h=model_h)
# Test 1
big_img = cv2.imdecode(np.fromfile(file=image_path, dtype=np.uint8), cv2.IMREAD_COLOR)
if big_img is None:
print(f"Error loading image: {image_path}")
else:
start_time = time.time()
for i in range(100):
res_loc = yolov.detect(big_img, label=0) # label=12 -> '声骸'
end_time = time.time()
print(f"Detection 1 time: {(end_time - start_time) * 1000:.2f} ms")
# Test 2
img2 = cv2.imread("tests/images/echo2.png")
if img2 is None:
print("Error loading image: tests/images/echo2.png")
else:
start_time = time.time()
for i in range(100):
res_loc = yolov.detect(img2, label=0)
end_time = time.time()
print(f"Detection 2 time: {(end_time - start_time) * 1000:.2f} ms")

View File

@ -196,7 +196,7 @@ class CombatCheck(BaseWWTask):
return self.wait_until(self.has_target, time_out=self.target_enemy_time_out,
pre_action=lambda: self.middle_click(interval=0.2))
def check_health_bar(self):
def has_health_bar(self):
if self._in_combat:
min_height = self.height_of_screen(12 / 2160)
max_height = min_height * 3
@ -221,8 +221,13 @@ class CombatCheck(BaseWWTask):
self.boss_health = self.boss_health_box.crop_frame(self.frame)
self.draw_boxes('boss_health', boxes, color='blue')
return True
return False
return self.find_boss_lv_text()
def check_health_bar(self):
if self.has_health_bar():
return True
else:
return self.find_boss_lv_text()
def find_boss_lv_text(self):
texts = self.ocr(box=self.box_of_screen(1269 / 3840, 10 / 2160, 2533 / 3840, 140 / 2160, hcenter=True),

View File

@ -5,7 +5,7 @@ import cv2
from PySide6.QtCore import Signal, QObject
from ok import Config, Logger, get_path_relative_to_exe
from src.OpenVinoYoloDetect import OpenVinoYoloDetect
from src.OpenVinoYolo8Detect import OpenVinoYolo8Detect
logger = Logger.get_logger(__name__)
@ -21,10 +21,10 @@ class Globals(QObject):
@property
def yolo_model(self):
if self._yolo_model is None:
self._yolo_model = OpenVinoYoloDetect(weights=get_path_relative_to_exe(os.path.join("assets","yolo", "yolov5s_320.onnx")))
self._yolo_model = OpenVinoYolo8Detect(weights=get_path_relative_to_exe(os.path.join("assets", "echo_model", "best.xml")))
return self._yolo_model
def yolo_detect(self, image, threshold=0.5, label=-1):
def yolo_detect(self, image, threshold=0.6, label=-1):
return self.yolo_model.detect(image, threshold=threshold, label=label)

View File

@ -20,7 +20,7 @@ class AutoCombatTask(BaseCombatTask, TriggerTask):
self.scene: WWScene | None = None
self.default_config.update({
'Auto Target': True,
'Auto Pick Echo After Combat': True,
'Auto Pick Echo After Combat': False,
})
self.config_description = {
'Auto Target': 'Turn off to enable auto combat only when manually target enemy using middle click'

View File

@ -116,7 +116,7 @@ class BaseCombatTask(CombatCheck):
self.wait_in_team_and_world(time_out=10)
self.sleep(1)
self.middle_click()
self.sleep(0.2)
self.sleep(1)
def run_in_circle_to_find_echo(self, circle_count=3):
directions = ['w', 'a', 's', 'd']

View File

@ -141,15 +141,59 @@ class BaseWWTask(BaseTask):
return None
return f
def walk_to_box(self, find_function, time_out=30, end_condition=None, y_offset=0.05, v_move_fix_time=0):
def walk_to_yolo_echo(self, time_out=15):
last_direction = None
start = time.time()
no_echo_start = 0
while time.time() - start < time_out:
self.next_frame()
if self.pick_echo():
return True
echos = self.find_echos()
if not echos:
if no_echo_start == 0:
no_echo_start = time.time()
elif time.time() - no_echo_start > 1.5:
self.log_debug(f'walk front to_echo, no echos found, break')
break
continue
else:
no_echo_start = 0
echo = echos[0]
center_distance = echo.center()[0] - self.width_of_screen(0.5)
threshold = 0.05 if not last_direction else 0.15
if abs(center_distance) < self.height_of_screen(threshold):
if echo.y + echo.height > self.height_of_screen(0.65):
next_direction = 's'
else:
next_direction = 'w'
elif center_distance > 0:
next_direction = 'd'
else:
next_direction = 'a'
last_direction = self._walk_direction(last_direction, next_direction)
self._stop_last_direction(last_direction)
def _walk_direction(self, last_direction, next_direction):
if next_direction != last_direction:
self._stop_last_direction(last_direction)
if next_direction:
self.send_key_down(next_direction)
return next_direction
def _stop_last_direction(self, last_direction):
if last_direction:
self.send_key_up(last_direction)
self.sleep(0.01)
return None
def walk_to_box(self, find_function, time_out=30, end_condition=None, y_offset=0.05):
if not find_function:
self.log_info('find_function not found, break')
return False
last_direction = None
v_fix_count = 0
original_y_offset = y_offset
start = time.time()
last_v_move = start
ended = False
last_target = None
while time.time() - start < time_out:
@ -166,27 +210,14 @@ class BaseWWTask(BaseTask):
treasure_icon = None
if treasure_icon:
last_target = treasure_icon
next_direction = None
if last_target is None:
if not end_condition:
self.log_info('find_function not found, break')
break
if 0 < v_move_fix_time < time.time() - last_v_move:
if v_fix_count < 5:
v_fix_count += 1
y_offset = original_y_offset - 0.05 * v_fix_count
else:
v_fix_count += 1
y_offset = original_y_offset + 0.05 * (v_fix_count - 4)
if next_direction is None:
next_direction = self.opposite_direction(last_direction)
self.log_info('find_function not found, change to opposite direction')
else:
x, y = last_target.center()
y = max(0, y - self.height_of_screen(y_offset))
next_direction = self.get_direction(x, y, self.width, self.height, current_direction=last_direction)
if next_direction == 'w' or next_direction == 's':
last_v_move = time.time()
if next_direction != last_direction:
if last_direction:
self.send_key_up(last_direction)
@ -405,7 +436,7 @@ class BaseWWTask(BaseTask):
result = self.executor.ocr_lib(image, use_det=True, use_cls=False, use_rec=True)
self.logger.info(f'ocr_result {result}')
def find_echo(self, threshold=0.5):
def find_echos(self, threshold=0.6):
"""
Main function to load ONNX model, perform inference, draw bounding boxes, and display the output image.
@ -417,12 +448,12 @@ class BaseWWTask(BaseTask):
list: List of dictionaries containing detection information such as class_id, class_name, confidence, etc.
"""
# Load the ONNX model
boxes = og.my_app.yolo_detect(self.frame, threshold=threshold, label=12)
ret = og.my_app.yolo_detect(self.frame, threshold=threshold, label=0)
ret = sorted(boxes, key=lambda detection: detection.x, reverse=True)
for box in ret:
box.y -= box.height * 2/3
box.y += box.height * 1/3
box.height = 1
self.draw_boxes("echo", ret)
return ret
def yolo_find_all(self, threshold=0.3):
@ -454,7 +485,7 @@ class BaseWWTask(BaseTask):
self.send_key('f')
return True
def yolo_find_echo(self, use_color=True, walk=True, turn=True):
def yolo_find_echo(self, use_color=False, turn=True):
if self.debug:
# self.draw_boxes('echo', echos)
self.screenshot('yolo_echo_start')
@ -467,14 +498,13 @@ class BaseWWTask(BaseTask):
for i in range(4):
if turn:
self.center_camera()
echos = self.find_echo()
if len(echos) > 0:
self.draw_boxes('yolo_echo', echos)
echos = self.find_echos()
max_echo_count = max(max_echo_count, len(echos))
self.log_debug(f'max_echo_count {max_echo_count}')
if echos:
self.log_info(f'yolo found echo {echos}')
return self.walk_to_box(self.find_echo, time_out=15, end_condition=self.pick_echo, v_move_fix_time=5), max_echo_count > 1
# return self.walk_to_box(self.find_echos, time_out=15, end_condition=self.pick_echo), max_echo_count > 1
return self.walk_to_yolo_echo(), max_echo_count > 1
if use_color:
color_percent = self.calculate_color_percentage(echo_color, front_box)
self.log_debug(f'pick_echo color_percent:{color_percent}')
@ -482,28 +512,26 @@ class BaseWWTask(BaseTask):
# if self.debug:
# self.screenshot('echo_color_picked')
self.log_debug(f'found color_percent {color_percent} > {color_threshold}, walk now')
return self.walk_find_echo(), max_echo_count > 1
if not turn and i==0:
#return self.walk_to_box(self.find_echos, time_out=15, end_condition=self.pick_echo), max_echo_count > 1
return self.walk_to_yolo_echo(), max_echo_count > 1
if not turn and i == 0:
return False, max_echo_count > 1
self.send_key('a', down_time=0.05)
self.sleep(0.5)
self.center_camera()
if walk:
picked = self.walk_find_echo()
return picked, max_echo_count > 1
return False, max_echo_count > 1
def center_camera(self):
self.click(0.5, 0.5, down_time=0.2, after_sleep=0.5, key='middle')
self.click(0.5, 0.5, down_time=0.2, after_sleep=1, key='middle')
def turn_direction(self, direction):
if direction != 'w':
self.send_key(direction, down_time=0.05, after_sleep=0.5)
self.center_camera()
def walk_find_echo(self, backward_time=1):
if self.walk_until_f(time_out=4, backward_time=backward_time, target_text=self.absorb_echo_text(),
def walk_find_echo(self, backward_time=1, time_out=4):
if self.walk_until_f(time_out=time_out, backward_time=backward_time, target_text=self.absorb_echo_text(),
raise_if_not_found=False): # find and pick echo
logger.debug(f'farm echo found echo move forward walk_until_f to find echo')
return True
@ -659,13 +687,13 @@ class BaseWWTask(BaseTask):
self.sleep(1)
def openF2Book(self, feature="gray_book_all_monsters"):
self.log_info('click f2 to open the book')
# self.send_key_down('alt')
# self.sleep(0.05)
# self.click_relative(0.77, 0.05)
# self.send_key_up('alt')
self.sleep(1)
self.send_key('f2')
self.log_info('click f2 to open the book')
self.send_key_down('alt')
self.sleep(0.05)
self.click_relative(0.77, 0.05)
self.send_key_up('alt')
# self.send_key('f2')
gray_book_boss = self.wait_book(feature)
if not gray_book_boss:
self.log_error("can't find gray_book_boss, make sure f2 is the hotkey for book", notify=True)

View File

@ -157,7 +157,7 @@ class BigMap(WWOneTimeTask, BaseCombatTask):
self.draw_boxes('me', in_big_map.scale(0.1), color='blue')
# self.screenshot('box_minimap', frame=frame, show_box=True)
# self.screenshot('template_minimap', frame=mat)
self.my_box = in_big_map.scale(1.25)
self.my_box = in_big_map.scale(1.3)
return in_big_map
def create_circle_mask_with_hole(image):
@ -201,6 +201,7 @@ class FarmMapTask(BigMap):
self.stuck_keys = [['space', 0.02], ['a',2], ['d',2], ['t', 0.02]]
self.stuck_index = 0
self.last_distance = 0
self._has_health_bar = False
@property
def star_move_distance_threshold(self):
@ -215,6 +216,8 @@ class FarmMapTask(BigMap):
def on_combat_check(self):
self.incr_drop(self.pick_f())
self.find_my_location()
if not self._has_health_bar:
self._has_health_bar = self.has_health_bar()
return True
def go_to_star(self):
@ -225,6 +228,7 @@ class FarmMapTask(BigMap):
while True:
self.sleep(0.01)
self.middle_click(interval=1, after_sleep=0.2)
self._has_health_bar = False
if self.in_combat():
self.sleep(2)
if current_direction is not None:
@ -234,14 +238,13 @@ class FarmMapTask(BigMap):
start = time.time()
self.combat_once()
duration = time.time() - start
if duration > 8:
self.my_box = self.my_box.scale(1.1)
while True:
dropped, has_more = self.yolo_find_echo(use_color=False, walk=False)
self.incr_drop(dropped)
self.sleep(0.5)
if not dropped or not has_more:
break
while True:
dropped, has_more = self.yolo_find_echo(use_color=False, turn=duration > 15 or self._has_health_bar)
self.incr_drop(dropped)
self.sleep(0.5)
if not dropped or not has_more:
break
star, distance, angle = self.find_direction_angle()
# self.draw_boxes('next_star', star, color='green')
if not star:

BIN
src/tests/images/echo.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.7 KiB

29
tests/TestCon.py Normal file
View File

@ -0,0 +1,29 @@
import unittest
from config import config
from ok.test.TaskTestCase import TaskTestCase
from src.task.AutoCombatTask import AutoCombatTask
config['debug'] = True
class TestCombatCheck(TaskTestCase):
task_class = AutoCombatTask
config = config
def test_con_full(self):
self.task.do_reset_to_false()
self.set_image('tests/images/con_full.png')
# in_combat = self.task.in_combat()
# self.assertTrue(in_combat)
#
# self.task.do_reset_to_false()
# self.set_image('tests/images/con_full.png')
# in_combat = self.task.in_combat()
# self.assertTrue(in_combat)
self.task.load_chars()
con_full = self.task.get_current_char().is_con_full()
self.assertTrue(con_full)
if __name__ == '__main__':
unittest.main()

30
tests/TestEcho.py Normal file
View File

@ -0,0 +1,30 @@
import time
import unittest
from config import config
from ok.test.TaskTestCase import TaskTestCase
from src.task.AutoCombatTask import AutoCombatTask
from src.task.DailyTask import DailyTask
config['debug'] = True
class TestEcho(TaskTestCase):
task_class = DailyTask
config = config
def test_find_echo(self):
self.set_image('tests/images/echo.png')
echos = self.task.find_echo()
self.assertEqual(1, len(echos))
time.sleep(1)
self.task.screenshot('echo1', show_box=True)
self.set_image('tests/images/echo2.png')
echos = self.task.find_echo()
time.sleep(1)
self.task.screenshot('echo2', show_box=True)
self.assertEqual(1, len(echos))
time.sleep(1)
if __name__ == '__main__':
unittest.main()

BIN
tests/images/con_full.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.7 MiB