wip

ai/compute.py

@@ -45,14 +45,15 @@ def perform_ocr(screenshot: bytes) -> list[dict]:
     for i in range(n):
         text = data['text'][i]
         if text and text.strip():
+            # Fix the center-point calculation (add first, then divide)
             results.append({
                 'text': text,
-                'x': data['width'][i] + data['left'][i] // 2,  # center x position
+                'x': data['left'][i] + data['width'][i] // 2,
-                'y': data['height'][i] + data['top'][i] // 2  # center y position
+                'y': data['top'][i] + data['height'][i] // 2
             })
     
     # check if debug is enabled
-    if logging.getLogger().isEnabledFor(logging.DEBUG):
+    if logger.isEnabledFor(logging.DEBUG):
         # take screenshot + put blue circle with x, y on screenshot for each component
         screenshot_with_circles = Image.open(io.BytesIO(screenshot))
         draw = ImageDraw.Draw(screenshot_with_circles)
@@ -66,9 +67,9 @@ def perform_ocr(screenshot: bytes) -> list[dict]:
             # vertical line
             draw.line((x, y - size, x, y + size), fill=color, width=width)
         screenshot_with_circles.save("screenshot_with_circles.png", format='PNG')
-    # save in a file
+        # save in a file
-    logger.debug("Debug, saving ocr results screenshot with circles")
+        logger.debug("Debug, saving ocr results screenshot with circles")
-    screenshot_with_circles.save("ocr_results.png", format='PNG')
+        screenshot_with_circles.save("ocr_results.png", format='PNG')
     return results
 
 def screenshot_to_base64(screenshot: bytes) -> str:
@@ -109,7 +110,7 @@ def wait(duration: float) -> None:
 def search_pc(query: str) -> None:
     """Presses the Windows key."""
     pyautogui.hotkey('win')
-    wait(2)
+    wait(4)
     press_keyboard(KeyboardInput(text=query))
 
 def reprompt(nextsteps: str, processor) -> None:

ai/processor.py

@@ -111,12 +111,13 @@ class AIProcessor:
                     try:
                         ocr_results = ai.compute.perform_ocr(img_bytes)
                     except Exception as e:
+                        traceback.print_exc()
                         logger.debug("OCR failed: %s", e)
 
                     self.session.messages.append(
                         aic.Message(
                             role="assistant",
-                            content=str(tool_calls),
+                            content=str(((tc.function.name, tc.function.arguments) for tc in tool_calls)),
                         )
                     )
 
@@ -134,7 +135,7 @@ class AIProcessor:
             output_text: str = response.choices[0].message.content  # type: ignore
             outputs.append(output_text)
             self.session.messages.append(
-                aic.Message(role="assistant", content=output_text)
+                aic.Message(role="assistant", content="Executed: " + (str(*outputs)))
             )
 
             return [*outputs]

objects/aic.py

@@ -31,7 +31,7 @@ FUNCTIONS = [
                     "click_type": {
                         "type": "string",
                         "enum": ["left", "double_left", "middle", "right"],
-                        "description": "The type of mouse click to perform. `double_left` is a double click."
+                        "description": "The type of mouse click to perform. `double_left` is a double click, used to open apps or files."
                     }
                 },
                 "required": ["click_type", "x", "y"],