from utils import ( create_response, show_image, pp, sanitize_message, check_blocklisted_url, ) import json from typing import Callable from typing import Protocol, List, Literal, Dict class Computer(Protocol): """Defines the 'shape' (methods/properties) our loop expects.""" @property def environment(self) -> Literal["windows", "mac", "linux", "browser"]: ... @property def dimensions(self) -> tuple[int, int]: ... def screenshot(self) -> str: ... def click(self, x: int, y: int, button: str = "left") -> None: ... def double_click(self, x: int, y: int) -> None: ... def scroll(self, x: int, y: int, scroll_x: int, scroll_y: int) -> None: ... def type(self, text: str) -> None: ... def wait(self, ms: int = 1000) -> None: ... def move(self, x: int, y: int) -> None: ... def keypress(self, keys: List[str]) -> None: ... def drag(self, path: List[Dict[str, int]]) -> None: ... def get_current_url() -> str: ... class Computer_Agent: """ A sample agent class that can be used to interact with a computer. (See simple_cua_loop.py for a simple example without an agent.) """ def __init__( self, model="computer-use-preview", computer: Computer = None, tools: list[dict] = [], acknowledge_safety_check_callback: Callable = lambda: False, ): self.model = model self.computer = computer self.tools = tools self.print_steps = True self.debug = False self.show_images = False self.acknowledge_safety_check_callback = acknowledge_safety_check_callback if computer: self.tools += [ { "type": "computer-preview", "display_width": computer.dimensions[0], "display_height": computer.dimensions[1], "environment": computer.environment, }, ] def debug_print(self, *args): if self.debug: pp(*args) def handle_item(self, item): """Handle each item; may cause a computer action + screenshot.""" if item["type"] == "message": if self.print_steps: print(item["content"][0]["text"]) if item["type"] == "function_call": name, args = item["name"], json.loads(item["arguments"]) if self.print_steps: print(f"{name}({args})") if hasattr(self.computer, name): # if function exists on computer, call it method = getattr(self.computer, name) method(**args) return [ { "type": "function_call_output", "call_id": item["call_id"], "output": "success", # hard-coded output for demo } ] if item["type"] == "computer_call": action = item["action"] action_type = action["type"] action_args = {k: v for k, v in action.items() if k != "type"} if self.print_steps: print(f"{action_type}({action_args})") method = getattr(self.computer, action_type) method(**action_args) screenshot_base64 = self.computer.screenshot() if self.show_images: show_image(screenshot_base64) # if user doesn't ack all safety checks exit with error pending_checks = item.get("pending_safety_checks", []) for check in pending_checks: message = check["message"] if not self.acknowledge_safety_check_callback(message): raise ValueError( f"Safety check failed: {message}. Cannot continue with unacknowledged safety checks." ) call_output = { "type": "computer_call_output", "call_id": item["call_id"], "acknowledged_safety_checks": pending_checks, "output": { "type": "input_image", "image_url": f"data:image/png;base64,{screenshot_base64}", }, } # additional URL safety checks for browser environments if self.computer.environment == "browser": current_url = self.computer.get_current_url() check_blocklisted_url(current_url) call_output["output"]["current_url"] = current_url return [call_output] return [] def run_full_turn( self, input_items, print_steps=True, debug=False, show_images=False ): self.print_steps = print_steps self.debug = debug self.show_images = show_images new_items = [] # keep looping until we get a final response while new_items[-1].get("role") != "assistant" if new_items else True: self.debug_print([sanitize_message(msg) for msg in input_items + new_items]) response = create_response( model=self.model, input=input_items + new_items, tools=self.tools, truncation="auto", ) self.debug_print(response) if "output" not in response and self.debug: print(response) raise ValueError("No output from model") else: new_items += response["output"] for item in response["output"]: new_items += self.handle_item(item) return new_items