From 17c4129f1f37ea0d107e5d1ef522d8d29214181b Mon Sep 17 00:00:00 2001 From: Timm Friebe Date: Sun, 27 Oct 2024 11:09:04 +0100 Subject: [PATCH 1/8] Implement realtime API --- .../com/openai/realtime/RealtimeApi.class.php | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 src/main/php/com/openai/realtime/RealtimeApi.class.php diff --git a/src/main/php/com/openai/realtime/RealtimeApi.class.php b/src/main/php/com/openai/realtime/RealtimeApi.class.php new file mode 100644 index 0000000..f737b27 --- /dev/null +++ b/src/main/php/com/openai/realtime/RealtimeApi.class.php @@ -0,0 +1,83 @@ +ws= new WebSocket($uri); + $this->headers= ['api-key' => $uri->user()]; + $this->marshalling= new Marshalling(); + } + + /** @param ?util.log.LogCategory $cat */ + public function setTrace($cat) { + $this->cat= $cat; + } + + /** Opens the underlying websocket, optionally passing headers */ + public function connect(array $headers= []): self { + $headers+= $this->headers; + $this->cat && $this->cat->info($this->ws->socket(), $this->ws->path(), $headers); + $this->ws->connect($headers); + return $this; + } + + /** Returns whether the underlying websocket is connected */ + public function connected(): bool { + return $this->ws->connected(); + } + + /** Closes the underlying websocket */ + public function close(): void { + $this->ws->close(); + } + + /** + * Sends a given payload. Doesn't wait for a response + * + * @param var $payload + * @return void + */ + public function send($payload): void { + $json= Json::of($this->marshalling->marshal($payload)); + $this->cat && $this->cat->debug('>>>', $json); + $this->ws->send($json); + } + + /** + * Receives an answer. Returns NULL if EOF is reached. + * + * @return var + */ + public function receive() { + $json= $this->ws->receive(); + $this->cat && $this->cat->debug('<<<', $json); + return null === $json ? null : $this->marshalling->unmarshal(Json::read($json)); + } + + /** + * Sends a given payload and returns the response to it. + * + * @param var $payload + * @return var + */ + public function transmit($payload) { + $this->send($payload); + return $this->receive(); + } + + /** Ensures socket is closed */ + public function __destruct() { + $this->ws && $this->ws->close(); + } +} \ No newline at end of file From 0ee4934cc3f46eaa74933402c48f21b97cebe3c2 Mon Sep 17 00:00:00 2001 From: Timm Friebe Date: Sun, 27 Oct 2024 12:45:44 +0100 Subject: [PATCH 2/8] Add tests for RealtimeApi implementation --- class.pth | 1 + .../com/openai/realtime/RealtimeApi.class.php | 24 ++++-- .../openai/unittest/RealtimeApiTest.class.php | 74 +++++++++++++++++++ .../openai/unittest/TestingSocket.class.php | 36 +++++++++ 4 files changed, 129 insertions(+), 6 deletions(-) create mode 100755 src/test/php/com/openai/unittest/RealtimeApiTest.class.php create mode 100755 src/test/php/com/openai/unittest/TestingSocket.class.php diff --git a/class.pth b/class.pth index d198bcb..9ff0daf 100644 --- a/class.pth +++ b/class.pth @@ -1,3 +1,4 @@ src/main/php/ src/test/php/ src/it/php/ +../websockets/src/main/php/ diff --git a/src/main/php/com/openai/realtime/RealtimeApi.class.php b/src/main/php/com/openai/realtime/RealtimeApi.class.php index f737b27..b709724 100644 --- a/src/main/php/com/openai/realtime/RealtimeApi.class.php +++ b/src/main/php/com/openai/realtime/RealtimeApi.class.php @@ -6,16 +6,28 @@ use util\log\Traceable; use websocket\WebSocket; -/** @see https://github.com/azure-samples/aoai-realtime-audio-sdk */ +/** + * OpenAI Realtime API enables you to build low-latency, multi-modal conversational + * experiences. It currently supports text and audio as both input and output, as + * well as function calling. + * + * @test com.openai.unittest.RealtimeApiTest + * @see https://platform.openai.com/docs/guides/realtime + */ class RealtimeApi implements Traceable { - private $ws, $headers; + private $ws, $headers, $marshalling; private $cat= null; - /** @param string|util.URI $endpoint */ + /** @param string|util.URI|websocket.WebSocket $endpoint */ public function __construct($endpoint) { - $uri= $endpoint instanceof URI ? $endpoint : new URI($endpoint); - $this->ws= new WebSocket($uri); - $this->headers= ['api-key' => $uri->user()]; + if ($endpoint instanceof WebSocket) { + $this->ws= $endpoint; + $this->headers= []; + } else { + $uri= $endpoint instanceof URI ? $endpoint : new URI($endpoint); + $this->ws= new WebSocket($uri); + $this->headers= ['api-key' => $uri->user()]; + } $this->marshalling= new Marshalling(); } diff --git a/src/test/php/com/openai/unittest/RealtimeApiTest.class.php b/src/test/php/com/openai/unittest/RealtimeApiTest.class.php new file mode 100755 index 0000000..291b894 --- /dev/null +++ b/src/test/php/com/openai/unittest/RealtimeApiTest.class.php @@ -0,0 +1,74 @@ +connected()); + } + + #[Test] + public function connect() { + $c= new RealtimeApi(new TestingSocket()); + $c->connect(); + + Assert::true($c->connected()); + } + + #[Test] + public function close() { + $c= new RealtimeApi(new TestingSocket()); + $c->connect(); + $c->close(); + + Assert::false($c->connected()); + } + + #[Test] + public function initial_handshake() { + $c= new RealtimeApi(new TestingSocket([ + '{"type": "session.created"}', + ])); + $c->connect(); + + Assert::equals(['type' => 'session.created'], $c->receive()); + } + + #[Test] + public function update_session() { + $c= new RealtimeApi(new TestingSocket([ + '{"type": "session.created"}', + '{"type": "session.update", "session": {"instructions": "You are TestGPT"}}', + '{"type": "session.updated"}', + ])); + $c->connect(); + $c->receive(); + $c->send(['type' => 'session.update', 'session' => ['instructions' => 'You are TestGPT']]); + + Assert::equals(['type' => 'session.updated'], $c->receive()); + } + + #[Test] + public function transmit() { + $c= new RealtimeApi(new TestingSocket([ + '{"type": "session.created"}', + '{"type": "conversation.item.create", "item": {"type": "message"}}', + '{"type": "conversation.item.created"}', + ])); + $c->connect(); + $c->receive(); + $response= $c->transmit(['type' => 'conversation.item.create', 'item' => ['type' => 'message']]); + + Assert::equals(['type' => 'conversation.item.created'], $response); + } +} \ No newline at end of file diff --git a/src/test/php/com/openai/unittest/TestingSocket.class.php b/src/test/php/com/openai/unittest/TestingSocket.class.php new file mode 100755 index 0000000..3843e3f --- /dev/null +++ b/src/test/php/com/openai/unittest/TestingSocket.class.php @@ -0,0 +1,36 @@ +messages= $messages; + } + + public function connected() { + return isset($this->connected); + } + + public function connect($headers= []) { + $this->connected= $headers; + } + + public function send($payload) { + $message= array_shift($this->messages); + if (json_decode($message, true) !== json_decode($payload, true)) { + throw new IllegalStateException('Unexpected '.$payload.', expecting '.$message); + } + } + + public function receive($timeout= null) { + return array_shift($this->messages); + } + + public function close($code= 1000, $reason = '') { + $this->connected= null; + } +} \ No newline at end of file From b601437c89f5372a5668a0089e12da34b0c626aa Mon Sep 17 00:00:00 2001 From: Timm Friebe Date: Sun, 27 Oct 2024 12:48:01 +0100 Subject: [PATCH 3/8] Remove incorrect reference --- class.pth | 1 - 1 file changed, 1 deletion(-) diff --git a/class.pth b/class.pth index 9ff0daf..d198bcb 100644 --- a/class.pth +++ b/class.pth @@ -1,4 +1,3 @@ src/main/php/ src/test/php/ src/it/php/ -../websockets/src/main/php/ From 7d5b686082e43a034194e052da46bece5c501b0c Mon Sep 17 00:00:00 2001 From: Timm Friebe Date: Sun, 27 Oct 2024 12:52:43 +0100 Subject: [PATCH 4/8] Add tests for authentication mechanisms --- .../com/openai/realtime/RealtimeApi.class.php | 8 ++------ .../openai/unittest/RealtimeApiTest.class.php | 18 +++++++++++++++++- .../openai/unittest/TestingSocket.class.php | 2 +- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/main/php/com/openai/realtime/RealtimeApi.class.php b/src/main/php/com/openai/realtime/RealtimeApi.class.php index b709724..ff849ce 100644 --- a/src/main/php/com/openai/realtime/RealtimeApi.class.php +++ b/src/main/php/com/openai/realtime/RealtimeApi.class.php @@ -15,18 +15,15 @@ * @see https://platform.openai.com/docs/guides/realtime */ class RealtimeApi implements Traceable { - private $ws, $headers, $marshalling; + private $ws, $marshalling; private $cat= null; /** @param string|util.URI|websocket.WebSocket $endpoint */ public function __construct($endpoint) { if ($endpoint instanceof WebSocket) { $this->ws= $endpoint; - $this->headers= []; } else { - $uri= $endpoint instanceof URI ? $endpoint : new URI($endpoint); - $this->ws= new WebSocket($uri); - $this->headers= ['api-key' => $uri->user()]; + $this->ws= new WebSocket((string)$endpoint); } $this->marshalling= new Marshalling(); } @@ -38,7 +35,6 @@ public function setTrace($cat) { /** Opens the underlying websocket, optionally passing headers */ public function connect(array $headers= []): self { - $headers+= $this->headers; $this->cat && $this->cat->info($this->ws->socket(), $this->ws->path(), $headers); $this->ws->connect($headers); return $this; diff --git a/src/test/php/com/openai/unittest/RealtimeApiTest.class.php b/src/test/php/com/openai/unittest/RealtimeApiTest.class.php index 291b894..b6c2f5f 100755 --- a/src/test/php/com/openai/unittest/RealtimeApiTest.class.php +++ b/src/test/php/com/openai/unittest/RealtimeApiTest.class.php @@ -1,10 +1,16 @@ 'test']]; + yield ['openai', ['Authorization' => 'Bearer test', 'OpenAI-Beta' => 'realtime=v1']]; + } + #[Test] public function can_create() { new RealtimeApi('wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview-2024-10-01'); @@ -25,6 +31,16 @@ public function connect() { Assert::true($c->connected()); } + #[Test, Values(from: 'authentications')] + public function passing_headers($kind, $headers) { + $s= new TestingSocket(); + + $c= new RealtimeApi($s); + $c->connect($headers); + + Assert::equals($headers, $s->connected); + } + #[Test] public function close() { $c= new RealtimeApi(new TestingSocket()); diff --git a/src/test/php/com/openai/unittest/TestingSocket.class.php b/src/test/php/com/openai/unittest/TestingSocket.class.php index 3843e3f..b3aef0d 100755 --- a/src/test/php/com/openai/unittest/TestingSocket.class.php +++ b/src/test/php/com/openai/unittest/TestingSocket.class.php @@ -5,7 +5,7 @@ class TestingSocket extends WebSocket { private $messages; - private $connected= null; + public $connected= null; public function __construct($messages= []) { $this->messages= $messages; From b224581a7f5e6d88d826b92529b077f2fcb98a48 Mon Sep 17 00:00:00 2001 From: Timm Friebe Date: Sun, 27 Oct 2024 12:54:52 +0100 Subject: [PATCH 5/8] Add xp-forge/websocket dependency --- composer.json | 1 + 1 file changed, 1 insertion(+) diff --git a/composer.json b/composer.json index b0cead9..021768c 100755 --- a/composer.json +++ b/composer.json @@ -11,6 +11,7 @@ "xp-framework/reflection": "^3.0 | ^2.0", "xp-forge/marshalling": "^2.0 | ^1.0", "xp-forge/rest-client": "^5.6", + "xp-forge/websocket": "^4.0", "php" : ">=7.4.0" }, "require-dev" : { From 72ec38ef41c137bdf770510b9d3c030c388aa377 Mon Sep 17 00:00:00 2001 From: Timm Friebe Date: Sun, 27 Oct 2024 12:55:56 +0100 Subject: [PATCH 6/8] Fix library name --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 021768c..63390a0 100755 --- a/composer.json +++ b/composer.json @@ -11,7 +11,7 @@ "xp-framework/reflection": "^3.0 | ^2.0", "xp-forge/marshalling": "^2.0 | ^1.0", "xp-forge/rest-client": "^5.6", - "xp-forge/websocket": "^4.0", + "xp-forge/websockets": "^4.0", "php" : ">=7.4.0" }, "require-dev" : { From 60c14a013317593726a33c2e508a3c542da865bd Mon Sep 17 00:00:00 2001 From: Timm Friebe Date: Sun, 27 Oct 2024 13:00:23 +0100 Subject: [PATCH 7/8] Fold if/else into ternary --- src/main/php/com/openai/realtime/RealtimeApi.class.php | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/php/com/openai/realtime/RealtimeApi.class.php b/src/main/php/com/openai/realtime/RealtimeApi.class.php index ff849ce..df6899c 100644 --- a/src/main/php/com/openai/realtime/RealtimeApi.class.php +++ b/src/main/php/com/openai/realtime/RealtimeApi.class.php @@ -20,11 +20,7 @@ class RealtimeApi implements Traceable { /** @param string|util.URI|websocket.WebSocket $endpoint */ public function __construct($endpoint) { - if ($endpoint instanceof WebSocket) { - $this->ws= $endpoint; - } else { - $this->ws= new WebSocket((string)$endpoint); - } + $this->ws= $endpoint instanceof WebSocket ? $endpoint : new WebSocket((string)$endpoint); $this->marshalling= new Marshalling(); } From f381478a478a96d95b2e9a16ee76e6077749f469 Mon Sep 17 00:00:00 2001 From: Timm Friebe Date: Sun, 27 Oct 2024 15:33:27 +0100 Subject: [PATCH 8/8] Receive `session.created` event once connected See https://github.com/xp-forge/openai/pull/15#issuecomment-2440043241 --- .../com/openai/realtime/RealtimeApi.class.php | 24 ++++++++++++--- .../openai/unittest/RealtimeApiTest.class.php | 30 +++++++++++-------- 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/src/main/php/com/openai/realtime/RealtimeApi.class.php b/src/main/php/com/openai/realtime/RealtimeApi.class.php index df6899c..44b6cf9 100644 --- a/src/main/php/com/openai/realtime/RealtimeApi.class.php +++ b/src/main/php/com/openai/realtime/RealtimeApi.class.php @@ -1,9 +1,10 @@ cat= $cat; } - /** Opens the underlying websocket, optionally passing headers */ - public function connect(array $headers= []): self { + /** + * Opens the underlying websocket, optionally passing headers + * + * Verifies a `session.created` event is received. This is sent by the server + * as soon as the connection is successfully established. Provides a connection- + * specific ID that may be useful for debugging or logging. + * + * @return var + * @throws lang.IllegalStateException + */ + public function connect(array $headers= []) { $this->cat && $this->cat->info($this->ws->socket(), $this->ws->path(), $headers); $this->ws->connect($headers); - return $this; + + $event= $this->receive(); + if ('session.created' === ($event['type'] ?? null)) return $event; + + $error= 'Unexpected handshake event "'.($event['type'] ?? '(null)').'"'; + $this->ws->close(4007, $error); + throw new IllegalStateException($error); } /** Returns whether the underlying websocket is connected */ diff --git a/src/test/php/com/openai/unittest/RealtimeApiTest.class.php b/src/test/php/com/openai/unittest/RealtimeApiTest.class.php index b6c2f5f..86b4355 100755 --- a/src/test/php/com/openai/unittest/RealtimeApiTest.class.php +++ b/src/test/php/com/openai/unittest/RealtimeApiTest.class.php @@ -1,9 +1,11 @@ connect(); Assert::true($c->connected()); @@ -33,7 +35,7 @@ public function connect() { #[Test, Values(from: 'authentications')] public function passing_headers($kind, $headers) { - $s= new TestingSocket(); + $s= new TestingSocket([self::SESSION_CREATED]); $c= new RealtimeApi($s); $c->connect($headers); @@ -43,7 +45,7 @@ public function passing_headers($kind, $headers) { #[Test] public function close() { - $c= new RealtimeApi(new TestingSocket()); + $c= new RealtimeApi(new TestingSocket([self::SESSION_CREATED])); $c->connect(); $c->close(); @@ -52,23 +54,26 @@ public function close() { #[Test] public function initial_handshake() { - $c= new RealtimeApi(new TestingSocket([ - '{"type": "session.created"}', - ])); - $c->connect(); + $c= new RealtimeApi(new TestingSocket([self::SESSION_CREATED])); + $session= $c->connect(); + + Assert::equals(['type' => 'session.created'], $session); + } - Assert::equals(['type' => 'session.created'], $c->receive()); + #[Test, Expect(class: IllegalStateException::class, message: 'Unexpected handshake event "error"')] + public function unexpected_handshake() { + $c= new RealtimeApi(new TestingSocket(['{"type":"error"}'])); + $c->connect(); } #[Test] public function update_session() { $c= new RealtimeApi(new TestingSocket([ - '{"type": "session.created"}', + self::SESSION_CREATED, '{"type": "session.update", "session": {"instructions": "You are TestGPT"}}', '{"type": "session.updated"}', ])); $c->connect(); - $c->receive(); $c->send(['type' => 'session.update', 'session' => ['instructions' => 'You are TestGPT']]); Assert::equals(['type' => 'session.updated'], $c->receive()); @@ -77,12 +82,11 @@ public function update_session() { #[Test] public function transmit() { $c= new RealtimeApi(new TestingSocket([ - '{"type": "session.created"}', + self::SESSION_CREATED, '{"type": "conversation.item.create", "item": {"type": "message"}}', '{"type": "conversation.item.created"}', ])); $c->connect(); - $c->receive(); $response= $c->transmit(['type' => 'conversation.item.create', 'item' => ['type' => 'message']]); Assert::equals(['type' => 'conversation.item.created'], $response);