words -> search -> short answer -> audio out.
*
* @package Aisentic\Api\Controllers
*/
declare( strict_types = 1 );
namespace Aisentic\Api\Controllers;
use Aisentic\Abstract\Base_Rest_Controller;
use Aisentic\Agent\Admin_Assistant_Agent;
use Aisentic\Agent\Utils\Admin_Pages;
use Aisentic\Agent\Utils\Screen_Id_To_Plugin_Mapper;
use Aisentic\Api\Repositories\Conversation_Repository;
use Aisentic\Api\Repositories\Message_Repository;
use Aisentic\Api\Models\Settings;
use Aisentic\Public\Policy;
use Aisentic\Util\Error_Handler;
use Aisentic\Util\Log;
use Aisentic\Util\Sanitization;
use Aisentic\Voice\Audio_Validator;
use Aisentic\Voice\Call_Window;
use Aisentic\Voice\Question_Kind;
use Aisentic\Voice\Speech_Chunker;
use Aisentic\Voice\Speech_To_Text;
use Aisentic\Voice\Stream_Speaker;
use Aisentic\Voice\Answer_Opening;
use Aisentic\Voice\Answer_Steps;
use Aisentic\Voice\Text_To_Speech;
use Aisentic\Voice\Turn_Timer;
use NeuronAI\Chat\Messages\Message;
use NeuronAI\Chat\Enums\MessageRole;
use NeuronAI\Chat\Messages\Stream\Chunks\TextChunk;
use NeuronAI\Chat\Messages\Stream\Chunks\ToolCallChunk;
use NeuronAI\Chat\Messages\Stream\Chunks\ToolResultChunk;
use WP_Error;
use WP_REST_Request;
defined( 'ABSPATH' ) || exit;
/**
* REST controller for the admin voice call.
*/
class Voice_Controller extends Base_Rest_Controller {
/**
* Option holding the engine the provider last answered with.
*
* @var string
*/
private const ENGINE_OPTION = 'aisentic_voice_engine';
/**
* Silence (ms) before we play a holding line.
*
* Two seconds is a normal pause between sentences. Filling it made the
* assistant sound like it talked to itself. A tool turn that is quiet for
* five seconds usually takes about twenty, so that one is worth covering.
*
* @var int
*/
private const COVER_SILENCE_AFTER_MS = 5000;
/**
* How long the browser holds the opening line before playing it.
*
* The line is sent at once but played later. If the real answer starts
* within this time, the browser drops the line. So only a slow turn hears
* it. See onSay() in useCallSession.ts.
*
* @var int
*/
private const HOLD_OPENING_LINE_MS = 700;
/**
* Times (ms) to play a "still working" line while the answer is not ready.
*
* The browser runs this clock, even while PHP waits on a slow tool or TTS.
* When the real answer starts, all lines still waiting are cancelled, so a
* fast turn hears none.
* Why so many: some tool answers take 50+ seconds. One line at 6.5 s left
* the caller in silence, so after 25 s we play one every 10 s, up to 75 s.
*
* @var int[]
*/
private const HOLD_FOLLOW_UP_LINES_MS = [ 6500, 15000, 25000, 35000, 45000, 55000, 65000, 75000 ];
/**
* How many times to ask the engine to read one of our lines properly.
*
* A bad reading is random, so a second try usually fixes it. These clips
* are kept for a week, and warm_up() runs after the call has closed, so
* extra tries cost the caller nothing.
*
* @var int
*/
private const SPEAK_ATTEMPTS = 3;
/**
* How many missing waiting lines one turn records after it ends.
*
* 24 clips at about 1.5 s each is 30 s of work. That would hold one PHP
* worker too long. Four per turn fills the set in the first few
* questions. See top_up_heard_lines().
*
* @var int
*/
private const WARM_PER_TURN = 4;
/**
* How many different waiting lines this site keeps recorded.
*
* Each one is a line the transcription model wrote on this site, so each
* site has its own list. There is a limit because each clip is stored in
* the options table, and rare lines are not worth keeping.
*
* @var int
*/
private const LINES_REMEMBERED = 40;
/**
* Where the lines this site has heard are kept.
*
* @var string
*/
private const HEARD_LINES_OPTION = 'aisentic_voice_heard_lines';
/**
* Number words, used to find figures in a spoken answer.
*
* The voice prompt asks the model to write for speech, so it spells
* numbers ("two hundred and fifty bookings"). A check for digits alone
* misses most of them.
*
* @var string
*/
private const NUMBER_WORDS = 'zero|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen|eighteen|nineteen|twenty|thirty|forty|fifty|sixty|seventy|eighty|ninety|hundred|thousand|million|billion|dozen';
/**
* Words that turn a number word into a count of something.
*
* @var string
*/
private const COUNTED_THINGS = 'dollars?|cents?|euros?|pounds?|taka|percent';
/**
* Words ending in s that are not a plural of anything.
*
* Without this list, "No one has registered" looks like a count (number
* word + word ending in s). It has no figure.
*
* @var string
*/
private const NOT_A_PLURAL = 'has|is|was|does|goes|says|this|these|those|its|yes|us|thus|plus|less|unless|across|always|perhaps|hers|theirs|whereas';
/**
* API namespace.
*
* @var string
*/
protected $namespace = 'aisentic/v1';
/**
* Milliseconds spent making audio during this turn.
*
* Kept on the object because say() is called from four places.
*
* @var int
*/
private $speech_ms = 0;
/**
* How many clips were sent during this turn.
*
* @var int
*/
private $clips = 0;
/**
* When the first sound of this turn went out, as a microtime( true ) mark.
*
* Set by the first event that carries audio, not when say() returns.
* say() returns only after the whole sentence is sent, which is seconds
* after the caller starts to hear it.
*
* @var float
*/
private $first_sound = 0.0;
/**
* When the caller last heard anything, as a microtime( true ) mark.
*
* Used to check if the caller is waiting in silence right now. Only then
* do we play a holding line. Without it, "One moment." and "Just a second,
* I'm looking that up." could play two seconds apart.
*
* @var float
*/
private $last_sound = 0.0;
/**
* Stopwatch for the turn being served right now.
*
* Kept on the object because emit() and say() need it, and they are far
* from where the turn starts.
*
* @var Turn_Timer|null
*/
private $timer = null;
/**
* A line with no clip yet, waiting to be made once the caller has gone.
*
* @var array{0:string,1:string}|null
*/
private $warm = null;
/**
* Whether the browser has a second acknowledgement waiting on its clock.
*
* @var bool
*/
private $follow_up_scheduled = false;
/**
* Whether a real sentence went out after the "still working" lines did.
*
* The browser drops all waiting lines when a real sentence arrives.
* See onSay() in useCallSession.ts.
*
* @var bool
*/
private $said_since_follow_ups = false;
/**
* The waiting line the transcription model wrote for this turn.
*
* @var string
*/
private $heard_line = '';
/**
* Whether this turn's 'done' event has been sent.
*
* @var bool
*/
private $done_sent = false;
/**
* When this call must stop, as a Unix time. 0 means no limit.
*
* Set when the turn starts. Checked again before each paid step, so a
* turn that starts just before the limit cannot run far past it.
*
* @var int
*/
private $deadline = 0;
/**
* Register the routes.
*
* @return void
*/
public function register_routes() {
register_rest_route(
$this->namespace,
'/voice/turn',
[
'methods' => 'POST',
'callback' => [ $this, 'handle_turn' ],
'permission_callback' => [ $this, 'check_voice_permissions' ],
]
);
// The same turn, sent back as it is made. The browser starts playing the
// first sentence while the model is still writing the second.
register_rest_route(
$this->namespace,
'/voice/turn/stream',
[
'methods' => 'POST',
'callback' => [ $this, 'stream_turn' ],
'permission_callback' => [ $this, 'check_voice_permissions' ],
]
);
register_rest_route(
$this->namespace,
'/voice/greet',
[
'methods' => 'POST',
'callback' => [ $this, 'handle_greet' ],
'permission_callback' => [ $this, 'check_voice_permissions' ],
]
);
register_rest_route(
$this->namespace,
'/voice/consent',
[
'methods' => 'POST',
'callback' => [ $this, 'record_consent' ],
'permission_callback' => [ $this, 'check_voice_permissions' ],
]
);
}
/**
* Only logged-in administrators may use voice, within the chat rate limit.
*
* There is no public voice channel, so only the capability is checked.
* The per-minute limit is the same one the text chat uses.
*
* @return bool|WP_Error
*/
public function check_voice_permissions() {
if ( ! is_user_logged_in() || ! current_user_can( 'manage_options' ) ) {
return new WP_Error(
'aisentic_voice_forbidden',
__( 'You are not allowed to use the voice assistant.', 'aisentic' ),
[ 'status' => 403 ]
);
}
$policy = new Policy();
$settings = new Settings();
if ( $policy->isRateLimited( $policy->getRequestIdentifier(), (int) $settings->get( 'general.rateLimitPerMinute', 20 ) ) ) {
return new WP_Error(
'aisentic_rate_limited',
__( 'Too many requests. Please wait a moment before sending another message.', 'aisentic' ),
[ 'status' => 429 ]
);
}
return true;
}
/**
* Record that the admin agreed to audio leaving the site.
*
* @return \WP_HTTP_Response|array
*/
public function record_consent() {
try {
$settings = new Settings();
$settings->merge_and_save( [ 'voice' => [ 'consentGiven' => true ] ] );
return $this->response( [ 'consentGiven' => true ], __( 'Thanks. Voice is ready.', 'aisentic' ) );
} catch ( \Throwable $e ) {
return Error_Handler::exceptionToRestError( $e );
}
}
/**
* Why this voice request must be refused, before any work is done.
*
* The same checks guard the greeting and both turn routes. The greeting
* skips the call-length check because it is what starts the call clock.
*
* @param Settings $settings Settings instance.
* @param bool $check_call_length Also refuse a call past its limit.
* @return \WP_HTTP_Response|null The error response, or null to carry on.
*/
private function refusal( Settings $settings, bool $check_call_length = true ) {
if ( ! $settings->get( 'voice.enabled', false ) ) {
return $this->error( __( 'Voice is turned off.', 'aisentic' ), 403, 'aisentic_voice_disabled' );
}
if ( ! $settings->get( 'voice.consentGiven', false ) ) {
return $this->error( __( 'Voice needs your agreement first.', 'aisentic' ), 428, 'aisentic_voice_consent_required' );
}
$limit = (int) $settings->get( 'voice.maxCallMinutes', Settings::VOICE_MAX_CALL_MINUTES );
if ( $check_call_length && Call_Window::too_long( get_current_user_id(), $limit ) ) {
return $this->error( __( 'This call reached its time limit. Start a new one.', 'aisentic' ), 403, 'aisentic_voice_call_too_long' );
}
$this->deadline = $check_call_length ? Call_Window::ends_at( get_current_user_id(), $limit ) : 0;
return null;
}
/**
* Has this call passed its time limit during the turn?
*
* @return bool
*/
private function out_of_time(): bool {
return $this->deadline > 0 && time() >= $this->deadline;
}
/**
* The error for a call that ran out of time during a turn.
*
* Also ends the call, so the next call starts a new clock.
*
* @return WP_Error
*/
private function time_up(): WP_Error {
Call_Window::end( get_current_user_id() );
return new WP_Error(
'aisentic_voice_call_too_long',
__( 'This call reached its time limit. Start a new one.', 'aisentic' ),
[ 'status' => 403 ]
);
}
/**
* Say hello before the person speaks.
*
* A call that starts in silence feels broken. The words are fixed here, not
* asked from the model, so there is no extra wait or cost. The hello is not
* saved to the conversation, because it is not a turn.
*
* @param WP_REST_Request $request Request object.
* @return \WP_HTTP_Response|array
*/
public function handle_greet( WP_REST_Request $request ) {
try {
$settings = new Settings();
$refused = $this->refusal( $settings, false );
if ( null !== $refused ) {
return $refused;
}
// Start the call clock again. Only this route starts it; the turn
// routes only read it.
Call_Window::start( get_current_user_id() );
$began = microtime( true );
$returning = (bool) $request->get_param( 'returning' );
$text = $this->greeting( $returning );
$tts = new Text_To_Speech( $settings );
$spoken = $tts->prepare( $text );
$at = microtime( true );
$audio = $this->remembered_hello( $tts, $spoken );
if ( is_wp_error( $audio ) ) {
return $this->from_wp_error( $audio );
}
Log::debug(
sprintf(
'Voice TIMING greet — tts %dms, total %dms',
$this->ms( $at ),
$this->ms( $began )
)
);
return $this->response(
[
'spoken' => $spoken,
'language' => '',
'tone' => 'friendly',
'audio' => (string) ( $audio['audio'] ?? '' ),
'mime' => (string) ( $audio['mime'] ?? '' ),
'speakHere' => ! empty( $audio['browser'] ),
'voiceDriver' => (string) ( $audio['driver'] ?? '' ),
'canSpeak' => ! isset( $audio['supported'] ) || (bool) $audio['supported'],
]
);
} catch ( \Throwable $e ) {
return Error_Handler::exceptionToRestError( $e );
}
}
/**
* Speak the hello once, then keep the audio.
*
* The hello lines never change, so making the audio on every call wastes
* 2-4 seconds. Each clip is kept for a week. Uses the 'friendly' tone, the
* same voice style as the opening lines.
*
* @param Text_To_Speech $tts Speech client.
* @param string $spoken The hello, ready to read out.
* @return array|WP_Error
*/
private function remembered_hello( Text_To_Speech $tts, string $spoken ) {
return $this->remembered_line( $tts, $spoken, 'friendly' );
}
/**
* Speak a fixed line once, then keep the audio for a week.
*
* Used for lines that do not change, like the hellos and the "one moment"
* lines. This saves about two seconds per line on later calls.
*
* @param Text_To_Speech $tts Speech client.
* @param string $spoken The line, ready to read out.
* @param string $tone One of Text_To_Speech::TONES.
* @return array|WP_Error
*/
private function remembered_line( Text_To_Speech $tts, string $spoken, string $tone ) {
$kept = $this->kept_line( $spoken, $tone );
if ( null !== $kept ) {
return $kept;
}
$audio = null;
for ( $try = 1; $try <= self::SPEAK_ATTEMPTS; $try++ ) {
// No language is sent, on purpose. The line may be translated, so the
// admin's locale may be wrong. Empty means "read what is written".
$audio = $tts->speak( $spoken, '', $tone );
if ( is_wp_error( $audio ) ) {
return $audio;
}
// Save which engine made this fresh clip. A kept clip only tells
// which engine made it back then. See clip_fits_engine().
$this->remember_engine( $audio );
if ( $this->faithful( $spoken, $audio ) ) {
$bytes = strlen( (string) ( $audio['audio'] ?? '' ) );
// Keep only clips with audio bytes, under 500 KB.
// A kokoro reply has no bytes (the browser makes the sound).
// Keeping it caused silence later on a browser without Kokoro.
// A very large clip would bloat the options table.
if ( $bytes > 0 && $bytes < 512000 ) {
set_transient( $this->line_key( $spoken, $tone ), $audio, WEEK_IN_SECONDS );
}
return $audio;
}
}
// Every try was wrong. Return the last one, marked 'drifted', and do
// not cache it. The caller decides: a greeting still plays, an opening
// line does not.
$audio['drifted'] = true;
return $audio;
}
/**
* Did the engine read the line, or answer it?
*
* The voice is a chat model. On a short line it sometimes replies instead
* of reading. Example: "One moment." became "Of course, no problem at all."
* In a test, 3 of 11 lines did this. It is random, so we check each clip.
*
* Compares the engine's transcript with our text, ignoring case and
* punctuation ("Okay." equals "Okay").
*
* @param string $spoken What we asked for.
* @param array $audio What came back.
* @return bool True also when the engine tells us nothing, because an
* engine that reports no transcript cannot be judged.
*/
private function faithful( string $spoken, array $audio ): bool {
$said = trim( (string) ( $audio['transcript'] ?? '' ) );
if ( '' === $said ) {
return true;
}
$squash = static fn( string $value ): string => mb_strtolower(
(string) preg_replace( '/[^\p{L}\p{N}]+/u', '', $value )
);
return $squash( $said ) === $squash( $spoken );
}
/**
* The transient name one of our own lines is kept under.
*
* The voice is set on the provider (VOICE_TTS_DRIVER and
* VOICE_TTS_OPENROUTER_VOICE), which this site cannot see. If you change
* it there, raise the number in the key to drop old clips.
*
* @param string $spoken The line, ready to read out.
* @param string $tone One of Text_To_Speech::TONES.
* @return string
*/
private function line_key( string $spoken, string $tone ): string {
// Raise the number to drop all old clips. Set to 4 for the Schedar
// voice, so old Sulafat clips are not reused.
return 'aisentic_voice_said_4_' . md5( $tone . '|' . $spoken );
}
/**
* The clip for a line, but only if it has already been made.
*
* Used where we must not wait. Making a new clip takes about two seconds,
* which would add to the silence the line is meant to cover. A line with
* no clip is skipped and made after the turn. See warm_up().
*
* @param string $spoken The line, ready to read out.
* @param string $tone One of Text_To_Speech::TONES.
* @return array|null Null when this line has never been spoken here.
*/
private function kept_line( string $spoken, string $tone ): ?array {
$kept = get_transient( $this->line_key( $spoken, $tone ) );
if ( ! is_array( $kept ) ) {
return null;
}
// The key does not include the engine, so check it here. A clip from
// another engine is not used.
return self::clip_fits_engine( $kept, $this->current_engine() ) ? $kept : null;
}
/**
* The engine the provider last answered with, or '' when none is known.
*
* @return string
*/
private function current_engine(): string {
return (string) get_option( self::ENGINE_OPTION, '' );
}
/**
* Write down which engine made this clip.
*
* Called only on fresh clips, not kept ones. So the option updates soon
* after the engine changes on the provider.
*
* @param array $audio What the speech client returned.
* @return void
*/
private function remember_engine( array $audio ): void {
$engine = (string) ( $audio['driver'] ?? '' );
if ( '' === $engine || $engine === $this->current_engine() ) {
return;
}
update_option( self::ENGINE_OPTION, $engine, false );
}
/**
* Is this kept clip still one this site would make today?
*
* True when the clip's engine matches the current engine, or when either
* one is unknown.
* Why: a kokoro clip (no bytes, the browser makes the sound) was served
* after the engine changed to openrouter. The browser had no Kokoro model,
* so the caller heard nothing.
*
* @param array $clip The kept clip.
* @param string $engine The engine the provider last answered with.
* @return bool
*/
private static function clip_fits_engine( array $clip, string $engine ): bool {
$made_by = (string) ( $clip['driver'] ?? '' );
if ( '' === $engine || '' === $made_by ) {
return true;
}
return $made_by === $engine;
}
/**
* Did this clip actually reach the caller's ears?
*
* The model is told "you have ALREADY said this out loud", so this must
* be true.
*
* Counts: a clip with audio bytes, or a browser clip (kokoro). Does not
* count: an empty reply, or an engine with no voice for this language.
*
* @param array $clip The clip.
* @return bool
*/
private static function was_really_said( array $clip ): bool {
if ( isset( $clip['supported'] ) && ! $clip['supported'] ) {
return false;
}
if ( '' !== (string) ( $clip['audio'] ?? '' ) ) {
return true;
}
return ! empty( $clip['browser'] );
}
/**
* Choose a line we can actually speak right now, and queue up one we cannot.
*
* Picks at random from the lines that already have a clip. Also queues one
* line with no clip, to be made after the turn (see warm_up()).
* Why: picking from all lines meant most picks had no clip yet, so they
* were skipped. Now the set fills up after a few turns.
*
* @param string[] $lines The lines that would fit, in no order.
* @param string $tone One of Text_To_Speech::TONES.
* @param string $avoid A line the caller just heard and must not hear again.
* @return array{0:string,1:array|null} The line and its clip, or ['', null]
* when none of them is ready yet.
*/
private function pick_said( array $lines, string $tone, string $avoid = '' ): array {
$ready = [];
$cold = [];
foreach ( $lines as $line ) {
if ( null === $this->kept_line( $line, $tone ) ) {
$cold[] = $line;
continue;
}
if ( $line !== $avoid ) {
$ready[] = $line;
}
}
// Queue one per turn, so no turn makes more than one clip.
if ( ! empty( $cold ) ) {
$this->warm = [ $cold[ wp_rand( 0, count( $cold ) - 1 ) ], $tone ];
}
if ( empty( $ready ) ) {
return [ '', null ];
}
$line = $ready[ wp_rand( 0, count( $ready ) - 1 ) ];
return [ $line, $this->kept_line( $line, $tone ) ];
}
/**
* The last short acknowledgement this admin heard.
*
* Kept per admin, so two admins do not affect each other. Kept for one
* day, which is longer than a call.
*
* @return string
*/
private function last_opening_line(): string {
return (string) get_transient( 'aisentic_voice_last_opening_' . get_current_user_id() );
}
/**
* Remember an acknowledgement only after its audio was sent.
*
* @param string $line The line the caller heard.
* @return void
*/
private function remember_opening_line( string $line ): void {
set_transient( 'aisentic_voice_last_opening_' . get_current_user_id(), $line, DAY_IN_SECONDS );
}
/**
* The opening line the transcription model wrote for THIS question.
*
* Speech to text also returns a short line about the question, with no
* extra request. Uses the cached clip, or makes one now and waits.
*
* Making a clip takes about 1.4-3.2 s, often longer than the silence it
* covers, and it delays the answer too. We accept this, because a line
* about the wrong subject is worse.
* Why: a cached line once said "Checking your ticket sales." to a caller
* who asked to add a speaker.
*
* If this returns nothing, say_opening_line() uses a hand-written line.
*
* @param Text_To_Speech $tts Speech client, for a line not made before.
* @param string $line The line the provider sent, or an empty string.
* @param string $tone One of Text_To_Speech::TONES.
* @return array{0:string,1:array|null} The line and its clip, or ['', null]
* when there is nothing to speak.
*/
private function pick_heard_line( Text_To_Speech $tts, string $line, string $tone ): array {
$line = trim( $line );
if ( '' === $line ) {
return [ '', null ];
}
// Counted after the turn in warm_up(), so the caller does not wait on
// a database write.
$this->heard_line = $line;
$kept = $this->kept_line( $line, $tone );
if ( null !== $kept && empty( $kept['drifted'] ) ) {
return [ $line, $kept ];
}
// Not made before. Make it now and wait for it.
$made = $this->remembered_line( $tts, $line, $tone );
if ( is_wp_error( $made ) || ! is_array( $made ) ) {
// Speech failed. Return nothing, so say_opening_line() uses a
// hand-written line instead.
return [ '', null ];
}
// A drifted clip comes back marked, and say_opening_line() stays
// silent for it.
return [ $line, $made ];
}
/**
* Make the clip for a line we skipped, now that nobody is waiting.
*
* Called after the answer is fully sent. On PHP-FPM the connection is
* closed first, so the caller does not wait. On other servers the
* microphone opens about two seconds later, once per new line.
*
* @param Text_To_Speech $tts Speech client.
* @return void
*/
private function warm_up( Text_To_Speech $tts ): void {
$queued = $this->warm;
$this->warm = null;
// The call is over, so no clip is needed soon. Do not pay for one.
if ( $this->out_of_time() ) {
return;
}
// Close the connection. The browser already has the whole answer.
if ( function_exists( 'fastcgi_finish_request' ) ) {
fastcgi_finish_request();
}
// The line this turn queued goes first. The caller is most likely to
// need it again soon.
if ( null !== $queued ) {
list( $line, $tone ) = $queued;
$this->remembered_line( $tts, $line, $tone );
}
$this->warm_waiting_line( $tts );
// Then this site's own heard lines, on every turn.
$this->remember_heard_line();
$this->top_up_heard_lines( $tts );
}
/**
* Cache one delayed acknowledgement after the caller has their answer.
*
* One per turn keeps the work small. After a few turns all are ready.
* Each clip is kept for a week.
*
* @param Text_To_Speech $tts Speech client.
* @return void
*/
private function warm_waiting_line( Text_To_Speech $tts ): void {
$cold = [];
foreach ( $this->waiting_lines() as $line ) {
if ( null === $this->kept_line( $line, 'friendly' ) ) {
$cold[] = $line;
}
}
if ( empty( $cold ) ) {
return;
}
$line = $cold[ wp_rand( 0, count( $cold ) - 1 ) ];
$this->remembered_line( $tts, $line, 'friendly' );
}
/**
* Count the line this turn heard, so the common ones can be found.
*
* There is no fixed list on purpose. The set is the lines this site really
* hears, sorted by how often.
*
* @return void
*/
private function remember_heard_line(): void {
$line = trim( $this->heard_line );
$this->heard_line = '';
if ( '' === $line ) {
return;
}
$seen = get_option( self::HEARD_LINES_OPTION, [] );
$seen = is_array( $seen ) ? $seen : [];
$seen[ $line ] = (int) ( $seen[ $line ] ?? 0 ) + 1;
// Most common first. Keep only the top LINES_REMEMBERED.
arsort( $seen );
update_option( self::HEARD_LINES_OPTION, array_slice( $seen, 0, self::LINES_REMEMBERED, true ), false );
}
/**
* Record a few of this site's own lines that have no clip yet.
*
* Making a clip takes about 1.5 s. Making it here, after the turn, means a
* later turn with the same line does not wait for it.
* At most WARM_PER_TURN clips per turn.
*
* Runs after fastcgi_finish_request(), so the caller does not wait.
*
* @param Text_To_Speech $tts Speech client.
* @return void
*/
private function top_up_heard_lines( Text_To_Speech $tts ): void {
$seen = get_option( self::HEARD_LINES_OPTION, [] );
if ( ! is_array( $seen ) || empty( $seen ) ) {
return;
}
$made = 0;
// All opening lines use one tone, so the voice and volume stay the same.
$tone = 'friendly';
foreach ( array_keys( $seen ) as $line ) {
if ( $made >= self::WARM_PER_TURN ) {
return;
}
if ( null !== $this->kept_line( (string) $line, $tone ) ) {
continue;
}
$this->remembered_line( $tts, (string) $line, $tone );
++$made;
}
}
/**
* Fill the silence while a tool runs.
*
* A tool turn can take about 11 s before the first sound. This does not
* make it faster. It lets the caller hear a voice after about 2 s.
*
* @param Stream_Speaker $speaker Speaker for this turn.
* @param Speech_Chunker $chunker Holds what the model wrote so far.
* @param Text_To_Speech $tts Speech client.
* @return string What was said out loud, or an empty string when nothing
* was. The words are returned, not just a yes, so the model
* can be told what the caller has already heard.
*/
private function models_own_line( Stream_Speaker $speaker, Speech_Chunker $chunker, Text_To_Speech $tts ): string {
// Use the model's own words ("I'm looking up the payment settings")
// because they name the subject. Only if there are 20+ characters.
$peek = $tts->clean( $chunker->peek() );
if ( mb_strlen( $peek ) < 20 ) {
return '';
}
// Skip it if it only says "wait" ("Let me check that now"). The caller
// already heard an opening line like that.
if ( $this->only_a_holding_phrase( $peek ) ) {
return '';
}
// Skip it if the sentence is not finished. Speaking half of it splits
// the answer in two.
// Example: "You're on US dollars." ... pause ... "- the dollar sign."
if ( ! self::ends_a_sentence( $peek ) ) {
return '';
}
$own = $chunker->rest();
return $this->say( $speaker, $own ) ? $own : '';
}
/**
* Has this text reached the end of a sentence?
*
* Ends with . ! ? or danda, maybe followed by a closing quote or bracket.
* Same marks as Speech_Chunker and Answer_Steps.
*
* @param string $text Cleaned text from the model.
* @return bool
*/
private static function ends_a_sentence( string $text ): bool {
return 1 === preg_match( '/[.!?\x{0964}][\"\'\)\]\x{201D}\x{2019}]*$/u', trim( $text ) );
}
/**
* Is this line nothing but "hold on" in one of its many shapes?
*
* Removes all "wait" phrases and checks what is left (under 12 letters
* means true). "I'm looking up your payment settings" is false, because
* "payment settings" is left. "Let me check that now" is true.
*
* @param string $text Cleaned text from the model.
* @return bool
*/
private function only_a_holding_phrase( string $text ): bool {
$phrases = [
'let me check',
'let me see',
'let me look',
'let me find',
'let me pull',
'let me have a look',
'let me get',
'let me grab',
'one moment',
'just a second',
'just a moment',
'give me a second',
'give me a moment',
'hold on',
'bear with me',
"i'll check",
"i'm checking",
"i'm looking",
'checking now',
'looking that up',
'pulling that up',
'pulling it up',
'for you',
'right now',
'now',
'that',
'this',
'it',
'up',
'on',
'the',
'your',
'a',
'ok',
'okay',
'sure',
'of course',
'right away',
'got it',
'happy to',
'absolutely',
];
$left = mb_strtolower( $text );
foreach ( $phrases as $phrase ) {
$left = str_replace( $phrase, ' ', $left );
}
// Keep only letters and digits. A real line keeps its subject.
$left = (string) preg_replace( '/[^\p{L}\p{N}]+/u', '', $left );
return mb_strlen( $left ) < 12;
}
/**
* Say one of our own holding lines, because the model has written nothing.
*
* Only ever used to cover real silence. See COVER_SILENCE_AFTER_MS.
*
* @param Text_To_Speech $tts Speech client.
* @return string What was said, or an empty string.
*/
private function canned_waiting_line( Text_To_Speech $tts ): string {
$at = microtime( true );
$line = $this->waiting_line();
$audio = $this->remembered_line( $tts, $line, 'friendly' );
if ( $this->timer ) {
// Normally cached. A slow time here means the cache was lost.
$this->timer->took( 'filler', $at );
}
$this->speech_ms += $this->ms( $at );
if ( is_wp_error( $audio ) ) {
Log::error( 'Voice: could not say "one moment" — ' . $audio->get_error_message() );
return '';
}
++$this->clips;
$this->emit(
[
'type' => 'say',
// Not part of the answer, so it is outside the numbering and
// never reaches the chat thread.
'index' => -1,
'filler' => true,
'text' => $line,
'audio' => (string) ( $audio['audio'] ?? '' ),
'mime' => (string) ( $audio['mime'] ?? '' ),
'speakHere' => ! empty( $audio['browser'] ),
'canSpeak' => ! isset( $audio['supported'] ) || (bool) $audio['supported'],
]
);
return $line;
}
/**
* Give the browser "still working" lines to play only if work stays slow.
*
* Uses only cached clips, because making one now would delay the answer.
* Missing clips are made after the turn. The line heard last time is
* skipped, so two slow turns do not start with the same words.
*
* @return bool True when at least one delayed line was sent to the browser.
*/
private function schedule_waiting_line(): bool {
$last = (string) get_transient( 'aisentic_voice_last_follow_up_' . get_current_user_id() );
$ready = [];
foreach ( $this->waiting_lines() as $line ) {
$audio = $this->kept_line( $line, 'friendly' );
if ( null !== $audio && $line !== $last ) {
$ready[] = [ $line, $audio ];
}
}
if ( empty( $ready ) ) {
return false;
}
shuffle( $ready );
$first = '';
foreach ( self::HOLD_FOLLOW_UP_LINES_MS as $i => $hold ) {
// Fewer lines than slots: start the list again. A line repeats
// back to back only when just one is ready.
list( $line, $audio ) = $ready[ $i % count( $ready ) ];
$first = '' === $first ? $line : $first;
++$this->clips;
$this->emit(
[
'type' => 'say',
'index' => -1,
'filler' => true,
'holdFor' => $hold,
'text' => $line,
'audio' => (string) ( $audio['audio'] ?? '' ),
'mime' => (string) ( $audio['mime'] ?? '' ),
'speakHere' => ! empty( $audio['browser'] ),
'canSpeak' => ! isset( $audio['supported'] ) || (bool) $audio['supported'],
]
);
}
set_transient( 'aisentic_voice_last_follow_up_' . get_current_user_id(), $first, DAY_IN_SECONDS );
return true;
}
/**
* How long the caller has heard nothing, in milliseconds.
*
* Counted from the last sound, or from the start of the turn when there
* has not been one.
*
* @return int
*/
private function silent_for(): int {
$since = $this->last_sound > 0.0
? $this->last_sound
: ( $this->timer ? $this->timer->began() : microtime( true ) );
return (int) round( ( microtime( true ) - $since ) * 1000 );
}
/**
* Did the answer say it was opening a page, and then not open one?
*
* The browser opens a page only from a link at the end of the answer
* (rule 14 of the voice prompt). An answer that says "I'm opening..." with
* no link leaves the caller on the same page. In a test, 2 of 8 did this.
*
* Only first-person claims count. A question like "Want me to open the
* payment settings?" needs no link.
*
* @param string $answer What the model wrote.
* @return bool
*/
private static function promised_a_page( string $answer ): bool {
if ( false !== stripos( $answer, 'class="aisentic-button"' ) ) {
return false;
}
return 1 === preg_match(
"/\b(i am|i'm|im)\s+(opening|taking you|pulling up)\b|\blet me (open|pull up)\b|\b(opening|taking you to) (the|your)\b/i",
$answer
);
}
/**
* Did this turn state facts without looking anything up?
*
* True when text is still held at the end and no tool was called. Then the
* model answered from memory, which does not know this site's data.
*
* @param bool $enabled voice.holdUnverifiedFacts.
* @param string $held What never got spoken.
* @param int $tool_calls How many tools this turn asked for.
* @return bool
*/
private static function needs_a_second_ask( bool $enabled, string $held, int $tool_calls ): bool {
return $enabled && '' !== trim( $held ) && 0 === $tool_calls;
}
/**
* Where does this sentence go: held back, onto the tail, or spoken now?
*
* Returns one of:
* hold it claims a figure and nothing has been looked up.
* tail the first sentence is already spoken, so this one joins the
* rest as one clip at the end (each clip costs about 2 s).
* speak it is the first sentence of the answer.
*
* hold must be tested first. Otherwise only sentence one is checked, and
* a made-up number in sentence two reaches the caller.
*
* @param bool $enabled voice.holdUnverifiedFacts.
* @param bool $grounded Is the model writing from a tool result now?
* @param string $held What is already waiting.
* @param bool $said_answer Has the first sentence been spoken?
* @param string $piece The sentence about to go somewhere.
* @return string 'hold', 'tail' or 'speak'.
*/
private static function route_piece( bool $enabled, bool $grounded, string $held, bool $said_answer, string $piece ): string {
if ( self::should_hold( $enabled, $grounded, $held, $piece ) ) {
return 'hold';
}
return $said_answer ? 'tail' : 'speak';
}
/**
* Should this sentence wait until a tool has confirmed it?
*
* Once one sentence is held, the rest wait behind it, so the caller never
* hears the answer arrive out of order.
*
* @param bool $enabled Is the unverified-facts guard turned on?
* @param bool $grounded Has a tool returned something this turn?
* @param string $held What is already waiting.
* @param string $piece The sentence about to go somewhere.
* @return bool
*/
private static function should_hold( bool $enabled, bool $grounded, string $held, string $piece ): bool {
if ( ! $enabled || $grounded ) {
return false;
}
return '' !== $held || self::states_a_fact( $piece );
}
/**
* Does this sentence tell the caller something that had to be looked up?
*
* A wrong number is hard for the caller to notice, because they have no
* screen. Example: it once said "two bookings are pending" when the real
* number was sixty-five.
*
* A sentence is a claim when it has (after removing [[tone]] tags and
* HTML tags, whose digits are not spoken):
* - a digit;
* - two or more number words ("thirty-seven million");
* - one number word counting something ("two bookings", "fifty percent").
* The last rule is narrow, so "not one single event" is not held.
*
* @param string $text One sentence, as the model wrote it.
* @return bool
*/
private static function states_a_fact( string $text ): bool {
$text = (string) preg_replace( '/\[\[[^\]]*\]\]/u', ' ', $text );
$text = (string) preg_replace( '/<[^>]*>/u', ' ', $text );
if ( 1 === preg_match( '/\d/u', $text ) ) {
return true;
}
$words = self::NUMBER_WORDS;
if ( preg_match_all( '/\b(?:' . $words . ')\b/iu', $text ) >= 2 ) {
return true;
}
// One number word, then within a couple of words something countable:
// a plural noun, or money.
return 1 === preg_match(
'/\b(?:' . $words . ')\b(?:[\s,\-]+\w+){0,2}[\s,\-]+(?:(?!(?:' . self::NOT_A_PLURAL . ')\b)\w+s\b|' . self::COUNTED_THINGS . ')/iu',
$text
);
}
/**
* Say a short line that fits the question, before the model is asked.
*
* Covers the ~3 s the model needs before it starts to answer. Like a
* person: "of course" to a request, "let me see" to a question.
* Question_Kind decides which.
*
* Sources, in order:
* 1. The line speech to text wrote for this question (if
* voice.dynamicOpeningLine is on). See pick_heard_line().
* 2. A hand-written line with a cached clip. See pick_said().
*
* @param Text_To_Speech $tts Speech client.
* @param Settings $settings Settings instance.
* @param array $heard The transcript result: text, language,
* and the line the provider wrote, if any.
* @param Turn_Timer $timer Stopwatch for this turn.
* @return string The line that was said, or an empty string. The words are
* returned so the model can be told what was already heard.
*/
private function say_opening_line( Text_To_Speech $tts, Settings $settings, array $heard, Turn_Timer $timer ): string {
$kind = Question_Kind::of( (string) $heard['text'] );
$lines = Question_Kind::opening_lines( $kind );
$timer->note( 'kind', $kind );
if ( empty( $lines ) ) {
return '';
}
// One tone for all short lines, so voice and volume match the answer.
$tone = 'friendly';
$at = $timer->mark();
$last_line = $this->last_opening_line();
// First choice: the line the transcription model wrote from this very
// question. Second choice: one of ours. See pick_heard_line().
$heard_line = (bool) $settings->get( 'voice.dynamicOpeningLine', false )
? (string) ( $heard['waiting'] ?? '' )
: '';
if ( $heard_line === $last_line ) {
$heard_line = '';
}
list( $line, $audio ) = $this->pick_heard_line( $tts, $heard_line, $tone );
$from_model = '' !== $line;
if ( ! $from_model ) {
// Keep any line already queued for warm_up(), so pick_said()
// does not replace it.
$queued = $this->warm;
list( $line, $audio ) = $this->pick_said( $lines, $tone, $last_line );
if ( null !== $queued ) {
$this->warm = $queued;
}
}
$timer->took( 'opening', $at );
$timer->note( 'openingFrom', $from_model ? 'heard' : 'list' );
if ( ! empty( $audio['drifted'] ) ) {
// The engine replied to the line instead of reading it. Say nothing.
$timer->note( 'openingDrifted', 1 );
return '';
}
if ( '' === $line || null === $audio ) {
// No line has a clip ready (or speech failed). Say nothing.
$timer->note( 'openingWarming', 1 );
return '';
}
if ( ! self::was_really_said( $audio ) ) {
// The clip has no sound. Do not send it, and do not tell the model
// it was said, because nobody heard it.
$timer->note( 'openingSilent', 1 );
return '';
}
++$this->clips;
$this->emit(
[
'type' => 'say',
// Not part of the answer, so it is outside the numbering and
// never reaches the chat thread.
'index' => -1,
'filler' => true,
'holdFor' => self::HOLD_OPENING_LINE_MS,
'text' => $line,
'audio' => (string) ( $audio['audio'] ?? '' ),
'mime' => (string) ( $audio['mime'] ?? '' ),
'speakHere' => ! empty( $audio['browser'] ),
'canSpeak' => ! isset( $audio['supported'] ) || (bool) $audio['supported'],
]
);
$this->remember_opening_line( $line );
$timer->at( 'openingSaid' );
return $line;
}
/**
* Answer a greeting here, without asking the model.
*
* Used for greetings during a call, like handle_greet(). A greeting has no
* question, so a model call would only add wait time.
* The turn is still saved, so the chat thread shows both sides.
*
* @param Text_To_Speech $tts Speech client.
* @param string[] $replies The lines any of which would do.
* @param array $heard {text, language} from speech to text.
* @param string $conversation_id Conversation UUID.
* @param Message_Repository $message_repo Message repository.
* @param Turn_Timer $timer Stopwatch for this turn.
* @return void
*/
private function answer_at_once( Text_To_Speech $tts, array $replies, array $heard, string $conversation_id, Message_Repository $message_repo, Turn_Timer $timer ): void {
$at = $timer->mark();
// Use a cached clip if one is ready. This is the answer, so if none is
// ready, make one and wait.
list( $reply, $audio ) = $this->pick_said( $replies, 'happy' );
if ( '' === $reply ) {
$reply = $replies[0];
$audio = $this->remembered_line( $tts, $reply, 'happy' );
$timer->note( 'smallTalkWarming', 1 );
}
$timer->took( 'smallTalk', $at );
if ( is_wp_error( $audio ) || ! is_array( $audio ) ) {
$this->emit_failure(
is_wp_error( $audio ) ? $audio : new WP_Error( 'aisentic_voice_no_audio', __( 'There was nothing to say.', 'aisentic' ), [ 'status' => 502 ] ),
$conversation_id
);
return;
}
++$this->clips;
$this->emit(
[
'type' => 'say',
'index' => 0,
'text' => $reply,
'audio' => (string) ( $audio['audio'] ?? '' ),
'mime' => (string) ( $audio['mime'] ?? '' ),
'speakHere' => ! empty( $audio['browser'] ),
'canSpeak' => ! isset( $audio['supported'] ) || (bool) $audio['supported'],
]
);
$save_at = $timer->mark();
$message_repo->create( $conversation_id, 'user', (string) $heard['text'], [ 'source' => 'voice' ] );
$message_repo->create( $conversation_id, 'assistant', $reply, [ 'source' => 'voice' ] );
$timer->took( 'db.save', $save_at );
$timer->note( 'answerChars', mb_strlen( $reply ) );
$timer->note( 'clips', $this->clips );
$timer->write( 'small-talk' );
$this->emit(
[
'type' => 'done',
'conversationId' => $conversation_id,
'heard' => (string) $heard['text'],
'language' => (string) $heard['language'],
'answer' => $reply,
'tone' => 'happy',
'link' => null,
]
);
// Must run after 'done'. warm_up() closes the connection, so nothing
// sent after it reaches the browser.
$this->warm_up( $tts );
}
/**
* Add a full stop if the model's text has no ending mark.
*
* Used when the text before a tool call was already spoken. What the model
* writes after the tool starts a new sentence.
*
* @param string $text The answer so far.
* @return string The same text, ending in a full stop and a space.
*/
private function close_sentence( string $text ): string {
$text = rtrim( $text );
if ( '' === $text ) {
return $text;
}
$last = mb_substr( $text, -1 );
if ( ! in_array( $last, [ '.', '!', '?', ':', ',', '—', "\u{0964}" ], true ) ) {
$text .= '.';
}
return $text . ' ';
}
/**
* A short line to say while the assistant looks something up.
*
* Vague on purpose. The tool has not answered yet, so it cannot promise a
* result.
*
* @return string
*/
private function waiting_line(): string {
$lines = $this->waiting_lines();
return $lines[ wp_rand( 0, count( $lines ) - 1 ) ];
}
/**
* Professional follow-ups for work that is taking longer than usual.
*
* @return string[]
*/
private function waiting_lines(): array {
return [
__( "I'm still checking that for you.", 'aisentic' ),
__( "This is taking a moment. I'm still working on it.", 'aisentic' ),
__( "I'm still working on that. Thank you for your patience.", 'aisentic' ),
__( "Just a little longer. I'm checking the details.", 'aisentic' ),
];
}
/**
* Milliseconds since a microtime( true ) mark.
*
* @param float $since Start mark.
* @return int
*/
private function ms( float $since ): int {
return (int) round( ( microtime( true ) - $since ) * 1000 );
}
/**
* Pick a hello, and never the same one twice in a row.
*
* The last hello is saved in user meta and skipped next time, so the
* assistant does not sound like a recording.
*
* @param bool $returning True when this is "Talk again" after a call ended.
* @return string
*/
private function greeting( bool $returning ): string {
$lines = $returning
? [
__( 'Welcome back! What shall we do next?', 'aisentic' ),
__( "Hi again — I'm listening. What do you need?", 'aisentic' ),
__( 'Good to have you back. What can I look up for you?', 'aisentic' ),
__( "Here I am! Go ahead, I'm all ears.", 'aisentic' ),
]
: [
__( "Hi there! I'm listening. What can I help you with?", 'aisentic' ),
__( 'Hello! Lovely to hear from you. What are we working on?', 'aisentic' ),
__( "Hey! I'm right here. Just tell me what you need.", 'aisentic' ),
__( 'Hi! Ready when you are — what would you like to do?', 'aisentic' ),
];
$key = 'aisentic_voice_last_greeting';
$user = get_current_user_id();
$last = (string) get_user_meta( $user, $key, true );
$fresh = array_values( array_filter( $lines, static fn( $line ) => $line !== $last ) );
$lines = empty( $fresh ) ? $lines : $fresh;
$chosen = $lines[ wp_rand( 0, count( $lines ) - 1 ) ];
update_user_meta( $user, $key, $chosen );
return $chosen;
}
/**
* Run one full voice turn.
*
* @param WP_REST_Request $request Request object.
* @return \WP_HTTP_Response|array
*/
public function handle_turn( WP_REST_Request $request ) {
$settings = new Settings();
$refused = $this->refusal( $settings );
if ( null !== $refused ) {
return $refused;
}
$began = microtime( true );
$files = $request->get_file_params();
$file = $files['audio'] ?? null;
$path = is_array( $file ) ? (string) ( $file['tmp_name'] ?? '' ) : '';
// Everything below runs inside try/finally so the recording is deleted
// even when a step throws. Audio is never kept, on any path out.
try {
$valid = ( new Audio_Validator( $settings ) )->validate( $file );
if ( is_wp_error( $valid ) ) {
return $this->from_wp_error( $valid );
}
// Voice and text share one conversation, so the chat box shows
// both. Checked before speech to text, so a bad id costs nothing.
$conversation_id = $this->conversation_id( $request );
if ( is_wp_error( $conversation_id ) ) {
return $this->from_wp_error( $conversation_id );
}
if ( $this->out_of_time() ) {
return $this->from_wp_error( $this->time_up() );
}
$at = microtime( true );
$heard = ( new Speech_To_Text( $settings ) )->transcribe( $path, (string) ( $file['name'] ?? 'clip.webm' ) );
if ( is_wp_error( $heard ) ) {
return $this->from_wp_error( $heard );
}
$stt_ms = $this->ms( $at );
$message_repo = new Message_Repository();
if ( $this->out_of_time() ) {
return $this->from_wp_error( $this->time_up() );
}
// Save nothing before the model answers. If the turn fails after
// the question is saved, every later turn fails with "expected
// role assistant", and the conversation is broken.
$at = microtime( true );
$answer = $this->ask_agent( $settings, $heard['text'], $heard['language'], $request, $conversation_id, $message_repo );
if ( is_wp_error( $answer ) ) {
return $this->from_wp_error( $answer );
}
$model_ms = $this->ms( $at );
// The prompt asks for the link last (rule 2), but the model does not
// always do it. So move it here.
$answer = $this->link_last( $answer );
// Read the tone tag, then remove it. It is never saved, shown or
// spoken.
$tone = Text_To_Speech::read_tone( $answer );
$answer = Text_To_Speech::strip_tone( $answer );
$answer = Answer_Opening::fix( $answer, $tone );
// Save the question and the answer together.
$message_repo->create( $conversation_id, 'user', $heard['text'], [ 'source' => 'voice' ] );
$message_repo->create( $conversation_id, 'assistant', $answer, [ 'source' => 'voice' ] );
// The answer is saved, so it shows in the chat. Only the speech
// is skipped.
if ( $this->out_of_time() ) {
return $this->from_wp_error( $this->time_up() );
}
$tts = new Text_To_Speech( $settings );
$spoken = $tts->prepare( $answer );
$at = microtime( true );
$audio = $tts->speak( $spoken, $heard['language'], $tone );
if ( is_wp_error( $audio ) ) {
return $this->from_wp_error( $audio );
}
// Log how long each step took, to find the slow one.
Log::debug(
sprintf(
'Voice TIMING stt %dms, model %dms, tts %dms, total %dms',
$stt_ms,
$model_ms,
$this->ms( $at ),
$this->ms( $began )
)
);
return $this->response(
[
'conversationId' => $conversation_id,
'heard' => $heard['text'],
'language' => $heard['language'],
'answer' => $answer,
'spoken' => $spoken,
'tone' => $tone,
'link' => $this->first_link( $answer, $request ),
// Empty when the browser does the speaking (Kokoro), or when
// the engine has no voice for this language. Neither is an
// error: the answer is still shown on screen.
'audio' => (string) ( $audio['audio'] ?? '' ),
'mime' => (string) ( $audio['mime'] ?? '' ),
'speakHere' => ! empty( $audio['browser'] ),
'voiceDriver' => (string) ( $audio['driver'] ?? '' ),
'canSpeak' => ! isset( $audio['supported'] ) || (bool) $audio['supported'],
]
);
} catch ( \Throwable $e ) {
return Error_Handler::exceptionToRestError( $e );
} finally {
$this->discard( $path );
}
}
/**
* Run one voice turn, speaking it before it is finished.
*
* Same work as handle_turn(), but streamed. Each finished sentence is
* spoken and sent at once, so the caller waits only for the first one.
* handle_turn() waits for the whole answer and one big clip.
*
* @param WP_REST_Request $request Request object.
* @return \WP_HTTP_Response|array|void Only returns when it fails before the stream opens.
*/
public function stream_turn( WP_REST_Request $request ) {
$settings = new Settings();
$refused = $this->refusal( $settings );
if ( null !== $refused ) {
return $refused;
}
$began = microtime( true );
// Start the clock when the request arrived, so WordPress boot time is
// counted too. REQUEST_TIME_FLOAT is set by PHP, not the browser, and
// is cast to float, so it needs no sanitising.
$timer = new Turn_Timer( (float) ( $_SERVER['REQUEST_TIME_FLOAT'] ?? $began ) ); // phpcs:ignore WordPress.Security.ValidatedSanitizedInput
$this->timer = $timer;
$timer->at( 'wpBoot' );
$files = $request->get_file_params();
$file = $files['audio'] ?? null;
$path = is_array( $file ) ? (string) ( $file['tmp_name'] ?? '' ) : '';
$conversation_id = '';
$streaming = false;
try {
$at = $timer->mark();
$valid = ( new Audio_Validator( $settings ) )->validate( $file );
if ( is_wp_error( $valid ) ) {
return $this->from_wp_error( $valid );
}
$timer->took( 'validate', $at );
// Found before speech to text, so a bad id costs no transcription.
$at = $timer->mark();
$conversation_id = $this->conversation_id( $request );
$timer->took( 'db.conversation', $at );
if ( is_wp_error( $conversation_id ) ) {
return $this->from_wp_error( $conversation_id );
}
if ( $this->out_of_time() ) {
return $this->from_wp_error( $this->time_up() );
}
$at = microtime( true );
$heard = ( new Speech_To_Text( $settings ) )->transcribe( $path, (string) ( $file['name'] ?? 'clip.webm' ), $timer );
if ( is_wp_error( $heard ) ) {
return $this->from_wp_error( $heard );
}
$stt_ms = $this->ms( $at );
$message_repo = new Message_Repository();
// After this, headers are sent. Errors go out as SSE error events,
// not JSON.
$this->open_stream();
$streaming = true;
$this->emit(
[
'type' => 'heard',
'conversationId' => $conversation_id,
'heard' => $heard['text'],
'language' => $heard['language'],
]
);
$timer->at( 'heardSent' );
$this->speak_while_writing( $settings, $request, $heard, $conversation_id, $message_repo, $stt_ms, $began, $timer );
} catch ( \Throwable $e ) {
if ( ! $streaming ) {
return Error_Handler::exceptionToRestError( $e );
}
$this->emit_exception( $e, (string) $conversation_id );
} finally {
$this->discard( $path );
}
exit; // Stops WP REST wrapping the stream in JSON.
}
/**
* Read the answer as it is written, and speak each finished sentence.
*
* @param Settings $settings Settings instance.
* @param WP_REST_Request $request Request object.
* @param array $heard {text, language} from speech to text.
* @param string $conversation_id Conversation UUID.
* @param Message_Repository $message_repo Message repository.
* @param int $stt_ms Milliseconds speech to text took.
* @param float $began Start mark for the whole turn.
* @param Turn_Timer|null $timer Stopwatch for this turn.
* @return void
*/
private function speak_while_writing( Settings $settings, WP_REST_Request $request, array $heard, string $conversation_id, Message_Repository $message_repo, int $stt_ms, float $began, ?Turn_Timer $timer = null ): void {
$timer = $timer ?? new Turn_Timer( $began );
$tts = new Text_To_Speech( $settings );
$tts->trace( $timer );
$this->speech_ms = 0;
$this->clips = 0;
$this->first_sound = 0.0;
$this->last_sound = 0.0;
$this->warm = null;
$this->follow_up_scheduled = false;
$this->said_since_follow_ups = false;
$this->done_sent = false;
// Small talk ("How are you") needs no model. A cached clip is used
// instead, which saves about four seconds.
$small_talk = Question_Kind::small_talk_replies( (string) $heard['text'] );
if ( ! empty( $small_talk ) ) {
$timer->note( 'kind', Question_Kind::SMALL_TALK );
$this->answer_at_once( $tts, $small_talk, $heard, $conversation_id, $message_repo, $timer );
return;
}
$speaker = new Stream_Speaker(
$tts,
(string) $heard['language'],
Text_To_Speech::DEFAULT_TONE,
(int) $settings->get( 'voice.maxAnswerChars', Settings::VOICE_MAX_ANSWER_CHARS )
);
$chunker = new Speech_Chunker();
$answer = '';
$tail = '';
// Sentences with facts, written before any tool ran. Not spoken yet.
// See states_a_fact().
$held = '';
$tone = '';
$first_ms = 0;
$word_ms = 0;
$model_at = microtime( true );
// One tool call is many chunks. Count the starts, not the chunks.
$in_tool = false;
$tool_calls = 0;
$told_to_wait = false;
// True after a second ask. Its tools run inside chat(), so $tool_calls
// does not count them. The steps check is skipped for a retried answer.
$retried = false;
// Has any part of the answer been spoken yet? Holding lines do not
// count ($first_ms counts any sound). The first answer sentence still
// needs its own clip.
$said_answer = false;
// Did any words arrive AFTER the tool answered? See the check below.
$tool_answered = false;
// Is the model writing from a tool result right now?
// Unlike $tool_answered, this goes back to false when a new tool call
// starts, because text written before that call is a guess again.
$grounded = false;
$spoke_after = false;
// Separate settings. The opening line plays at the start of each turn.
// The holding lines play only after real silence.
$opening_on = (bool) $settings->get( 'voice.openingLine', true );
$filler_on = (bool) $settings->get( 'voice.fillerLine', true );
$hold_on = (bool) $settings->get( 'voice.holdUnverifiedFacts', true );
// Say a short line before the model is asked. It covers the ~3 s the
// model needs to read its prompt. Speech to text is done, so the line
// can fit the question (see Question_Kind). Small talk was handled
// above and never gets here.
$waiting_line = $opening_on ? $this->say_opening_line( $tts, $settings, $heard, $timer ) : '';
// Schedule the waiting lines even if no opening line was said.
// Why: without this, one turn had 29 s of silence before the first word.
if ( $filler_on ) {
$this->follow_up_scheduled = $this->schedule_waiting_line();
}
$told_to_wait = $this->follow_up_scheduled;
if ( '' !== $waiting_line ) {
$first_ms = $this->sound_ms( $began );
}
try {
$at = $timer->mark();
$agent = $this->voice_agent( $settings, $request );
$timer->took( 'agent.build', $at );
// Build the system prompt here so its time and size are logged.
// It is the largest part of every request.
$at = $timer->mark();
$prompt = $agent->instructions();
$timer->took( 'agent.prompt', $at );
$timer->note( 'promptChars', strlen( $prompt ) );
$timer->note( 'tools', count( $agent->getTools() ) );
// For questions about this site's figures, tell the model to use a
// tool on the first ask. See Question_Kind::asks_to_be_looked_up().
// Why: without this the model answered from memory, the answer was
// dropped, and the second ask took 30 s with no sound.
$look_first = Question_Kind::asks_to_be_looked_up( (string) $heard['text'] );
$timer->note( 'lookFirst', $look_first ? 1 : 0 );
$at = $timer->mark();
$messages = $this->turn_messages( $message_repo, $conversation_id, (string) $heard['text'], (string) $heard['language'], $waiting_line, $look_first ? 'first' : '' );
$timer->took( 'db.history', $at );
$timer->note( 'history', count( $messages ) );
if ( $this->out_of_time() ) {
$this->emit_failure( $this->time_up(), $conversation_id );
return;
}
$timer->at( 'modelAsked' );
$tool_at = 0.0;
foreach ( $agent->stream( $messages )->events() as $chunk ) {
// Mark the first chunk. Time before it is network and prompt
// reading; time after it is the model writing.
$timer->at( 'modelFirstChunk' );
// The caller hung up. Stop paying for an answer nobody hears.
if ( 1 === connection_aborted() ) {
break;
}
// The call ran out of time while the model was writing.
if ( $this->out_of_time() ) {
$this->emit_failure( $this->time_up(), $conversation_id );
return;
}
// After a tool call, cover the wait if nothing else is planned.
// The canned line plays only after real silence
// (COVER_SILENCE_AFTER_MS), not as soon as a tool starts.
// Why: two canned lines two seconds apart sounded strange.
if ( $filler_on && ! $told_to_wait && ! $said_answer && $tool_calls > 0 ) {
// First choice: the model's own words before the tool,
// because they name the subject.
$own = $this->models_own_line( $speaker, $chunker, $tts );
if ( '' !== $own ) {
$told_to_wait = true;
$first_ms = 0 === $first_ms ? $this->sound_ms( $began ) : $first_ms;
// Add a full stop, or the chat thread reads
// "Let me checkYou've got 2,013 events".
$answer = $this->close_sentence( $answer );
} elseif ( $this->silent_for() >= self::COVER_SILENCE_AFTER_MS ) {
$told_to_wait = true;
if ( '' !== $this->canned_waiting_line( $tts ) ) {
$first_ms = 0 === $first_ms ? $this->sound_ms( $began ) : $first_ms;
}
}
}
if ( $chunk instanceof ToolResultChunk ) {
if ( $in_tool && $tool_at > 0.0 ) {
$timer->took( 'tool.run', $tool_at );
$tool_at = 0.0;
}
$in_tool = false;
$tool_answered = true;
$grounded = true;
$timer->at( 'toolDone' );
// A fact was held before the tool ran, so it was a guess.
// Drop it from speech and from the saved answer. A wrong
// number in history gets repeated in later turns.
if ( '' !== $held ) {
$timer->note( 'droppedGuess', 1 );
// Clear the tail and the chunker too. They hold parts
// of the same guess and would be spoken later.
$held = '';
$tail = '';
$answer = '';
$chunker->forget();
}
continue;
}
if ( $chunk instanceof ToolCallChunk ) {
if ( $in_tool ) {
continue;
}
$in_tool = true;
// A new tool call. Text from here is unchecked again.
$grounded = false;
$tool_at = $timer->mark();
$timer->at( 'toolAsked' );
++$tool_calls;
// The model spoke since the waiting lines were sent ("Let me
// try a different search"), so the browser dropped them.
// Send them again, because this tool may take 30 s.
if ( $filler_on && $this->said_since_follow_ups ) {
$this->said_since_follow_ups = false;
$this->follow_up_scheduled = $this->schedule_waiting_line() || $this->follow_up_scheduled;
}
continue;
}
if ( ! $chunk instanceof TextChunk ) {
continue;
}
$text = (string) $chunk->content;
if ( '' === $text ) {
continue;
}
$answer .= $text;
if ( $tool_answered ) {
$spoke_after = true;
}
if ( 0 === $word_ms ) {
$word_ms = $this->ms( $began );
$timer->at( 'firstWord' );
}
// The tone tag is the first thing the model writes, so it is
// nearly always known before the first sentence is ready.
if ( '' === $tone && false !== strpos( $answer, ']]' ) ) {
$tone = Text_To_Speech::read_tone( $answer );
$speaker->set_tone( $tone );
}
foreach ( $chunker->add( $text ) as $piece ) {
// No tone tag yet. Guess the tone from the first sentence
// before it is spoken. Many models (e.g. gpt-4.1-nano)
// never write the tag, so this is common.
if ( '' === $tone ) {
$tone = Text_To_Speech::read_tone( $piece );
$speaker->set_tone( $tone );
}
// Only the first sentence gets its own clip. Each clip
// costs about 2 s no matter the length (5 chars 1967 ms,
// 175 chars 2393 ms), so the rest goes as one clip.
// Decide: hold, tail, or speak. See route_piece().
$road = self::route_piece( $hold_on, $grounded, $held, $said_answer, $piece );
if ( 'hold' === $road ) {
$held .= ' ' . $piece;
continue;
}
if ( 'tail' === $road ) {
$tail .= ' ' . $piece;
continue;
}
// Fix the opening word before speaking, not only in the
// saved text. Otherwise the caller may hear "No worries"
// before a refusal.
if ( ! $said_answer ) {
$piece = Answer_Opening::fix( $piece, $tone );
}
if ( $this->say( $speaker, $piece ) ) {
$said_answer = true;
if ( 0 === $first_ms ) {
$first_ms = $this->sound_ms( $began );
}
}
}
}
// Facts were written but no tool was called, so they are guesses.
// Do not speak them. Ask again below and tell the model to look
// it up first.
// Why: such answers were wrong (e.g. revenue "three thousand two
// hundred dollars" when the real figure was 37,766,754.90).
$unchecked = self::needs_a_second_ask( $hold_on, $held, $tool_calls );
if ( $unchecked ) {
$timer->note( 'uncheckedFacts', 1 );
// Clear all three buffers, not only $held: the tail and the
// last unfinished sentence in the chunker are part of the same
// guess. Otherwise $rest below speaks part of it, and it is
// saved to history.
$held = '';
$tail = '';
$answer = '';
$chunker->forget();
}
// Held text goes first, because it is the start of the answer.
$rest = trim( $held . ' ' . $tail . ' ' . $chunker->rest() );
// If no tool ran and a page opens, cut made-up how-to steps
// after the page name. Done here before speaking, and again on the
// saved answer below. See Answer_Steps::fix().
$rest = Answer_Steps::fix( $rest, $tool_calls, $said_answer, $this->settings_on_linked_page( $answer, $request ) );
if ( '' !== $rest && $this->say( $speaker, $rest ) && 0 === $first_ms ) {
$first_ms = $this->sound_ms( $began );
}
// Ask once more, without streaming, when the turn has no useful
// answer:
// 1. Nothing came back (about 1 turn in 15).
// 2. The model ran a tool and then stopped without answering.
// 3. It answered from memory and the facts were dropped above.
$stopped_after_tool = $tool_calls > 0 && $tool_answered && ! $spoke_after;
if ( '' === trim( $answer ) || $stopped_after_tool ) {
if ( $this->out_of_time() ) {
$this->emit_failure( $this->time_up(), $conversation_id );
return;
}
$retry_at = $timer->mark();
// chat() runs all tools before it answers, so it is more
// likely to return real data.
$retry = $this->ask_agent( $settings, (string) $heard['text'], (string) $heard['language'], $request, $conversation_id, $message_repo, $unchecked );
$timer->took( 'model.retry', $retry_at );
$timer->count( 'retries' );
if ( is_wp_error( $retry ) ) {
$this->emit_failure( $retry, $conversation_id );
return;
}
// Save only the retry. The text from the first try may
// already be spoken, but it is not saved.
// Why: joining both put two answers in one message, and
// sometimes they did not agree.
$answer = $retry;
$retried = true;
$tone = Text_To_Speech::read_tone( $retry );
$speaker->set_tone( $tone );
if ( $this->say( $speaker, Text_To_Speech::strip_tone( $retry ) ) && 0 === $first_ms ) {
$first_ms = $this->sound_ms( $began );
}
}
} catch ( \Throwable $e ) {
$this->emit_exception( $e, $conversation_id );
return;
}
$timer->at( 'modelDone' );
$model_ms = $this->ms( $model_at );
$answer = $this->link_last( $answer );
$tone = '' !== $tone ? $tone : Text_To_Speech::read_tone( $answer );
$answer = Text_To_Speech::strip_tone( $answer );
$answer = Answer_Opening::fix( $answer, $tone );
$answer = $retried ? $answer : Answer_Steps::fix( $answer, $tool_calls, false, $this->settings_on_linked_page( $answer, $request ) );
// Save the question and the answer together.
$save_at = $timer->mark();
$message_repo->create( $conversation_id, 'user', (string) $heard['text'], [ 'source' => 'voice' ] );
$message_repo->create( $conversation_id, 'assistant', $answer, [ 'source' => 'voice' ] );
$timer->took( 'db.save', $save_at );
// "first sound" is how long the caller waited in silence. The other
// numbers (first word, tools, speech) show which part was slow.
Log::debug(
sprintf(
'Voice TIMING stream — stt %dms, first word %dms, first sound %dms, tools %d, speech %dms over %d clips, model and speech %dms, total %dms',
$stt_ms,
$word_ms,
$first_ms,
$tool_calls,
$this->speech_ms,
$this->clips,
$model_ms,
$this->ms( $began )
)
);
// Said it was opening a page but gave no link. Only counted, not
// fixed: asking again would speak the whole answer a second time.
if ( self::promised_a_page( $answer ) ) {
$timer->note( 'brokenPromise', 1 );
}
$timer->note( 'answerChars', mb_strlen( $answer ) );
$timer->note( 'toolCalls', $tool_calls );
$timer->note( 'clips', $this->clips );
$timer->write( 'stream' );
$this->emit(
[
'type' => 'done',
'conversationId' => $conversation_id,
'heard' => (string) $heard['text'],
'language' => (string) $heard['language'],
'answer' => $answer,
'tone' => $tone,
'link' => $this->first_link( $answer, $request ),
]
);
$this->warm_up( $tts );
}
/**
* Speak one piece and send it.
*
* If a piece cannot be spoken, it is logged and skipped. The call goes on,
* and the words still appear in the chat thread.
*
* @param Stream_Speaker $speaker Speaker for this turn.
* @param string $piece Raw text from the model.
* @return bool True when something was sent.
*/
private function say( Stream_Speaker $speaker, string $piece ): bool {
// Out of time: no more paid speech. The loop above sends the error.
if ( $this->out_of_time() ) {
return false;
}
$at = microtime( true );
if ( $this->timer ) {
$this->timer->at( 'firstSayStarted' );
}
// The speaker sends its own events (text, audio parts, end), because
// only it knows how many there are.
$sent = $speaker->say( $piece, fn( array $event ) => $this->emit( $event ) );
$this->speech_ms += $this->ms( $at );
if ( is_wp_error( $sent ) ) {
Log::error( 'Voice: a sentence could not be spoken — ' . $sent->get_error_message() );
return false;
}
if ( true !== $sent ) {
return false;
}
++$this->clips;
return true;
}
/**
* Start a Server-Sent Events response.
*
* @return void
*/
private function open_stream(): void {
if ( function_exists( 'apache_setenv' ) ) {
apache_setenv( 'no-gzip', '1' ); // phpcs:ignore WordPress.PHP.DiscouragedPHPFunctions.runtime_configuration_apache_setenv
}
while ( ob_get_level() > 0 ) {
ob_end_clean();
}
ob_implicit_flush( true );
// Stop when the caller hangs up. PHP notices on the next write to the
// stream and ends the request, so no more model or speech calls are
// paid for. The unfinished turn is not saved: nobody heard it.
ignore_user_abort( false );
header( 'Content-Type: text/event-stream; charset=UTF-8' );
header( 'Cache-Control: no-cache, no-store' );
header( 'Connection: keep-alive' );
header( 'X-Accel-Buffering: no' );
}
/**
* Send one event down the open stream.
*
* @param array $event Event payload.
* @return void
*/
private function emit( array $event ): void {
if ( 'done' === ( $event['type'] ?? '' ) ) {
$this->done_sent = true;
}
if ( $this->carries_sound( $event ) ) {
$this->last_sound = microtime( true );
if ( 0.0 === $this->first_sound ) {
$this->first_sound = $this->last_sound;
if ( $this->timer ) {
// The moment the first sound leaves the server.
$this->timer->at( 'firstSoundOut' );
}
}
}
if ( 'say' === ( $event['type'] ?? '' ) ) {
if ( empty( $event['filler'] ) ) {
$this->said_since_follow_ups = true;
}
}
echo 'data: ' . wp_json_encode( $event ) . "\n\n"; // phpcs:ignore WordPress.Security.EscapeOutput.OutputNotEscaped -- JSON for an SSE stream, not HTML.
flush();
}
/**
* Is this the event that makes a noise come out of the speaker?
*
* Either a finished clip on a 'say', or the first piece of a streamed one.
*
* @param array $event Event payload.
* @return bool
*/
private function carries_sound( array $event ): bool {
if ( 'audio' === ( $event['type'] ?? '' ) ) {
return true;
}
return 'say' === ( $event['type'] ?? '' ) && '' !== (string) ( $event['audio'] ?? '' );
}
/**
* Milliseconds from the start of the turn to the first sound.
*
* Never zero, because zero means "nothing spoken yet".
*
* @param float $began Start mark for the whole turn.
* @return int
*/
private function sound_ms( float $began ): int {
$at = $this->first_sound > 0.0 ? $this->first_sound : microtime( true );
return max( 1, (int) round( ( $at - $began ) * 1000 ) );
}
/**
* Report a failure that happened after the stream was already open.
*
* Always sends a 'done' event after the error, so the browser knows the
* turn ended.
*
* @param WP_Error $error What went wrong.
* @param string $conversation_id Conversation UUID.
* @return void
*/
private function emit_failure( WP_Error $error, string $conversation_id ): void {
$data = $error->get_error_data();
$status = is_array( $data ) && isset( $data['status'] ) ? (int) $data['status'] : 502;
Log::error(
'Voice FAILED — ' . $error->get_error_code(),
[
'status' => $status,
'message' => $error->get_error_message(),
]
);
$this->emit(
[
'type' => 'error',
'code' => $error->get_error_code(),
'message' => $error->get_error_message(),
]
);
$this->emit(
[
'type' => 'done',
'conversationId' => $conversation_id,
'heard' => '',
'language' => '',
'answer' => '',
'tone' => Text_To_Speech::DEFAULT_TONE,
'link' => null,
]
);
}
/**
* Report an exception that happened after the stream was already open.
*
* The real message is logged. The caller gets a general message, because
* the real one may contain provider details or file paths.
*
* @param \Throwable $e The caught exception.
* @param string $conversation_id Conversation UUID.
* @return void
*/
private function emit_exception( \Throwable $e, string $conversation_id ): void {
Log::error( 'Voice stream failed: ' . $e->getMessage() );
// 'done' was already sent. Do not send a second error and 'done'.
if ( $this->done_sent ) {
return;
}
$this->emit_failure(
new WP_Error(
'aisentic_voice_agent_failed',
__( 'Something went wrong while generating the response. Please try again.', 'aisentic' ),
[ 'status' => 502 ]
),
$conversation_id
);
}
/**
* The agent that answers a call.
*
* The same agent as the text chat, with voice mode on. Voice mode removes
* all tools that can write and adds the rules for spoken answers.
*
* @param Settings $settings Settings instance.
* @param WP_REST_Request $request Request object.
* @return Admin_Assistant_Agent
*/
private function voice_agent( Settings $settings, WP_REST_Request $request ): Admin_Assistant_Agent {
$current_page = sanitize_text_field( (string) ( $request->get_param( 'current_page' ) ?? '' ) );
return new Admin_Assistant_Agent( $settings, $current_page, null, null, [], true );
}
/**
* The history of this conversation, with this turn's question on the end.
*
* @param Message_Repository $message_repo Message repository.
* @param string $conversation_id Conversation UUID.
* @param string $text What the admin said.
* @param string $language Detected language code.
* @param string $already_said A holding line the caller has
* already heard, or ''.
* @param string $look_it_up '' for a normal turn. 'first'
* when the question is about this
* site's figures. 'again' when the
* first try answered from memory.
* @return array
*/
private function turn_messages( Message_Repository $message_repo, string $conversation_id, string $text, string $language, string $already_said = '', string $look_it_up = '' ): array {
$prompt = $text;
if ( '' !== $language ) {
$prompt = "[The user spoke in language code: {$language}. Answer in that same language.]\n\n" . $text;
}
// The caller already heard a short line, so tell the model not to
// start with another one ("Of course", "Let me check"). This
// overrides rule 2 of the voice prompt. It is added last, because the
// model follows the last instruction best.
if ( '' !== $already_said ) {
$prompt .= "\n\n[You have ALREADY said \"{$already_said}\" out loud and they heard it. So for THIS answer only, open with the first REAL word of the answer. Do not say Of course, Sure, Got it, Right away, Happy to, Absolutely, and do not say Let me check, Let me see, Let me look, One moment, Just a second, I'm looking that up, Checking now, or anything like any of them. They heard that from you two seconds ago; saying it again is the same nothing twice.]";
}
// Tell the model to use a tool, not its memory. Added last, because
// the model follows the last instruction best.
// 'again': says the first answer came from memory and was dropped.
// 'first': no first answer yet, so it only says the question is about
// this site (see Question_Kind::asks_to_be_looked_up()).
if ( '' !== $look_it_up ) {
$opening = 'again' === $look_it_up
? 'You just answered this from memory and the figures were wrong, so they were not spoken.'
: 'This question is about this site\'s own figures and settings.';
$prompt .= "\n\n[{$opening} You do NOT know anything about this site that a tool has not told you in THIS answer. Call the tool that holds it, read the result, and answer from that. If no tool can tell you, say plainly that you cannot see it and offer to open the page — that is a good answer. What you remember about this site is not.]";
}
$messages = $this->recent_history( $message_repo, $conversation_id );
$messages[] = Message::make( MessageRole::USER, $prompt );
return $messages;
}
/**
* Ask the voice agent for a short answer.
*
* @param Settings $settings Settings instance.
* @param string $text What the admin said.
* @param string $language Detected language code.
* @param WP_REST_Request $request Request object.
* @param string $conversation_id Conversation UUID this call belongs to.
* @param Message_Repository $message_repo Message repository.
* @param bool $must_look_it_up Tell the model to use a tool
* rather than its own memory.
* @return string|WP_Error
*/
private function ask_agent( Settings $settings, string $text, string $language, WP_REST_Request $request, string $conversation_id, Message_Repository $message_repo, bool $must_look_it_up = false ) {
try {
$agent = $this->voice_agent( $settings, $request );
// Earlier turns, then this question with its language tag.
$messages = $this->turn_messages( $message_repo, $conversation_id, $text, $language, '', $must_look_it_up ? 'again' : '' );
$reply = $agent->chat( $messages )->getMessage()->getContent();
// About 1 turn in 15 comes back empty. Try once more with a new
// agent, so the caller does not get silence and a 502.
if ( '' === trim( (string) $reply ) ) {
$retry = $this->voice_agent( $settings, $request );
$reply = $retry->chat( $messages )->getMessage()->getContent();
}
} catch ( \Throwable $e ) {
Log::error( 'Voice agent failed: ' . $e->getMessage() );
return new WP_Error(
'aisentic_voice_agent_failed',
__( 'Something went wrong while generating the response. Please try again.', 'aisentic' ),
[ 'status' => 502 ]
);
}
$reply = is_string( $reply ) ? trim( $reply ) : '';
if ( '' === $reply ) {
return new WP_Error( 'aisentic_voice_empty_answer', __( 'I could not find an answer for that.', 'aisentic' ), [ 'status' => 502 ] );
}
return $reply;
}
/**
* Find the conversation this call belongs to, or start one.
*
* Voice uses the admin channel like the text chat, so a call continues an
* open conversation. An id that is not this admin's own conversation is
* refused. A new one is not started in its place.
*
* @param WP_REST_Request $request Request object.
* @return string|WP_Error Conversation UUID.
*/
private function conversation_id( WP_REST_Request $request ) {
$repo = new Conversation_Repository();
$sent = (string) ( $request->get_param( 'conversationId' ) ?? '' );
if ( '' !== $sent ) {
$sent = Sanitization::sanitizeConversationId( $sent );
if ( '' === $sent ) {
return Error_Handler::createRestError(
'aisentic_invalid_conversation_id',
__( 'Invalid conversation ID.', 'aisentic' ),
null,
400
);
}
$conversation = $repo->findByUuid( $sent );
if ( ! $conversation ) {
return Error_Handler::createRestError(
'aisentic_conversation_not_found',
__( 'Conversation not found.', 'aisentic' ),
null,
404
);
}
if ( 'admin' !== $conversation->channel || get_current_user_id() !== (int) $conversation->user_id ) {
return Error_Handler::createRestError(
'aisentic_forbidden',
__( 'You do not have permission to access this conversation.', 'aisentic' ),
null,
403
);
}
return $sent;
}
$created = $repo->create( get_current_user_id(), 'admin' );
if ( ! $created ) {
return Error_Handler::createRestError(
'aisentic_conversation_create_failed',
__( 'Failed to create conversation.', 'aisentic' ),
null,
500
);
}
return $created->uuid;
}
/**
* The last few turns of this conversation, as agent messages.
*
* Six turns by default (filter: aisentic_voice_history_turns). Enough for
* follow-ups like "and how do I add a speaker to it?". Long history also
* makes the model write long answers, which is bad for speech.
*
* Why six, not two: a benchmark said fewer messages is faster, but on a
* real call two turns was not faster (first byte stayed about 4.9 s). The
* benchmark repeated the same request, so it hit a warm cache.
*
* Tool results from earlier turns are not sent. The model runs the tool
* again if it needs to.
*
* @param Message_Repository $message_repo Message repository.
* @param string $conversation_id Conversation UUID.
* @return array
*/
private function recent_history( Message_Repository $message_repo, string $conversation_id ): array {
$limit = (int) apply_filters( 'aisentic_voice_history_turns', 6 );
$rows = $message_repo->get_recent_by_conversation( $conversation_id, max( 1, $limit * 2 ) );
$out = [];
foreach ( $rows as $row ) {
$role = MessageRole::tryFrom( (string) $row->role );
// Keep only user and assistant rows. Tool rows would need a
// matching call and result pair.
if ( MessageRole::USER !== $role && MessageRole::ASSISTANT !== $role ) {
continue;
}
$content = trim( (string) $row->content );
if ( '' === $content ) {
continue;
}
$out[] = [ $role, $content ];
}
return $this->alternating( $out );
}
/**
* Force the history into strict user, assistant, user, assistant order.
*
* The provider refuses any other order ("expected role assistant").
* Old broken conversations are still in the database, and a person can
* send two messages before the first is answered.
*
* Two rows in a row with the same role are joined into one, so no words
* are lost.
*
* @param array $rows List of [MessageRole, string] pairs, oldest first.
* @return array Message objects, ready to send.
*/
private function alternating( array $rows ): array {
$merged = [];
foreach ( $rows as $row ) {
$last = count( $merged ) - 1;
if ( $last >= 0 && $merged[ $last ][0] === $row[0] ) {
$merged[ $last ][1] .= "\n\n" . $row[1];
continue;
}
$merged[] = $row;
}
// Must start on a question...
while ( ! empty( $merged ) && MessageRole::USER !== $merged[0][0] ) {
array_shift( $merged );
}
// ...and end on an answer, because this turn's question comes next.
$last = end( $merged );
while ( false !== $last && MessageRole::ASSISTANT !== $last[0] ) {
array_pop( $merged );
$last = end( $merged );
}
return array_map(
static fn( $row ) => Message::make( $row[0], $row[1] ),
$merged
);
}
/**
* Put the page link at the end of the answer, where it belongs.
*
* If the link comes first, the chat shows a button above the text, which
* looks wrong. Only the first link is kept, because a call opens one page.
*
* @param string $answer Answer as the model wrote it.
* @return string
*/
private function link_last( string $answer ): string {
if ( ! preg_match_all( '/]*>.*?<\/a>/is', $answer, $m ) ) {
return $answer;
}
$first = $m[0][0];
$words = trim( (string) preg_replace( '/]*>.*?<\/a>/is', ' ', $answer ) );
$words = trim( (string) preg_replace( '/\s+/u', ' ', $words ) );
// Only a link, no words. Return just the link.
if ( '' === $words ) {
return $first;
}
return $words . "\n\n" . $first;
}
/**
* Pull the first admin link out of the answer, for the on-screen button.
*
* The agent writes links as Text.
* Only links to this site, to a real plugin page, are returned. So a
* made-up external URL cannot send the browser off-site.
*
* @param string $answer Agent answer, possibly containing HTML.
* @param WP_REST_Request $request The turn, for the screen the caller is on.
* @return array|null {url: string, label: string}
*/
private function first_link( string $answer, WP_REST_Request $request ): ?array {
if ( ! preg_match( '/]*href=["\']([^"\']+)["\'][^>]*>(.*?)<\/a>/is', $answer, $m ) ) {
return null;
}
$url = self::same_site_url( html_entity_decode( trim( $m[1] ), ENT_QUOTES, 'UTF-8' ) );
$label = trim( wp_strip_all_tags( $m[2] ) );
if ( null === $url ) {
return null;
}
$url = esc_url_raw( $url );
if ( '' === $url ) {
return null;
}
if ( ! $this->real_plugin_page( $url, $request ) ) {
return null;
}
return [
'url' => $url,
'label' => '' !== $label ? $label : __( 'Open page', 'aisentic' ),
];
}
/**
* Keep a link only if it opens a page on this site.
*
* A path ("/wp-admin/...", "admin.php?page=x") is ours. A full address
* must have the same scheme, host and port as home_url().
* These are dropped:
* - "//evil.example" (no scheme, but the browser opens another site).
* - "https://evil.example" and "http://site:8080" (another host or port).
* - "javascript:", "data:" and every other scheme.
*
* @param string $url The address the model wrote.
* @return string|null The address, or null when it leaves the site.
*/
private static function same_site_url( string $url ): ?string {
// Browsers treat "\" like "/", so "/\evil.example" is "//evil.example".
$url = str_replace( '\\', '/', $url );
if ( '' === $url || 0 === strpos( $url, '//' ) ) {
return null;
}
$parts = wp_parse_url( $url );
if ( false === $parts ) {
return null;
}
// No scheme and no host: a path on this site.
if ( ! isset( $parts['scheme'] ) && ! isset( $parts['host'] ) ) {
// "javascript:alert(1)" has no "//", so wp_parse_url() may not see
// the scheme. A colon before the first "/", "?" or "#" is a scheme.
if ( preg_match( '#^[^/?\#]*:#', $url ) ) {
return null;
}
return $url;
}
$home = wp_parse_url( home_url() );
if ( ! is_array( $home ) || empty( $home['host'] ) || empty( $parts['host'] ) ) {
return null;
}
$scheme = strtolower( (string) ( $parts['scheme'] ?? '' ) );
if ( ! in_array( $scheme, [ 'http', 'https' ], true ) ) {
return null;
}
$port = (int) ( $parts['port'] ?? ( 'https' === $scheme ? 443 : 80 ) );
$home_port = (int) ( $home['port'] ?? ( 'https' === strtolower( (string) ( $home['scheme'] ?? 'http' ) ) ? 443 : 80 ) );
if ( 0 !== strcasecmp( (string) $parts['host'], (string) $home['host'] ) || $port !== $home_port ) {
return null;
}
return $url;
}
/**
* Is this address a screen the open plugin really has?
*
* The prompt says to copy addresses from the page list, but the model
* sometimes makes them up (6 of 12 links in one test). So we check here.
*
* Only admin.php?page= links are checked, against the open plugin's list.
* These pass without a check:
* - WordPress core pages (post-new.php, options-general.php, ...).
* - Any link when there is no plugin, or the page list is empty (the
* sitemap service may be down; we do not want to block all links).
*
* @param string $url The address the model wrote.
* @param WP_REST_Request $request The turn, for the screen the caller is on.
* @return bool
*/
private function real_plugin_page( string $url, WP_REST_Request $request ): bool {
if ( ! preg_match( '#(^|/)admin\.php\?#i', $url ) ) {
return true;
}
$current = sanitize_text_field( (string) ( $request->get_param( 'current_page' ) ?? '' ) );
$plugin = (string) Screen_Id_To_Plugin_Mapper::get( $current );
if ( '' === $plugin || Screen_Id_To_Plugin_Mapper::WORDPRESS_CORE_KEY === $plugin ) {
return true;
}
$pages = new Admin_Pages( new Settings() );
if ( empty( $pages->all( $plugin ) ) ) {
return true;
}
return $pages->knows( $url, $plugin );
}
/**
* The names of the settings on the page this answer opens.
*
* The prompt shows the model these same names, so a how-to sentence that
* uses them is not a guess. See Answer_Steps.
*
* @param string $answer What the model wrote, link included.
* @param WP_REST_Request $request The turn, for the screen the caller is on.
* @return string[] Empty when there is no link or nothing is known.
*/
private function settings_on_linked_page( string $answer, WP_REST_Request $request ): array {
if ( ! preg_match_all( '/]*href=["\']([^"\']+)["\']/i', $answer, $m ) ) {
return [];
}
$current = sanitize_text_field( (string) ( $request->get_param( 'current_page' ) ?? '' ) );
$plugin = (string) Screen_Id_To_Plugin_Mapper::get( $current );
if ( '' === $plugin || Screen_Id_To_Plugin_Mapper::WORDPRESS_CORE_KEY === $plugin ) {
return [];
}
$url = html_entity_decode( (string) end( $m[1] ), ENT_QUOTES, 'UTF-8' );
return ( new Admin_Pages( new Settings() ) )->holds_for( $url, $plugin );
}
/**
* Delete the uploaded recording.
*
* Audio is never kept: not in uploads, not in the media library, and not
* in the temp folder.
*
* @param string $path Temp path, may be empty.
* @return void
*/
private function discard( string $path ): void {
if ( '' === $path || ! file_exists( $path ) ) {
return;
}
wp_delete_file( $path );
}
/**
* Convert a WP_Error into this controller's error response shape.
*
* @param WP_Error $error Error to convert.
* @return \WP_HTTP_Response
*/
private function from_wp_error( WP_Error $error ) {
$data = $error->get_error_data();
$status = is_array( $data ) && isset( $data['status'] ) ? (int) $data['status'] : 422;
// Log every failed turn, so it can be fixed. Otherwise the browser
// shows 502 and the log has no reason.
Log::error(
'Voice FAILED — ' . $error->get_error_code(),
[
'status' => $status,
'message' => $error->get_error_message(),
]
);
return $this->error( $error->get_error_message(), $status, $error->get_error_code() );
}
}