ULLMAIBlueprintLibrary and are the recommended approach for flexibility.
UFUNCTION(BlueprintCallable, Category = "LLMAI|AI", BlueprintPure)
static TArray<FString> GetAvailableAIProviders();
// Returns: ["OpenAI", "Grok", "LocalAI"]
// Future: Additional providers (Claude, Gemini, etc.)
UFUNCTION(BlueprintCallable, Category = "LLMAI|AI", BlueprintPure)
static FString GetDefaultAIProvider();
UFUNCTION(BlueprintCallable, Category = "LLMAI|AI")
static void SetDefaultAIProvider(const FString& Provider);
UFUNCTION(BlueprintCallable, Category = "LLMAI|AI", BlueprintPure)
static TArray<FString> GetKnownAIModels(const FString& Provider);
UFUNCTION(BlueprintCallable, Category = "LLMAI|AI", BlueprintPure)
static bool IsValidAIModel(const FString& Provider, const FString& Model);
UFUNCTION(BlueprintCallable, Category = "LLMAI|AI", BlueprintPure)
static FString GetDefaultAIModel(const FString& Provider);
UFUNCTION(BlueprintCallable, Category = "LLMAI|AI")
static void SetDefaultAIModel(const FString& Provider, const FString& Model);
UFUNCTION(BlueprintCallable, Category = "LLMAI|AI", BlueprintPure)
static TArray<FString> GetKnownAIVoices(const FString& Provider);
UFUNCTION(BlueprintCallable, Category = "LLMAI|AI", BlueprintPure)
static bool IsValidAIVoice(const FString& Provider, const FString& Voice);
UFUNCTION(BlueprintCallable, Category = "LLMAI|AI", BlueprintPure)
static FString GetDefaultAIVoice(const FString& Provider);
UFUNCTION(BlueprintCallable, Category = "LLMAI|AI")
static void SetDefaultAIVoice(const FString& Provider, const FString& Voice);
The main component for AI communication and session management.
UFUNCTION(BlueprintCallable, Category = "LLMAI|AI")
void ConnectToAI(const FString& Provider,
const FString& Model,
const FString& Instructions,
const TArray<FString>& Modalities,
float Temperature = 0.8f,
const FString& Voice = "");
// Provider: "OpenAI", "LocalAI", or "Grok"
// Modalities: ["text"], ["audio"], or ["text", "audio"]
// Voice: REQUIRED for Grok (locks after first session update)
// Optional for OpenAI/LocalAI (can change via StartVoiceMode)
For Grok provider, the Voice parameter is REQUIRED and cannot be changed without reconnecting. Grok locks the voice after the first session update. If not specified, defaults to "Ara".
OpenAI and LocalAI do not have this limitation - voice can be changed mid-session via StartVoiceMode().
UFUNCTION(BlueprintCallable, Category = "LLMAI|AI", BlueprintPure)
bool IsAISession() const;
// True if connected to any AI provider
UFUNCTION(BlueprintCallable, Category = "LLMAI|AI", BlueprintPure)
bool IsAISessionReady() const;
// True if session is ready for communication
UFUNCTION(BlueprintCallable, Category = "LLMAI|Socket", BlueprintPure)
ELLMAISessionState GetCurrentSessionState() const;
UFUNCTION(BlueprintCallable, Category = "LLMAI|Socket", BlueprintPure)
bool IsAIActivelyResponding() const;
UFUNCTION(BlueprintCallable, Category = "LLMAI|Socket")
void Connect(const FString& URL, const FString& Protocol = TEXT(""));
UFUNCTION(BlueprintCallable, Category = "LLMAI|Socket")
void Disconnect();
UFUNCTION(BlueprintCallable, Category = "LLMAI|Socket", BlueprintPure)
bool IsConnected() const;
UFUNCTION(BlueprintCallable, Category = "LLMAI|Socket", BlueprintPure)
bool IsConnecting() const;
// These are called internally by ConnectToAI
void ConnectToOpenAI(const FString& Model,
const FString& Instructions,
const TArray<FString>& Modalities,
float Temperature = 0.8f,
const FString& Voice = "");
void ConnectToLocalAI(const FString& Model,
const FString& Instructions,
const TArray<FString>& Modalities,
float Temperature = 0.8f,
const FString& Voice = "");
void ConnectToGrok(const FString& Model,
const FString& Instructions,
const TArray<FString>& Modalities,
float Temperature = 0.8f,
const FString& Voice = "Ara");
| Feature |
|
LocalAI
|
Grok
|
|---|---|---|---|
| Voice Options | 6+ voices | Custom (Chatterbox) | 5 voices (Ara, Rex, Sal, Eve, Leo) |
| Change Voice Mid-Session | ✅ Yes | ✅ Yes | ❌ No - Must reconnect |
| VAD Type Support | Server VAD, Semantic VAD | Server VAD only | Server VAD only |
| Text-Only Mode | ✅ Supported | ✅ Supported | ⚠️ Client-side filtered |
| Web Search | ❌ No | ❌ No | ✅ Yes (enable_search) |
| Input Transcription Streaming | ✅ Delta events | ✅ Delta events | ❌ Completed only |
| Function Calling | ✅ Supported | ✅ Supported | ✅ Supported |
UENUM(BlueprintType)
enum class ELLMAISessionState : uint8
{
Disconnected,
Connecting,
Connected,
SessionCreating,
SessionReady,
VoiceStarting,
VoiceActive,
Error
};
UFUNCTION(BlueprintCallable, Category = "LLMAI|Voice")
bool StartVoiceMode(const FString& Voice = TEXT(""),
const FString& TTSModel = TEXT("")); // v2.0: TTSModel param
UFUNCTION(BlueprintCallable, Category = "LLMAI|Voice")
bool EndVoiceMode(bool bClearConversation = true);
UFUNCTION(BlueprintCallable, Category = "LLMAI|AI")
void SendAudioToAI(const TArray<float>& AudioSamples);
UFUNCTION(BlueprintCallable, Category = "LLMAI|Voice")
void CommitAudioBuffer();
UFUNCTION(BlueprintCallable, Category = "LLMAI|Voice")
void TruncateAudioBuffer(int32 AudioEndMs = 0);
UFUNCTION(BlueprintCallable, Category = "LLMAI|Voice")
void SetMicrophoneMuted(bool bMuted);
UFUNCTION(BlueprintCallable, Category = "LLMAI|Voice")
void ToggleMicrophoneMute();
UFUNCTION(BlueprintPure, Category = "LLMAI|Voice")
bool IsMicrophoneMuted() const;
UFUNCTION(BlueprintPure, Category = "LLMAI|Voice")
bool IsMicrophoneAutoGated() const;
// Returns true when mic is auto-muted because AI is speaking
UFUNCTION(BlueprintCallable, Category = "LLMAI|Voice")
void StopBufferedAudioPlayback();
// Stop playback when AI finished but audio still buffered
UFUNCTION(BlueprintCallable, Category = "LLMAI|AI")
void SendTextToAI(const FString& Text,
bool bUseTextModality = true,
bool bUseAudioModality = true);
UFUNCTION(BlueprintCallable, Category = "LLMAI|AI")
void AddContextToAI(const FString& ContextText);
UFUNCTION(BlueprintCallable, Category = "LLMAI|Socket")
void TriggerAIResponse(const FString& Instructions = TEXT(""));
UFUNCTION(BlueprintCallable, Category = "LLMAI|Socket")
void InterruptAIResponse();
UFUNCTION(BlueprintCallable, Category = "LLMAI|Socket")
void CancelResponse();
UFUNCTION(BlueprintCallable, Category = "LLMAI|Socket")
void TriggerAIResponseWithModalities(
const TArray<FString>& Modalities,
const FString& Instructions = TEXT(""));
UFUNCTION(BlueprintCallable, Category = "LLMAI|Socket")
void TriggerTextOnlyResponse(const FString& Instructions = TEXT(""));
UFUNCTION(BlueprintCallable, Category = "LLMAI|Voice")
void TriggerAudioOnlyResponse(const FString& Instructions = TEXT(""));
// These still work but use generic versions above instead
void SendTextToOpenAI(const FString& Text,
bool bUseTextModality = true,
bool bUseAudioModality = true);
void AddContextToOpenAI(const FString& ContextText);
void SendTextToLocalAI(...); // Same signature
void AddContextToLocalAI(...); // Same signature
UFUNCTION(BlueprintCallable, Category = "LLMAI|AI Functions")
void RegisterAIFunction(const FLLMFunctionDefinition& FunctionDefinition);
UFUNCTION(BlueprintCallable, Category = "LLMAI|AI Functions")
void UnregisterAIFunction(const FString& FunctionName);
UFUNCTION(BlueprintCallable, Category = "LLMAI|AI Functions")
void SendAIFunctionCallResult(const FString& CallId,
const FString& Result);
UFUNCTION(BlueprintCallable, Category = "LLMAI|AI Functions", BlueprintPure)
TArray<FString> GetRegisteredAIFunctionNames() const;
UFUNCTION(BlueprintCallable, Category = "LLMAI|AI Function Profiles")
void RegisterFunctionProfile(ULLMAIFunctionProfile* Profile);
UFUNCTION(BlueprintCallable, Category = "LLMAI|AI Function Profiles")
void RegisterFunctionDefinitionAsset(ULLMAIFunctionDefinitionAsset* FunctionAsset);
UFUNCTION(BlueprintCallable, Category = "LLMAI|AI Function Profiles", BlueprintPure)
bool IsFunctionProfileLoaded(const FString& ProfileName) const;
UFUNCTION(BlueprintCallable, Category = "LLMAI|AI Function Profiles", BlueprintPure)
TArray<FString> GetLoadedProfileNames() const;
UFUNCTION(BlueprintCallable, Category = "LLMAI|Voice")
void SetupVoiceCapture(UAudioCaptureComponent* MicComponent = nullptr);
UFUNCTION(BlueprintCallable, Category = "LLMAI|Voice")
void SetupAudioPlayback(UAudioComponent* AudioComponent,
bool bIgnoreFlushing = true);
UFUNCTION(BlueprintCallable, Category = "LLMAI|Audio Stream")
void SetAudioStreamComponent(ULLMAIAudioStreamComponent* AudioComponent);
UFUNCTION(BlueprintCallable, Category = "LLMAI|Audio Stream")
void StartAudioStreaming();
UFUNCTION(BlueprintCallable, Category = "LLMAI|Audio Stream")
void StopAudioStreaming();
UFUNCTION(BlueprintCallable, Category = "LLMAI|Audio Stream", BlueprintPure)
bool IsAudioStreaming() const;
UFUNCTION(BlueprintCallable, Category = "LLMAI|Audio Stream", BlueprintPure)
bool IsAudioStreamAutoCreated() const;
UFUNCTION(BlueprintCallable, Category = "LLMAI|Audio Stream", BlueprintPure)
bool IsAudioPlaybackAutoCreated() const;
UFUNCTION(BlueprintCallable, Category = "LLMAI|Voice", BlueprintPure)
bool IsAudioPlaying() const;
UFUNCTION(BlueprintCallable, Category = "LLMAI|Socket", BlueprintPure)
FString GetCurrentSessionIdentifier() const;
UFUNCTION(BlueprintCallable, Category = "LLMAI|Socket", BlueprintPure)
FString GetCurrentResponseId() const;
UPROPERTY(BlueprintAssignable, Category = "LLMAI|Socket")
FOnSocketConnected OnConnected;
UPROPERTY(BlueprintAssignable, Category = "LLMAI|Socket")
FOnSocketDisconnected OnDisconnected;
UPROPERTY(BlueprintAssignable, Category = "LLMAI|Socket")
FOnSocketError OnError;
UPROPERTY(BlueprintAssignable, Category = "LLMAI|Voice")
FOnAISessionReady OnAISessionReady;
UPROPERTY(BlueprintAssignable, Category = "LLMAI|Voice")
FOnVoiceModeActivated OnVoiceModeActivated;
UPROPERTY(BlueprintAssignable, Category = "LLMAI|Voice")
FOnVoiceSessionComplete OnVoiceSessionComplete;
UPROPERTY(BlueprintAssignable, Category = "LLMAI|Text Response")
FOnAITextResponseBegin OnAITextResponseBegin;
// (ResponseId)
UPROPERTY(BlueprintAssignable, Category = "LLMAI|Text Response")
FOnAITextResponseDelta OnAITextResponseDelta;
// (DeltaText) - streaming text chunks
UPROPERTY(BlueprintAssignable, Category = "LLMAI|Text Response")
FOnAITextComplete OnAITextComplete;
// (FullText) - complete text when done
UPROPERTY(BlueprintAssignable, Category = "LLMAI|Text Response")
FOnAIResponseComplete OnAIResponseComplete;
// (ResponseId) - fires after text/audio complete
UPROPERTY(BlueprintAssignable, Category = "LLMAI|Transcription")
FOnInputAudioTranscriptionDelta OnInputAudioTranscriptionDelta;
// (ItemId, DeltaText)
UPROPERTY(BlueprintAssignable, Category = "LLMAI|Transcription")
FOnInputAudioTranscriptionCompleted OnInputAudioTranscriptionCompleted;
// (ItemId, FullTranscript)
UPROPERTY(BlueprintAssignable, Category = "LLMAI|Conversation")
FOnConversationItemCreated OnConversationItemCreated;
// (ItemId, Role, ItemType)
UPROPERTY(BlueprintAssignable, Category = "LLMAI|Voice")
FOnVoiceSessionComplete OnVoiceSessionComplete;
// (GeneratedAudioFile) - USoundWave* with recorded audio
UPROPERTY(BlueprintAssignable, Category = "LLMAI|Voice")
FOnVoiceSessionProgress OnVoiceSessionProgress;
// (StatusMessage)
UPROPERTY(BlueprintAssignable, Category = "LLMAI|Voice")
FOnMicrophoneMuted OnMicrophoneMuted;
UPROPERTY(BlueprintAssignable, Category = "LLMAI|Voice")
FOnMicrophoneUnmuted OnMicrophoneUnmuted;
UPROPERTY(BlueprintAssignable, Category = "LLMAI|Voice")
FOnMicrophoneAutoGated OnMicrophoneAutoGated;
UPROPERTY(BlueprintAssignable, Category = "LLMAI|Voice")
FOnMicrophoneAutoUngated OnMicrophoneAutoUngated;
UPROPERTY(BlueprintAssignable, Category = "LLMAI|Voice")
FOnVoiceOutputBegin OnVoiceOutputBegin;
// Fires when AI starts speaking
UPROPERTY(BlueprintAssignable, Category = "LLMAI|Voice")
FOnVoiceOutputEnd OnVoiceOutputEnd;
// Fires when AI finishes speaking
UPROPERTY(BlueprintAssignable, Category = "LLMAI|Voice")
FOnServerVADSpeechStarted OnServerVADSpeechStarted;
UPROPERTY(BlueprintAssignable, Category = "LLMAI|Voice")
FOnServerVADSpeechStopped OnServerVADSpeechStopped;
UPROPERTY(BlueprintAssignable, Category = "LLMAI|Voice")
FOnServerVADAudioCommitted OnServerVADAudioCommitted;
UPROPERTY(BlueprintAssignable, Category = "LLMAI|Reasoning")
FOnAIOutputItemAdded OnAIOutputItemAdded;
// (ItemId)
UPROPERTY(BlueprintAssignable, Category = "LLMAI|Reasoning")
FOnAIContentPartAdded OnAIContentPartAdded;
// (ItemId, ContentIndex, ContentType)
UPROPERTY(BlueprintAssignable, Category = "LLMAI|Reasoning")
FOnAIOutputItemComplete OnAIOutputItemComplete;
// (ItemId, ContentJson)
UPROPERTY(BlueprintAssignable, Category = "LLMAI|Reasoning")
FOnAIReasoningDelta OnAIReasoningDelta;
// (ItemId, ReasoningDelta)
UPROPERTY(BlueprintAssignable, Category = "LLMAI|Reasoning")
FOnAIReasoningComplete OnAIReasoningComplete;
// (ItemId, FullReasoning)
UPROPERTY(BlueprintAssignable, Category = "LLMAI|AI Functions")
FOnAIFunctionCallRequested OnAIFunctionCallRequested;
UPROPERTY(BlueprintAssignable, Category = "LLMAI|AI Functions")
FOnAIFunctionCallResult OnAIFunctionCallResult;
// v2.0: Function Profile Events
UPROPERTY(BlueprintAssignable, Category = "LLMAI|Function Profiles")
FOnAIFunctionProfileLoaded OnFunctionProfileLoaded;
UPROPERTY(BlueprintAssignable, Category = "LLMAI|Function Profiles")
FOnAIFunctionProfileLoadFailed OnFunctionProfileLoadFailed;
UPROPERTY(BlueprintAssignable, Category = "LLMAI|OpenAI")
FOnAIServerError OnAIServerError;
// (ErrorType, ErrorCode, ErrorMessage, EventId)
UPROPERTY(BlueprintAssignable, Category = "LLMAI|OpenAI")
FOnAIRateLimitExceeded OnAIRateLimitExceeded;
// (RateLimitType, ResetSeconds, Remaining)
UPROPERTY(BlueprintAssignable, Category = "LLMAI|LocalAI")
FOnContextCompacted OnContextCompacted;
// (TokensAfter, ContextUsagePercent, Message)
// Fires when LocalAI compacts context to free memory
USTRUCT(BlueprintType)
struct FLLMFunctionDefinition
{
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Function Definition")
FString Name;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Function Definition")
FString Description;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Function Definition")
TArray<FLLMFunctionParameter> Parameters;
};
USTRUCT(BlueprintType)
struct FLLMFunctionParameter
{
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Function Parameter")
FString Name;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Function Parameter")
ELLMFunctionParameterType Type; // String, Number, Integer, Boolean, Array, Object
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Function Parameter")
FString Description;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Function Parameter")
bool bIsRequired;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Function Parameter")
FString DefaultValue;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Function Parameter")
TArray<FString> AllowedValues; // Optional enum values
};
UENUM(BlueprintType)
enum class ELLMFunctionParameterType : uint8
{
String,
Number,
Integer,
Boolean,
Array,
Object
};
UENUM(BlueprintType)
enum class EAIProvider : uint8
{
None, // Not connected to any provider
OpenAI, // OpenAI cloud service
LocalAI, // LocalAI local/offline service
Grok // Grok (xAI) cloud service with web search
};
USTRUCT(BlueprintType)
struct FLLMFunctionCall
{
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Function Call")
FString CallId;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Function Call")
FString Name;
UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = "Function Call")
FString ArgumentsJson;
};
UFUNCTION(BlueprintCallable, Category = "LLMAI|Audio Utils")
TArray<float> ConvertPCM16ToFloat(const TArray<uint8>& PCM16Data);
UFUNCTION(BlueprintCallable, Category = "LLMAI|Audio Utils")
TArray<uint8> ConvertFloatToPCM16(const TArray<float>& FloatData);
UFUNCTION(BlueprintCallable, Category = "LLMAI|Audio Utils")
float CalculateAudioLevel(const TArray<float>& AudioSamples);
UFUNCTION(BlueprintCallable, Category = "LLMAI|Audio Utils")
TArray<float> ApplyBasicNoiseReduction(const TArray<float>& AudioSamples,
float NoiseFloor = 0.02f);
UFUNCTION(BlueprintCallable, Category = "LLMAI|Audio Generation")
USoundWave* CreateSoundWaveFromPCM16(
const TArray<uint8>& PCM16Data,
const FString& SoundName = TEXT("GeneratedAudio"));
UFUNCTION(BlueprintCallable, Category = "LLMAI|Audio Filters")
void SetupAudioFilters(float LowPassFreq = 0.0f,
float HighPassFreq = 0.0f,
bool bEnableFilters = true);
UFUNCTION(BlueprintCallable, Category = "LLMAI|Audio Filters")
void SetFilterFrequencies(float LowPassFreq, float HighPassFreq);
UFUNCTION(BlueprintCallable, Category = "LLMAI|Audio Filters")
void EnableAudioFilters(bool bEnabled);
UPROPERTY(BlueprintAssignable, Category = "LLMAI|LiveLink")
FOnLiveLinkAudioInitialized OnLiveLinkAudioInitialized;
// (SubjectName) - Fires when LiveLink audio source is ready
UFUNCTION(BlueprintCallable, Category = "LLMAI|Audio Input")
void ReceiveExternalAudioData(const TArray<float>& AudioSamples);
// For UE 5.5.4 compatibility - feed audio from external sources
UFUNCTION(BlueprintCallable, BlueprintPure, Category = "LLMAI|Settings")
FString GetDefaultInstructions(const FString& Provider) const;
UFUNCTION(BlueprintCallable, BlueprintPure, Category = "LLMAI|Settings")
FString GetDefaultOpenAIAPIKey() const;
UFUNCTION(BlueprintCallable, BlueprintPure, Category = "LLMAI|Settings")
FString GetDefaultInputAudioFormat() const;
UFUNCTION(BlueprintCallable, BlueprintPure, Category = "LLMAI|Settings")
FString GetDefaultOutputAudioFormat() const;
UFUNCTION(BlueprintCallable, BlueprintPure, Category = "LLMAI|Utilities")
bool CanSafelyUpdateUI() const;
// Returns true if safe to update UI (world exists, not tearing down)
// Use before updating widgets in event handlers
Static utility functions for common operations.
UFUNCTION(BlueprintCallable, Category = "LLMAI|Audio")
static ULLMAIAudioStreamComponent* CreateAudioStreamComponent(
AActor* Actor, const FString& ComponentName);
UFUNCTION(BlueprintCallable, Category = "LLMAI|Socket")
static ULLMAIClientComponent* CreateSocketClientComponent(
AActor* Actor, const FString& ComponentName);
UFUNCTION(BlueprintCallable, Category = "LLMAI|Audio", BlueprintPure)
static ULLMAIAudioStreamComponent* GetAudioStreamComponent(AActor* Actor);
UFUNCTION(BlueprintCallable, Category = "LLMAI|Socket", BlueprintPure)
static ULLMAIClientComponent* GetLLMAIClientComponent(AActor* Actor);