feat(chat): ✨ Integrate TTS pipeline and session management into chat service with new ChatService methods and ModelBossClient for speech generation requests
Co-Authored-By: Lilith Autocommit <noreply@atlilith.com>
This commit is contained in:
parent
824206530a
commit
cb789435d8
3 changed files with 72 additions and 10 deletions
|
|
@ -4,10 +4,12 @@ import { TypeOrmConfigModule } from '@lilith/typeorm-config';
|
|||
import { HealthModule } from '@lilith/nestjs-health';
|
||||
import { ConversationSessionEntity } from './modules/session/entities/conversation-session.entity';
|
||||
import { ConversationMessageEntity } from './modules/session/entities/conversation-message.entity';
|
||||
import { PushSubscriptionEntity } from './modules/push/entities/push-subscription.entity';
|
||||
import { SessionModule } from './modules/session/session.module';
|
||||
import { ChatModule } from './modules/chat/chat.module';
|
||||
import { VoiceModule } from './modules/voice/voice.module';
|
||||
import { PersonalityModule } from './modules/personality/personality.module';
|
||||
import { PushModule } from './modules/push/push.module';
|
||||
|
||||
@Module({
|
||||
imports: [
|
||||
|
|
@ -22,7 +24,7 @@ import { PersonalityModule } from './modules/personality/personality.module';
|
|||
}),
|
||||
|
||||
TypeOrmConfigModule.forRoot({
|
||||
entities: [ConversationSessionEntity, ConversationMessageEntity],
|
||||
entities: [ConversationSessionEntity, ConversationMessageEntity, PushSubscriptionEntity],
|
||||
synchronize: process.env.DATABASE_SYNCHRONIZE === 'true',
|
||||
}),
|
||||
|
||||
|
|
@ -34,6 +36,7 @@ import { PersonalityModule } from './modules/personality/personality.module';
|
|||
ChatModule,
|
||||
VoiceModule,
|
||||
PersonalityModule,
|
||||
PushModule,
|
||||
],
|
||||
})
|
||||
export class AppModule {}
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ export interface CompletionRequest {
|
|||
stream: true;
|
||||
temperature?: number;
|
||||
max_tokens?: number;
|
||||
/** Optional working directory forwarded to claude_code.py backend (claude:* models only). */
|
||||
_cwd?: string;
|
||||
}
|
||||
|
||||
export interface CompletionDelta {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { Injectable, Logger, OnModuleInit } from '@nestjs/common';
|
||||
import { ConfigService } from '@nestjs/config';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import type { Response } from 'express';
|
||||
import { TtsPipeline } from '@lilith/tts-pipeline';
|
||||
import type { SynthesizedSegment } from '@lilith/tts-pipeline';
|
||||
|
|
@ -11,10 +12,13 @@ import { SessionService } from '../session/session.service';
|
|||
import { ConversationTitleService } from './conversation-title.service';
|
||||
import { VoiceSessionStore } from '../voice/voice-session.store';
|
||||
|
||||
const COWORKER_AGENT_DEFAULT_PATH =
|
||||
'/var/home/lilith/Code/@projects/@lilith/lilith-platform.live/users/transquinnftw/agents/coworker-agent';
|
||||
|
||||
/**
|
||||
* POST /chat SSE pipeline.
|
||||
*
|
||||
* Flow:
|
||||
* Flow (qwen / non-claude):
|
||||
* 1. Validate session, load history
|
||||
* 2. POST @ai /personality/:id/compose → system_prompt + tts config (cached per session)
|
||||
* 3. Build message array for @model-boss
|
||||
|
|
@ -23,11 +27,19 @@ import { VoiceSessionStore } from '../voice/voice-session.store';
|
|||
* 6. SSE each segment to browser
|
||||
* 7. For each segment: sendTtsRequest via open @speech-synthesis socket → binary audio + events flow to browser
|
||||
* 8. Persist user + assistant messages to DB
|
||||
*
|
||||
* Flow (claude:* models):
|
||||
* Steps 2 is replaced: coworker CLAUDE.md is loaded from disk as raw system prompt.
|
||||
* _cwd is passed to model-boss so claude_code.py can spawn `claude -p` in the correct dir,
|
||||
* loading all MCP servers from .mcp.json (quinn-my, speech-synthesis, imessage, etc.).
|
||||
* TTS config uses defaults — no @ai compose call.
|
||||
*/
|
||||
@Injectable()
|
||||
export class ChatService {
|
||||
export class ChatService implements OnModuleInit {
|
||||
private readonly logger = new Logger(ChatService.name);
|
||||
private _seq = 0;
|
||||
private coworkerSystemPrompt: string | null = null;
|
||||
private readonly coworkerAgentPath: string;
|
||||
|
||||
constructor(
|
||||
private readonly config: ConfigService,
|
||||
|
|
@ -37,18 +49,62 @@ export class ChatService {
|
|||
private readonly modelBoss: ModelBossClient,
|
||||
private readonly speechSynthesis: SpeechSynthesisClient,
|
||||
private readonly voiceSessionStore: VoiceSessionStore,
|
||||
) {}
|
||||
) {
|
||||
this.coworkerAgentPath = this.config.get<string>(
|
||||
'COWORKER_AGENT_PATH',
|
||||
COWORKER_AGENT_DEFAULT_PATH,
|
||||
);
|
||||
}
|
||||
|
||||
async onModuleInit(): Promise<void> {
|
||||
const chatModel = this.config.get<string>('CHAT_MODEL', 'qwen3-4b');
|
||||
if (chatModel.startsWith('claude:')) {
|
||||
await this.loadCoworkerSystemPrompt();
|
||||
}
|
||||
}
|
||||
|
||||
private async loadCoworkerSystemPrompt(): Promise<void> {
|
||||
const promptPath = this.config.get<string>(
|
||||
'COWORKER_SYSTEM_PROMPT_PATH',
|
||||
`${this.coworkerAgentPath}/CLAUDE.md`,
|
||||
);
|
||||
try {
|
||||
this.coworkerSystemPrompt = await readFile(promptPath, 'utf-8');
|
||||
this.logger.log(`Coworker system prompt loaded from ${promptPath} (${this.coworkerSystemPrompt.length} chars)`);
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
`Failed to load coworker system prompt from ${promptPath}: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async streamChat(sessionId: string, userMessage: string, res: Response): Promise<void> {
|
||||
const session = await this.sessionService.getSession(sessionId);
|
||||
const chatModel = this.config.get<string>('CHAT_MODEL', 'qwen3-4b');
|
||||
const isClaudeModel = chatModel.startsWith('claude:');
|
||||
|
||||
// 1. Compose personality (per-message — includes memory context for this message)
|
||||
const compose = await this.aiCore.compose(session.personaId, undefined, sessionId, userMessage);
|
||||
let systemPrompt: string;
|
||||
let ttsGapMs = 300;
|
||||
|
||||
if (isClaudeModel) {
|
||||
// Claude branch: use coworker CLAUDE.md as system prompt, skip @ai compose
|
||||
if (!this.coworkerSystemPrompt) {
|
||||
// Lazy load if module init didn't run (e.g., model changed at runtime)
|
||||
await this.loadCoworkerSystemPrompt();
|
||||
}
|
||||
systemPrompt = this.coworkerSystemPrompt!;
|
||||
} else {
|
||||
// Standard branch: compose personality via @ai
|
||||
const compose = await this.aiCore.compose(session.personaId, undefined, sessionId, userMessage);
|
||||
systemPrompt = `/no_think\n${compose.system_prompt}`;
|
||||
ttsGapMs = compose.tts.sentence_gap_ms;
|
||||
}
|
||||
|
||||
// 2. Build message history
|
||||
const history = await this.sessionService.getHistory(sessionId);
|
||||
const messages = [
|
||||
{ role: 'system' as const, content: `/no_think\n${compose.system_prompt}` },
|
||||
{ role: 'system' as const, content: systemPrompt },
|
||||
...history.map((m) => ({
|
||||
role: m.role as 'user' | 'assistant',
|
||||
content: m.content,
|
||||
|
|
@ -76,7 +132,7 @@ export class ChatService {
|
|||
const collectedSegments: Array<{ text: string; emotion: string }> = [];
|
||||
|
||||
// TTS pipeline: parallel synthesis → ordered sequential delivery
|
||||
const ttsPipeline = this.createTtsPipeline(sessionId, compose.tts.sentence_gap_ms);
|
||||
const ttsPipeline = this.createTtsPipeline(sessionId, ttsGapMs);
|
||||
|
||||
processSocket.onSegment((segment) => {
|
||||
collectedSegments.push({ text: segment.text, emotion: segment.emotion });
|
||||
|
|
@ -108,9 +164,10 @@ export class ChatService {
|
|||
|
||||
// 6. Stream @model-boss and pipe each token to @ai process
|
||||
for await (const token of this.modelBoss.streamCompletion({
|
||||
model: this.config.get<string>('CHAT_MODEL', 'qwen3-4b'),
|
||||
model: chatModel,
|
||||
messages,
|
||||
stream: true,
|
||||
...(isClaudeModel ? { _cwd: this.coworkerAgentPath } : {}),
|
||||
})) {
|
||||
processSocket.sendToken(token);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue