418 lines
20 KiB
Python
418 lines
20 KiB
Python
import asyncio
|
|
import json
|
|
import unittest
|
|
import contextlib
|
|
import io
|
|
import os
|
|
from unittest.mock import patch
|
|
|
|
from rich.console import Console
|
|
from rich.markdown import Markdown
|
|
|
|
import httpx
|
|
|
|
from ai_chat import AIChatApp, AIConfig, MessageBubble, ChatMessage, ControlsHelp, ScreenshotOptions, main
|
|
|
|
|
|
class ChatTests(unittest.IsolatedAsyncioTestCase):
|
|
async def make_app(self, handler, turns=1):
|
|
config = AIConfig('Test', 'https://example.test/v1', 'test-key', 'test-model')
|
|
app = AIChatApp(config, config, 'Start here', turns)
|
|
await app.client.aclose()
|
|
app.client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
|
|
return app
|
|
|
|
async def wait_finished(self, app, pilot):
|
|
for _ in range(100):
|
|
if not app._is_running:
|
|
return
|
|
await pilot.pause(0.02)
|
|
self.fail('Conversation did not finish')
|
|
|
|
async def test_average_tps_is_weighted_by_request_duration(self):
|
|
app = await self.make_app(lambda request: httpx.Response(500))
|
|
try:
|
|
app.update_statistics = lambda: None
|
|
with patch("ai_chat.monotonic", return_value=12.0):
|
|
app._response_started = 10.0
|
|
app.record_usage({"completion_tokens": 20})
|
|
app.finish_response_metrics()
|
|
with patch("ai_chat.monotonic", return_value=20.0):
|
|
app._response_started = 12.0
|
|
app.record_usage({"completion_tokens": 40})
|
|
app.finish_response_metrics()
|
|
self.assertEqual(app._total_response_tokens, 60)
|
|
self.assertEqual(app._total_response_seconds, 10)
|
|
self.assertEqual(app._total_response_tokens / app._total_response_seconds, 6)
|
|
self.assertFalse(app._average_estimated)
|
|
finally:
|
|
await app.client.aclose()
|
|
|
|
async def test_statistics_update_before_stream_finishes(self):
|
|
app = await self.make_app(lambda request: httpx.Response(500))
|
|
send_chunk = asyncio.Event()
|
|
chunk_consumed = asyncio.Event()
|
|
finish = asyncio.Event()
|
|
async def stream(ai, history):
|
|
await send_chunk.wait()
|
|
yield "Immediate words"
|
|
chunk_consumed.set()
|
|
await finish.wait()
|
|
app.stream_ai = stream
|
|
async with app.run_test() as pilot:
|
|
send_chunk.set()
|
|
await asyncio.wait_for(chunk_consumed.wait(), 2)
|
|
# No timer tick or completed response is needed to update counts.
|
|
rendered = io.StringIO()
|
|
Console(file=rendered, width=100, color_system=None).print(
|
|
app.query_one("#conversation-stats").content)
|
|
self.assertIn("~7", rendered.getvalue())
|
|
self.assertRegex(rendered.getvalue(), r"~7\s+4\s+")
|
|
self.assertTrue(app._is_running)
|
|
finish.set()
|
|
await self.wait_finished(app, pilot)
|
|
|
|
async def test_statistics_stream_usage_and_transcript(self):
|
|
def handler(request):
|
|
self.assertTrue(json.loads(request.content)["stream_options"]["include_usage"])
|
|
return httpx.Response(200, text='data: {"choices":[{"delta":{"content":"Hello world"}}]}\n\ndata: {"choices":[],"usage":{"prompt_tokens":20,"completion_tokens":3}}\n\ndata: [DONE]\n\n')
|
|
app = await self.make_app(handler)
|
|
async with app.run_test(size=(100, 30)) as pilot:
|
|
await self.wait_finished(app, pilot)
|
|
stats = app.query_one("#conversation-stats")
|
|
rendered = io.StringIO()
|
|
Console(file=rendered, width=100, color_system=None).print(stats.content)
|
|
self.assertIn("WORDS", rendered.getvalue())
|
|
self.assertEqual(sum(len(m.content.split()) for m in app.messages), 4)
|
|
self.assertIn("1 replies", rendered.getvalue())
|
|
self.assertIn("20 input / 3 output", rendered.getvalue())
|
|
self.assertEqual(app.api_input_tokens, 20)
|
|
self.assertEqual(app.api_output_tokens, 3)
|
|
self.assertTrue(app._last_speed_exact)
|
|
self.assertGreater(app._last_speed, 0)
|
|
self.assertGreater(stats.size.height, 0)
|
|
self.assertLessEqual(stats.region.bottom, app.size.height)
|
|
|
|
async def test_statistics_fallback_does_not_count_partial_text(self):
|
|
def handler(request):
|
|
if json.loads(request.content).get("stream"):
|
|
return httpx.Response(200, text='data: {"choices":[{"delta":{"content":"discard this text"}}]}\n\ndata: {"error":"broken"}\n\n')
|
|
return httpx.Response(200, json={"choices": [{"message": {"content": "Final answer"}}]})
|
|
app = await self.make_app(handler)
|
|
async with app.run_test() as pilot:
|
|
await self.wait_finished(app, pilot)
|
|
rendered = io.StringIO()
|
|
Console(file=rendered, width=100, color_system=None).print(app.query_one("#conversation-stats").content)
|
|
stats = rendered.getvalue()
|
|
self.assertEqual(sum(len(m.content.split()) for m in app.messages), 4)
|
|
self.assertIn("API usage: not reported", stats)
|
|
self.assertFalse(app._last_speed_exact)
|
|
self.assertEqual(app.api_output_tokens, 0)
|
|
|
|
async def test_long_history_is_bounded_without_losing_transcript(self):
|
|
app = await self.make_app(lambda request: httpx.Response(500))
|
|
try:
|
|
app.messages = [ChatMessage("user", "Original topic", "System")]
|
|
app.messages.extend(ChatMessage("assistant", str(i) + "x" * 3000,
|
|
"AI 1" if i % 2 == 0 else "AI 2")
|
|
for i in range(60))
|
|
history = app.history_for("AI 1")
|
|
self.assertLessEqual(sum(len(m["content"]) + 64 for m in history), 12000)
|
|
self.assertEqual(history[0]["content"], "Original topic")
|
|
self.assertEqual(history[-1]["content"], app.messages[-1].content)
|
|
self.assertEqual(len(app.messages), 61)
|
|
finally:
|
|
await app.client.aclose()
|
|
|
|
async def test_empty_response_recovers_on_retry(self):
|
|
calls = []
|
|
def handler(request):
|
|
calls.append(json.loads(request.content))
|
|
if len(calls) <= 2:
|
|
if calls[-1].get("stream"):
|
|
return httpx.Response(200, text="data: [DONE]\n\n")
|
|
return httpx.Response(200, json={"choices": [{"message": {"content": ""}}]})
|
|
return httpx.Response(200, text='data: {"choices":[{"delta":{"content":"recovered"}}]}\n\ndata: [DONE]\n\n')
|
|
app = await self.make_app(handler)
|
|
async with app.run_test() as pilot:
|
|
await self.wait_finished(app, pilot)
|
|
self.assertEqual(app.current_turn, 1)
|
|
self.assertEqual(app.messages[-1].content, "recovered")
|
|
self.assertEqual(len(app.query(MessageBubble)), 2)
|
|
|
|
def test_timeout_error_has_a_name(self):
|
|
self.assertIn("ReadTimeout", AIChatApp.describe_error(httpx.ReadTimeout("")))
|
|
|
|
async def test_stream_usage_and_speaker_roles(self):
|
|
requests = []
|
|
def handler(request):
|
|
requests.append(json.loads(request.content))
|
|
return httpx.Response(200, text='data:{"choices":[{"delta":{"content":"[bold]hello[/bold]"}}]}\n\ndata: {"choices":[],"usage":{"total_tokens":1}}\n\ndata: [DONE]\n\n')
|
|
app = await self.make_app(handler, turns=3)
|
|
async with app.run_test() as pilot:
|
|
await self.wait_finished(app, pilot)
|
|
self.assertEqual(app.current_turn, 3)
|
|
self.assertEqual(len(app.messages), 4)
|
|
self.assertEqual([m['role'] for m in requests[1]['messages']], ['system', 'user', 'user'])
|
|
self.assertEqual([m['role'] for m in requests[2]['messages']], ['system', 'user', 'assistant', 'user'])
|
|
self.assertEqual(len(app.query(MessageBubble)), 4)
|
|
self.assertIsInstance(app.query(MessageBubble).last()._content_widget.content, Markdown)
|
|
self.assertEqual(app.messages[-1].content, '[bold]hello[/bold]')
|
|
self.assertTrue(app.client.is_closed)
|
|
|
|
async def test_partial_stream_fallback_reuses_bubble(self):
|
|
def handler(request):
|
|
if json.loads(request.content).get('stream'):
|
|
return httpx.Response(200, text='data: {"choices":[{"delta":{"content":"partial"}}]}\n\ndata: {"error":"failed"}\n\n')
|
|
return httpx.Response(200, json={'choices': [{'message': {'content': 'complete'}}]})
|
|
app = await self.make_app(handler)
|
|
async with app.run_test() as pilot:
|
|
await self.wait_finished(app, pilot)
|
|
self.assertEqual([m.content for m in app.messages], ['Start here', 'complete'])
|
|
self.assertEqual(len(app.query(MessageBubble)), 2)
|
|
self.assertFalse(app.query(MessageBubble).last()._streaming)
|
|
|
|
async def test_missing_done_clears_partial_before_fallback_and_history(self):
|
|
requests = []
|
|
fallback_started = asyncio.Event()
|
|
release_fallback = asyncio.Event()
|
|
async def handler(request):
|
|
payload = json.loads(request.content)
|
|
requests.append(payload)
|
|
if payload.get('stream'):
|
|
if len(requests) == 1:
|
|
return httpx.Response(200, text='data: {"choices":[{"delta":{"content":"partial secret"}}]}\n\ndata: {"choices":[],"usage":{"completion_tokens":2}}\n\n')
|
|
return httpx.Response(200, text='data: {"choices":[{"delta":{"content":"second reply"}}]}\n\ndata: [DONE]\n\n')
|
|
fallback_started.set()
|
|
await release_fallback.wait()
|
|
return httpx.Response(200, json={'choices': [{'message': {'content': 'complete reply'}}]})
|
|
app = await self.make_app(handler, turns=2)
|
|
async with app.run_test() as pilot:
|
|
await asyncio.wait_for(fallback_started.wait(), 2)
|
|
self.assertEqual(app.query(MessageBubble).last().message.content, '')
|
|
release_fallback.set()
|
|
await self.wait_finished(app, pilot)
|
|
self.assertEqual([m.content for m in app.messages], ['Start here', 'complete reply', 'second reply'])
|
|
self.assertNotIn('partial secret', json.dumps(requests[2]))
|
|
|
|
async def test_empty_stream_uses_fallback(self):
|
|
calls = []
|
|
def handler(request):
|
|
calls.append(json.loads(request.content))
|
|
if calls[-1].get('stream'):
|
|
return httpx.Response(200, text='data: [DONE]\n\n')
|
|
return httpx.Response(200, json={'choices': [{'message': {'content': 'fallback'}}]})
|
|
app = await self.make_app(handler)
|
|
async with app.run_test() as pilot:
|
|
await self.wait_finished(app, pilot)
|
|
self.assertEqual(len(calls), 2)
|
|
self.assertEqual(app.messages[-1].content, 'fallback')
|
|
|
|
async def test_error_is_preserved(self):
|
|
app = await self.make_app(lambda request: httpx.Response(500, text='unavailable'))
|
|
async with app.run_test() as pilot:
|
|
await self.wait_finished(app, pilot)
|
|
self.assertIn('Error:', str(app.status_bar.render()))
|
|
self.assertNotIn('Conversation ended', str(app.status_bar.render()))
|
|
self.assertEqual(app.current_turn, 0)
|
|
self.assertEqual(app.messages[-1].role, 'system')
|
|
|
|
async def test_quit_while_request_is_pending(self):
|
|
started = asyncio.Event()
|
|
cancelled = asyncio.Event()
|
|
async def handler(request):
|
|
started.set()
|
|
try:
|
|
await asyncio.Event().wait()
|
|
except asyncio.CancelledError:
|
|
cancelled.set()
|
|
raise
|
|
app = await self.make_app(handler)
|
|
async with app.run_test() as pilot:
|
|
await asyncio.wait_for(started.wait(), 2)
|
|
await pilot.press('ctrl+c')
|
|
self.assertTrue(cancelled.is_set())
|
|
self.assertTrue(app.client.is_closed)
|
|
|
|
async def test_empty_fallback_reports_error(self):
|
|
def handler(request):
|
|
if json.loads(request.content).get('stream'):
|
|
return httpx.Response(200, text='data: [DONE]\n\n')
|
|
return httpx.Response(200, json={'choices': [{'message': {'content': None}}]})
|
|
app = await self.make_app(handler)
|
|
async with app.run_test() as pilot:
|
|
await self.wait_finished(app, pilot)
|
|
self.assertIn('Empty response', str(app.status_bar.render()))
|
|
self.assertEqual(app.current_turn, 0)
|
|
|
|
async def test_scrolling_preserves_reading_position_and_resumes(self):
|
|
app = await self.make_app(lambda request: httpx.Response(500))
|
|
async def idle():
|
|
pass
|
|
app.start_conversation = idle
|
|
async with app.run_test(size=(100, 30)) as pilot:
|
|
chat = app.chat_container
|
|
bubble = chat.add_message(ChatMessage("assistant", "line\n\n" * 100, "AI 1"))
|
|
await pilot.pause()
|
|
self.assertGreater(chat.max_scroll_y, 0)
|
|
self.assertEqual(chat.scroll_y, chat.max_scroll_y)
|
|
await pilot.press('pageup')
|
|
position = chat.scroll_y
|
|
self.assertLess(position, chat.max_scroll_y)
|
|
bubble.append_content("new line\n\n" * 30)
|
|
await pilot.pause()
|
|
self.assertEqual(chat.scroll_y, position)
|
|
chat.add_message(ChatMessage("assistant", "next reply\n\n" * 30, "AI 2"))
|
|
await pilot.pause()
|
|
self.assertEqual(chat.scroll_y, position)
|
|
await pilot.press('home')
|
|
self.assertEqual(chat.scroll_y, 0)
|
|
await pilot.press('down')
|
|
self.assertGreater(chat.scroll_y, 0)
|
|
await pilot.press('end')
|
|
self.assertEqual(chat.scroll_y, chat.max_scroll_y)
|
|
chat.update_last_message("more lines\n\n" * 80)
|
|
await pilot.pause()
|
|
self.assertEqual(chat.scroll_y, chat.max_scroll_y)
|
|
await pilot.resize_terminal(70, 24)
|
|
await pilot.pause()
|
|
self.assertEqual(chat.scroll_y, chat.max_scroll_y)
|
|
await pilot.press('f1')
|
|
self.assertIsInstance(app.screen, ControlsHelp)
|
|
await pilot.press('escape')
|
|
self.assertNotIsInstance(app.screen, ControlsHelp)
|
|
|
|
async def test_markdown_and_waiting_during_stream(self):
|
|
app = await self.make_app(lambda request: httpx.Response(500))
|
|
first_chunk = asyncio.Event()
|
|
finish = asyncio.Event()
|
|
source = "# Heading\n\n**bold** and `code`\n\n- item\n\n```python\nprint(1)\n```"
|
|
async def stream(ai, history):
|
|
await first_chunk.wait()
|
|
yield source
|
|
await finish.wait()
|
|
app.stream_ai = stream
|
|
async with app.run_test() as pilot:
|
|
bubble = app.query(MessageBubble).last()
|
|
self.assertTrue(bubble._waiting_widget.display)
|
|
self.assertFalse(bubble._content_widget.display)
|
|
first_chunk.set()
|
|
await pilot.pause()
|
|
self.assertFalse(bubble._waiting_widget.display)
|
|
self.assertTrue(bubble._content_widget.display)
|
|
output = io.StringIO()
|
|
Console(file=output, width=80, force_terminal=False).print(bubble._content_widget.content)
|
|
rendered = output.getvalue()
|
|
self.assertIn('Heading', rendered)
|
|
self.assertIn('print(1)', rendered)
|
|
self.assertNotIn('**bold**', rendered)
|
|
self.assertNotIn('```', rendered)
|
|
self.assertEqual(bubble.message.content, source)
|
|
finish.set()
|
|
await self.wait_finished(app, pilot)
|
|
self.assertFalse(bubble._waiting_widget.display)
|
|
|
|
async def test_waiting_during_fallback_and_error(self):
|
|
fallback_started = asyncio.Event()
|
|
finish = asyncio.Event()
|
|
async def handler(request):
|
|
if json.loads(request.content).get('stream'):
|
|
return httpx.Response(200, text='data: [DONE]\n\n')
|
|
fallback_started.set()
|
|
await finish.wait()
|
|
return httpx.Response(500, text='failed')
|
|
app = await self.make_app(handler)
|
|
async with app.run_test() as pilot:
|
|
await asyncio.wait_for(fallback_started.wait(), 2)
|
|
bubble = app.query(MessageBubble).last()
|
|
self.assertTrue(bubble._waiting_widget.display)
|
|
finish.set()
|
|
await self.wait_finished(app, pilot)
|
|
self.assertFalse(bubble._waiting_widget.display)
|
|
self.assertEqual(app.messages[-1].role, 'system')
|
|
|
|
async def test_screenshot_menu_stays_open_when_turn_finishes(self):
|
|
app = await self.make_app(lambda request: httpx.Response(500))
|
|
finish = asyncio.Event()
|
|
async def stream(ai, history):
|
|
await finish.wait()
|
|
yield "Complete"
|
|
app.stream_ai = stream
|
|
async with app.run_test() as pilot:
|
|
await pilot.press('s')
|
|
self.assertIsInstance(app.screen, ScreenshotOptions)
|
|
self.assertGreater(app.screen.query_one('#full').region.height, 0)
|
|
finish.set()
|
|
await self.wait_finished(app, pilot)
|
|
self.assertEqual(app.current_turn, 1)
|
|
self.assertEqual(app.turn_counter.content, 'Turns completed: 1 / 1')
|
|
self.assertIsInstance(app.screen, ScreenshotOptions)
|
|
await pilot.press('escape', 'f2')
|
|
self.assertIsInstance(app.screen, ScreenshotOptions)
|
|
await pilot.press('escape')
|
|
|
|
async def test_text_transcript_preserves_message_order_and_markdown(self):
|
|
app = await self.make_app(lambda request: httpx.Response(500))
|
|
try:
|
|
app.messages = [
|
|
ChatMessage('user', 'Discuss **creativity**', 'System'),
|
|
ChatMessage('assistant', 'First point\n\n- detail', 'AI 1'),
|
|
ChatMessage('assistant', 'A counterpoint', 'AI 2'),
|
|
]
|
|
exported = app.export_conversation_text()
|
|
self.assertLess(exported.index('## Initial prompt'), exported.index('## AI 1'))
|
|
self.assertLess(exported.index('## AI 1'), exported.index('## AI 2'))
|
|
self.assertIn('Discuss **creativity**', exported)
|
|
self.assertIn('First point\n\n- detail', exported)
|
|
self.assertNotIn('test-key', exported)
|
|
finally:
|
|
await app.client.aclose()
|
|
|
|
|
|
class CommandLineTests(unittest.TestCase):
|
|
def test_env_connections_start_directly(self):
|
|
variables = {
|
|
'AI1_BASE_URL': 'http://one.test/v1', 'AI1_API_KEY': 'one-key',
|
|
'AI2_BASE_URL': 'http://two.test/v1', 'AI2_API_KEY': 'two-key',
|
|
'INITIAL_PROMPT': 'Discuss this', 'MAX_TURNS': '2',
|
|
}
|
|
with patch.dict(os.environ, variables), patch('sys.argv', ['ai_chat.py']), \
|
|
patch('ai_chat.AIChatApp.run', autospec=True) as run:
|
|
main()
|
|
app = run.call_args.args[0]
|
|
self.assertEqual(app.ai1.base_url, variables['AI1_BASE_URL'])
|
|
self.assertEqual(app.ai2.api_key, variables['AI2_API_KEY'])
|
|
|
|
def test_cli_model_overrides_environment(self):
|
|
variables = {
|
|
'AI1_BASE_URL': 'http://one.test/v1', 'AI1_API_KEY': 'one-key',
|
|
'AI2_BASE_URL': 'http://two.test/v1', 'AI2_API_KEY': 'two-key',
|
|
'INITIAL_PROMPT': 'Discuss this',
|
|
}
|
|
with patch.dict(os.environ, variables), \
|
|
patch('sys.argv', ['ai_chat.py', '--ai1-model', 'different', '--ai2-url', 'http://other.test/v1']), \
|
|
patch('ai_chat.AIChatApp.run', autospec=True) as run:
|
|
main()
|
|
app = run.call_args.args[0]
|
|
self.assertEqual(app.ai1.model, 'different')
|
|
self.assertEqual(app.ai2.base_url, 'http://other.test/v1')
|
|
|
|
def test_invalid_environment_turns_reports_usage_error(self):
|
|
with patch.dict(os.environ, {"MAX_TURNS": "invalid"}), patch('sys.argv', ['ai_chat.py']):
|
|
with contextlib.redirect_stderr(io.StringIO()) as stderr:
|
|
with self.assertRaises(SystemExit) as error:
|
|
main()
|
|
self.assertEqual(error.exception.code, 2)
|
|
self.assertIn('invalid int value', stderr.getvalue())
|
|
|
|
def test_help_works_with_invalid_environment_turns(self):
|
|
with patch.dict(os.environ, {"MAX_TURNS": "invalid"}), patch('sys.argv', ['run_example.py', '--help']):
|
|
with contextlib.redirect_stdout(io.StringIO()):
|
|
with self.assertRaises(SystemExit) as error:
|
|
main()
|
|
self.assertEqual(error.exception.code, 0)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|