Files
sloppy/test_ai_chat.py

418 lines
20 KiB
Python

import asyncio
import json
import unittest
import contextlib
import io
import os
from unittest.mock import patch
from rich.console import Console
from rich.markdown import Markdown
import httpx
from ai_chat import AIChatApp, AIConfig, MessageBubble, ChatMessage, ControlsHelp, ScreenshotOptions, main
class ChatTests(unittest.IsolatedAsyncioTestCase):
async def make_app(self, handler, turns=1):
config = AIConfig('Test', 'https://example.test/v1', 'test-key', 'test-model')
app = AIChatApp(config, config, 'Start here', turns)
await app.client.aclose()
app.client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
return app
async def wait_finished(self, app, pilot):
for _ in range(100):
if not app._is_running:
return
await pilot.pause(0.02)
self.fail('Conversation did not finish')
async def test_average_tps_is_weighted_by_request_duration(self):
app = await self.make_app(lambda request: httpx.Response(500))
try:
app.update_statistics = lambda: None
with patch("ai_chat.monotonic", return_value=12.0):
app._response_started = 10.0
app.record_usage({"completion_tokens": 20})
app.finish_response_metrics()
with patch("ai_chat.monotonic", return_value=20.0):
app._response_started = 12.0
app.record_usage({"completion_tokens": 40})
app.finish_response_metrics()
self.assertEqual(app._total_response_tokens, 60)
self.assertEqual(app._total_response_seconds, 10)
self.assertEqual(app._total_response_tokens / app._total_response_seconds, 6)
self.assertFalse(app._average_estimated)
finally:
await app.client.aclose()
async def test_statistics_update_before_stream_finishes(self):
app = await self.make_app(lambda request: httpx.Response(500))
send_chunk = asyncio.Event()
chunk_consumed = asyncio.Event()
finish = asyncio.Event()
async def stream(ai, history):
await send_chunk.wait()
yield "Immediate words"
chunk_consumed.set()
await finish.wait()
app.stream_ai = stream
async with app.run_test() as pilot:
send_chunk.set()
await asyncio.wait_for(chunk_consumed.wait(), 2)
# No timer tick or completed response is needed to update counts.
rendered = io.StringIO()
Console(file=rendered, width=100, color_system=None).print(
app.query_one("#conversation-stats").content)
self.assertIn("~7", rendered.getvalue())
self.assertRegex(rendered.getvalue(), r"~7\s+4\s+")
self.assertTrue(app._is_running)
finish.set()
await self.wait_finished(app, pilot)
async def test_statistics_stream_usage_and_transcript(self):
def handler(request):
self.assertTrue(json.loads(request.content)["stream_options"]["include_usage"])
return httpx.Response(200, text='data: {"choices":[{"delta":{"content":"Hello world"}}]}\n\ndata: {"choices":[],"usage":{"prompt_tokens":20,"completion_tokens":3}}\n\ndata: [DONE]\n\n')
app = await self.make_app(handler)
async with app.run_test(size=(100, 30)) as pilot:
await self.wait_finished(app, pilot)
stats = app.query_one("#conversation-stats")
rendered = io.StringIO()
Console(file=rendered, width=100, color_system=None).print(stats.content)
self.assertIn("WORDS", rendered.getvalue())
self.assertEqual(sum(len(m.content.split()) for m in app.messages), 4)
self.assertIn("1 replies", rendered.getvalue())
self.assertIn("20 input / 3 output", rendered.getvalue())
self.assertEqual(app.api_input_tokens, 20)
self.assertEqual(app.api_output_tokens, 3)
self.assertTrue(app._last_speed_exact)
self.assertGreater(app._last_speed, 0)
self.assertGreater(stats.size.height, 0)
self.assertLessEqual(stats.region.bottom, app.size.height)
async def test_statistics_fallback_does_not_count_partial_text(self):
def handler(request):
if json.loads(request.content).get("stream"):
return httpx.Response(200, text='data: {"choices":[{"delta":{"content":"discard this text"}}]}\n\ndata: {"error":"broken"}\n\n')
return httpx.Response(200, json={"choices": [{"message": {"content": "Final answer"}}]})
app = await self.make_app(handler)
async with app.run_test() as pilot:
await self.wait_finished(app, pilot)
rendered = io.StringIO()
Console(file=rendered, width=100, color_system=None).print(app.query_one("#conversation-stats").content)
stats = rendered.getvalue()
self.assertEqual(sum(len(m.content.split()) for m in app.messages), 4)
self.assertIn("API usage: not reported", stats)
self.assertFalse(app._last_speed_exact)
self.assertEqual(app.api_output_tokens, 0)
async def test_long_history_is_bounded_without_losing_transcript(self):
app = await self.make_app(lambda request: httpx.Response(500))
try:
app.messages = [ChatMessage("user", "Original topic", "System")]
app.messages.extend(ChatMessage("assistant", str(i) + "x" * 3000,
"AI 1" if i % 2 == 0 else "AI 2")
for i in range(60))
history = app.history_for("AI 1")
self.assertLessEqual(sum(len(m["content"]) + 64 for m in history), 12000)
self.assertEqual(history[0]["content"], "Original topic")
self.assertEqual(history[-1]["content"], app.messages[-1].content)
self.assertEqual(len(app.messages), 61)
finally:
await app.client.aclose()
async def test_empty_response_recovers_on_retry(self):
calls = []
def handler(request):
calls.append(json.loads(request.content))
if len(calls) <= 2:
if calls[-1].get("stream"):
return httpx.Response(200, text="data: [DONE]\n\n")
return httpx.Response(200, json={"choices": [{"message": {"content": ""}}]})
return httpx.Response(200, text='data: {"choices":[{"delta":{"content":"recovered"}}]}\n\ndata: [DONE]\n\n')
app = await self.make_app(handler)
async with app.run_test() as pilot:
await self.wait_finished(app, pilot)
self.assertEqual(app.current_turn, 1)
self.assertEqual(app.messages[-1].content, "recovered")
self.assertEqual(len(app.query(MessageBubble)), 2)
def test_timeout_error_has_a_name(self):
self.assertIn("ReadTimeout", AIChatApp.describe_error(httpx.ReadTimeout("")))
async def test_stream_usage_and_speaker_roles(self):
requests = []
def handler(request):
requests.append(json.loads(request.content))
return httpx.Response(200, text='data:{"choices":[{"delta":{"content":"[bold]hello[/bold]"}}]}\n\ndata: {"choices":[],"usage":{"total_tokens":1}}\n\ndata: [DONE]\n\n')
app = await self.make_app(handler, turns=3)
async with app.run_test() as pilot:
await self.wait_finished(app, pilot)
self.assertEqual(app.current_turn, 3)
self.assertEqual(len(app.messages), 4)
self.assertEqual([m['role'] for m in requests[1]['messages']], ['system', 'user', 'user'])
self.assertEqual([m['role'] for m in requests[2]['messages']], ['system', 'user', 'assistant', 'user'])
self.assertEqual(len(app.query(MessageBubble)), 4)
self.assertIsInstance(app.query(MessageBubble).last()._content_widget.content, Markdown)
self.assertEqual(app.messages[-1].content, '[bold]hello[/bold]')
self.assertTrue(app.client.is_closed)
async def test_partial_stream_fallback_reuses_bubble(self):
def handler(request):
if json.loads(request.content).get('stream'):
return httpx.Response(200, text='data: {"choices":[{"delta":{"content":"partial"}}]}\n\ndata: {"error":"failed"}\n\n')
return httpx.Response(200, json={'choices': [{'message': {'content': 'complete'}}]})
app = await self.make_app(handler)
async with app.run_test() as pilot:
await self.wait_finished(app, pilot)
self.assertEqual([m.content for m in app.messages], ['Start here', 'complete'])
self.assertEqual(len(app.query(MessageBubble)), 2)
self.assertFalse(app.query(MessageBubble).last()._streaming)
async def test_missing_done_clears_partial_before_fallback_and_history(self):
requests = []
fallback_started = asyncio.Event()
release_fallback = asyncio.Event()
async def handler(request):
payload = json.loads(request.content)
requests.append(payload)
if payload.get('stream'):
if len(requests) == 1:
return httpx.Response(200, text='data: {"choices":[{"delta":{"content":"partial secret"}}]}\n\ndata: {"choices":[],"usage":{"completion_tokens":2}}\n\n')
return httpx.Response(200, text='data: {"choices":[{"delta":{"content":"second reply"}}]}\n\ndata: [DONE]\n\n')
fallback_started.set()
await release_fallback.wait()
return httpx.Response(200, json={'choices': [{'message': {'content': 'complete reply'}}]})
app = await self.make_app(handler, turns=2)
async with app.run_test() as pilot:
await asyncio.wait_for(fallback_started.wait(), 2)
self.assertEqual(app.query(MessageBubble).last().message.content, '')
release_fallback.set()
await self.wait_finished(app, pilot)
self.assertEqual([m.content for m in app.messages], ['Start here', 'complete reply', 'second reply'])
self.assertNotIn('partial secret', json.dumps(requests[2]))
async def test_empty_stream_uses_fallback(self):
calls = []
def handler(request):
calls.append(json.loads(request.content))
if calls[-1].get('stream'):
return httpx.Response(200, text='data: [DONE]\n\n')
return httpx.Response(200, json={'choices': [{'message': {'content': 'fallback'}}]})
app = await self.make_app(handler)
async with app.run_test() as pilot:
await self.wait_finished(app, pilot)
self.assertEqual(len(calls), 2)
self.assertEqual(app.messages[-1].content, 'fallback')
async def test_error_is_preserved(self):
app = await self.make_app(lambda request: httpx.Response(500, text='unavailable'))
async with app.run_test() as pilot:
await self.wait_finished(app, pilot)
self.assertIn('Error:', str(app.status_bar.render()))
self.assertNotIn('Conversation ended', str(app.status_bar.render()))
self.assertEqual(app.current_turn, 0)
self.assertEqual(app.messages[-1].role, 'system')
async def test_quit_while_request_is_pending(self):
started = asyncio.Event()
cancelled = asyncio.Event()
async def handler(request):
started.set()
try:
await asyncio.Event().wait()
except asyncio.CancelledError:
cancelled.set()
raise
app = await self.make_app(handler)
async with app.run_test() as pilot:
await asyncio.wait_for(started.wait(), 2)
await pilot.press('ctrl+c')
self.assertTrue(cancelled.is_set())
self.assertTrue(app.client.is_closed)
async def test_empty_fallback_reports_error(self):
def handler(request):
if json.loads(request.content).get('stream'):
return httpx.Response(200, text='data: [DONE]\n\n')
return httpx.Response(200, json={'choices': [{'message': {'content': None}}]})
app = await self.make_app(handler)
async with app.run_test() as pilot:
await self.wait_finished(app, pilot)
self.assertIn('Empty response', str(app.status_bar.render()))
self.assertEqual(app.current_turn, 0)
async def test_scrolling_preserves_reading_position_and_resumes(self):
app = await self.make_app(lambda request: httpx.Response(500))
async def idle():
pass
app.start_conversation = idle
async with app.run_test(size=(100, 30)) as pilot:
chat = app.chat_container
bubble = chat.add_message(ChatMessage("assistant", "line\n\n" * 100, "AI 1"))
await pilot.pause()
self.assertGreater(chat.max_scroll_y, 0)
self.assertEqual(chat.scroll_y, chat.max_scroll_y)
await pilot.press('pageup')
position = chat.scroll_y
self.assertLess(position, chat.max_scroll_y)
bubble.append_content("new line\n\n" * 30)
await pilot.pause()
self.assertEqual(chat.scroll_y, position)
chat.add_message(ChatMessage("assistant", "next reply\n\n" * 30, "AI 2"))
await pilot.pause()
self.assertEqual(chat.scroll_y, position)
await pilot.press('home')
self.assertEqual(chat.scroll_y, 0)
await pilot.press('down')
self.assertGreater(chat.scroll_y, 0)
await pilot.press('end')
self.assertEqual(chat.scroll_y, chat.max_scroll_y)
chat.update_last_message("more lines\n\n" * 80)
await pilot.pause()
self.assertEqual(chat.scroll_y, chat.max_scroll_y)
await pilot.resize_terminal(70, 24)
await pilot.pause()
self.assertEqual(chat.scroll_y, chat.max_scroll_y)
await pilot.press('f1')
self.assertIsInstance(app.screen, ControlsHelp)
await pilot.press('escape')
self.assertNotIsInstance(app.screen, ControlsHelp)
async def test_markdown_and_waiting_during_stream(self):
app = await self.make_app(lambda request: httpx.Response(500))
first_chunk = asyncio.Event()
finish = asyncio.Event()
source = "# Heading\n\n**bold** and `code`\n\n- item\n\n```python\nprint(1)\n```"
async def stream(ai, history):
await first_chunk.wait()
yield source
await finish.wait()
app.stream_ai = stream
async with app.run_test() as pilot:
bubble = app.query(MessageBubble).last()
self.assertTrue(bubble._waiting_widget.display)
self.assertFalse(bubble._content_widget.display)
first_chunk.set()
await pilot.pause()
self.assertFalse(bubble._waiting_widget.display)
self.assertTrue(bubble._content_widget.display)
output = io.StringIO()
Console(file=output, width=80, force_terminal=False).print(bubble._content_widget.content)
rendered = output.getvalue()
self.assertIn('Heading', rendered)
self.assertIn('print(1)', rendered)
self.assertNotIn('**bold**', rendered)
self.assertNotIn('```', rendered)
self.assertEqual(bubble.message.content, source)
finish.set()
await self.wait_finished(app, pilot)
self.assertFalse(bubble._waiting_widget.display)
async def test_waiting_during_fallback_and_error(self):
fallback_started = asyncio.Event()
finish = asyncio.Event()
async def handler(request):
if json.loads(request.content).get('stream'):
return httpx.Response(200, text='data: [DONE]\n\n')
fallback_started.set()
await finish.wait()
return httpx.Response(500, text='failed')
app = await self.make_app(handler)
async with app.run_test() as pilot:
await asyncio.wait_for(fallback_started.wait(), 2)
bubble = app.query(MessageBubble).last()
self.assertTrue(bubble._waiting_widget.display)
finish.set()
await self.wait_finished(app, pilot)
self.assertFalse(bubble._waiting_widget.display)
self.assertEqual(app.messages[-1].role, 'system')
async def test_screenshot_menu_stays_open_when_turn_finishes(self):
app = await self.make_app(lambda request: httpx.Response(500))
finish = asyncio.Event()
async def stream(ai, history):
await finish.wait()
yield "Complete"
app.stream_ai = stream
async with app.run_test() as pilot:
await pilot.press('s')
self.assertIsInstance(app.screen, ScreenshotOptions)
self.assertGreater(app.screen.query_one('#full').region.height, 0)
finish.set()
await self.wait_finished(app, pilot)
self.assertEqual(app.current_turn, 1)
self.assertEqual(app.turn_counter.content, 'Turns completed: 1 / 1')
self.assertIsInstance(app.screen, ScreenshotOptions)
await pilot.press('escape', 'f2')
self.assertIsInstance(app.screen, ScreenshotOptions)
await pilot.press('escape')
async def test_text_transcript_preserves_message_order_and_markdown(self):
app = await self.make_app(lambda request: httpx.Response(500))
try:
app.messages = [
ChatMessage('user', 'Discuss **creativity**', 'System'),
ChatMessage('assistant', 'First point\n\n- detail', 'AI 1'),
ChatMessage('assistant', 'A counterpoint', 'AI 2'),
]
exported = app.export_conversation_text()
self.assertLess(exported.index('## Initial prompt'), exported.index('## AI 1'))
self.assertLess(exported.index('## AI 1'), exported.index('## AI 2'))
self.assertIn('Discuss **creativity**', exported)
self.assertIn('First point\n\n- detail', exported)
self.assertNotIn('test-key', exported)
finally:
await app.client.aclose()
class CommandLineTests(unittest.TestCase):
def test_env_connections_start_directly(self):
variables = {
'AI1_BASE_URL': 'http://one.test/v1', 'AI1_API_KEY': 'one-key',
'AI2_BASE_URL': 'http://two.test/v1', 'AI2_API_KEY': 'two-key',
'INITIAL_PROMPT': 'Discuss this', 'MAX_TURNS': '2',
}
with patch.dict(os.environ, variables), patch('sys.argv', ['ai_chat.py']), \
patch('ai_chat.AIChatApp.run', autospec=True) as run:
main()
app = run.call_args.args[0]
self.assertEqual(app.ai1.base_url, variables['AI1_BASE_URL'])
self.assertEqual(app.ai2.api_key, variables['AI2_API_KEY'])
def test_cli_model_overrides_environment(self):
variables = {
'AI1_BASE_URL': 'http://one.test/v1', 'AI1_API_KEY': 'one-key',
'AI2_BASE_URL': 'http://two.test/v1', 'AI2_API_KEY': 'two-key',
'INITIAL_PROMPT': 'Discuss this',
}
with patch.dict(os.environ, variables), \
patch('sys.argv', ['ai_chat.py', '--ai1-model', 'different', '--ai2-url', 'http://other.test/v1']), \
patch('ai_chat.AIChatApp.run', autospec=True) as run:
main()
app = run.call_args.args[0]
self.assertEqual(app.ai1.model, 'different')
self.assertEqual(app.ai2.base_url, 'http://other.test/v1')
def test_invalid_environment_turns_reports_usage_error(self):
with patch.dict(os.environ, {"MAX_TURNS": "invalid"}), patch('sys.argv', ['ai_chat.py']):
with contextlib.redirect_stderr(io.StringIO()) as stderr:
with self.assertRaises(SystemExit) as error:
main()
self.assertEqual(error.exception.code, 2)
self.assertIn('invalid int value', stderr.getvalue())
def test_help_works_with_invalid_environment_turns(self):
with patch.dict(os.environ, {"MAX_TURNS": "invalid"}), patch('sys.argv', ['run_example.py', '--help']):
with contextlib.redirect_stdout(io.StringIO()):
with self.assertRaises(SystemExit) as error:
main()
self.assertEqual(error.exception.code, 0)
if __name__ == '__main__':
unittest.main()