Files
bob16/bob64/kernel.c
T
2026-10-09 20:06:51 +02:00

3599 lines
168 KiB
C

#include "kernel.h"
#include "interrupts.h"
#include "paging.h"
#include "console.h"
#include "heap.h"
#include "runtime.h"
#include "keyboard.h"
#include "mouse.h"
#include "shell.h"
#include "filesystem.h"
#include "exec.h"
#include "process.h"
#include "syscall.h"
#include "compiler.h"
#include "firmware_store.h"
#include "pci.h"
#include "xhci.h"
#include "usb.h"
#include "block_device.h"
#include "partition.h"
#include "disk_store.h"
#define BOB64_KERNEL_HEAP_BASE 0xffff900000000000ULL
#define BOB64_KERNEL_HEAP_LIMIT (64ULL*1024ULL*1024ULL)
#define BOB64_KERNEL_PRIVILEGE_STACK_SIZE (16ULL*1024ULL)
#define BOB64_XHCI_MMIO_MAP_SIZE (4ULL*BOB64_PAGE_SIZE)
#define BOB64_XHCI_RING_TRBS 256u
#define BOB64_XHCI_MAX_SCRATCHPADS 1023u
#define BOB64_XHCI_USBCMD_RUN 0x00000001u
#define BOB64_XHCI_USBSTS_HALTED 0x00000001u
#define BOB64_XHCI_USBSTS_NOT_READY 0x00000800u
#define BOB64_XHCI_TRB_TYPE_SHIFT 10u
#define BOB64_XHCI_TRB_TYPE_LINK 6u
#define BOB64_XHCI_TRB_TYPE_ENABLE_SLOT 9u
#define BOB64_XHCI_TRB_TYPE_DISABLE_SLOT 10u
#define BOB64_XHCI_TRB_TYPE_NOOP_COMMAND 23u
#define BOB64_XHCI_TRB_TYPE_COMMAND_COMPLETION 33u
#define BOB64_XHCI_TRB_TYPE_PORT_STATUS_CHANGE 34u
#define BOB64_XHCI_TRB_TYPE_TRANSFER_EVENT 32u
#define BOB64_XHCI_TRB_COMPLETION_SUCCESS 1u
#define KERNEL_XHCI_STORAGE_DEVICE_LIMIT 8u
static BOB64_HEAP kernel_heap;
static BOB64_FILESYSTEM kernel_filesystem;
static u64 kernel_next_window_owner=1;
static BOB64_SHELL kernel_shell;
static BOB64_PCI_DEVICE kernel_xhci_device;
static int kernel_xhci_found;
#define KERNEL_XHCI_HID_DEVICE_LIMIT BOB64_USB_INPUT_DEVICE_LIMIT
static u8 kernel_xhci_connected_ports[KERNEL_XHCI_HID_DEVICE_LIMIT];
static u8 kernel_xhci_connected_speeds[KERNEL_XHCI_HID_DEVICE_LIMIT];
typedef struct {
BOB64_FILESYSTEM *Filesystem;
u8 NxSupported,PhysicalAddressBits;
} KERNEL_APP_CONTEXT;
static KERNEL_APP_CONTEXT kernel_app_context;
static usize kernel_heap_mapped;
static usize kernel_heap_high_water;
static void *kernel_privilege_stack;
static char kernel_user_output[1024];
static usize kernel_user_output_length;
static u8 kernel_user_output_truncated;
static u8 kernel_compiler_output[BOB64_COMPILER_IMAGE_LIMIT];
static void kernel_write(const char *text);
static void kernel_hex64(u64 value);
static int kernel_pci_read_config(void *context,u8 bus,u8 device,
u8 function,u8 offset,u32 *value) {
(void)context;
return bob64_pci_read_config32(bus,device,function,offset,value);
}
typedef struct {
volatile u8 *Base;
usize Size;
u8 CapabilityLength;
} KERNEL_XHCI_MMIO;
typedef struct {
u64 Parameter;
u32 Status;
u32 Control;
} KERNEL_XHCI_TRB;
typedef struct {
u64 RingBase;
u32 RingSize;
u32 Reserved;
} KERNEL_XHCI_ERST_ENTRY;
typedef struct {
volatile KERNEL_XHCI_TRB *CommandRing;
volatile KERNEL_XHCI_TRB *EventRing;
volatile u64 *DeviceContextArray;
u64 CommandPhysical,EventPhysical;
u32 CommandIndex,EventIndex;
u8 EventCycle;
KERNEL_XHCI_TRB DeferredEvents[32];
u8 DeferredHead,DeferredCount;
} KERNEL_XHCI_RINGS;
static KERNEL_XHCI_RINGS kernel_xhci_rings;
static KERNEL_XHCI_MMIO kernel_xhci_live_mmio;
static BOB64_XHCI_CAPABILITIES kernel_xhci_live_capabilities;
typedef struct {
u32 RouteString;
u8 RootPort,ParentSlot,ParentPort,Speed,Depth,TtHubSlot,TtPort;
} KERNEL_XHCI_TOPOLOGY;
static KERNEL_XHCI_TOPOLOGY kernel_xhci_hub_children[32];
static u8 kernel_xhci_hub_child_count;
typedef struct {
volatile KERNEL_XHCI_TRB *Ring;
volatile u8 *Report;
u64 RingPhysical,ReportPhysical;
u32 NextIndex,PendingIndex;
u8 Slot,Dci,Kind,ReportLength,Cycle,Active,Pending,TransferErrors,RootPort;
u8 LoggedReport,LoggedInput,LoggedWheel;
BOB64_KEYBOARD_HID_STATE KeyboardState;
KERNEL_XHCI_TOPOLOGY Topology;
} KERNEL_XHCI_HID_DEVICE;
typedef struct {
volatile KERNEL_XHCI_TRB *Ring;
volatile u8 *Report;
volatile KERNEL_XHCI_TRB *ControlRing;
volatile u8 *ControlBuffer;
u64 RingPhysical,ReportPhysical;
u64 ControlRingPhysical,ControlBufferPhysical;
u32 NextIndex,PendingIndex,ControlNextIndex;
u8 Slot,Dci,ReportLength,Cycle,Active,Pending,TransferErrors,RootPort;
u8 LoggedChange,ControlCycle,PortCount;
KERNEL_XHCI_TOPOLOGY Topology;
} KERNEL_XHCI_HUB_DEVICE;
typedef struct {
volatile KERNEL_XHCI_TRB *BulkInRing,*BulkOutRing;
volatile u8 *Scratch;
u64 BulkInRingPhysical,BulkOutRingPhysical;
u64 ScratchPhysical,BlockCount;
u16 BulkInMaxPacketSize,BulkOutMaxPacketSize;
u32 BulkInNext,BulkOutNext,NextTag,BlockSize;
u8 Slot,BulkInDci,BulkOutDci,BulkInCycle,BulkOutCycle,Active;
KERNEL_XHCI_TOPOLOGY Topology;
BOB64_BLOCK_DEVICE BlockDevice;
BOB64_PARTITION DataPartition;
u8 HasDataPartition,DisconnectTestReadPending;
} KERNEL_XHCI_STORAGE_DEVICE;
static KERNEL_XHCI_HID_DEVICE kernel_xhci_hid_devices[KERNEL_XHCI_HID_DEVICE_LIMIT];
static u8 kernel_xhci_hid_device_count;
static KERNEL_XHCI_HUB_DEVICE kernel_xhci_hub_devices[KERNEL_XHCI_HID_DEVICE_LIMIT];
static u8 kernel_xhci_hub_device_count;
static KERNEL_XHCI_STORAGE_DEVICE
kernel_xhci_storage_devices[KERNEL_XHCI_STORAGE_DEVICE_LIMIT];
static u8 kernel_xhci_storage_device_count;
static int kernel_xhci_storage_probe(KERNEL_XHCI_STORAGE_DEVICE *device);
static int kernel_xhci_hub_status_submit(KERNEL_XHCI_HUB_DEVICE *device);
static int kernel_xhci_delay_ticks(u64 ticks);
static void kernel_xhci_hub_process_changes(KERNEL_XHCI_HUB_DEVICE *hub);
static void kernel_xhci_address_hub_children(KERNEL_XHCI_MMIO *mmio,
const BOB64_XHCI_CAPABILITIES *capabilities,u32 start_index);
static void kernel_xhci_hid_reconnect(u8 port);
_Static_assert(sizeof(KERNEL_XHCI_TRB)==16,"xHCI TRB is 16 bytes");
_Static_assert(sizeof(KERNEL_XHCI_ERST_ENTRY)==16,"xHCI ERST entry is 16 bytes");
static int kernel_xhci_read32(void *opaque,u32 offset,u32 *value) {
KERNEL_XHCI_MMIO *mmio=(KERNEL_XHCI_MMIO *)opaque;
if(!mmio||!value||(offset&3u)||offset>mmio->Size||
mmio->Size-offset<sizeof(u32))return -1;
*value=*(volatile u32 *)(volatile void *)(mmio->Base+offset);
return 0;
}
static int kernel_xhci_write32(void *opaque,u32 offset,u32 value) {
KERNEL_XHCI_MMIO *mmio=(KERNEL_XHCI_MMIO *)opaque;
if(!mmio||(offset&3u)||offset>mmio->Size||
mmio->Size-offset<sizeof(u32))return -1;
*(volatile u32 *)(volatile void *)(mmio->Base+offset)=value;
return 0;
}
static int kernel_xhci_dma_alloc(u64 pages,u64 *physical,volatile u8 **virtual) {
u64 bytes;
u64 base;
u64 flags=BOB64_PAGE_WRITE;
if(!pages||!physical||!virtual||pages>~(u64)0/BOB64_PAGE_SIZE)return -1;
bytes=pages*BOB64_PAGE_SIZE;
base=bob64_page_alloc(&bob64_boot_page_allocator,pages);
if(!base||base>=0x0000800000000000ULL||
bytes>0x0000800000000000ULL-base)return -1;
if(bob64_bootstrap_space.NxSupported)flags|=BOB64_PAGE_NX;
for(u64 offset=0;offset<bytes;offset+=BOB64_PAGE_SIZE) {
u64 translated=0;
int result=bob64_page_translate(&bob64_bootstrap_space.PageTable,
base+offset,&translated,0);
if(result==0)
result=bob64_page_map(&bob64_bootstrap_space.PageTable,base+offset,
base+offset,flags);
else if(result==1&&translated==base+offset)
result=bob64_page_protect(&bob64_bootstrap_space.PageTable,
base+offset,flags);
else return -1;
if(result)return -1;
__asm__ volatile("invlpg (%0)"::"r"((void *)(uintptr_t)(base+offset)):
"memory");
}
volatile u8 *memory=(volatile u8 *)(uintptr_t)base;
for(u64 i=0;i<bytes;i++)memory[i]=0;
*physical=base;*virtual=memory;
return 0;
}
static int kernel_xhci_write64(KERNEL_XHCI_MMIO *mmio,u32 offset,u64 value) {
if(!mmio||(offset&7u)||offset>mmio->Size||
mmio->Size-offset<sizeof(value))return -1;
if(kernel_xhci_write32(mmio,offset,(u32)value)||
kernel_xhci_write32(mmio,offset+4,(u32)(value>>32)))return -1;
return 0;
}
static int kernel_xhci_advance_event(KERNEL_XHCI_MMIO *mmio,
const BOB64_XHCI_CAPABILITIES *capabilities) {
u32 runtime_offset;
if(!mmio||!capabilities||kernel_xhci_rings.EventIndex>=
BOB64_XHCI_RING_TRBS)return -1;
kernel_xhci_rings.EventIndex++;
if(kernel_xhci_rings.EventIndex==BOB64_XHCI_RING_TRBS) {
kernel_xhci_rings.EventIndex=0;
kernel_xhci_rings.EventCycle^=1u;
}
runtime_offset=capabilities->RuntimeOffset-mmio->CapabilityLength;
return kernel_xhci_write64(mmio,runtime_offset+0x38,
kernel_xhci_rings.EventPhysical+
kernel_xhci_rings.EventIndex*sizeof(KERNEL_XHCI_TRB)+8u);
}
static int kernel_xhci_event_take(KERNEL_XHCI_MMIO *mmio,
const BOB64_XHCI_CAPABILITIES *capabilities,u32 expected_type,
KERNEL_XHCI_TRB *result) {
if(!mmio||!capabilities||!result||!kernel_xhci_rings.EventRing)return -1;
for(u32 offset=0;offset<kernel_xhci_rings.DeferredCount;offset++) {
u32 index=(kernel_xhci_rings.DeferredHead+offset)%
(u32)(sizeof(kernel_xhci_rings.DeferredEvents)/
sizeof(kernel_xhci_rings.DeferredEvents[0]));
KERNEL_XHCI_TRB *queued=&kernel_xhci_rings.DeferredEvents[index];
u32 type=(queued->Control>>BOB64_XHCI_TRB_TYPE_SHIFT)&0x3fu;
if(!expected_type||type==expected_type) {
*result=*queued;
for(u32 move=offset;move+1<kernel_xhci_rings.DeferredCount;move++) {
u32 to=(kernel_xhci_rings.DeferredHead+move)%32u;
u32 from=(kernel_xhci_rings.DeferredHead+move+1u)%32u;
kernel_xhci_rings.DeferredEvents[to]=
kernel_xhci_rings.DeferredEvents[from];
}
kernel_xhci_rings.DeferredCount--;
return 1;
}
}
volatile KERNEL_XHCI_TRB *event=
&kernel_xhci_rings.EventRing[kernel_xhci_rings.EventIndex];
u32 control=event->Control;
if((control&1u)!=kernel_xhci_rings.EventCycle)return 0;
result->Parameter=event->Parameter;
result->Status=event->Status;
result->Control=control;
if(kernel_xhci_advance_event(mmio,capabilities))return -2;
u32 type=(control>>BOB64_XHCI_TRB_TYPE_SHIFT)&0x3fu;
if(!expected_type||type==expected_type)return 1;
if(kernel_xhci_rings.DeferredCount>=32u)return -3;
u32 tail=(kernel_xhci_rings.DeferredHead+
kernel_xhci_rings.DeferredCount)%32u;
kernel_xhci_rings.DeferredEvents[tail]=*result;
kernel_xhci_rings.DeferredCount++;
return 0;
}
static int kernel_xhci_wait_event(KERNEL_XHCI_MMIO *mmio,
const BOB64_XHCI_CAPABILITIES *capabilities,u32 expected_type,
u32 poll_limit,KERNEL_XHCI_TRB *result) {
if(!mmio||!capabilities||!expected_type||!poll_limit||!result||
!kernel_xhci_rings.EventRing)return -1;
for(u32 wait=0;wait<poll_limit;wait++) {
int taken=kernel_xhci_event_take(mmio,capabilities,expected_type,result);
if(taken<0)return taken;
if(taken)return 0;
__asm__ volatile("pause");
}
return -4;
}
static int kernel_xhci_wait_transfer(KERNEL_XHCI_MMIO *mmio,
const BOB64_XHCI_CAPABILITIES *capabilities,u8 slot,u8 dci,
u64 trb_physical,u32 poll_limit,KERNEL_XHCI_TRB *completion) {
KERNEL_XHCI_TRB deferred[32];
u32 deferred_count=0;
int found=0;
int storage_removed=0;
#ifdef BOB64_STORAGE_ACTIVE_DISCONNECT_TEST
static u8 active_disconnect_test_armed;
#endif
if(!mmio||!capabilities||!slot||dci<2u||dci>31u||!poll_limit||
!completion||!kernel_xhci_rings.EventRing)return -1;
#ifdef BOB64_STORAGE_ACTIVE_DISCONNECT_TEST
if(!active_disconnect_test_armed) {
for(u32 i=0;i<kernel_xhci_storage_device_count;i++) {
if(kernel_xhci_storage_devices[i].Active&&
kernel_xhci_storage_devices[i].Slot==slot&&
kernel_xhci_storage_devices[i].DisconnectTestReadPending) {
active_disconnect_test_armed=1;
kernel_write("bob64 kernel: USB MSC READ(10) data transfer in flight\r\n");
(void)kernel_xhci_delay_ticks(150);
break;
}
}
}
#endif
while(kernel_xhci_rings.DeferredCount&&deferred_count<32u) {
deferred[deferred_count++]=kernel_xhci_rings.DeferredEvents[
kernel_xhci_rings.DeferredHead];
kernel_xhci_rings.DeferredHead=(u8)((kernel_xhci_rings.DeferredHead+1u)%32u);
kernel_xhci_rings.DeferredCount--;
}
for(u32 wait=0;wait<poll_limit;wait++) {
volatile KERNEL_XHCI_TRB *event=
&kernel_xhci_rings.EventRing[kernel_xhci_rings.EventIndex];
u32 control=event->Control;
if((control&1u)!=kernel_xhci_rings.EventCycle) {
__asm__ volatile("pause");
continue;
}
KERNEL_XHCI_TRB current={event->Parameter,event->Status,control};
if(kernel_xhci_advance_event(mmio,capabilities))break;
u32 type=(control>>BOB64_XHCI_TRB_TYPE_SHIFT)&0x3fu;
if(type==BOB64_XHCI_TRB_TYPE_TRANSFER_EVENT&&
(u8)(control>>24)==slot&&((control>>16)&0x1fu)==dci&&
current.Parameter==trb_physical) {
*completion=current;
found=1;
break;
}
if(type==BOB64_XHCI_TRB_TYPE_PORT_STATUS_CHANGE) {
u8 port=(u8)(current.Parameter>>24);
u32 port_status=0;
for(u32 i=0;i<kernel_xhci_storage_device_count;i++) {
KERNEL_XHCI_STORAGE_DEVICE *device=
&kernel_xhci_storage_devices[i];
if(device->Active&&device->Slot==slot&&
device->Topology.RootPort==port&&port&&
port<=capabilities->MaxPorts&&
!kernel_xhci_read32(mmio,0x400u+((u32)port-1u)*0x10u,
&port_status)&&!(port_status&1u)) {
storage_removed=1;
break;
}
}
if(storage_removed) {
kernel_write("bob64 kernel: USB MSC transfer aborted after disconnect\r\n");
if(deferred_count>=32u)break;
deferred[deferred_count++]=current;
break;
}
}
if(deferred_count>=32u)break;
deferred[deferred_count++]=current;
}
for(u32 i=0;i<deferred_count;i++) {
if(kernel_xhci_rings.DeferredCount>=32u)return -2;
u32 tail=(kernel_xhci_rings.DeferredHead+
kernel_xhci_rings.DeferredCount)%32u;
kernel_xhci_rings.DeferredEvents[tail]=deferred[i];
kernel_xhci_rings.DeferredCount++;
}
return found?0:(storage_removed?-4:-3);
}
static int kernel_xhci_disable_slot(u8 slot_id) {
if(!slot_id||!kernel_xhci_rings.CommandRing||
kernel_xhci_rings.CommandIndex>=BOB64_XHCI_RING_TRBS-1)return -1;
u32 command_index=kernel_xhci_rings.CommandIndex;
volatile KERNEL_XHCI_TRB *command=
&kernel_xhci_rings.CommandRing[command_index];
KERNEL_XHCI_TRB completion;
command->Parameter=0;
command->Status=0;
command->Control=(BOB64_XHCI_TRB_TYPE_DISABLE_SLOT<<
BOB64_XHCI_TRB_TYPE_SHIFT)|((u32)slot_id<<24)|1u;
__asm__ volatile("sfence" ::: "memory");
u32 doorbell=kernel_xhci_live_capabilities.DoorbellOffset-
kernel_xhci_live_mmio.CapabilityLength;
if(kernel_xhci_write32(&kernel_xhci_live_mmio,doorbell,0))return -2;
int result=kernel_xhci_wait_event(&kernel_xhci_live_mmio,
&kernel_xhci_live_capabilities,
BOB64_XHCI_TRB_TYPE_COMMAND_COMPLETION,10000000u,&completion);
if(result)return -3+result;
if(completion.Parameter!=kernel_xhci_rings.CommandPhysical+
command_index*sizeof(KERNEL_XHCI_TRB)||
(completion.Status>>24)!=BOB64_XHCI_TRB_COMPLETION_SUCCESS||
(u8)(completion.Control>>24)!=slot_id)return -4;
kernel_xhci_rings.CommandIndex++;
if(kernel_xhci_rings.DeviceContextArray)
kernel_xhci_rings.DeviceContextArray[slot_id]=0;
return 0;
}
static int kernel_xhci_usb_hid_submit(KERNEL_XHCI_HID_DEVICE *device) {
if(!device||!device->Active||device->Pending||!device->Ring)return -1;
for(u32 i=0;i<device->ReportLength;i++)device->Report[i]=0;
if(device->NextIndex==BOB64_XHCI_RING_TRBS-1) {
volatile KERNEL_XHCI_TRB *link=
&device->Ring[BOB64_XHCI_RING_TRBS-1];
link->Parameter=device->RingPhysical;link->Status=0;
link->Control=(BOB64_XHCI_TRB_TYPE_LINK<<BOB64_XHCI_TRB_TYPE_SHIFT)|
2u|device->Cycle;
device->NextIndex=0;device->Cycle^=1u;
}
u32 index=device->NextIndex++;
volatile KERNEL_XHCI_TRB *transfer=&device->Ring[index];
transfer->Parameter=device->ReportPhysical;
transfer->Status=device->ReportLength;
transfer->Control=(1u<<BOB64_XHCI_TRB_TYPE_SHIFT)|(1u<<5)|device->Cycle;
__asm__ volatile("sfence":::"memory");
u32 doorbell=kernel_xhci_live_capabilities.DoorbellOffset-
kernel_xhci_live_mmio.CapabilityLength+(u32)device->Slot*4u;
if(kernel_xhci_write32(&kernel_xhci_live_mmio,doorbell,device->Dci))return -2;
device->PendingIndex=index;device->Pending=1;
return 0;
}
static int kernel_xhci_hub_status_submit(KERNEL_XHCI_HUB_DEVICE *device) {
if(!device||!device->Active||device->Pending||!device->Ring)return -1;
for(u32 i=0;i<device->ReportLength;i++)device->Report[i]=0;
if(device->NextIndex==BOB64_XHCI_RING_TRBS-1) {
volatile KERNEL_XHCI_TRB *link=
&device->Ring[BOB64_XHCI_RING_TRBS-1];
link->Parameter=device->RingPhysical;link->Status=0;
link->Control=(BOB64_XHCI_TRB_TYPE_LINK<<BOB64_XHCI_TRB_TYPE_SHIFT)|
2u|device->Cycle;
device->NextIndex=0;device->Cycle^=1u;
}
u32 index=device->NextIndex++;
volatile KERNEL_XHCI_TRB *transfer=&device->Ring[index];
transfer->Parameter=device->ReportPhysical;
transfer->Status=device->ReportLength;
transfer->Control=(1u<<BOB64_XHCI_TRB_TYPE_SHIFT)|(1u<<5)|device->Cycle;
__asm__ volatile("sfence" ::: "memory");
u32 doorbell=kernel_xhci_live_capabilities.DoorbellOffset-
kernel_xhci_live_mmio.CapabilityLength+(u32)device->Slot*4u;
if(kernel_xhci_write32(&kernel_xhci_live_mmio,doorbell,device->Dci))return -2;
device->PendingIndex=index;device->Pending=1;
return 0;
}
static void kernel_xhci_hid_disconnect(KERNEL_XHCI_HID_DEVICE *device,u32 index) {
static const u8 released_report[8]={0};
if(!device||!device->Active)return;
device->Active=0;device->Pending=0;
if(device->Kind==1)
(void)bob64_keyboard_hid_report_device(&device->KeyboardState,
released_report);
else if(device->Kind==2)
(void)bob64_mouse_usb_report_device(index,0,0,0,0);
if(kernel_xhci_disable_slot(device->Slot))
kernel_write("bob64 kernel: USB HID slot disable failed\r\n");
kernel_write("bob64 kernel: USB HID transfer stopped; held input released\r\n");
}
static void kernel_xhci_usb_hid_poll(void) {
if((!kernel_xhci_hid_device_count&&!kernel_xhci_hub_device_count&&
!kernel_xhci_storage_device_count)||
!kernel_xhci_rings.EventRing)return;
for(u32 count=0;count<8;count++) {
KERNEL_XHCI_TRB event;
int taken=kernel_xhci_event_take(&kernel_xhci_live_mmio,
&kernel_xhci_live_capabilities,0,&event);
if(taken<=0)return;
u32 type=(event.Control>>BOB64_XHCI_TRB_TYPE_SHIFT)&0x3fu;
if(type==BOB64_XHCI_TRB_TYPE_PORT_STATUS_CHANGE) {
u8 port=(u8)(event.Parameter>>24);
u32 port_status=0;
if(port&&port<=kernel_xhci_live_capabilities.MaxPorts&&
!kernel_xhci_read32(&kernel_xhci_live_mmio,
0x400u+((u32)port-1u)*0x10u,&port_status)) {
if(port_status&1u)kernel_xhci_hid_reconnect(port);
else {
for(u32 i=0;i<kernel_xhci_hid_device_count;i++)
if(kernel_xhci_hid_devices[i].Active&&
kernel_xhci_hid_devices[i].RootPort==port)
kernel_xhci_hid_disconnect(&kernel_xhci_hid_devices[i],i);
for(u32 i=0;i<kernel_xhci_hub_device_count;i++) {
KERNEL_XHCI_HUB_DEVICE *hub=&kernel_xhci_hub_devices[i];
if(hub->Active&&hub->RootPort==port) {
hub->Active=0;hub->Pending=0;
(void)kernel_xhci_disable_slot(hub->Slot);
}
}
for(u32 i=0;i<kernel_xhci_storage_device_count;i++) {
KERNEL_XHCI_STORAGE_DEVICE *storage=
&kernel_xhci_storage_devices[i];
if(storage->Active&&storage->Topology.RootPort==port) {
storage->Active=0;
(void)kernel_xhci_disable_slot(storage->Slot);
kernel_write("bob64 kernel: USB mass-storage device disconnected\r\n");
}
}
}
}
} else if(type==BOB64_XHCI_TRB_TYPE_TRANSFER_EVENT) {
u32 completion=(event.Status>>24)&0xffu;
for(u32 i=0;i<kernel_xhci_hid_device_count;i++) {
KERNEL_XHCI_HID_DEVICE *device=&kernel_xhci_hid_devices[i];
if(!device->Pending||
(u8)(event.Control>>24)!=device->Slot||
((event.Control>>16)&0x1fu)!=device->Dci||
event.Parameter!=device->RingPhysical+
device->PendingIndex*sizeof(KERNEL_XHCI_TRB))continue;
device->Pending=0;
if(completion==BOB64_XHCI_TRB_COMPLETION_SUCCESS||completion==13u) {
device->TransferErrors=0;
u8 report[8]={0};
for(u32 j=0;j<device->ReportLength&&j<sizeof(report);j++)
report[j]=device->Report[j];
if(!device->LoggedReport) {
device->LoggedReport=1;
kernel_write("bob64 kernel: USB HID transfer report received\r\n");
}
if(device->Kind==1) {
int has_key=0;
(void)bob64_keyboard_hid_report_device(
&device->KeyboardState,report);
for(u32 j=2;j<sizeof(report);j++)if(report[j])has_key=1;
if(has_key&&!device->LoggedInput) {
device->LoggedInput=1;
kernel_write("bob64 kernel: USB HID keyboard input queued\r\n");
}
} else if(device->Kind==2) {
s8 wheel=device->ReportLength>=4?(s8)report[3]:0;
int mouse_result=bob64_mouse_usb_report_device(i,report[0],
(s8)report[1],(s8)report[2],wheel);
if(!mouse_result&&!device->LoggedInput) {
device->LoggedInput=1;
kernel_write("bob64 kernel: USB HID mouse input queued\r\n");
}
if(!mouse_result&&wheel&&!device->LoggedWheel) {
device->LoggedWheel=1;
kernel_write("bob64 kernel: USB HID mouse wheel input queued\r\n");
}
}
(void)kernel_xhci_usb_hid_submit(device);
} else if(++device->TransferErrors>=3u) {
kernel_xhci_hid_disconnect(device,i);
} else {
(void)kernel_xhci_usb_hid_submit(device);
}
break;
}
for(u32 i=0;i<kernel_xhci_hub_device_count;i++) {
KERNEL_XHCI_HUB_DEVICE *hub=&kernel_xhci_hub_devices[i];
if(!hub->Active||!hub->Pending||
(u8)(event.Control>>24)!=hub->Slot||
((event.Control>>16)&0x1fu)!=hub->Dci||
event.Parameter!=hub->RingPhysical+
hub->PendingIndex*sizeof(KERNEL_XHCI_TRB))continue;
hub->Pending=0;
if(completion==BOB64_XHCI_TRB_COMPLETION_SUCCESS||completion==13u) {
int changed=0;
hub->TransferErrors=0;
for(u32 byte=0;byte<hub->ReportLength;byte++)
if(hub->Report[byte])changed=1;
if(changed&&!hub->LoggedChange) {
hub->LoggedChange=1;
kernel_write("bob64 kernel: USB hub downstream change report received slot=");
kernel_hex64(hub->Slot);kernel_write(" bitmap=");
for(u32 byte=0;byte<hub->ReportLength;byte++) {
kernel_hex64(hub->Report[byte]);kernel_write(" ");
}
kernel_write("\r\n");
}
if(changed)kernel_xhci_hub_process_changes(hub);
(void)kernel_xhci_hub_status_submit(hub);
} else if(++hub->TransferErrors<3u)
(void)kernel_xhci_hub_status_submit(hub);
else hub->Active=0;
break;
}
}
}
}
static int kernel_xhci_start_command_ring(KERNEL_XHCI_MMIO *mmio,
const BOB64_XHCI_CAPABILITIES *capabilities) {
u64 dcbaa_physical,command_physical,event_physical,erst_physical;
u64 scratch_array_physical=0;
u64 scratchpad_physical;
volatile u8 *dcbaa_bytes,*command_bytes,*event_bytes,*erst_bytes;
u64 scratch_array_pages;
u32 status,pagesize,command;
if(!mmio||!capabilities||capabilities->MaxScratchpadBuffers>
BOB64_XHCI_MAX_SCRATCHPADS)return -1;
if(kernel_xhci_read32(mmio,8,&pagesize)||!(pagesize&1u))return -2;
if(kernel_xhci_dma_alloc(1,&dcbaa_physical,&dcbaa_bytes)||
kernel_xhci_dma_alloc(1,&command_physical,&command_bytes)||
kernel_xhci_dma_alloc(1,&event_physical,&event_bytes)||
kernel_xhci_dma_alloc(1,&erst_physical,&erst_bytes))return -3;
if(capabilities->MaxScratchpadBuffers) {
scratch_array_pages=(capabilities->MaxScratchpadBuffers*sizeof(u64)+
BOB64_PAGE_SIZE-1)/BOB64_PAGE_SIZE;
volatile u8 *scratch_array_bytes;
if(kernel_xhci_dma_alloc(scratch_array_pages,&scratch_array_physical,
&scratch_array_bytes))return -4;
volatile u64 *scratch_array=(volatile u64 *)(volatile void *)scratch_array_bytes;
for(u32 i=0;i<capabilities->MaxScratchpadBuffers;i++) {
volatile u8 *scratch_bytes;
if(kernel_xhci_dma_alloc(1,&scratchpad_physical,&scratch_bytes))
return -5;
scratch_array[i]=scratchpad_physical;
}
*(volatile u64 *)(volatile void *)dcbaa_bytes=scratch_array_physical;
}
volatile KERNEL_XHCI_TRB *command_ring=
(volatile KERNEL_XHCI_TRB *)(volatile void *)command_bytes;
volatile KERNEL_XHCI_TRB *event_ring=
(volatile KERNEL_XHCI_TRB *)(volatile void *)event_bytes;
volatile KERNEL_XHCI_ERST_ENTRY *erst=
(volatile KERNEL_XHCI_ERST_ENTRY *)(volatile void *)erst_bytes;
kernel_xhci_rings.CommandRing=command_ring;
kernel_xhci_rings.EventRing=event_ring;
kernel_xhci_rings.CommandPhysical=command_physical;
kernel_xhci_rings.EventPhysical=event_physical;
kernel_xhci_rings.DeviceContextArray=
(volatile u64 *)(volatile void *)dcbaa_bytes;
kernel_xhci_rings.CommandIndex=0;
kernel_xhci_rings.EventIndex=0;
kernel_xhci_rings.EventCycle=1;
command_ring[BOB64_XHCI_RING_TRBS-1].Parameter=command_physical;
command_ring[BOB64_XHCI_RING_TRBS-1].Status=0;
command_ring[BOB64_XHCI_RING_TRBS-1].Control=
(BOB64_XHCI_TRB_TYPE_LINK<<BOB64_XHCI_TRB_TYPE_SHIFT)|3u;
command_ring[0].Parameter=0;
command_ring[0].Status=0;
command_ring[0].Control=
(BOB64_XHCI_TRB_TYPE_NOOP_COMMAND<<BOB64_XHCI_TRB_TYPE_SHIFT)|1u;
erst[0].RingBase=event_physical;
erst[0].RingSize=BOB64_XHCI_RING_TRBS;
if(kernel_xhci_write64(mmio,0x30,dcbaa_physical)||
kernel_xhci_write64(mmio,0x18,command_physical|1u))return -6;
if(kernel_xhci_write32(mmio,0x38,1u))return -7;
u32 runtime_offset=capabilities->RuntimeOffset-mmio->CapabilityLength;
if(runtime_offset>mmio->Size||mmio->Size-runtime_offset<0x40)return -8;
if(kernel_xhci_write32(mmio,runtime_offset+0x28,1u)||
kernel_xhci_write64(mmio,runtime_offset+0x30,erst_physical)||
kernel_xhci_write64(mmio,runtime_offset+0x38,event_physical))return -9;
__asm__ volatile("sfence" ::: "memory");
if(kernel_xhci_read32(mmio,0,&command)||
kernel_xhci_write32(mmio,0,command|BOB64_XHCI_USBCMD_RUN))return -10;
for(u32 i=0;i<1000000u;i++) {
if(kernel_xhci_read32(mmio,4,&status))return -11;
if(!(status&BOB64_XHCI_USBSTS_NOT_READY)&&
!(status&BOB64_XHCI_USBSTS_HALTED))break;
if(i==999999u)return -12;
}
u32 doorbell_offset=capabilities->DoorbellOffset-mmio->CapabilityLength;
if(kernel_xhci_write32(mmio,doorbell_offset,0))return -16;
KERNEL_XHCI_TRB completion;
if(kernel_xhci_wait_event(mmio,capabilities,
BOB64_XHCI_TRB_TYPE_COMMAND_COMPLETION,1000000u,&completion))return -15;
if(completion.Parameter!=command_physical||
(completion.Status>>24)!=BOB64_XHCI_TRB_COMPLETION_SUCCESS)return -13;
kernel_xhci_rings.CommandIndex=1;
return 0;
}
static int kernel_xhci_enable_slot(KERNEL_XHCI_MMIO *mmio,
const BOB64_XHCI_CAPABILITIES *capabilities,u8 *slot_id) {
u32 doorbell_offset;
if(!mmio||!capabilities||!slot_id||
!kernel_xhci_rings.CommandRing||!kernel_xhci_rings.EventRing||
kernel_xhci_rings.CommandIndex>=BOB64_XHCI_RING_TRBS-1||
kernel_xhci_rings.EventIndex>=BOB64_XHCI_RING_TRBS)return -1;
u32 command_index=kernel_xhci_rings.CommandIndex;
volatile KERNEL_XHCI_TRB *command=&kernel_xhci_rings.CommandRing[command_index];
KERNEL_XHCI_TRB completion;
command->Parameter=0;
command->Status=0;
command->Control=(BOB64_XHCI_TRB_TYPE_ENABLE_SLOT<<
BOB64_XHCI_TRB_TYPE_SHIFT)|1u;
__asm__ volatile("sfence" ::: "memory");
doorbell_offset=capabilities->DoorbellOffset-mmio->CapabilityLength;
if(kernel_xhci_write32(mmio,doorbell_offset,0))return -2;
int result=kernel_xhci_wait_event(mmio,capabilities,
BOB64_XHCI_TRB_TYPE_COMMAND_COMPLETION,10000000u,&completion);
if(result)return -5+result;
u8 result_slot=(u8)(completion.Control>>24);
if(completion.Parameter!=kernel_xhci_rings.CommandPhysical+
command_index*sizeof(KERNEL_XHCI_TRB)||
(completion.Status>>24)!=BOB64_XHCI_TRB_COMPLETION_SUCCESS||
!result_slot||result_slot>capabilities->MaxSlots)return -3;
*slot_id=result_slot;
kernel_xhci_rings.CommandIndex++;
return 0;
}
static int kernel_xhci_control_in_data(KERNEL_XHCI_MMIO *mmio,
const BOB64_XHCI_CAPABILITIES *capabilities,u8 slot_id,
volatile KERNEL_XHCI_TRB *transfer_ring,u64 transfer_ring_physical,
u64 buffer_physical,u8 request_type,u8 request,u16 value,u16 index,
u16 length,u32 ring_index) {
u32 doorbell_offset;
KERNEL_XHCI_TRB completion;
if(!mmio||!capabilities||!slot_id||!transfer_ring||!length||
ring_index+2>=BOB64_XHCI_RING_TRBS-1)return -1;
transfer_ring[ring_index].Parameter=bob64_usb_control_setup(
request_type,request,value,index,length);
transfer_ring[ring_index].Status=8;
transfer_ring[ring_index].Control=(2u<<BOB64_XHCI_TRB_TYPE_SHIFT)|
(3u<<16)|(1u<<6)|(1u<<4)|1u;
transfer_ring[ring_index+1].Parameter=buffer_physical;
transfer_ring[ring_index+1].Status=length;
transfer_ring[ring_index+1].Control=(3u<<BOB64_XHCI_TRB_TYPE_SHIFT)|
(1u<<16)|(1u<<4)|(1u<<2)|1u;
transfer_ring[ring_index+2].Parameter=0;
transfer_ring[ring_index+2].Status=0;
transfer_ring[ring_index+2].Control=(4u<<BOB64_XHCI_TRB_TYPE_SHIFT)|
(1u<<5)|1u;
__asm__ volatile("sfence" ::: "memory");
doorbell_offset=capabilities->DoorbellOffset-mmio->CapabilityLength+
(u32)slot_id*4u;
if(kernel_xhci_write32(mmio,doorbell_offset,1u))return -2;
int result=kernel_xhci_wait_event(mmio,capabilities,
BOB64_XHCI_TRB_TYPE_TRANSFER_EVENT,10000000u,&completion);
if(result)return -3+result;
if((completion.Status>>24)!=BOB64_XHCI_TRB_COMPLETION_SUCCESS||
(u8)(completion.Control>>24)!=slot_id||
((completion.Control>>16)&0x1fu)!=1u||
completion.Parameter!=transfer_ring_physical+
(ring_index+2)*sizeof(KERNEL_XHCI_TRB))return -4;
return 0;
}
static int kernel_xhci_get_descriptor(KERNEL_XHCI_MMIO *mmio,
const BOB64_XHCI_CAPABILITIES *capabilities,u8 slot_id,
volatile KERNEL_XHCI_TRB *transfer_ring,u64 transfer_ring_physical,
u64 buffer_physical,u8 descriptor_type,u16 length,u32 ring_index) {
return kernel_xhci_control_in_data(mmio,capabilities,slot_id,
transfer_ring,transfer_ring_physical,buffer_physical,0x80,6,
(u16)((u16)descriptor_type<<8),0,length,ring_index);
}
static int kernel_xhci_control_no_data(KERNEL_XHCI_MMIO *mmio,
const BOB64_XHCI_CAPABILITIES *capabilities,u8 slot_id,
volatile KERNEL_XHCI_TRB *transfer_ring,u64 transfer_ring_physical,
u32 ring_index,u8 request_type,u8 request,u16 value,u16 index) {
KERNEL_XHCI_TRB completion;
if(!mmio||!capabilities||!slot_id||!transfer_ring||
ring_index+1>=BOB64_XHCI_RING_TRBS-1)return -1;
transfer_ring[ring_index].Parameter=bob64_usb_control_setup(
request_type,request,value,index,0);
transfer_ring[ring_index].Status=8;
transfer_ring[ring_index].Control=(2u<<BOB64_XHCI_TRB_TYPE_SHIFT)|
(1u<<6)|(1u<<4)|1u;
transfer_ring[ring_index+1].Parameter=0;
transfer_ring[ring_index+1].Status=0;
transfer_ring[ring_index+1].Control=(4u<<BOB64_XHCI_TRB_TYPE_SHIFT)|
(1u<<16)|(1u<<5)|1u;
__asm__ volatile("sfence" ::: "memory");
u32 doorbell_offset=capabilities->DoorbellOffset-mmio->CapabilityLength+
(u32)slot_id*4u;
if(kernel_xhci_write32(mmio,doorbell_offset,1u))return -2;
int result=kernel_xhci_wait_event(mmio,capabilities,
BOB64_XHCI_TRB_TYPE_TRANSFER_EVENT,10000000u,&completion);
if(result)return -3+result;
if((completion.Status>>24)!=BOB64_XHCI_TRB_COMPLETION_SUCCESS||
(u8)(completion.Control>>24)!=slot_id||
((completion.Control>>16)&0x1fu)!=1u||
completion.Parameter!=transfer_ring_physical+
(ring_index+1)*sizeof(KERNEL_XHCI_TRB))return -4;
return 0;
}
static int kernel_xhci_delay_ticks(u64 ticks) {
u64 start=bob64_timer_ticks();
if(!ticks)return 0;
for(u32 wait=0;wait<200000000u;wait++) {
if(bob64_timer_ticks()-start>=ticks)return 0;
__asm__ volatile("pause");
}
return -1;
}
static int kernel_xhci_update_hub_context(KERNEL_XHCI_MMIO *mmio,
const BOB64_XHCI_CAPABILITIES *capabilities,u8 slot_id,
volatile u8 *device_context_bytes,u32 context_size,u8 port_count,
u8 tt_think_time) {
u64 input_context_physical;
volatile u8 *input_context_bytes;
KERNEL_XHCI_TRB completion;
u32 command_index=kernel_xhci_rings.CommandIndex;
if(!mmio||!capabilities||!slot_id||!device_context_bytes||
!context_size||!port_count||command_index>=BOB64_XHCI_RING_TRBS-1)
return -1;
if(kernel_xhci_dma_alloc(1,&input_context_physical,
&input_context_bytes))return -2;
volatile u32 *input=(volatile u32 *)(volatile void *)input_context_bytes;
volatile u32 *slot=(volatile u32 *)(volatile void *)(input_context_bytes+
context_size);
volatile u32 *live_slot=(volatile u32 *)(volatile void *)device_context_bytes;
input[1]=1u; /* Update the Slot Context. */
for(u32 word=0;word<context_size/sizeof(u32);word++)slot[word]=live_slot[word];
slot[0]|=1u<<26; /* Hub */
slot[1]=(slot[1]&0x00ffffffu)|((u32)port_count<<24);
u32 speed=(live_slot[0]>>20)&0xfu;
slot[2]=(slot[2]&~(3u<<16))|
(speed==3u?((u32)(tt_think_time&3u)<<16):0u);
volatile KERNEL_XHCI_TRB *command=
&kernel_xhci_rings.CommandRing[command_index];
command->Parameter=input_context_physical;
command->Status=0;
command->Control=(12u<<BOB64_XHCI_TRB_TYPE_SHIFT)|
((u32)slot_id<<24)|1u;
__asm__ volatile("sfence" ::: "memory");
u32 doorbell_offset=capabilities->DoorbellOffset-mmio->CapabilityLength;
if(kernel_xhci_write32(mmio,doorbell_offset,0))return -3;
int result=kernel_xhci_wait_event(mmio,capabilities,
BOB64_XHCI_TRB_TYPE_COMMAND_COMPLETION,10000000u,&completion);
if(result)return -4+result;
if(completion.Parameter!=kernel_xhci_rings.CommandPhysical+
command_index*sizeof(KERNEL_XHCI_TRB)||
(completion.Status>>24)!=BOB64_XHCI_TRB_COMPLETION_SUCCESS||
(u8)(completion.Control>>24)!=slot_id)return -5;
kernel_xhci_rings.CommandIndex++;
return 0;
}
static int kernel_xhci_configure_usb2_hub(KERNEL_XHCI_MMIO *mmio,
const BOB64_XHCI_CAPABILITIES *capabilities,u8 slot_id,
const KERNEL_XHCI_TOPOLOGY *topology,
volatile KERNEL_XHCI_TRB *transfer_ring,u64 transfer_ring_physical,
u64 descriptor_physical,volatile u8 *descriptor_bytes,
volatile u8 *device_context_bytes,u32 context_size,u8 configuration,
u8 *port_count_out,u32 *control_index_out) {
BOB64_USB_HUB_DESCRIPTOR hub;
u32 connected=0,reset=0,control_index=20;
if(!mmio||!capabilities||!slot_id||!topology||!transfer_ring||
!port_count_out||!control_index_out||
!descriptor_bytes)
return -1;
int result=kernel_xhci_control_no_data(mmio,capabilities,slot_id,
transfer_ring,transfer_ring_physical,12,0x00,9,configuration,0);
if(result)return -2+result;
result=kernel_xhci_control_in_data(mmio,capabilities,slot_id,
transfer_ring,transfer_ring_physical,descriptor_physical,0xa0,6,
0x2900,0,7,14);
if(result)return -6+result;
if(descriptor_bytes[0]<7||descriptor_bytes[1]!=0x29)return -10;
result=kernel_xhci_control_in_data(mmio,capabilities,slot_id,
transfer_ring,transfer_ring_physical,descriptor_physical,0xa0,6,
0x2900,0,descriptor_bytes[0],17);
if(result)return -11+result;
if(bob64_usb_parse_hub_descriptor((const void *)(uintptr_t)descriptor_bytes,
descriptor_bytes[0],&hub)) {
kernel_write("bob64 kernel: invalid USB hub descriptor len=");
kernel_hex64(descriptor_bytes[0]);kernel_write(" type=");
kernel_hex64(descriptor_bytes[1]);kernel_write(" ports=");
kernel_hex64(descriptor_bytes[2]);kernel_write(" raw=");
for(u32 i=0;i<12&&i<descriptor_bytes[0];i++) {
kernel_hex64(descriptor_bytes[i]);kernel_write(" ");
}
kernel_write("\r\n");
return -15;
}
/* Keep this one-page EP0 ring within its link TRB while scanning ports. */
if(hub.PortCount>16)return -16;
*port_count_out=hub.PortCount;
result=kernel_xhci_update_hub_context(mmio,capabilities,slot_id,
device_context_bytes,context_size,hub.PortCount,
(u8)((hub.Characteristics>>5)&3u));
if(result)return -20+result;
if((hub.Characteristics&3u)<2u) {
for(u32 port=1;port<=hub.PortCount;port++) {
result=kernel_xhci_control_no_data(mmio,capabilities,slot_id,
transfer_ring,transfer_ring_physical,control_index,0x23,3,8,
(u16)port);
if(result)return -25+result;
control_index+=2;
}
u64 delay_ms=(u64)hub.PowerOnToGood*2u;
u64 delay_ticks=(delay_ms+9u)/10u;
if(kernel_xhci_delay_ticks(delay_ticks))return -26;
}
control_index=20u+((hub.Characteristics&3u)<2u?
(u32)hub.PortCount*2u:0u);
for(u32 port=1;port<=hub.PortCount;port++) {
result=kernel_xhci_control_in_data(mmio,capabilities,slot_id,
transfer_ring,transfer_ring_physical,descriptor_physical,0xa3,0,
0,(u16)port,4,control_index);
if(result)return -30+result;
u16 port_status=(u16)(descriptor_bytes[0]|((u16)descriptor_bytes[1]<<8));
control_index+=3;
if(port_status&1u) {
u16 port_change=(u16)(descriptor_bytes[2]|((u16)descriptor_bytes[3]<<8));
connected++;
result=kernel_xhci_control_no_data(mmio,capabilities,slot_id,
transfer_ring,transfer_ring_physical,control_index,0x23,3,4,
(u16)port);
if(result)return -31+result;
control_index+=2;
if(kernel_xhci_delay_ticks(2))return -32;
result=kernel_xhci_control_in_data(mmio,capabilities,slot_id,
transfer_ring,transfer_ring_physical,descriptor_physical,
0xa3,0,0,(u16)port,4,control_index);
if(result)return -33+result;
port_status=(u16)(descriptor_bytes[0]|((u16)descriptor_bytes[1]<<8));
port_change=(u16)(descriptor_bytes[2]|((u16)descriptor_bytes[3]<<8));
control_index+=3;
if((port_status&(1u<<4))||!(port_status&(1u<<1)))return -34;
reset++;
if(port_change&1u) {
result=kernel_xhci_control_no_data(mmio,capabilities,slot_id,
transfer_ring,transfer_ring_physical,control_index,
0x23,1,16,(u16)port);
if(result)return -35+result;
control_index+=2;
}
if(port_change&(1u<<4)) {
result=kernel_xhci_control_no_data(mmio,capabilities,slot_id,
transfer_ring,transfer_ring_physical,control_index,
0x23,1,20,(u16)port);
if(result)return -36+result;
control_index+=2;
}
if(kernel_xhci_hub_child_count<
sizeof(kernel_xhci_hub_children)/
sizeof(kernel_xhci_hub_children[0])&&
topology->Depth<5u) {
u32 route=0;
u16 port_speed_bits=(u16)((port_status>>9)&3u);
u8 child_speed=port_speed_bits==2u?3u:
port_speed_bits==1u?2u:1u;
if(!bob64_usb_route_string_append(topology->RouteString,
topology->Depth,(u8)port,&route)) {
KERNEL_XHCI_TOPOLOGY *child=
&kernel_xhci_hub_children[
kernel_xhci_hub_child_count++];
child->RouteString=route;
child->RootPort=topology->RootPort;
child->ParentSlot=slot_id;
child->ParentPort=(u8)port;
child->Speed=child_speed;
child->Depth=(u8)(topology->Depth+1u);
child->TtHubSlot=topology->TtHubSlot;
child->TtPort=topology->TtPort;
if(topology->Speed==3u&&child_speed<3u) {
child->TtHubSlot=slot_id;
child->TtPort=(u8)port;
}
}
}
}
}
kernel_write("bob64 kernel: USB 2 hub configured ports=");
kernel_hex64(hub.PortCount);kernel_write(" connected=");
kernel_hex64(connected);kernel_write(" reset=");kernel_hex64(reset);
kernel_write("\r\n");
*control_index_out=control_index;
return 0;
}
static int kernel_xhci_configure_hub_status_endpoint(KERNEL_XHCI_MMIO *mmio,
const BOB64_XHCI_CAPABILITIES *capabilities,u8 slot_id,
const KERNEL_XHCI_TOPOLOGY *topology,
const BOB64_USB_HUB_INTERFACE *hub_interface,u8 port_count,
volatile u8 *device_context_bytes,u32 context_size,
volatile KERNEL_XHCI_TRB *control_ring,u64 control_ring_physical,
volatile u8 *control_buffer,u64 control_buffer_physical,
u32 control_next_index) {
KERNEL_XHCI_HUB_DEVICE *hub=0;
u64 ring_physical,report_physical,input_physical;
volatile u8 *ring_bytes,*report_bytes,*input_bytes;
u32 dci,report_length,interval,command_index,doorbell_offset;
KERNEL_XHCI_TRB completion;
if(!mmio||!capabilities||!slot_id||!topology||!hub_interface||
!port_count||!device_context_bytes||!context_size||!control_ring||
!control_buffer||!control_buffer_physical||
!(hub_interface->EndpointAddress&0x80u)||
(hub_interface->EndpointAttributes&3u)!=3u)return -1;
dci=((hub_interface->EndpointAddress&0x0fu)<<1)|1u;
report_length=((u32)port_count+1u+7u)/8u;
if(dci<2u||dci>31u||!report_length||report_length>16u||
report_length>hub_interface->EndpointMaxPacketSize)return -2;
for(u32 i=0;i<kernel_xhci_hub_device_count;i++)
if(!kernel_xhci_hub_devices[i].Active) { hub=&kernel_xhci_hub_devices[i];break; }
if(!hub&&kernel_xhci_hub_device_count<KERNEL_XHCI_HID_DEVICE_LIMIT)
hub=&kernel_xhci_hub_devices[kernel_xhci_hub_device_count++];
if(!hub)return -3;
if(kernel_xhci_dma_alloc(1,&ring_physical,&ring_bytes)||
kernel_xhci_dma_alloc(1,&report_physical,&report_bytes)||
kernel_xhci_dma_alloc(1,&input_physical,&input_bytes))return -4;
volatile KERNEL_XHCI_TRB *ring=
(volatile KERNEL_XHCI_TRB *)(volatile void *)ring_bytes;
ring[BOB64_XHCI_RING_TRBS-1].Parameter=ring_physical;
ring[BOB64_XHCI_RING_TRBS-1].Status=0;
ring[BOB64_XHCI_RING_TRBS-1].Control=
(BOB64_XHCI_TRB_TYPE_LINK<<BOB64_XHCI_TRB_TYPE_SHIFT)|3u;
volatile u32 *input=(volatile u32 *)(volatile void *)input_bytes;
volatile u32 *slot=(volatile u32 *)(volatile void *)(input_bytes+context_size);
volatile u32 *endpoint=(volatile u32 *)(volatile void *)(input_bytes+
((dci+1u)*context_size));
volatile u32 *live_slot=(volatile u32 *)(volatile void *)device_context_bytes;
input[1]=1u|(1u<<dci);
for(u32 word=0;word<context_size/sizeof(u32);word++)slot[word]=live_slot[word];
slot[0]=(slot[0]&~(31u<<27))|(dci<<27);
if(topology->Speed>=3u)
interval=hub_interface->EndpointInterval?
(u32)hub_interface->EndpointInterval-1u:0u;
else {
u32 period=hub_interface->EndpointInterval?
hub_interface->EndpointInterval:1u;
interval=3u;
while(period>1u&&interval<15u) { period=(period+1u)>>1;interval++; }
}
endpoint[0]=interval<<16;
endpoint[1]=(3u<<1)|(7u<<3)|
((u32)hub_interface->EndpointMaxPacketSize<<16);
endpoint[2]=(u32)(ring_physical|1u);
endpoint[3]=(u32)(ring_physical>>32);
endpoint[4]=report_length|((u32)hub_interface->EndpointMaxPacketSize<<16);
command_index=kernel_xhci_rings.CommandIndex;
if(command_index>=BOB64_XHCI_RING_TRBS-1)return -5;
volatile KERNEL_XHCI_TRB *command=
&kernel_xhci_rings.CommandRing[command_index];
command->Parameter=input_physical;command->Status=0;
command->Control=(12u<<BOB64_XHCI_TRB_TYPE_SHIFT)|
((u32)slot_id<<24)|1u;
__asm__ volatile("sfence" ::: "memory");
doorbell_offset=capabilities->DoorbellOffset-mmio->CapabilityLength;
if(kernel_xhci_write32(mmio,doorbell_offset,0))return -6;
int result=kernel_xhci_wait_event(mmio,capabilities,
BOB64_XHCI_TRB_TYPE_COMMAND_COMPLETION,10000000u,&completion);
if(result)return -7+result;
if(completion.Parameter!=kernel_xhci_rings.CommandPhysical+
command_index*sizeof(KERNEL_XHCI_TRB)||
(completion.Status>>24)!=BOB64_XHCI_TRB_COMPLETION_SUCCESS||
(u8)(completion.Control>>24)!=slot_id)return -8;
kernel_xhci_rings.CommandIndex++;
hub->Ring=ring;hub->RingPhysical=ring_physical;
hub->Report=report_bytes;hub->ReportPhysical=report_physical;
hub->NextIndex=0;hub->PendingIndex=0;hub->Slot=slot_id;hub->Dci=(u8)dci;
hub->ReportLength=(u8)report_length;hub->Cycle=1;hub->Pending=0;
hub->TransferErrors=0;hub->RootPort=topology->RootPort;
hub->Topology=*topology;hub->LoggedChange=0;hub->Active=1;
hub->PortCount=port_count;
hub->ControlRing=control_ring;
hub->ControlRingPhysical=control_ring_physical;
hub->ControlBuffer=control_buffer;
hub->ControlBufferPhysical=control_buffer_physical;
hub->ControlNextIndex=control_next_index;
hub->ControlCycle=1;
kernel_write("bob64 kernel: USB hub status endpoint configured; monitoring port changes\r\n");
return kernel_xhci_hub_status_submit(hub);
}
static int kernel_xhci_configure_storage_endpoints(KERNEL_XHCI_MMIO *mmio,
const BOB64_XHCI_CAPABILITIES *capabilities,u8 slot_id,
const KERNEL_XHCI_TOPOLOGY *topology,
const BOB64_USB_MASS_STORAGE *storage,
volatile u8 *device_context_bytes,u32 context_size) {
KERNEL_XHCI_STORAGE_DEVICE *device=0;
u64 input_physical,in_ring_physical,out_ring_physical,scratch_physical;
volatile u8 *input_bytes,*in_ring_bytes,*out_ring_bytes,*scratch_bytes;
KERNEL_XHCI_TRB completion;
u8 in_dci,out_dci;
u32 command_index,doorbell_offset;
if(!mmio||!capabilities||!slot_id||!topology||!storage||
!device_context_bytes||(context_size!=32u&&context_size!=64u)||
!(storage->BulkInEndpoint&0x80u)||
(storage->BulkOutEndpoint&0x80u)||
!storage->BulkInMaxPacketSize||!storage->BulkOutMaxPacketSize)
return -1;
in_dci=(u8)(((storage->BulkInEndpoint&0x0fu)<<1)|1u);
out_dci=(u8)((storage->BulkOutEndpoint&0x0fu)<<1);
if(in_dci<2u||out_dci<2u||in_dci>31u||out_dci>31u||in_dci==out_dci)
return -2;
for(u32 i=0;i<kernel_xhci_storage_device_count;i++)
if(!kernel_xhci_storage_devices[i].Active) {
device=&kernel_xhci_storage_devices[i];break;
}
if(!device&&kernel_xhci_storage_device_count<
KERNEL_XHCI_STORAGE_DEVICE_LIMIT)
device=&kernel_xhci_storage_devices[kernel_xhci_storage_device_count++];
if(!device)return -3;
if(kernel_xhci_dma_alloc(1,&in_ring_physical,&in_ring_bytes)||
kernel_xhci_dma_alloc(1,&out_ring_physical,&out_ring_bytes)||
kernel_xhci_dma_alloc(1,&input_physical,&input_bytes)||
kernel_xhci_dma_alloc(2,&scratch_physical,&scratch_bytes))return -4;
volatile KERNEL_XHCI_TRB *in_ring=
(volatile KERNEL_XHCI_TRB *)(volatile void *)in_ring_bytes;
volatile KERNEL_XHCI_TRB *out_ring=
(volatile KERNEL_XHCI_TRB *)(volatile void *)out_ring_bytes;
in_ring[BOB64_XHCI_RING_TRBS-1].Parameter=in_ring_physical;
in_ring[BOB64_XHCI_RING_TRBS-1].Control=
(BOB64_XHCI_TRB_TYPE_LINK<<BOB64_XHCI_TRB_TYPE_SHIFT)|3u;
out_ring[BOB64_XHCI_RING_TRBS-1].Parameter=out_ring_physical;
out_ring[BOB64_XHCI_RING_TRBS-1].Control=
(BOB64_XHCI_TRB_TYPE_LINK<<BOB64_XHCI_TRB_TYPE_SHIFT)|3u;
volatile u32 *input=(volatile u32 *)(volatile void *)input_bytes;
volatile u32 *slot=(volatile u32 *)(volatile void *)(input_bytes+
context_size);
volatile u32 *live_slot=(volatile u32 *)(volatile void *)device_context_bytes;
volatile u32 *in_endpoint=(volatile u32 *)(volatile void *)(input_bytes+
(in_dci+1u)*context_size);
volatile u32 *out_endpoint=(volatile u32 *)(volatile void *)(input_bytes+
(out_dci+1u)*context_size);
input[1]=1u|(1u<<in_dci)|(1u<<out_dci);
for(u32 word=0;word<context_size/sizeof(u32);word++)
slot[word]=live_slot[word];
u32 max_dci=in_dci>out_dci?in_dci:out_dci;
slot[0]=(slot[0]&~(31u<<27))|(max_dci<<27);
in_endpoint[1]=(3u<<1)|(6u<<3)|
((u32)storage->BulkInMaxPacketSize<<16);
in_endpoint[2]=(u32)(in_ring_physical|1u);
in_endpoint[3]=(u32)(in_ring_physical>>32);
in_endpoint[4]=1024u;
out_endpoint[1]=(3u<<1)|(2u<<3)|
((u32)storage->BulkOutMaxPacketSize<<16);
out_endpoint[2]=(u32)(out_ring_physical|1u);
out_endpoint[3]=(u32)(out_ring_physical>>32);
out_endpoint[4]=1024u;
command_index=kernel_xhci_rings.CommandIndex;
if(command_index>=BOB64_XHCI_RING_TRBS-1)return -5;
volatile KERNEL_XHCI_TRB *command=
&kernel_xhci_rings.CommandRing[command_index];
command->Parameter=input_physical;command->Status=0;
command->Control=(12u<<BOB64_XHCI_TRB_TYPE_SHIFT)|
((u32)slot_id<<24)|1u;
__asm__ volatile("sfence" ::: "memory");
doorbell_offset=capabilities->DoorbellOffset-mmio->CapabilityLength;
if(kernel_xhci_write32(mmio,doorbell_offset,0))return -6;
int result=kernel_xhci_wait_event(mmio,capabilities,
BOB64_XHCI_TRB_TYPE_COMMAND_COMPLETION,10000000u,&completion);
if(result)return -7+result;
if(completion.Parameter!=kernel_xhci_rings.CommandPhysical+
command_index*sizeof(KERNEL_XHCI_TRB)||
(completion.Status>>24)!=BOB64_XHCI_TRB_COMPLETION_SUCCESS||
(u8)(completion.Control>>24)!=slot_id)return -8;
kernel_xhci_rings.CommandIndex++;
device->BulkInRing=in_ring;device->BulkOutRing=out_ring;
device->BulkInRingPhysical=in_ring_physical;
device->BulkOutRingPhysical=out_ring_physical;
device->BulkInMaxPacketSize=storage->BulkInMaxPacketSize;
device->BulkOutMaxPacketSize=storage->BulkOutMaxPacketSize;
device->Slot=slot_id;device->BulkInDci=in_dci;device->BulkOutDci=out_dci;
device->BulkInNext=0;device->BulkOutNext=0;
device->BulkInCycle=1;device->BulkOutCycle=1;device->NextTag=0;
device->Scratch=scratch_bytes;device->ScratchPhysical=scratch_physical;
device->BlockSize=0;device->BlockCount=0;device->HasDataPartition=0;
device->Topology=*topology;device->Active=1;
kernel_write("bob64 kernel: USB mass-storage bulk endpoints configured slot=");
kernel_hex64(slot_id);kernel_write(" in=0x");
kernel_hex64(storage->BulkInEndpoint);kernel_write(" out=0x");
kernel_hex64(storage->BulkOutEndpoint);kernel_write(" packet=");
kernel_hex64(storage->BulkInMaxPacketSize);kernel_write("\r\n");
int probe_result=kernel_xhci_storage_probe(device);
if(probe_result) {
kernel_write("bob64 kernel: USB MSC command probe failed result=");
kernel_hex64((u64)(s64)probe_result);kernel_write("\r\n");
}
return 0;
}
static int kernel_xhci_storage_bulk(KERNEL_XHCI_STORAGE_DEVICE *device,
int input,u64 buffer_physical,u32 length,u32 *actual_length) {
volatile KERNEL_XHCI_TRB *ring;
u64 ring_physical;
u32 *next;
u8 *cycle;
u8 dci;
if(!device||!device->Active||!length||length>BOB64_PAGE_SIZE||
!actual_length)return -1;
if(input) {
ring=device->BulkInRing;ring_physical=device->BulkInRingPhysical;
next=&device->BulkInNext;cycle=&device->BulkInCycle;dci=device->BulkInDci;
} else {
ring=device->BulkOutRing;ring_physical=device->BulkOutRingPhysical;
next=&device->BulkOutNext;cycle=&device->BulkOutCycle;dci=device->BulkOutDci;
}
if(!ring||!ring_physical||*next>=BOB64_XHCI_RING_TRBS)return -2;
if(*next==BOB64_XHCI_RING_TRBS-1u) {
volatile KERNEL_XHCI_TRB *link=&ring[*next];
link->Parameter=ring_physical;link->Status=0;
link->Control=(BOB64_XHCI_TRB_TYPE_LINK<<BOB64_XHCI_TRB_TYPE_SHIFT)|
2u|*cycle;
*next=0;*cycle^=1u;
}
u32 index=(*next)++;
volatile KERNEL_XHCI_TRB *transfer=&ring[index];
transfer->Parameter=buffer_physical;
transfer->Status=length;
transfer->Control=(1u<<BOB64_XHCI_TRB_TYPE_SHIFT)|(1u<<5)|*cycle;
__asm__ volatile("sfence" ::: "memory");
u32 doorbell=kernel_xhci_live_capabilities.DoorbellOffset-
kernel_xhci_live_mmio.CapabilityLength+(u32)device->Slot*4u;
if(kernel_xhci_write32(&kernel_xhci_live_mmio,doorbell,dci))return -3;
KERNEL_XHCI_TRB completion;
int result=kernel_xhci_wait_transfer(&kernel_xhci_live_mmio,
&kernel_xhci_live_capabilities,device->Slot,dci,
ring_physical+(u64)index*sizeof(KERNEL_XHCI_TRB),10000000u,
&completion);
if(result)return -4+result;
u32 code=completion.Status>>24;
u32 residual=completion.Status&0x00ffffffu;
if((code!=BOB64_XHCI_TRB_COMPLETION_SUCCESS&&code!=13u)||
residual>length)return -8;
*actual_length=length-residual;
return 0;
}
static int kernel_xhci_storage_command_internal(
KERNEL_XHCI_STORAGE_DEVICE *device,
const u8 *cdb,u8 cdb_length,int device_to_host,u32 data_length,
void *data,u32 *actual_data,int allow_check_condition_recovery);
static int kernel_xhci_storage_command_internal(
KERNEL_XHCI_STORAGE_DEVICE *device,
const u8 *cdb,u8 cdb_length,int device_to_host,u32 data_length,
void *data,u32 *actual_data,int allow_check_condition_recovery) {
if(!device||!device->Active||!device->Scratch||!device->ScratchPhysical||
!cdb||!data||!actual_data||data_length>BOB64_PAGE_SIZE)return -1;
volatile u8 *scratch=device->Scratch;
u32 tag=++device->NextTag;
if(!tag)tag=++device->NextTag;
if(bob64_usb_msc_build_cbw((void *)(uintptr_t)scratch,
BOB64_USB_MSC_CBW_SIZE,tag,data_length,device_to_host,0,cdb,cdb_length))
return -2;
u32 actual=0,data_actual=0;
int result=kernel_xhci_storage_bulk(device,0,device->ScratchPhysical,
BOB64_USB_MSC_CBW_SIZE,&actual);
if(result||actual!=BOB64_USB_MSC_CBW_SIZE) {
kernel_write("bob64 kernel: USB MSC CBW transfer failed result=");
kernel_hex64((u64)(s64)result);kernel_write(" bytes=");
kernel_hex64(actual);kernel_write("\r\n");
return -3;
}
if(data_length) {
if(!device_to_host)
for(u32 i=0;i<data_length;i++)scratch[i]=((const u8 *)data)[i];
actual=0;
#ifdef BOB64_STORAGE_ACTIVE_DISCONNECT_TEST
if(device_to_host&&(cdb[0]==0x28u||cdb[0]==0x88u)&&
data_length==device->BlockSize)
device->DisconnectTestReadPending=1;
#endif
result=kernel_xhci_storage_bulk(device,device_to_host,
device->ScratchPhysical,data_length,&actual);
#ifdef BOB64_STORAGE_ACTIVE_DISCONNECT_TEST
device->DisconnectTestReadPending=0;
#endif
if(result) {
kernel_write("bob64 kernel: USB MSC data transfer failed result=");
kernel_hex64((u64)(s64)result);kernel_write(" input=");
kernel_hex64((u64)(device_to_host!=0));kernel_write("\r\n");
return -4;
}
data_actual=actual;
}
actual=0;
result=kernel_xhci_storage_bulk(device,1,
device->ScratchPhysical+BOB64_PAGE_SIZE,BOB64_USB_MSC_CSW_SIZE,&actual);
if(result||actual!=BOB64_USB_MSC_CSW_SIZE) {
kernel_write("bob64 kernel: USB MSC CSW transfer failed result=");
kernel_hex64((u64)(s64)result);kernel_write(" bytes=");
kernel_hex64(actual);kernel_write("\r\n");
return -5;
}
u32 residue=0;
u8 status=0xff;
if(bob64_usb_msc_parse_csw((const void *)(uintptr_t)
(scratch+BOB64_PAGE_SIZE),BOB64_USB_MSC_CSW_SIZE,tag,data_length,
&residue,&status)) {
kernel_write("bob64 kernel: USB MSC CSW validation failed tag=");
kernel_hex64(tag);kernel_write(" status=");kernel_hex64(status);
kernel_write(" residue=");kernel_hex64(residue);kernel_write("\r\n");
return -6;
}
if(status!=0) {
kernel_write("bob64 kernel: USB MSC SCSI command status=");
kernel_hex64(status);kernel_write(" opcode=");
kernel_hex64(cdb[0]);kernel_write(" residue=");
kernel_hex64(residue);kernel_write("\r\n");
if(status==1u&&allow_check_condition_recovery&&cdb[0]!=0x03u) {
u8 sense_cdb[6]={0x03,0,0,0,18,0};
u8 sense[18]={0};
u32 sense_actual=0;
int sense_result=kernel_xhci_storage_command_internal(device,
sense_cdb,sizeof(sense_cdb),1,sizeof(sense),sense,
&sense_actual,0);
if(sense_result||sense_actual<14u) {
kernel_write("bob64 kernel: USB MSC REQUEST SENSE failed result=");
kernel_hex64((u64)(s64)sense_result);kernel_write(" bytes=");
kernel_hex64(sense_actual);kernel_write("\r\n");
return -6;
}
kernel_write("bob64 kernel: USB MSC sense key=");
kernel_hex64(sense[2]&0x0fu);kernel_write(" asc=");
kernel_hex64(sense[12]);kernel_write(" ascq=");
kernel_hex64(sense[13]);kernel_write("; retrying once\r\n");
return kernel_xhci_storage_command_internal(device,cdb,cdb_length,
device_to_host,data_length,data,actual_data,0);
}
return -6;
}
if(data_length&&data_actual+residue!=data_length) {
kernel_write("bob64 kernel: USB MSC data residue mismatch actual=");
kernel_hex64(data_actual);kernel_write(" residue=");
kernel_hex64(residue);kernel_write(" expected=");
kernel_hex64(data_length);kernel_write("\r\n");
return -7;
}
if(data_length&&device_to_host) {
for(u32 i=0;i<data_actual;i++)((u8 *)data)[i]=scratch[i];
}
*actual_data=data_actual;
return 0;
}
static int kernel_xhci_storage_command(KERNEL_XHCI_STORAGE_DEVICE *device,
const u8 *cdb,u8 cdb_length,int device_to_host,u32 data_length,
void *data,u32 *actual_data) {
return kernel_xhci_storage_command_internal(device,cdb,cdb_length,
device_to_host,data_length,data,actual_data,1);
}
static int kernel_xhci_storage_read_blocks_raw(KERNEL_XHCI_STORAGE_DEVICE *device,
u64 lba,u32 block_count,void *buffer) {
u8 cdb[16],cdb_length=0;
u32 length,actual=0;
if(!device||!device->BlockSize||!block_count||!buffer||
(u64)block_count*device->BlockSize>BOB64_PAGE_SIZE||
lba>=device->BlockCount||block_count>device->BlockCount-lba||
bob64_scsi_build_read_write_cdb(lba,block_count,0,cdb,&cdb_length))
return -1;
length=(u32)block_count*device->BlockSize;
int result=kernel_xhci_storage_command(device,cdb,cdb_length,1,
length,buffer,&actual);
return result?result:(actual==length?0:-2);
}
static int kernel_xhci_storage_write_blocks_raw(KERNEL_XHCI_STORAGE_DEVICE *device,
u64 lba,u32 block_count,void *buffer) {
u8 cdb[16],cdb_length=0;
u32 length,actual=0;
if(!device||!device->BlockSize||!block_count||!buffer||
(u64)block_count*device->BlockSize>BOB64_PAGE_SIZE||
lba>=device->BlockCount||block_count>device->BlockCount-lba||
bob64_scsi_build_read_write_cdb(lba,block_count,1,cdb,&cdb_length))
return -1;
length=(u32)block_count*device->BlockSize;
int result=kernel_xhci_storage_command(device,cdb,cdb_length,0,
length,buffer,&actual);
return result?result:(actual==length?0:-2);
}
static int kernel_xhci_storage_sync_raw(KERNEL_XHCI_STORAGE_DEVICE *device) {
u8 cdb[10]={0x35};
u8 no_data=0;
u32 actual=0;
if(!device||!device->Active)return -1;
return kernel_xhci_storage_command(device,cdb,sizeof(cdb),1,0,
&no_data,&actual);
}
static int kernel_xhci_storage_block_read(void *context,u64 lba,u32 count,
void *buffer) {
KERNEL_XHCI_STORAGE_DEVICE *device=(KERNEL_XHCI_STORAGE_DEVICE *)context;
return kernel_xhci_storage_read_blocks_raw(device,lba,count,buffer);
}
static int kernel_xhci_storage_block_write(void *context,u64 lba,u32 count,
const void *buffer) {
KERNEL_XHCI_STORAGE_DEVICE *device=(KERNEL_XHCI_STORAGE_DEVICE *)context;
return kernel_xhci_storage_write_blocks_raw(device,lba,count,(void *)buffer);
}
static int kernel_xhci_storage_block_flush(void *context) {
return kernel_xhci_storage_sync_raw(
(KERNEL_XHCI_STORAGE_DEVICE *)context);
}
static int kernel_xhci_storage_probe(KERNEL_XHCI_STORAGE_DEVICE *device) {
u8 inquiry_cdb[6]={0x12,0,0,0,36,0};
u8 capacity_cdb[16]={0x25};
u8 inquiry[36],capacity[32];
u64 block_count=0;
u32 block_size=0;
int needs_capacity16=0;
u32 actual=0;
if(!device||!device->Active||device->BlockSize)return -1;
int result=kernel_xhci_storage_command(device,inquiry_cdb,sizeof(inquiry_cdb),
1,sizeof(inquiry),inquiry,&actual);
if(result||actual!=sizeof(inquiry))return -2;
result=kernel_xhci_storage_command(device,capacity_cdb,10,1,8,capacity,
&actual);
if(result||actual!=8)return -3;
if(bob64_scsi_parse_read_capacity10(capacity,8,&block_count,&block_size,
&needs_capacity16))return -4;
if(needs_capacity16) {
bob64_scsi_build_read_capacity16_cdb(capacity_cdb);
result=kernel_xhci_storage_command(device,capacity_cdb,16,1,32,
capacity,&actual);
if(result||actual!=32||bob64_scsi_parse_read_capacity16(capacity,32,
&block_count,&block_size))return -3;
}
if(!block_size||block_size>BOB64_PAGE_SIZE)return -4;
device->BlockSize=block_size;device->BlockCount=block_count;
#ifdef BOB64_STORAGE_WRITE_TEST
int initial_read_only=0;
#else
int initial_read_only=1;
#endif
if(bob64_block_device_init(&device->BlockDevice,device,device->BlockCount,
block_size,initial_read_only,kernel_xhci_storage_block_read,
kernel_xhci_storage_block_write,kernel_xhci_storage_block_flush))
return -4;
kernel_write("bob64 kernel: USB block device registered sectors=");
kernel_hex64(device->BlockCount);kernel_write(" sector-bytes=");
kernel_hex64(device->BlockSize);kernel_write(" access=");
kernel_write(device->BlockDevice.ReadOnly?"read-only\r\n":"read-write\r\n");
BOB64_PARTITION partition;
int partition_result=bob64_gpt_find_partition(&device->BlockDevice,
bob64_gpt_bob_data_type_guid,&partition);
if(!partition_result) {
device->DataPartition=partition;device->HasDataPartition=1;
if(bob64_block_device_init(&device->BlockDevice,device,
device->BlockCount,block_size,0,kernel_xhci_storage_block_read,
kernel_xhci_storage_block_write,kernel_xhci_storage_block_flush))
return -4;
kernel_write("bob64 kernel: dedicated data partition enables writes\r\n");
kernel_write("bob64 kernel: GPT bob64 data partition found first-lba=");
kernel_hex64(partition.FirstLba);kernel_write(" blocks=");
kernel_hex64(partition.BlockCount);kernel_write("\r\n");
} else if(partition_result==1) {
kernel_write("bob64 kernel: GPT bob64 data partition absent\r\n");
} else {
kernel_write("bob64 kernel: GPT data partition table invalid result=");
kernel_hex64((u64)(s64)partition_result);kernel_write("\r\n");
}
if(bob64_block_read(&device->BlockDevice,device->BlockCount,1,
(void *)(uintptr_t)device->Scratch)!=-1)return -5;
result=bob64_block_read(&device->BlockDevice,0,1,
(void *)(uintptr_t)device->Scratch);
if(result)return -6;
actual=block_size;
kernel_write("bob64 kernel: USB MSC READ CAPACITY passed blocks=");
kernel_hex64(device->BlockCount);kernel_write(" sector-bytes=");
kernel_hex64(device->BlockSize);kernel_write("\r\n");
kernel_write("bob64 kernel: USB MSC sector-read passed lba=0 bytes=");
kernel_hex64(actual);kernel_write("\r\n");
#ifdef BOB64_STORAGE_LBA64_TEST
if(device->BlockCount<=0x100000000ULL)return -13;
result=bob64_block_read(&device->BlockDevice,0x100000000ULL,1,
(void *)(uintptr_t)device->Scratch);
if(result)return -14;
for(u32 i=0;i<block_size;i++)
if(device->Scratch[i]!=(u8)(i*53u+0xa7u))return -15;
kernel_write("bob64 kernel: USB MSC high-LBA READ(16) passed lba=0000000100000000\r\n");
#endif
#ifdef BOB64_STORAGE_WRITE_TEST
if(device->BlockCount<2u||block_size>BOB64_PAGE_SIZE)return -7;
volatile u8 *write_buffer=device->Scratch+BOB64_PAGE_SIZE;
volatile u8 *read_buffer=device->Scratch;
for(u32 i=0;i<block_size;i++)write_buffer[i]=(u8)(i*37u+0x5au);
result=bob64_block_write(&device->BlockDevice,1,1,
(void *)(uintptr_t)write_buffer);
if(result)return -8;
result=bob64_block_flush(&device->BlockDevice);
if(result)return -12;
if(bob64_block_write(&device->BlockDevice,device->BlockCount,1,
(void *)(uintptr_t)write_buffer)!=-1)return -11;
result=bob64_block_read(&device->BlockDevice,1,1,
(void *)(uintptr_t)read_buffer);
if(result)return -9;
for(u32 i=0;i<block_size;i++)
if(read_buffer[i]!=(u8)(i*37u+0x5au))return -10;
kernel_write("bob64 kernel: USB MSC sector-write/flush/readback passed lba=1 bytes=");
kernel_hex64(block_size);kernel_write("\r\n");
#endif
return 0;
}
static u32 kernel_xhci_hub_control_reserve(KERNEL_XHCI_HUB_DEVICE *hub,
u32 trb_count) {
if(!hub||!hub->ControlRing||trb_count>3u||
hub->ControlNextIndex>=BOB64_XHCI_RING_TRBS)return 0xffffffffu;
if(hub->ControlNextIndex+trb_count>BOB64_XHCI_RING_TRBS-1u) {
volatile KERNEL_XHCI_TRB *link=
&hub->ControlRing[hub->ControlNextIndex];
link->Parameter=hub->ControlRingPhysical;link->Status=0;
link->Control=(BOB64_XHCI_TRB_TYPE_LINK<<BOB64_XHCI_TRB_TYPE_SHIFT)|
2u|hub->ControlCycle;
hub->ControlNextIndex=0;
hub->ControlCycle^=1u;
}
u32 index=hub->ControlNextIndex;
hub->ControlNextIndex+=trb_count;
return index;
}
static int kernel_xhci_hub_control_in(KERNEL_XHCI_HUB_DEVICE *hub,
u8 request_type,u8 request,u16 value,u16 index,u16 length) {
KERNEL_XHCI_TRB completion;
if(!hub||!hub->Active||!length||length>BOB64_PAGE_SIZE)return -1;
u32 ring_index=kernel_xhci_hub_control_reserve(hub,3u);
if(ring_index==0xffffffffu)return -2;
u32 cycle=hub->ControlCycle;
volatile KERNEL_XHCI_TRB *ring=hub->ControlRing;
ring[ring_index].Parameter=bob64_usb_control_setup(request_type,request,
value,index,length);
ring[ring_index].Status=8;
ring[ring_index].Control=(2u<<BOB64_XHCI_TRB_TYPE_SHIFT)|
(3u<<16)|(1u<<6)|(1u<<4)|cycle;
ring[ring_index+1].Parameter=hub->ControlBufferPhysical;
ring[ring_index+1].Status=length;
ring[ring_index+1].Control=(3u<<BOB64_XHCI_TRB_TYPE_SHIFT)|
(1u<<16)|(1u<<4)|(1u<<2)|cycle;
ring[ring_index+2].Parameter=0;ring[ring_index+2].Status=0;
ring[ring_index+2].Control=(4u<<BOB64_XHCI_TRB_TYPE_SHIFT)|
(1u<<5)|cycle;
__asm__ volatile("sfence" ::: "memory");
u32 doorbell=kernel_xhci_live_capabilities.DoorbellOffset-
kernel_xhci_live_mmio.CapabilityLength+(u32)hub->Slot*4u;
if(kernel_xhci_write32(&kernel_xhci_live_mmio,doorbell,1u))return -3;
int result=kernel_xhci_wait_event(&kernel_xhci_live_mmio,
&kernel_xhci_live_capabilities,BOB64_XHCI_TRB_TYPE_TRANSFER_EVENT,
10000000u,&completion);
if(result)return -4+result;
if((completion.Status>>24)!=BOB64_XHCI_TRB_COMPLETION_SUCCESS||
(u8)(completion.Control>>24)!=hub->Slot||
((completion.Control>>16)&0x1fu)!=1u||
completion.Parameter!=hub->ControlRingPhysical+
(ring_index+2u)*sizeof(KERNEL_XHCI_TRB))return -8;
return 0;
}
static int kernel_xhci_hub_control_no_data(KERNEL_XHCI_HUB_DEVICE *hub,
u8 request_type,u8 request,u16 value,u16 index) {
KERNEL_XHCI_TRB completion;
if(!hub||!hub->Active)return -1;
u32 ring_index=kernel_xhci_hub_control_reserve(hub,2u);
if(ring_index==0xffffffffu)return -2;
u32 cycle=hub->ControlCycle;
volatile KERNEL_XHCI_TRB *ring=hub->ControlRing;
ring[ring_index].Parameter=bob64_usb_control_setup(request_type,request,
value,index,0);
ring[ring_index].Status=8;
ring[ring_index].Control=(2u<<BOB64_XHCI_TRB_TYPE_SHIFT)|
(1u<<6)|(1u<<4)|cycle;
ring[ring_index+1].Parameter=0;ring[ring_index+1].Status=0;
ring[ring_index+1].Control=(4u<<BOB64_XHCI_TRB_TYPE_SHIFT)|
(1u<<16)|(1u<<5)|cycle;
__asm__ volatile("sfence" ::: "memory");
u32 doorbell=kernel_xhci_live_capabilities.DoorbellOffset-
kernel_xhci_live_mmio.CapabilityLength+(u32)hub->Slot*4u;
if(kernel_xhci_write32(&kernel_xhci_live_mmio,doorbell,1u))return -3;
int result=kernel_xhci_wait_event(&kernel_xhci_live_mmio,
&kernel_xhci_live_capabilities,BOB64_XHCI_TRB_TYPE_TRANSFER_EVENT,
10000000u,&completion);
if(result)return -4+result;
if((completion.Status>>24)!=BOB64_XHCI_TRB_COMPLETION_SUCCESS||
(u8)(completion.Control>>24)!=hub->Slot||
((completion.Control>>16)&0x1fu)!=1u||
completion.Parameter!=hub->ControlRingPhysical+
(ring_index+1u)*sizeof(KERNEL_XHCI_TRB))return -8;
return 0;
}
static int kernel_xhci_address_and_read_descriptor(KERNEL_XHCI_MMIO *mmio,
const BOB64_XHCI_CAPABILITIES *capabilities,u8 slot_id,
const KERNEL_XHCI_TOPOLOGY *topology) {
u64 device_context_physical,input_context_physical;
u64 transfer_ring_physical,descriptor_physical;
volatile u8 *device_context_bytes,*input_context_bytes;
volatile u8 *transfer_ring_bytes,*descriptor_bytes;
KERNEL_XHCI_HID_DEVICE *hid_device;
u32 context_size=(capabilities->HccParameters1&(1u<<2))?64u:32u;
u32 command_index=kernel_xhci_rings.CommandIndex;
u32 doorbell_offset;
u16 max_packet;
u8 hub_port_count=0;
u32 hub_control_index=0;
KERNEL_XHCI_TRB completion;
if(!mmio||!capabilities||!slot_id||!topology||!topology->RootPort||
topology->Speed<1||topology->Speed>5||
!kernel_xhci_rings.DeviceContextArray||
command_index>=BOB64_XHCI_RING_TRBS-1)return -1;
if(topology->Speed==1)max_packet=8;
else if(topology->Speed==2)max_packet=8;
else if(topology->Speed==3)max_packet=64;
else max_packet=512;
if(kernel_xhci_dma_alloc(1,&device_context_physical,
&device_context_bytes)||
kernel_xhci_dma_alloc(1,&input_context_physical,
&input_context_bytes)||
kernel_xhci_dma_alloc(1,&transfer_ring_physical,
&transfer_ring_bytes)||
kernel_xhci_dma_alloc(1,&descriptor_physical,&descriptor_bytes))return -2;
kernel_xhci_rings.DeviceContextArray[slot_id]=device_context_physical;
volatile u32 *input=(volatile u32 *)(volatile void *)input_context_bytes;
volatile u32 *slot=(volatile u32 *)(volatile void *)(input_context_bytes+
context_size);
volatile u32 *endpoint0=(volatile u32 *)(volatile void *)(input_context_bytes+
2u*context_size);
input[1]=(1u<<0)|(1u<<1); /* Add the Slot and Endpoint 0 contexts. */
slot[0]=(topology->RouteString&0x000fffffu)|
((u32)topology->Speed<<20)|(1u<<27);
slot[1]=(u32)topology->RootPort<<16;
if(topology->TtHubSlot&&topology->TtPort&&topology->Speed<3u) {
slot[2]=(u32)topology->TtHubSlot|
((u32)topology->TtPort<<8);
}
endpoint0[1]=(3u<<1)|(4u<<3)|((u32)max_packet<<16);
endpoint0[2]=(u32)(transfer_ring_physical|1u);
endpoint0[3]=(u32)(transfer_ring_physical>>32);
endpoint0[4]=8u;
volatile KERNEL_XHCI_TRB *command=
&kernel_xhci_rings.CommandRing[command_index];
command->Parameter=input_context_physical;
command->Status=0;
command->Control=(11u<<BOB64_XHCI_TRB_TYPE_SHIFT)|
((u32)slot_id<<24)|1u;
__asm__ volatile("sfence" ::: "memory");
doorbell_offset=capabilities->DoorbellOffset-mmio->CapabilityLength;
if(kernel_xhci_write32(mmio,doorbell_offset,0))return -3;
int result=kernel_xhci_wait_event(mmio,capabilities,
BOB64_XHCI_TRB_TYPE_COMMAND_COMPLETION,10000000u,&completion);
if(result)return -4+result;
if(completion.Parameter!=kernel_xhci_rings.CommandPhysical+
command_index*sizeof(KERNEL_XHCI_TRB)||
(completion.Status>>24)!=BOB64_XHCI_TRB_COMPLETION_SUCCESS||
(u8)(completion.Control>>24)!=slot_id)return -5;
kernel_xhci_rings.CommandIndex++;
volatile KERNEL_XHCI_TRB *transfer_ring=
(volatile KERNEL_XHCI_TRB *)(volatile void *)transfer_ring_bytes;
transfer_ring[BOB64_XHCI_RING_TRBS-1].Parameter=transfer_ring_physical;
transfer_ring[BOB64_XHCI_RING_TRBS-1].Status=0;
transfer_ring[BOB64_XHCI_RING_TRBS-1].Control=
(BOB64_XHCI_TRB_TYPE_LINK<<BOB64_XHCI_TRB_TYPE_SHIFT)|3u;
/* GET_DESCRIPTOR(Device, 8 bytes) on the newly addressed EP0. */
transfer_ring[0].Parameter=0x0008000001000680ULL;
transfer_ring[0].Status=8u;
transfer_ring[0].Control=(2u<<BOB64_XHCI_TRB_TYPE_SHIFT)|
(3u<<16)|(1u<<6)|(1u<<4)|1u;
transfer_ring[1].Parameter=descriptor_physical;
transfer_ring[1].Status=8u;
transfer_ring[1].Control=(3u<<BOB64_XHCI_TRB_TYPE_SHIFT)|
(1u<<16)|(1u<<4)|(1u<<2)|1u;
transfer_ring[2].Parameter=0;
transfer_ring[2].Status=0;
transfer_ring[2].Control=(4u<<BOB64_XHCI_TRB_TYPE_SHIFT)|
(1u<<5)|1u;
__asm__ volatile("sfence" ::: "memory");
u32 slot_doorbell=doorbell_offset+(u32)slot_id*4u;
if(kernel_xhci_write32(mmio,slot_doorbell,1u))return -6;
result=kernel_xhci_wait_event(mmio,capabilities,
BOB64_XHCI_TRB_TYPE_TRANSFER_EVENT,10000000u,
&completion);
if(result)return -7+result;
if((completion.Status>>24)!=BOB64_XHCI_TRB_COMPLETION_SUCCESS||
(u8)(completion.Control>>24)!=slot_id||
((completion.Control>>16)&0x1fu)!=1u||
completion.Parameter!=transfer_ring_physical+
2u*sizeof(KERNEL_XHCI_TRB))return -8;
if(descriptor_bytes[0]<8||descriptor_bytes[1]!=1||
descriptor_bytes[7]!=(topology->Speed>=4u?9u:max_packet))return -9;
transfer_ring[3].Parameter=0x0012000001000680ULL;
transfer_ring[3].Status=8u;
transfer_ring[3].Control=(2u<<BOB64_XHCI_TRB_TYPE_SHIFT)|
(3u<<16)|(1u<<6)|(1u<<4)|1u;
transfer_ring[4].Parameter=descriptor_physical;
transfer_ring[4].Status=18u;
transfer_ring[4].Control=(3u<<BOB64_XHCI_TRB_TYPE_SHIFT)|
(1u<<16)|(1u<<4)|(1u<<2)|1u;
transfer_ring[5].Parameter=0;
transfer_ring[5].Status=0;
transfer_ring[5].Control=(4u<<BOB64_XHCI_TRB_TYPE_SHIFT)|
(1u<<5)|1u;
__asm__ volatile("sfence" ::: "memory");
if(kernel_xhci_write32(mmio,slot_doorbell,1u))return -10;
result=kernel_xhci_wait_event(mmio,capabilities,
BOB64_XHCI_TRB_TYPE_TRANSFER_EVENT,10000000u,&completion);
if(result)return -11+result;
if((completion.Status>>24)!=BOB64_XHCI_TRB_COMPLETION_SUCCESS||
(u8)(completion.Control>>24)!=slot_id||
((completion.Control>>16)&0x1fu)!=1u||
completion.Parameter!=transfer_ring_physical+
5u*sizeof(KERNEL_XHCI_TRB))return -12;
if(descriptor_bytes[0]!=18||descriptor_bytes[1]!=1||
descriptor_bytes[7]!=(topology->Speed>=4u?9u:max_packet))return -13;
u16 vendor_id=(u16)(descriptor_bytes[8]|((u16)descriptor_bytes[9]<<8));
u16 product_id=(u16)(descriptor_bytes[10]|((u16)descriptor_bytes[11]<<8));
kernel_write("bob64 kernel: USB device addressed; descriptor type=");
kernel_hex64(descriptor_bytes[1]);kernel_write(" EP0 packet=");
kernel_hex64(descriptor_bytes[7]);kernel_write(" VID=0x");
kernel_hex64(vendor_id);
kernel_write(" PID=0x");
kernel_hex64(product_id);
kernel_write("\r\n");
result=kernel_xhci_get_descriptor(mmio,capabilities,slot_id,transfer_ring,
transfer_ring_physical,descriptor_physical,2,9,6);
if(result)return -14+result;
if(descriptor_bytes[0]<9||descriptor_bytes[1]!=2)return -18;
u16 configuration_length=(u16)(descriptor_bytes[2]|((u16)descriptor_bytes[3]<<8));
if(configuration_length<9||configuration_length>BOB64_PAGE_SIZE)return -19;
result=kernel_xhci_get_descriptor(mmio,capabilities,slot_id,transfer_ring,
transfer_ring_physical,descriptor_physical,2,configuration_length,9);
if(result)return -20+result;
BOB64_USB_HUB_INTERFACE hub_interface;
result=bob64_usb_find_hub_interface(
(const void *)(uintptr_t)descriptor_bytes,configuration_length,
&hub_interface);
if(result<0)return -21+result;
if(!result) {
u8 child_count_before=kernel_xhci_hub_child_count;
result=kernel_xhci_configure_usb2_hub(mmio,capabilities,slot_id,
topology,transfer_ring,transfer_ring_physical,descriptor_physical,
descriptor_bytes,device_context_bytes,context_size,
hub_interface.ConfigurationValue,&hub_port_count,
&hub_control_index);
if(result) {
kernel_xhci_hub_child_count=child_count_before;
kernel_write("bob64 kernel: USB hub initialization failed result=");
kernel_hex64((u64)(s64)result);kernel_write("\r\n");
} else {
result=kernel_xhci_configure_hub_status_endpoint(mmio,capabilities,
slot_id,topology,&hub_interface,hub_port_count,
device_context_bytes,context_size,transfer_ring,
transfer_ring_physical,descriptor_bytes,descriptor_physical,
hub_control_index);
if(result) {
kernel_write("bob64 kernel: USB hub change monitoring unavailable result=");
kernel_hex64((u64)(s64)result);kernel_write("\r\n");
}
}
return 0;
}
BOB64_USB_MASS_STORAGE storage;
result=bob64_usb_find_mass_storage(
(const void *)(uintptr_t)descriptor_bytes,configuration_length,
&storage);
if(result<0)return -23+result;
if(!result) {
kernel_write("bob64 kernel: USB mass-storage BOT interface found config=");
kernel_hex64(storage.ConfigurationValue);kernel_write(" interface=");
kernel_hex64(storage.InterfaceNumber);kernel_write(" in=0x");
kernel_hex64(storage.BulkInEndpoint);kernel_write(" out=0x");
kernel_hex64(storage.BulkOutEndpoint);kernel_write("\r\n");
result=kernel_xhci_control_no_data(mmio,capabilities,slot_id,
transfer_ring,transfer_ring_physical,12,0x00,9,
storage.ConfigurationValue,0);
if(result) {
kernel_write("bob64 kernel: USB storage SET_CONFIGURATION failed result=");
kernel_hex64((u64)(s64)result);kernel_write("\r\n");
return -24+result;
}
result=kernel_xhci_configure_storage_endpoints(mmio,capabilities,
slot_id,topology,&storage,device_context_bytes,context_size);
if(result) {
kernel_write("bob64 kernel: USB storage endpoint setup failed result=");
kernel_hex64((u64)(s64)result);kernel_write("\r\n");
return -25+result;
}
return 0;
}
BOB64_USB_HID_BOOT_DEVICE hid;
u8 endpoint_dci,hid_kind,report_length;
result=bob64_usb_find_hid_boot_keyboard(
(const void *)(uintptr_t)descriptor_bytes,configuration_length,&hid);
if(result<0)return -25+result;
if(!result) { hid_kind=1;report_length=8; }
else {
result=bob64_usb_find_hid_boot_mouse(
(const void *)(uintptr_t)descriptor_bytes,configuration_length,&hid);
if(result)return -27+result;
hid_kind=2;
report_length=hid.EndpointMaxPacketSize>=4?4:3;
}
u32 hid_device_index=kernel_xhci_hid_device_count;
for(u32 i=0;i<kernel_xhci_hid_device_count;i++)
if(!kernel_xhci_hid_devices[i].Active) { hid_device_index=i;break; }
if(hid_device_index>=KERNEL_XHCI_HID_DEVICE_LIMIT)return -28;
hid_device=&kernel_xhci_hid_devices[hid_device_index];
endpoint_dci=(u8)(((hid.EndpointAddress&0x0fu)<<1)|
((hid.EndpointAddress&0x80u)?1u:0u));
if(endpoint_dci<2||endpoint_dci>31)return -26;
kernel_write("bob64 kernel: USB HID boot ");
kernel_write(hid_kind==1?"keyboard":"mouse");
kernel_write(" endpoint found address=0x");
kernel_hex64(hid.EndpointAddress);kernel_write(" packet=");
kernel_hex64(hid.EndpointMaxPacketSize);kernel_write(" interval=");
kernel_hex64(hid.EndpointInterval);kernel_write(" config=");
kernel_hex64(hid.ConfigurationValue);kernel_write("\r\n");
/* Select the USB configuration and ask the HID interface for boot reports. */
result=kernel_xhci_control_no_data(mmio,capabilities,slot_id,transfer_ring,
transfer_ring_physical,12,0x00,9,hid.ConfigurationValue,0);
if(result) {
kernel_write("bob64 kernel: USB SET_CONFIGURATION failed result=");
kernel_hex64((u64)(s64)result);kernel_write("\r\n");
return -30+result;
}
result=kernel_xhci_control_no_data(mmio,capabilities,slot_id,transfer_ring,
transfer_ring_physical,14,0x21,0x0b,0,hid.InterfaceNumber);
if(result) {
kernel_write("bob64 kernel: USB HID SET_PROTOCOL failed result=");
kernel_hex64((u64)(s64)result);kernel_write("\r\n");
return -35+result;
}
/* Add the interrupt-IN endpoint (EP1 IN is DCI 3) to the live device. */
u64 endpoint_ring_physical;
volatile u8 *endpoint_ring_bytes;
if(kernel_xhci_dma_alloc(1,&endpoint_ring_physical,&endpoint_ring_bytes))
return -40;
volatile KERNEL_XHCI_TRB *endpoint_ring=
(volatile KERNEL_XHCI_TRB *)(volatile void *)endpoint_ring_bytes;
endpoint_ring[BOB64_XHCI_RING_TRBS-1].Parameter=endpoint_ring_physical;
endpoint_ring[BOB64_XHCI_RING_TRBS-1].Status=0;
endpoint_ring[BOB64_XHCI_RING_TRBS-1].Control=
(BOB64_XHCI_TRB_TYPE_LINK<<BOB64_XHCI_TRB_TYPE_SHIFT)|3u;
if(kernel_xhci_dma_alloc(1,&input_context_physical,&input_context_bytes))
return -41;
input=(volatile u32 *)(volatile void *)input_context_bytes;
slot=(volatile u32 *)(volatile void *)(input_context_bytes+context_size);
volatile u32 *endpoint=(volatile u32 *)(volatile void *)(input_context_bytes+
((u32)endpoint_dci+1u)*context_size);
input[1]=(1u<<0)|(1u<<endpoint_dci);
volatile u32 *live_slot=(volatile u32 *)(volatile void *)device_context_bytes;
for(u32 word=0;word<context_size/sizeof(u32);word++)slot[word]=live_slot[word];
slot[0]=(slot[0]&~(31u<<27))|((u32)endpoint_dci<<27);
u32 interval;
if(topology->Speed>=3)interval=hid.EndpointInterval?
(u32)hid.EndpointInterval-1u:0u;
else {
u32 period=hid.EndpointInterval?hid.EndpointInterval:1u;
interval=3;
while(period>1u&&interval<15u) { period=(period+1u)>>1;interval++; }
}
endpoint[0]=interval<<16;
endpoint[1]=(3u<<1)|(7u<<3)|
((u32)hid.EndpointMaxPacketSize<<16);
endpoint[2]=(u32)(endpoint_ring_physical|1u);
endpoint[3]=(u32)(endpoint_ring_physical>>32);
endpoint[4]=(u32)report_length|((u32)hid.EndpointMaxPacketSize<<16);
command_index=kernel_xhci_rings.CommandIndex;
if(command_index>=BOB64_XHCI_RING_TRBS-1)return -42;
command=&kernel_xhci_rings.CommandRing[command_index];
command->Parameter=input_context_physical;
command->Status=0;
command->Control=(12u<<BOB64_XHCI_TRB_TYPE_SHIFT)|
((u32)slot_id<<24)|1u;
__asm__ volatile("sfence" ::: "memory");
doorbell_offset=capabilities->DoorbellOffset-mmio->CapabilityLength;
if(kernel_xhci_write32(mmio,doorbell_offset,0))return -43;
result=kernel_xhci_wait_event(mmio,capabilities,
BOB64_XHCI_TRB_TYPE_COMMAND_COMPLETION,10000000u,&completion);
if(result)return -44+result;
if(completion.Parameter!=kernel_xhci_rings.CommandPhysical+
command_index*sizeof(KERNEL_XHCI_TRB)||
(completion.Status>>24)!=BOB64_XHCI_TRB_COMPLETION_SUCCESS||
(u8)(completion.Control>>24)!=slot_id)return -45;
kernel_xhci_rings.CommandIndex++;
u64 report_physical;
volatile u8 *report_bytes;
if(kernel_xhci_dma_alloc(1,&report_physical,&report_bytes))return -46;
hid_device->Ring=endpoint_ring;
hid_device->RingPhysical=endpoint_ring_physical;
hid_device->Report=report_bytes;
hid_device->ReportPhysical=report_physical;
hid_device->Slot=slot_id;hid_device->Dci=endpoint_dci;
hid_device->Kind=hid_kind;hid_device->ReportLength=report_length;
hid_device->Cycle=1;hid_device->NextIndex=0;
hid_device->Pending=0;hid_device->Active=1;
hid_device->TransferErrors=0;
hid_device->RootPort=topology->RootPort;
hid_device->Topology=*topology;
hid_device->LoggedReport=hid_device->LoggedInput=hid_device->LoggedWheel=0;
hid_device->KeyboardState=(BOB64_KEYBOARD_HID_STATE){0};
if(kernel_xhci_hid_device_count<=hid_device_index)
kernel_xhci_hid_device_count=(u8)(hid_device_index+1u);
kernel_write("bob64 kernel: USB HID boot ");
kernel_write(hid_kind==1?"keyboard":"mouse");
kernel_write(" endpoint configured; waiting for input report\r\n");
return 0;
}
static void kernel_xhci_address_hub_children(
KERNEL_XHCI_MMIO *mmio,
const BOB64_XHCI_CAPABILITIES *capabilities,u32 start_index) {
u32 index=start_index,limit=capabilities->MaxSlots;
if(limit>KERNEL_XHCI_HID_DEVICE_LIMIT)limit=KERNEL_XHCI_HID_DEVICE_LIMIT;
while(index<kernel_xhci_hub_child_count&&index<limit) {
KERNEL_XHCI_TOPOLOGY topology=kernel_xhci_hub_children[index++];
if(!topology.RootPort)continue;
u8 slot_id=0;
int result=kernel_xhci_enable_slot(mmio,capabilities,&slot_id);
if(result) {
kernel_write("bob64 kernel: USB hub child slot allocation failed result=");
kernel_hex64((u64)(s64)result);kernel_write("\r\n");
break;
}
kernel_write("bob64 kernel: xHCI USB hub child slot enabled id=");
kernel_hex64(slot_id);kernel_write(" route=0x");
kernel_hex64(topology.RouteString);kernel_write(" speed=");
kernel_hex64(topology.Speed);kernel_write("\r\n");
result=kernel_xhci_address_and_read_descriptor(mmio,capabilities,
slot_id,&topology);
if(result) {
(void)kernel_xhci_disable_slot(slot_id);
kernel_write("bob64 kernel: USB hub child unsupported root port ");
kernel_hex64(topology.RootPort);kernel_write(" port ");
kernel_hex64(topology.ParentPort);kernel_write(" result=");
kernel_hex64((u64)(s64)result);kernel_write("\r\n");
}
}
if(index<kernel_xhci_hub_child_count) {
kernel_write("bob64 kernel: USB hub child limit reached count=");
kernel_hex64(kernel_xhci_hub_child_count);kernel_write("\r\n");
}
}
static int kernel_xhci_topology_in_subtree(
const KERNEL_XHCI_TOPOLOGY *topology,u8 root_port,u32 route,
u8 depth) {
u32 mask;
if(!topology||!depth||depth>5u||topology->RootPort!=root_port||
topology->Depth<depth)return 0;
mask=(1u<<((u32)depth*4u))-1u;
return (topology->RouteString&mask)==route;
}
static void kernel_xhci_hub_remove_subtree(KERNEL_XHCI_HUB_DEVICE *parent,
u32 route,u8 depth) {
if(!parent)return;
for(u32 i=0;i<kernel_xhci_hid_device_count;i++) {
KERNEL_XHCI_HID_DEVICE *device=&kernel_xhci_hid_devices[i];
if(device->Active&&kernel_xhci_topology_in_subtree(&device->Topology,
parent->RootPort,route,depth))
kernel_xhci_hid_disconnect(device,i);
}
for(u32 i=0;i<kernel_xhci_hub_device_count;i++) {
KERNEL_XHCI_HUB_DEVICE *hub=&kernel_xhci_hub_devices[i];
if(hub->Active&&kernel_xhci_topology_in_subtree(&hub->Topology,
parent->RootPort,route,depth)) {
hub->Active=0;hub->Pending=0;
(void)kernel_xhci_disable_slot(hub->Slot);
}
}
for(u32 i=0;i<kernel_xhci_hub_child_count;i++)
if(kernel_xhci_topology_in_subtree(&kernel_xhci_hub_children[i],
parent->RootPort,route,depth))
kernel_xhci_hub_children[i].RootPort=0;
}
static void kernel_xhci_hub_process_changes(KERNEL_XHCI_HUB_DEVICE *hub) {
if(!hub||!hub->Active)return;
for(u32 port=1;port<=hub->PortCount;port++) {
u32 byte=port>>3,bit=port&7u;
if(byte>=hub->ReportLength||!(hub->Report[byte]&(1u<<bit)))continue;
int result=kernel_xhci_hub_control_in(hub,0xa3,0,0,(u16)port,4);
if(result) {
kernel_write("bob64 kernel: USB hub port status read failed port=");
kernel_hex64(port);kernel_write(" result=");
kernel_hex64((u64)(s64)result);kernel_write("\r\n");
continue;
}
u16 status=(u16)(hub->ControlBuffer[0]|((u16)hub->ControlBuffer[1]<<8));
u16 change=(u16)(hub->ControlBuffer[2]|((u16)hub->ControlBuffer[3]<<8));
if(change) {
kernel_write("bob64 kernel: USB hub changed port=");kernel_hex64(port);
kernel_write(" status=0x");kernel_hex64(status);
kernel_write(" change=0x");kernel_hex64(change);kernel_write("\r\n");
}
u32 child_route=0;
u8 child_depth=(u8)(hub->Topology.Depth+1u);
if((change&1u)&&!bob64_usb_route_string_append(hub->Topology.RouteString,
hub->Topology.Depth,(u8)port,&child_route)) {
kernel_xhci_hub_remove_subtree(hub,child_route,child_depth);
if(status&1u) {
result=kernel_xhci_hub_control_no_data(hub,0x23,3,4,(u16)port);
if(!result)result=kernel_xhci_delay_ticks(2);
if(!result)result=kernel_xhci_hub_control_in(hub,
0xa3,0,0,(u16)port,4);
if(result) {
kernel_write("bob64 kernel: USB hub port reset failed port=");
kernel_hex64(port);kernel_write(" result=");
kernel_hex64((u64)(s64)result);kernel_write("\r\n");
} else {
status=(u16)(hub->ControlBuffer[0]|
((u16)hub->ControlBuffer[1]<<8));
change|=(u16)(hub->ControlBuffer[2]|
((u16)hub->ControlBuffer[3]<<8));
if(!(status&(1u<<1))) {
kernel_write("bob64 kernel: USB hub port did not enable after reset port=");
kernel_hex64(port);kernel_write("\r\n");
} else {
u16 speed_bits=(u16)((status>>9)&3u);
KERNEL_XHCI_TOPOLOGY child={0};
child.RootPort=hub->RootPort;
child.ParentSlot=hub->Slot;
child.ParentPort=(u8)port;
child.RouteString=child_route;
child.Depth=child_depth;
child.Speed=speed_bits==2u?3u:
speed_bits==1u?2u:1u;
child.TtHubSlot=hub->Topology.TtHubSlot;
child.TtPort=hub->Topology.TtPort;
if(hub->Topology.Speed==3u&&child.Speed<3u) {
child.TtHubSlot=hub->Slot;
child.TtPort=(u8)port;
}
u32 queue_start=kernel_xhci_hub_child_count;
u8 slot_id=0;
result=kernel_xhci_enable_slot(&kernel_xhci_live_mmio,
&kernel_xhci_live_capabilities,&slot_id);
if(!result)result=kernel_xhci_address_and_read_descriptor(
&kernel_xhci_live_mmio,&kernel_xhci_live_capabilities,
slot_id,&child);
if(result) {
if(slot_id)(void)kernel_xhci_disable_slot(slot_id);
kernel_write("bob64 kernel: USB hub hotplug enumeration failed port=");
kernel_hex64(port);kernel_write(" result=");
kernel_hex64((u64)(s64)result);kernel_write("\r\n");
} else {
kernel_xhci_address_hub_children(
&kernel_xhci_live_mmio,
&kernel_xhci_live_capabilities,queue_start);
for(u32 i=0;i<kernel_xhci_hid_device_count;i++)
if(kernel_xhci_hid_devices[i].Active&&
!kernel_xhci_hid_devices[i].Pending)
(void)kernel_xhci_usb_hid_submit(
&kernel_xhci_hid_devices[i]);
kernel_write("bob64 kernel: USB hub hotplug device enumerated port=");
kernel_hex64(port);kernel_write("\r\n");
}
}
}
}
}
/* Clear every reported change feature to prevent a persistent bitmap. */
for(u32 change_bit=0;change_bit<5u;change_bit++)
if(change&(1u<<change_bit))
(void)kernel_xhci_hub_control_no_data(hub,0x23,1,
(u16)(16u+change_bit),(u16)port);
}
}
static int kernel_xhci_scan_connected_ports(KERNEL_XHCI_MMIO *mmio,
const BOB64_XHCI_CAPABILITIES *capabilities,u32 *connected_count) {
const u32 port_power=1u<<9;
const u32 port_enabled=1u<<1;
const u32 port_reset=1u<<4;
const u32 port_warm_reset=1u<<31;
const u32 port_warm_reset_change=1u<<19;
const u32 port_reset_change=1u<<21;
const u32 port_change_mask=0x00fe0000u;
u32 connected=0,reset=0;
if(!mmio||!capabilities||!connected_count)return -1;
for(u32 port=0;port<capabilities->MaxPorts;port++) {
u32 port_status;
u32 offset=0x400u+port*0x10u;
if(kernel_xhci_read32(mmio,offset,&port_status))return -2;
if(port_status&1u) {
connected++;
if(connected<=KERNEL_XHCI_HID_DEVICE_LIMIT) {
kernel_xhci_connected_ports[connected-1]=(u8)(port+1);
kernel_xhci_connected_speeds[connected-1]=
(u8)((port_status>>10)&0xfu);
}
kernel_write("bob64 kernel: xHCI connected root port ");
kernel_hex64(port+1);kernel_write(" status=0x");
kernel_hex64(port_status);kernel_write("\r\n");
if(!(port_status&port_enabled)) {
u32 speed=(port_status>>10)&0xfu;
if(speed>=4u) {
if(port_status&port_warm_reset_change)
(void)kernel_xhci_write32(mmio,offset,
(port_status&port_power)|port_warm_reset_change);
if(kernel_xhci_write32(mmio,offset,
(port_status&port_power)|port_warm_reset))return -4;
int warm_completed=0;
for(u32 wait=0;wait<20000000u;wait++) {
if(kernel_xhci_read32(mmio,offset,&port_status))return -5;
if(port_status&port_warm_reset_change) {
if(!(port_status&port_enabled))return -6;
(void)kernel_xhci_write32(mmio,offset,
(port_status&port_power)|
(port_status&port_change_mask));
warm_completed=1;
break;
}
__asm__ volatile("pause");
}
if(!warm_completed)return -7;
reset++;
kernel_write("bob64 kernel: xHCI root port warm reset passed ");
kernel_hex64(port+1);kernel_write("\r\n");
continue;
}
if(kernel_xhci_write32(mmio,offset,
(port_status&port_power)|port_reset))return -8;
int completed=0;
for(u32 wait=0;wait<20000000u;wait++) {
if(kernel_xhci_read32(mmio,offset,&port_status))return -9;
if(!(port_status&port_reset)) {
if(!(port_status&port_enabled))return -10;
completed=1;
break;
}
__asm__ volatile("pause");
}
if(!completed)return -11;
if(port_status&port_reset_change)
(void)kernel_xhci_write32(mmio,offset,
(port_status&port_power)|
(port_status&port_change_mask));
reset++;
kernel_write("bob64 kernel: xHCI root port reset passed ");
kernel_hex64(port+1);kernel_write("\r\n");
}
}
}
*connected_count=connected;
if(reset) {
kernel_write("bob64 kernel: xHCI root-port resets=");
kernel_hex64(reset);kernel_write("\r\n");
}
return 0;
}
static void kernel_xhci_hid_reconnect(u8 port) {
u32 port_status;
const u32 port_enabled=1u<<1;
const u32 port_power=1u<<9;
const u32 port_change_mask=0x00fe0000u;
if(!port||port>kernel_xhci_live_capabilities.MaxPorts)return;
for(u32 i=0;i<kernel_xhci_hid_device_count;i++)
if(kernel_xhci_hid_devices[i].Active&&
kernel_xhci_hid_devices[i].RootPort==port)return;
for(u32 i=0;i<kernel_xhci_storage_device_count;i++)
if(kernel_xhci_storage_devices[i].Active&&
kernel_xhci_storage_devices[i].Topology.RootPort==port)return;
if(kernel_xhci_read32(&kernel_xhci_live_mmio,
0x400u+((u32)port-1u)*0x10u,&port_status)||!(port_status&1u))return;
/* A replacement device can arrive while the root port still appears
enabled for the removed device. Disable it so the scan performs the
required USB2 reset or SuperSpeed warm reset before addressing. */
if(port_status&port_enabled) {
if(kernel_xhci_write32(&kernel_xhci_live_mmio,
0x400u+((u32)port-1u)*0x10u,
(port_status&port_power)|port_enabled|
(port_status&port_change_mask))) {
kernel_write("bob64 kernel: USB reconnect port disable failed\r\n");
return;
}
}
if(!kernel_xhci_read32(&kernel_xhci_live_mmio,
0x400u+((u32)port-1u)*0x10u,&port_status)&&
(port_status&port_enabled)) {
u32 speed=(port_status>>10)&0xfu;
u32 reset_request=speed>=4u?(1u<<31):(1u<<4);
u32 reset_change=speed>=4u?(1u<<19):(1u<<21);
if(kernel_xhci_write32(&kernel_xhci_live_mmio,
0x400u+((u32)port-1u)*0x10u,
(port_status&port_power)|reset_request|
(port_status&port_change_mask))) {
kernel_write("bob64 kernel: USB reconnect port reset request failed\r\n");
return;
}
int reset_completed=0;
for(u32 wait=0;wait<20000000u;wait++) {
if(kernel_xhci_read32(&kernel_xhci_live_mmio,
0x400u+((u32)port-1u)*0x10u,&port_status))return;
if(port_status&reset_change) {
if(!(port_status&port_enabled))break;
(void)kernel_xhci_write32(&kernel_xhci_live_mmio,
0x400u+((u32)port-1u)*0x10u,
(port_status&port_power)|
(port_status&port_change_mask));
reset_completed=1;
break;
}
}
if(!reset_completed) {
kernel_write("bob64 kernel: USB reconnect port reset timed out\r\n");
return;
}
}
u32 connected=0;
if(kernel_xhci_scan_connected_ports(&kernel_xhci_live_mmio,
&kernel_xhci_live_capabilities,&connected)) {
kernel_write("bob64 kernel: USB reconnect port reset failed\r\n");
return;
}
if(kernel_xhci_read32(&kernel_xhci_live_mmio,
0x400u+((u32)port-1u)*0x10u,&port_status)||!(port_status&1u))return;
u8 speed=(u8)((port_status>>10)&0xfu),slot_id=0;
if(speed<1u||speed>5u||kernel_xhci_enable_slot(
&kernel_xhci_live_mmio,&kernel_xhci_live_capabilities,&slot_id)) {
kernel_write("bob64 kernel: USB reconnect slot allocation failed\r\n");
return;
}
KERNEL_XHCI_TOPOLOGY topology={0};
topology.RootPort=port;topology.Speed=speed;
int result=kernel_xhci_address_and_read_descriptor(
&kernel_xhci_live_mmio,&kernel_xhci_live_capabilities,slot_id,
&topology);
if(result) {
(void)kernel_xhci_disable_slot(slot_id);
kernel_write("bob64 kernel: USB reconnect device unsupported result=");
kernel_hex64((u64)(s64)result);kernel_write("\r\n");
return;
}
kernel_xhci_hub_child_count=0;
kernel_xhci_address_hub_children(&kernel_xhci_live_mmio,
&kernel_xhci_live_capabilities,0);
for(u32 i=0;i<kernel_xhci_hid_device_count;i++)
if(kernel_xhci_hid_devices[i].Active&&
kernel_xhci_hid_devices[i].RootPort==port) {
if(kernel_xhci_usb_hid_submit(&kernel_xhci_hid_devices[i]))
kernel_xhci_hid_disconnect(&kernel_xhci_hid_devices[i],i);
break;
}
kernel_write("bob64 kernel: USB device reconnected on root port ");
kernel_hex64(port);kernel_write("\r\n");
}
static void kernel_pci_report_usb(void *context,const BOB64_PCI_DEVICE *device) {
(void)context;
if(!bob64_pci_is_usb_controller(device))return;
if(bob64_pci_is_xhci_controller(device))
kernel_write("bob64 kernel: xHCI PCI controller found at ");
else kernel_write("bob64 kernel: USB PCI controller found at ");
kernel_hex64(((u64)device->Bus<<16)|((u64)device->Device<<8)|
device->Function);
kernel_write(" vendor=0x");kernel_hex64(device->VendorId);
kernel_write(" device=0x");kernel_hex64(device->DeviceId);
kernel_write(" BAR0=0x");kernel_hex64(device->Bar[0]);
kernel_write("\r\n");
if(bob64_pci_is_xhci_controller(device)) {
kernel_xhci_device=*device;kernel_xhci_found=1;
}
}
static int kernel_xhci_probe_capabilities(const BOB64_KERNEL_BOOT_INFO *info) {
const u64 virtual_address=0xffffa00000000000ULL;
BOB64_XHCI_CAPABILITIES capabilities;
KERNEL_XHCI_MMIO mmio;
u64 physical;
int result;
if(!kernel_xhci_found||!info||!kernel_xhci_device.BarValid[0]||
kernel_xhci_device.BarIsIo[0])return -10;
if(bob64_pci_enable_memory(&kernel_xhci_device))return -12;
physical=kernel_xhci_device.Bar[0];
if(physical&(BOB64_PAGE_SIZE-1))return -11;
result=bob64_page_map_range(&bob64_bootstrap_space.PageTable,virtual_address,
physical,BOB64_XHCI_MMIO_MAP_SIZE,BOB64_PAGE_WRITE|BOB64_PAGE_PCD|
BOB64_PAGE_PWT|(info->NxSupported?BOB64_PAGE_NX:0));
if(result)return -100+result;
for(u64 offset=0;offset<BOB64_XHCI_MMIO_MAP_SIZE;
offset+=BOB64_PAGE_SIZE)
__asm__ volatile("invlpg (%0)"::"r"((void *)(uintptr_t)(virtual_address+offset)):
"memory");
if(bob64_xhci_parse_capabilities((const volatile void *)(uintptr_t)virtual_address,
BOB64_XHCI_MMIO_MAP_SIZE,&capabilities)) {
kernel_write("bob64 kernel: xHCI caps invalid len=");
kernel_hex64(capabilities.CapabilityLength);kernel_write(" version=");
kernel_hex64(capabilities.Version);kernel_write(" slots=");
kernel_hex64(capabilities.MaxSlots);kernel_write(" intr=");
kernel_hex64(capabilities.MaxInterrupters);kernel_write(" ports=");
kernel_hex64(capabilities.MaxPorts);kernel_write(" db=");
kernel_hex64(capabilities.DoorbellOffset);kernel_write(" rt=");
kernel_hex64(capabilities.RuntimeOffset);kernel_write("\r\n");
for(u64 offset=0;offset<BOB64_XHCI_MMIO_MAP_SIZE;
offset+=BOB64_PAGE_SIZE) {
(void)bob64_page_unmap(&bob64_bootstrap_space.PageTable,
virtual_address+offset,0,0);
__asm__ volatile("invlpg (%0)"::"r"((void *)(uintptr_t)(virtual_address+offset)):
"memory");
}
return -20;
}
kernel_write("bob64 kernel: xHCI capabilities mapped, version=0x");
kernel_hex64(capabilities.Version);kernel_write(" slots=");
kernel_hex64(capabilities.MaxSlots);kernel_write(" ports=");
kernel_hex64(capabilities.MaxPorts);kernel_write(" scratchpads=");
kernel_hex64(capabilities.MaxScratchpadBuffers);kernel_write("\r\n");
mmio.Base=(volatile u8 *)(uintptr_t)(virtual_address+
capabilities.CapabilityLength);
mmio.Size=(usize)(BOB64_XHCI_MMIO_MAP_SIZE-capabilities.CapabilityLength);
mmio.CapabilityLength=capabilities.CapabilityLength;
kernel_xhci_live_mmio=mmio;
kernel_xhci_live_capabilities=capabilities;
result=bob64_xhci_halt_reset(kernel_xhci_read32,kernel_xhci_write32,
&mmio,1000000u,0);
if(result)return -30+result;
kernel_write("bob64 kernel: xHCI controller halted and reset\r\n");
result=kernel_xhci_start_command_ring(&mmio,&capabilities);
if(result)return -40+result;
kernel_write("bob64 kernel: xHCI command ring completion passed\r\n");
u32 connected_ports=0,device_limit=capabilities.MaxSlots;
if(device_limit>KERNEL_XHCI_HID_DEVICE_LIMIT)
device_limit=KERNEL_XHCI_HID_DEVICE_LIMIT;
if(!device_limit)return -59;
result=kernel_xhci_scan_connected_ports(&mmio,&capabilities,
&connected_ports);
if(result)return -60+result;
kernel_write("bob64 kernel: xHCI connected ports=");
kernel_hex64(connected_ports);kernel_write("\r\n");
kernel_xhci_hid_device_count=0;
kernel_xhci_hub_device_count=0;
kernel_xhci_hub_child_count=0;
for(u32 port_index=0;port_index<connected_ports&&
port_index<device_limit;port_index++) {
u8 slot_id=0;
result=kernel_xhci_enable_slot(&mmio,&capabilities,&slot_id);
if(result)return -70+result;
kernel_write("bob64 kernel: xHCI USB slot enabled id=");
kernel_hex64(slot_id);kernel_write("\r\n");
KERNEL_XHCI_TOPOLOGY topology={0};
topology.RootPort=kernel_xhci_connected_ports[port_index];
topology.Speed=kernel_xhci_connected_speeds[port_index];
result=kernel_xhci_address_and_read_descriptor(&mmio,&capabilities,
slot_id,&topology);
if(result) {
kernel_write("bob64 kernel: USB HID device unsupported at port ");
kernel_hex64(kernel_xhci_connected_ports[port_index]);
kernel_write(" result=");kernel_hex64((u64)(s64)result);
kernel_write("\r\n");
continue;
}
kernel_write("bob64 kernel: USB device descriptor read passed\r\n");
}
kernel_xhci_address_hub_children(&mmio,&capabilities,0);
for(u32 i=0;i<kernel_xhci_hid_device_count;i++) {
if(kernel_xhci_usb_hid_submit(&kernel_xhci_hid_devices[i]))return -90;
}
return 0;
}
static int kernel_text_contains(const char *text,const char *needle) {
if(!text||!needle)return 0;
for(usize i=0;text[i];i++) {
usize j=0;
while(needle[j]&&text[i+j]&&text[i+j]==needle[j])j++;
if(!needle[j])return 1;
}
return 0;
}
static int kernel_interrupts_enabled(void) {
u64 flags;
__asm__ volatile("pushfq; pop %0":"=r"(flags));
return (flags&0x200)!=0;
}
static void kernel_user_write_character(void *context,u8 character);
static s64 kernel_run_application(void *context,const char *name,
usize argument_count,
const char *const *arguments,int *started);
static int kernel_app_run_application(void *context,const char *name,
usize argument_count,
const char *const *arguments,
s64 *exit_status);
static int kernel_copy_user_bytes(void *context,u64 address,void *destination,
usize length);
static int kernel_process_fault_smoke_test(BOB64_PROCESS *process,
BOB64_PAGE_TABLE *process_table,const BOB64_PROCESS_OPERATIONS *operations,
const u8 *executable,usize executable_size,u64 expected_vector,
u64 expected_rip,u64 error_mask,u64 error_value,int diagnose,
const char *success_message);
static int kernel_wait_for_input_event(void *context,BOB64_EVENT *event);
static int kernel_snapshot_save(void *context,const void *snapshot,
usize snapshot_size);
static int kernel_snapshot_restore(void *context,BOB64_FILESYSTEM *filesystem);
extern const u8 bob64_embedded_app[];
extern const u8 bob64_embedded_app_end[];
extern const u8 bob64_embedded_display[];
extern const u8 bob64_embedded_display_end[];
extern const u8 bob64_embedded_gui[];
extern const u8 bob64_embedded_gui_end[];
extern const u8 bob64_embedded_nested_app[];
extern const u8 bob64_embedded_nested_app_end[];
extern const u8 bob64_embedded_echo[];
extern const u8 bob64_embedded_echo_end[];
extern const u8 bob64_embedded_ls[];
extern const u8 bob64_embedded_ls_end[];
extern const u8 bob64_embedded_cat[];
extern const u8 bob64_embedded_cat_end[];
extern const u8 bob64_embedded_notes[];
extern const u8 bob64_embedded_notes_end[];
extern const u8 bob64_embedded_info[];
extern const u8 bob64_embedded_info_end[];
extern const u8 bob64_embedded_mouse_smoke[];
extern const u8 bob64_embedded_mouse_smoke_end[];
extern const u8 bob64_embedded_launcher[];
extern const u8 bob64_embedded_launcher_end[];
extern const u8 bob64_embedded_source[];
extern const u8 bob64_embedded_source_end[];
typedef struct {
BOB64_PAGE_TABLE *PageTable;
} KERNEL_PROCESS_CONTEXT;
static u64 read_cr3(void);
static u64 process_allocate_page(void *context) {
(void)context;
return bob64_page_alloc(&bob64_boot_page_allocator,1);
}
static int process_free_page(void *context,u64 physical_address) {
(void)context;
return bob64_page_free(&bob64_boot_page_allocator,physical_address,1);
}
static int process_map_page(void *context,u64 virtual_address,u64 physical_address,
u64 flags,u8 **writable_address) {
KERNEL_PROCESS_CONTEXT *process=(KERNEL_PROCESS_CONTEXT *)context;
if(!process||!process->PageTable||!writable_address||
bob64_page_map(process->PageTable,
virtual_address,physical_address,flags))return -1;
__asm__ volatile("invlpg (%0)"::"r"((void *)(uintptr_t)virtual_address):"memory");
*writable_address=(u8 *)(uintptr_t)virtual_address;
return 0;
}
static int process_protect_page(void *context,u64 virtual_address,u64 flags) {
KERNEL_PROCESS_CONTEXT *process=(KERNEL_PROCESS_CONTEXT *)context;
if(!process||!process->PageTable||
bob64_page_protect(process->PageTable,virtual_address,flags))return -1;
__asm__ volatile("invlpg (%0)"::"r"((void *)(uintptr_t)virtual_address):"memory");
return 0;
}
static int process_unmap_page(void *context,u64 virtual_address) {
KERNEL_PROCESS_CONTEXT *process=(KERNEL_PROCESS_CONTEXT *)context;
if(!process||!process->PageTable||
bob64_page_unmap(process->PageTable,virtual_address,0,0))return -1;
__asm__ volatile("invlpg (%0)"::"r"((void *)(uintptr_t)virtual_address):"memory");
return 0;
}
static void kernel_write16(u8 *bytes,u16 value) {
bytes[0]=(u8)value;bytes[1]=(u8)(value>>8);
}
static void kernel_write32(u8 *bytes,u32 value) {
for(u32 i=0;i<4;i++)bytes[i]=(u8)(value>>(i*8));
}
static void kernel_write64(u8 *bytes,u64 value) {
kernel_write32(bytes,(u32)value);kernel_write32(bytes+4,(u32)(value>>32));
}
static usize kernel_emit_test_syscall(u8 *code,usize offset,u32 number,u32 argument) {
code[offset++]=0xb8;kernel_write32(code+offset,number);offset+=4;
code[offset++]=0xb9;kernel_write32(code+offset,argument);offset+=4;
code[offset++]=0xcd;code[offset++]=0x80;
return offset;
}
static usize kernel_emit_test_buffer_syscall(u8 *code,usize offset,
const char *text,usize length) {
usize instruction_start=offset,lea_end,buffer_offset;
code[offset++]=0xb8;kernel_write32(code+offset,BOB64_SYSCALL_WRITE_BUFFER);offset+=4;
code[offset++]=0x48;code[offset++]=0x8d;code[offset++]=0x0d;
lea_end=offset+4;
buffer_offset=instruction_start+24;
kernel_write32(code+offset,(u32)(buffer_offset-lea_end));offset+=4;
code[offset++]=0xba;kernel_write32(code+offset,(u32)length);offset+=4;
code[offset++]=0xcd;code[offset++]=0x80;
code[offset++]=0xe9;
kernel_write32(code+offset,(u32)length);offset+=4;
for(usize i=0;i<length;i++)code[offset++]=(u8)text[i];
return offset;
}
static u32 kernel_crc32(const u8 *bytes,usize length) {
u32 value=0xffffffffu;
for(usize i=0;i<length;i++) {
value^=bytes[i];
for(u32 bit=0;bit<8;bit++)value=(value>>1)^(0xedb88320u&-(value&1u));
}
return ~value;
}
static int kernel_bytes_equal(const char *left,const char *right) {
while(*left&&*left==*right){left++;right++;}
return *left==*right;
}
static void kernel_build_bob_application(u8 *executable) {
static const char text[]="bob!";
usize code_size=0;
for(usize i=0;i<BOB64_EXEC_HEADER_SIZE+BOB64_PAGE_SIZE;i++)executable[i]=0;
executable[0]='B';executable[1]='6';executable[2]='4';executable[3]='E';
kernel_write16(executable+4,BOB64_EXEC_VERSION);
kernel_write16(executable+6,BOB64_EXEC_HEADER_SIZE);
kernel_write32(executable+8,BOB64_APP_ABI_VERSION);
kernel_write64(executable+16,BOB64_PAGE_SIZE);
kernel_write64(executable+24,2*BOB64_PAGE_SIZE);
kernel_write64(executable+32,0);
kernel_write64(executable+40,BOB64_PAGE_SIZE);
for(usize i=0;i<BOB64_PAGE_SIZE;i++)
executable[BOB64_EXEC_HEADER_SIZE+i]=0x90;
code_size=kernel_emit_test_syscall(executable+BOB64_EXEC_HEADER_SIZE,
code_size,BOB64_SYSCALL_QUERY_ABI,0);
code_size=kernel_emit_test_buffer_syscall(executable+BOB64_EXEC_HEADER_SIZE,
code_size,text,sizeof(text)-1);
(void)kernel_emit_test_syscall(executable+BOB64_EXEC_HEADER_SIZE,
code_size,BOB64_SYSCALL_EXIT,0);
kernel_write32(executable+48,kernel_crc32(executable+BOB64_EXEC_HEADER_SIZE,
(usize)BOB64_PAGE_SIZE));
}
static u64 kernel_build_exception_diagnostic_application(u8 *executable) {
static const char text[]="bob!";
u8 *code=executable+BOB64_EXEC_HEADER_SIZE;
usize offset=0;
kernel_build_bob_application(executable);
offset=kernel_emit_test_syscall(code,offset,BOB64_SYSCALL_QUERY_ABI,0);
offset=kernel_emit_test_buffer_syscall(code,offset,text,sizeof(text)-1);
code[offset++]=0x49;code[offset++]=0xbf; /* mov r15, imm64 */
kernel_write64(code+offset,0x1234567887654321ULL);offset+=8;
u64 fault_rip=BOB64_PROCESS_IMAGE_BASE+offset;
code[offset++]=0x0f;code[offset++]=0x0b; /* ud2 */
kernel_write32(executable+48,kernel_crc32(executable+BOB64_EXEC_HEADER_SIZE,
(usize)BOB64_PAGE_SIZE));
return fault_rip;
}
static void kernel_build_code_write_fault_application(u8 *executable) {
static const char text[]="bob!";
u8 *code=executable+BOB64_EXEC_HEADER_SIZE;
usize offset;
kernel_build_bob_application(executable);
offset=kernel_emit_test_buffer_syscall(code,0,text,sizeof(text)-1);
code[offset++]=0xc6;code[offset++]=0x05;
kernel_write32(code+offset,(u32)(0-(offset+5)));offset+=4;
code[offset++]=0x90;
(void)offset;
kernel_write32(executable+48,kernel_crc32(executable+BOB64_EXEC_HEADER_SIZE,
(usize)BOB64_PAGE_SIZE));
}
static void kernel_build_nx_fault_application(u8 *executable) {
u8 *code=executable+BOB64_EXEC_HEADER_SIZE;
usize offset;
kernel_build_bob_application(executable);
offset=kernel_emit_test_buffer_syscall(code,0,"bob!",4);
code[offset++]=0xe9;
kernel_write32(code+offset,(u32)((u64)BOB64_PAGE_SIZE-(offset+4)));offset+=4;
executable[BOB64_EXEC_HEADER_SIZE+BOB64_PAGE_SIZE]=0x0f;
executable[BOB64_EXEC_HEADER_SIZE+BOB64_PAGE_SIZE+1]=0x0b;
kernel_write64(executable+16,2*BOB64_PAGE_SIZE);
kernel_write64(executable+24,2*BOB64_PAGE_SIZE);
kernel_write32(executable+48,kernel_crc32(executable+BOB64_EXEC_HEADER_SIZE,
(usize)(2*BOB64_PAGE_SIZE)));
}
static int kernel_install_bob_application(BOB64_FILESYSTEM *filesystem) {
usize length=(usize)(bob64_embedded_app_end-bob64_embedded_app);
if(length<BOB64_EXEC_HEADER_SIZE)return -1;
return bob64_fs_write(filesystem,"bob.b64e",bob64_embedded_app,length);
}
static int kernel_install_display_application(BOB64_FILESYSTEM *filesystem) {
usize length=(usize)(bob64_embedded_display_end-bob64_embedded_display);
if(length<BOB64_EXEC_HEADER_SIZE)return -1;
return bob64_fs_write(filesystem,"display.b64e",bob64_embedded_display,length);
}
static int kernel_install_gui_application(BOB64_FILESYSTEM *filesystem) {
usize length=(usize)(bob64_embedded_gui_end-bob64_embedded_gui);
if(!length)return -1;
return bob64_fs_write(filesystem,"desktop.b64e",bob64_embedded_gui,length);
}
static int kernel_install_nested_application(BOB64_FILESYSTEM *filesystem) {
usize length=(usize)(bob64_embedded_nested_app_end-
bob64_embedded_nested_app);
if(length<BOB64_EXEC_HEADER_SIZE)return -1;
return bob64_fs_write(filesystem,"nested.b64e",bob64_embedded_nested_app,
length);
}
static int kernel_install_echo_application(BOB64_FILESYSTEM *filesystem) {
usize length=(usize)(bob64_embedded_echo_end-bob64_embedded_echo);
if(length<BOB64_EXEC_HEADER_SIZE)return -1;
return bob64_fs_write(filesystem,"echo.b64e",bob64_embedded_echo,length);
}
static int kernel_install_ls_application(BOB64_FILESYSTEM *filesystem) {
usize length=(usize)(bob64_embedded_ls_end-bob64_embedded_ls);
if(length<BOB64_EXEC_HEADER_SIZE)return -1;
return bob64_fs_write(filesystem,"ls.b64e",bob64_embedded_ls,length);
}
static int kernel_install_cat_application(BOB64_FILESYSTEM *filesystem) {
usize length=(usize)(bob64_embedded_cat_end-bob64_embedded_cat);
if(length<BOB64_EXEC_HEADER_SIZE)return -1;
return bob64_fs_write(filesystem,"cat.b64e",bob64_embedded_cat,length);
}
static int kernel_install_notes_application(BOB64_FILESYSTEM *filesystem) {
usize length=(usize)(bob64_embedded_notes_end-bob64_embedded_notes);
if(length<BOB64_EXEC_HEADER_SIZE)return -1;
return bob64_fs_write(filesystem,"notes.b64e",bob64_embedded_notes,length);
}
static int kernel_install_info_application(BOB64_FILESYSTEM *filesystem) {
usize length=(usize)(bob64_embedded_info_end-bob64_embedded_info);
if(length<BOB64_EXEC_HEADER_SIZE)return -1;
return bob64_fs_write(filesystem,"info.b64e",bob64_embedded_info,length);
}
static int kernel_install_mouse_smoke_application(BOB64_FILESYSTEM *filesystem) {
usize length=(usize)(bob64_embedded_mouse_smoke_end-
bob64_embedded_mouse_smoke);
if(length<BOB64_EXEC_HEADER_SIZE)return -1;
return bob64_fs_write(filesystem,"mousesmoke.b64e",
bob64_embedded_mouse_smoke,length);
}
static int kernel_install_launcher_application(BOB64_FILESYSTEM *filesystem) {
usize length=(usize)(bob64_embedded_launcher_end-
bob64_embedded_launcher);
if(length<BOB64_EXEC_HEADER_SIZE)return -1;
return bob64_fs_write(filesystem,"launcher.b64e",bob64_embedded_launcher,
length);
}
static int kernel_install_bob_source(BOB64_FILESYSTEM *filesystem) {
usize length=(usize)(bob64_embedded_source_end-bob64_embedded_source);
return bob64_fs_write(filesystem,"bob.c",bob64_embedded_source,length);
}
static int kernel_process_smoke_test(const BOB64_KERNEL_BOOT_INFO *info) {
static const char *arguments[]={"bob!"};
static u8 executable[BOB64_EXEC_HEADER_SIZE+BOB64_PAGE_SIZE];
static u8 nx_executable[BOB64_EXEC_HEADER_SIZE+2*BOB64_PAGE_SIZE];
BOB64_PAGE_TABLE process_table;
KERNEL_PROCESS_CONTEXT process_context={&process_table};
BOB64_PROCESS_OPERATIONS operations={process_allocate_page,process_free_page,
process_map_page,process_protect_page,process_unmap_page,&process_context,0};
BOB64_PROCESS process;
BOB64_PROCESS_PAGE *pages;
BOB64_APP_STARTUP *startup;
u64 physical,flags,kernel_root,table_checkpoint;
usize allocated_before,heap_checkpoint;
if(!info->NxSupported)return 1;
pages=(BOB64_PROCESS_PAGE *)bob64_heap_calloc(&kernel_heap,
(usize)BOB64_PROCESS_PAGE_LIMIT,sizeof(*pages));
if(!pages)return -1;
kernel_root=read_cr3();
table_checkpoint=bob64_bootstrap_space.TablePoolUsed;
heap_checkpoint=kernel_heap_high_water;
allocated_before=bob64_boot_page_allocator.AllocatedCount;
operations.NxSupported=info->NxSupported;
if(bob64_process_init(&process,pages,(usize)BOB64_PROCESS_PAGE_LIMIT,&operations)) {
bob64_heap_free(&kernel_heap,pages);return -1;
}
if(bob64_page_table_init(&process_table,info->PhysicalAddressBits,
bob64_bootstrap_space.PageTable.Allocate,bob64_bootstrap_space.PageTable.Access,
bob64_bootstrap_space.PageTable.Context)||
bob64_page_table_clone_isolated(&process_table,&bob64_bootstrap_space.PageTable,
BOB64_PROCESS_IMAGE_BASE,BOB64_PROCESS_ISOLATED_SIZE)) {
if(kernel_heap_high_water==heap_checkpoint)
bob64_bootstrap_space.TablePoolUsed=table_checkpoint;
bob64_heap_free(&kernel_heap,pages);return -1;
}
__asm__ volatile("mov %0,%%cr3"::"r"(process_table.RootPhysical):"memory");
kernel_build_bob_application(executable);
int result=bob64_process_load(&process,executable,sizeof(executable),1,arguments);
if(result) {
bob64_process_unload(&process);
__asm__ volatile("mov %0,%%cr3"::"r"(kernel_root):"memory");
if(kernel_heap_high_water==heap_checkpoint)
bob64_bootstrap_space.TablePoolUsed=table_checkpoint;
bob64_heap_free(&kernel_heap,pages);return -1;
}
startup=(BOB64_APP_STARTUP *)(uintptr_t)process.StartupAddress;
result=process.EntryAddress==BOB64_PROCESS_IMAGE_BASE&&
startup->AbiVersion==BOB64_APP_ABI_VERSION&&startup->ArgumentCount==1&&
startup->Arguments&&kernel_bytes_equal(startup->Arguments[0],"bob!")&&
bob64_page_translate(&process_table,
BOB64_PROCESS_IMAGE_BASE,&physical,&flags)==1&&physical&&
(flags&BOB64_PAGE_USER)&&!(flags&BOB64_PAGE_WRITE)&&!(flags&BOB64_PAGE_NX)&&
bob64_page_translate(&process_table,
BOB64_PROCESS_IMAGE_BASE+BOB64_PAGE_SIZE,&physical,&flags)==1&&
(flags&BOB64_PAGE_USER)&&(flags&BOB64_PAGE_WRITE)&&(flags&BOB64_PAGE_NX)&&
bob64_page_translate(&process_table,
BOB64_PROCESS_STACK_GUARD,&physical,&flags)==0;
if(result) {
kernel_user_output_length=0;
kernel_user_output_truncated=0;
bob64_syscall_set_address_space(&process_table);
bob64_syscall_set_read_user(kernel_copy_user_bytes,0);
bob64_syscall_set_write(kernel_user_write_character,0);
bob64_syscall_set_wait_event(kernel_wait_for_input_event,0);
s64 app_status=bob64_enter_user(process.EntryAddress,process.StartupAddress,
process.InitialStackPointer);
bob64_syscall_set_wait_event(0,0);
bob64_syscall_set_write(0,0);
bob64_syscall_set_read_user(0,0);
bob64_syscall_set_address_space(0);
kernel_user_output[kernel_user_output_length]=0;
if(app_status||kernel_user_output_truncated||
kernel_user_output_length!=4||
!kernel_bytes_equal(kernel_user_output,"bob!"))result=0;
else kernel_write("\r\n");
}
int unload_result=bob64_process_unload(&process);
if(unload_result||process.PageCount||process.Loaded||
bob64_boot_page_allocator.AllocatedCount!=allocated_before)result=-1;
else result=result?0:-1;
if(!result) {
u64 exception_rip=kernel_build_exception_diagnostic_application(executable);
if(kernel_process_fault_smoke_test(&process,&process_table,&operations,
executable,sizeof(executable),6,exception_rip,0,0,1,
"bob64 kernel: full-width user exception diagnostics passed\r\n"))
result=-1;
}
if(!result) {
kernel_build_code_write_fault_application(executable);
if(kernel_process_fault_smoke_test(&process,&process_table,&operations,
executable,sizeof(executable),14,BOB64_PROCESS_IMAGE_BASE+28,0x7,0x7,1,
"bob64 kernel: read-only code protection passed\r\n"))result=-1;
}
if(!result) {
kernel_build_nx_fault_application(nx_executable);
if(kernel_process_fault_smoke_test(&process,&process_table,&operations,
nx_executable,sizeof(nx_executable),14,
BOB64_PROCESS_IMAGE_BASE+BOB64_PAGE_SIZE,0x15,0x15,0,
"bob64 kernel: non-executable data protection passed\r\n"))result=-1;
}
__asm__ volatile("mov %0,%%cr3"::"r"(kernel_root):"memory");
if(kernel_heap_high_water==heap_checkpoint)
bob64_bootstrap_space.TablePoolUsed=table_checkpoint;
if(bob64_boot_page_allocator.AllocatedCount!=allocated_before)result=-1;
if(bob64_heap_free(&kernel_heap,pages))result=-1;
return result;
}
static u64 read_cr3(void) {
u64 value;
__asm__ volatile("mov %%cr3,%0":"=r"(value));
return value&BOB64_PAGE_ADDRESS_MASK;
}
static __attribute__((noreturn)) void kernel_halt(void) {
__asm__ volatile("cli");
for(;;)__asm__ volatile("hlt");
}
static void kernel_write(const char *text) {
bob64_early_console_write(text);
bob64_framebuffer_write(text);
}
static void kernel_user_write_character(void *context,u8 character) {
char text[2]={(char)character,0};
(void)context;
if(kernel_user_output_length+1<sizeof(kernel_user_output))
kernel_user_output[kernel_user_output_length++]=(char)character;
else kernel_user_output_truncated=1;
kernel_write(text);
}
static int kernel_copy_user_bytes(void *context,u64 address,void *destination,
usize length) {
const volatile u8 *source=(const volatile u8 *)(uintptr_t)address;
u8 *target=(u8 *)destination;
(void)context;
if((!source&&length)||(!target&&length)||
length>BOB64_SYSCALL_COPY_CHUNK_BYTES)
return -1;
for(usize i=0;i<length;i++)target[i]=source[i];
return 0;
}
static int kernel_process_fault_smoke_test(BOB64_PROCESS *process,
BOB64_PAGE_TABLE *process_table,const BOB64_PROCESS_OPERATIONS *operations,
const u8 *executable,usize executable_size,u64 expected_vector,
u64 expected_rip,u64 error_mask,u64 error_value,int diagnose,
const char *success_message) {
static const char *arguments[]={"bob!"};
s64 status;
int passed;
if(bob64_process_init(process,process->Pages,process->PageCapacity,operations)||
bob64_process_load(process,executable,executable_size,1,arguments)) {
(void)bob64_process_unload(process);
return -1;
}
kernel_user_output_length=0;
kernel_user_output_truncated=0;
bob64_user_exception_vector=~(u64)0;
bob64_user_exception_error=~(u64)0;
bob64_user_exception_rip=~(u64)0;
bob64_user_expected_exception_vector=diagnose?~(u64)0:expected_vector;
bob64_user_expected_exception_error_mask=error_mask;
bob64_user_expected_exception_error_value=error_value;
bob64_syscall_set_address_space(process_table);
bob64_syscall_set_read_user(kernel_copy_user_bytes,0);
bob64_syscall_set_write(kernel_user_write_character,0);
status=bob64_enter_user(process->EntryAddress,process->StartupAddress,
process->InitialStackPointer);
bob64_user_expected_exception_vector=~(u64)0;
bob64_syscall_set_write(0,0);
bob64_syscall_set_read_user(0,0);
bob64_syscall_set_address_space(0);
kernel_user_output[kernel_user_output_length]=0;
passed=status==-(s64)(128+expected_vector)&&
bob64_user_exception_vector==expected_vector&&
bob64_user_exception_rip==expected_rip&&
(bob64_user_exception_error&error_mask)==error_value&&
!kernel_user_output_truncated&&kernel_user_output_length==4&&
kernel_bytes_equal(kernel_user_output,"bob!");
if(bob64_process_unload(process))return -1;
if(!passed)return -1;
kernel_write(success_message);
return 0;
}
static int kernel_wait_for_input_event(void *context,BOB64_EVENT *event) {
(void)context;
if(!event)return -1;
/* int 0x80 enters through an interrupt gate, which clears IF. */
__asm__ volatile("sti":::"memory");
for(;;) {
kernel_xhci_usb_hid_poll();
if(!bob64_mouse_poll_event(event)||!bob64_keyboard_poll_event(event))break;
__asm__ volatile("pause");
}
/* Keep the syscall handler's state until iretq restores the user flags. */
__asm__ volatile("cli":::"memory");
return 0;
}
static int kernel_write_user_bytes(void *context,u64 address,const void *source,
usize length) {
volatile u8 *target=(volatile u8 *)(uintptr_t)address;
const u8 *bytes=(const u8 *)source;
(void)context;
if((!target&&length)||(!bytes&&length)||length>BOB64_SYSCALL_MAX_BUFFER)
return -1;
for(usize i=0;i<length;i++)target[i]=bytes[i];
return 0;
}
static s64 kernel_app_read_file(void *context,const char *name,u8 *buffer,
usize capacity) {
KERNEL_APP_CONTEXT *app=(KERNEL_APP_CONTEXT *)context;
const char *data;
usize length;
if(!app||!app->Filesystem||!name||(!buffer&&capacity))return -22;
if(bob64_fs_read(app->Filesystem,name,&data,&length))return -2;
if(length>capacity)return -28;
for(usize i=0;i<length;i++)buffer[i]=(u8)data[i];
return (s64)length;
}
static s64 kernel_app_write_file(void *context,const char *name,const u8 *buffer,
usize length) {
KERNEL_APP_CONTEXT *app=(KERNEL_APP_CONTEXT *)context;
if(!app||!app->Filesystem||!name||(!buffer&&length))return -22;
if(bob64_fs_write(app->Filesystem,name,buffer,length))return -28;
return (s64)length;
}
static s64 kernel_app_open_stream(void *context,const char *name,u32 flags) {
KERNEL_APP_CONTEXT *app=(KERNEL_APP_CONTEXT *)context;
const char *data;usize length;
if(!app||!app->Filesystem||!name)return -22;
if(bob64_fs_read(app->Filesystem,name,&data,&length)) {
if(!(flags&BOB64_FILE_OPEN_CREATE))return -2;
if(bob64_fs_write(app->Filesystem,name,0,0))return -28;
length=0;
} else if(flags&BOB64_FILE_OPEN_TRUNCATE) {
if(bob64_fs_write(app->Filesystem,name,0,0))return -28;
length=0;
}
if(length>0x7fffffffffffffffULL)return -75;
return (s64)length;
}
static s64 kernel_app_read_stream(void *context,const char *name,u64 offset,
u8 *buffer,usize capacity) {
KERNEL_APP_CONTEXT *app=(KERNEL_APP_CONTEXT *)context;
usize count=0;
if(!app||!app->Filesystem||!name||(!buffer&&capacity))return -22;
if(bob64_fs_read_at(app->Filesystem,name,offset,buffer,capacity,&count))return -2;
return (s64)count;
}
static s64 kernel_app_write_stream(void *context,const char *name,u64 offset,
const u8 *buffer,usize length) {
KERNEL_APP_CONTEXT *app=(KERNEL_APP_CONTEXT *)context;
if(!app||!app->Filesystem||!name||(!buffer&&length))return -22;
if(bob64_fs_write_at(app->Filesystem,name,offset,buffer,length))return -28;
return (s64)length;
}
typedef struct {
BOB64_FILE_INFO *Entries;
usize Capacity,Count;
int Full;
} KERNEL_FILE_LIST_CONTEXT;
static int kernel_app_list_file(void *context,const char *name,usize size) {
KERNEL_FILE_LIST_CONTEXT *list=(KERNEL_FILE_LIST_CONTEXT *)context;
usize length=0;
BOB64_FILE_INFO *entry;
while(name[length]&&length<sizeof(list->Entries[0].Name)-1)length++;
if(name[length]||list->Count>=list->Capacity) {
list->Full=1;return 1;
}
entry=&list->Entries[list->Count++];
for(usize i=0;i<sizeof(*entry);i++)((u8 *)entry)[i]=0;
for(usize i=0;i<length;i++)entry->Name[i]=name[i];
entry->Size=(u64)size;
return 0;
}
static s64 kernel_app_list_files(void *context,BOB64_FILE_INFO *entries,
usize capacity) {
KERNEL_APP_CONTEXT *app=(KERNEL_APP_CONTEXT *)context;
KERNEL_FILE_LIST_CONTEXT list={entries,capacity,0,0};
if(!app||!app->Filesystem||!entries||!capacity)return -22;
bob64_fs_list(app->Filesystem,kernel_app_list_file,&list);
return list.Full?-28:(s64)list.Count;
}
static s64 kernel_app_delete_file(void *context,const char *name) {
KERNEL_APP_CONTEXT *app=(KERNEL_APP_CONTEXT *)context;
if(!app||!app->Filesystem||!name)return -22;
return bob64_fs_delete(app->Filesystem,name)?-2:0;
}
static s64 kernel_run_application(void *context,const char *name,usize argument_count,
const char *const *arguments,int *started) {
BOB64_PAGE_TABLE process_table;
KERNEL_PROCESS_CONTEXT process_context={&process_table};
BOB64_PROCESS_OPERATIONS operations;
BOB64_PROCESS process;
BOB64_PROCESS_PAGE *pages;
const char *image;
usize image_size;
u64 kernel_root,table_checkpoint;
usize heap_checkpoint;
s64 status=-1;
KERNEL_APP_CONTEXT *app=(KERNEL_APP_CONTEXT *)context;
void *application_privilege_stack=0;
BOB64_WINDOW_SERVER *window_server=0;
u64 previous_privilege_stack=bob64_kernel_tss.Rsp0;
int entered=0,cleanup_error=0,syscall_context_saved=0;
int window_server_ready=0;
u64 window_owner=0;
if(started)*started=0;
if(!app||!app->Filesystem||!app->NxSupported||!started||
!app->PhysicalAddressBits||
bob64_user_depth>=BOB64_USER_ENTRY_MAX_DEPTH||
bob64_fs_read(app->Filesystem,name,&image,&image_size))return -1;
operations=(BOB64_PROCESS_OPERATIONS){process_allocate_page,process_free_page,
process_map_page,process_protect_page,process_unmap_page,&process_context,
app->NxSupported};
pages=(BOB64_PROCESS_PAGE *)bob64_heap_calloc(&kernel_heap,
(usize)BOB64_PROCESS_PAGE_LIMIT,sizeof(*pages));
if(!pages)return -1;
kernel_root=read_cr3();
table_checkpoint=bob64_bootstrap_space.TablePoolUsed;
heap_checkpoint=kernel_heap_high_water;
if(bob64_process_init(&process,pages,(usize)BOB64_PROCESS_PAGE_LIMIT,&operations))
goto done;
if(bob64_page_table_init(&process_table,app->PhysicalAddressBits,
bob64_bootstrap_space.PageTable.Allocate,bob64_bootstrap_space.PageTable.Access,
bob64_bootstrap_space.PageTable.Context))goto done;
if(bob64_page_table_clone_isolated(&process_table,&bob64_bootstrap_space.PageTable,
BOB64_PROCESS_IMAGE_BASE,BOB64_PROCESS_ISOLATED_SIZE))goto done;
__asm__ volatile("mov %0,%%cr3"::"r"(process_table.RootPhysical):"memory");
if(bob64_process_load(&process,image,image_size,argument_count,arguments))goto restore;
application_privilege_stack=bob64_heap_alloc(&kernel_heap,
(usize)BOB64_KERNEL_PRIVILEGE_STACK_SIZE);
if(!application_privilege_stack) {
if(bob64_process_unload(&process))cleanup_error=1;
goto restore;
}
for(usize i=0;i<(usize)BOB64_KERNEL_PRIVILEGE_STACK_SIZE;i++)
((u8 *)application_privilege_stack)[i]=0;
u64 application_stack_top=(u64)(uintptr_t)application_privilege_stack+
BOB64_KERNEL_PRIVILEGE_STACK_SIZE;
if(bob64_tss_set_rsp0(&bob64_kernel_tss,application_stack_top)||
bob64_kernel_tss.Rsp0!=application_stack_top) {
if(bob64_process_unload(&process))cleanup_error=1;
goto restore;
}
if(bob64_syscall_context_push()) {
if(bob64_process_unload(&process))cleanup_error=1;
goto restore;
}
syscall_context_saved=1;
kernel_user_output_length=0;
kernel_user_output_truncated=0;
u32 display_width,display_height;
if(!bob64_framebuffer_resolution(&display_width,&display_height)) {
window_owner=kernel_next_window_owner++;
if(!window_owner)window_owner=kernel_next_window_owner++;
window_server=(BOB64_WINDOW_SERVER *)bob64_heap_calloc(&kernel_heap,1,
sizeof(*window_server));
if(window_server&&!bob64_window_server_init(window_server,&kernel_heap,
window_owner,display_width,display_height))window_server_ready=1;
else if(window_server) {
bob64_window_server_close(window_server);
if(bob64_heap_free(&kernel_heap,window_server))cleanup_error=1;
window_server=0;
}
}
bob64_syscall_set_address_space(&process_table);
bob64_syscall_set_read_user(kernel_copy_user_bytes,0);
bob64_syscall_set_write_user(kernel_write_user_bytes,0);
bob64_syscall_set_filesystem(kernel_app_read_file,kernel_app_write_file,app);
bob64_syscall_set_file_stream(kernel_app_open_stream,kernel_app_read_stream,
kernel_app_write_stream,app);
bob64_syscall_set_file_manager(kernel_app_list_files,kernel_app_delete_file,app);
bob64_syscall_set_app_runner(kernel_app_run_application,app);
bob64_syscall_set_write(kernel_user_write_character,0);
bob64_syscall_set_wait_event(kernel_wait_for_input_event,0);
bob64_syscall_set_window_server(window_server_ready?window_server:0,
window_owner);
*started=1;entered=1;
status=bob64_enter_user(process.EntryAddress,process.StartupAddress,
process.InitialStackPointer);
kernel_user_output[kernel_user_output_length]=0;
if(bob64_tss_set_rsp0(&bob64_kernel_tss,previous_privilege_stack)||
bob64_kernel_tss.Rsp0!=previous_privilege_stack)cleanup_error=1;
if(bob64_heap_free(&kernel_heap,application_privilege_stack))cleanup_error=1;
application_privilege_stack=0;
if(window_server_ready)bob64_window_server_close(window_server);
if(window_server&&bob64_heap_free(&kernel_heap,window_server))cleanup_error=1;
if(syscall_context_saved&&bob64_syscall_context_pop())cleanup_error=1;
kernel_write("\r\n");
if(bob64_process_unload(&process))cleanup_error=1;
restore:
if(application_privilege_stack) {
if(bob64_tss_set_rsp0(&bob64_kernel_tss,previous_privilege_stack))
cleanup_error=1;
if(bob64_heap_free(&kernel_heap,application_privilege_stack))cleanup_error=1;
application_privilege_stack=0;
}
__asm__ volatile("mov %0,%%cr3"::"r"(kernel_root):"memory");
done:
/* A heap peak can leave shared page-table levels mapped after the heap
later shrinks. Preserve those table pages across this address space. */
if(kernel_heap_high_water==heap_checkpoint)
bob64_bootstrap_space.TablePoolUsed=table_checkpoint;
if(bob64_heap_free(&kernel_heap,pages)||cleanup_error)status=-1;
if(!entered)*started=0;
return status;
}
static int kernel_app_run_application(void *context,const char *name,
usize argument_count,
const char *const *arguments,
s64 *exit_status) {
int started=0;
s64 status;
if(!exit_status)return -22;
status=kernel_run_application(context,name,argument_count,arguments,&started);
if(!started)return -2;
*exit_status=status;
return 0;
}
static int kernel_compile_application(void *context,const char *source_name,
const char *output_name,usize *error_offset) {
KERNEL_APP_CONTEXT *app=(KERNEL_APP_CONTEXT *)context;
const char *source;
usize source_length,output_length;
int result;
if(!app||!app->Filesystem||!source_name||!output_name||
bob64_fs_read(app->Filesystem,source_name,&source,&source_length))return -1;
result=bob64_compile_c(source,source_length,kernel_compiler_output,
sizeof(kernel_compiler_output),&output_length,error_offset);
if(result)return result;
if(bob64_fs_write(app->Filesystem,output_name,kernel_compiler_output,output_length))
return -1;
return 0;
}
static void kernel_hex64(u64 value) {
static const char digits[]="0123456789abcdef";
char text[17];
for(u32 i=0;i<16;i++)text[i]=digits[(value>>(60-i*4))&15];
text[16]=0;
kernel_write(text);
}
static int kernel_heap_grow(void *context,void **region,usize *region_size) {
const BOB64_KERNEL_BOOT_INFO *info=(const BOB64_KERNEL_BOOT_INFO *)context;
u64 physical,virtual_address;
int map_result;
if(!info||!region||!region_size||kernel_heap_mapped>=BOB64_KERNEL_HEAP_LIMIT)
return 0;
physical=bob64_page_alloc(&bob64_boot_page_allocator,1);
if(!physical) {
kernel_write("bob64 kernel: heap physical page allocation failed\r\n");
return 0;
}
virtual_address=BOB64_KERNEL_HEAP_BASE+kernel_heap_mapped;
map_result=bob64_page_map(&bob64_bootstrap_space.PageTable,virtual_address,
physical,BOB64_PAGE_WRITE|(info->NxSupported?BOB64_PAGE_NX:0));
if(map_result) {
kernel_write("bob64 kernel: heap page-table mapping failed (result=0x");
kernel_hex64((u64)(s64)map_result);
kernel_write(" tables=0x");kernel_hex64(bob64_bootstrap_space.TablePoolUsed);
kernel_write("/0x");kernel_hex64(bob64_bootstrap_space.TablePoolPages);
kernel_write(")\r\n");
bob64_page_free(&bob64_boot_page_allocator,physical,1);
return 0;
}
__asm__ volatile("invlpg (%0)"::"r"((void *)(uintptr_t)virtual_address):"memory");
volatile u64 *words=(volatile u64 *)(uintptr_t)virtual_address;
for(usize i=0;i<EFI_PAGE_SIZE/sizeof(u64);i++)words[i]=0;
kernel_heap_mapped+=(usize)EFI_PAGE_SIZE;
if(kernel_heap_mapped>kernel_heap_high_water)
kernel_heap_high_water=kernel_heap_mapped;
*region=(void *)(uintptr_t)virtual_address;
*region_size=(usize)EFI_PAGE_SIZE;
return 1;
}
static int kernel_heap_shrink(void *context,void *region,usize region_size) {
const BOB64_KERNEL_BOOT_INFO *info=(const BOB64_KERNEL_BOOT_INFO *)context;
u64 expected;
if(!info||!region||!region_size||
(region_size&(usize)(EFI_PAGE_SIZE-1))||
region_size>kernel_heap_mapped||
region_size>~(u64)0-(u64)(uintptr_t)BOB64_KERNEL_HEAP_BASE)
return -1;
expected=BOB64_KERNEL_HEAP_BASE+kernel_heap_mapped-region_size;
if((u64)(uintptr_t)region!=expected)return -1;
for(usize offset=0;offset<region_size;offset+=(usize)EFI_PAGE_SIZE) {
u64 translated,flags;
if(bob64_page_translate(&bob64_bootstrap_space.PageTable,
expected+offset,&translated,&flags)!=1||
(translated&(EFI_PAGE_SIZE-1))||
(flags&(BOB64_PAGE_PRESENT|BOB64_PAGE_WRITE))!=
(BOB64_PAGE_PRESENT|BOB64_PAGE_WRITE))return -1;
}
for(usize offset=0;offset<region_size;offset+=(usize)EFI_PAGE_SIZE) {
u64 unmapped_physical;
if(bob64_page_unmap(&bob64_bootstrap_space.PageTable,expected+offset,
&unmapped_physical,0))return -1;
__asm__ volatile("invlpg (%0)"::"r"((void *)(uintptr_t)(expected+offset)):
"memory");
if(bob64_page_free(&bob64_boot_page_allocator,unmapped_physical,1))
return -1;
}
kernel_heap_mapped-=region_size;
return 0;
}
static int kernel_heap_smoke_test(const BOB64_KERNEL_BOOT_INFO *info) {
char *first,*large,*zero;
if(bob64_heap_init(&kernel_heap,(usize)BOB64_KERNEL_HEAP_LIMIT,
kernel_heap_grow,(void *)info))return -1;
bob64_heap_set_shrink(&kernel_heap,kernel_heap_shrink);
first=(char *)bob64_heap_alloc(&kernel_heap,64);
large=(char *)bob64_heap_alloc(&kernel_heap,5000);
zero=(char *)bob64_heap_calloc(&kernel_heap,32,2);
if(!first||!large||!zero||((uintptr_t)first&15)||((uintptr_t)large&15)||
((uintptr_t)zero&15)||!((uintptr_t)first>=BOB64_KERNEL_HEAP_BASE))return -1;
first[0]='b';first[1]='o';first[2]='b';first[3]=0;
large[0]='6';large[1]='4';large[4999]='!';
for(usize i=0;i<64;i++)if(zero[i])return -1;
if(first[0]!='b'||first[1]!='o'||first[2]!='b'||large[0]!='6'||
large[1]!='4'||large[4999]!='!')return -1;
if(bob64_heap_free(&kernel_heap,large)||bob64_heap_free(&kernel_heap,first)||
bob64_heap_free(&kernel_heap,zero)||bob64_heap_free(&kernel_heap,zero)!=-1)
return -1;
if(bob64_heap_mapped_bytes(&kernel_heap)!=EFI_PAGE_SIZE||
kernel_heap_mapped!=EFI_PAGE_SIZE)return -1;
return 0;
}
static int kernel_table_pool_smoke_test(const BOB64_KERNEL_BOOT_INFO *info) {
BOB64_PAGE_TABLE table;
u64 checkpoint,initial_pages,physical;
if(!info||!bob64_bootstrap_space.TablePoolPages)return -1;
checkpoint=bob64_bootstrap_space.TablePoolUsed;
initial_pages=bob64_bootstrap_space.TablePoolPages;
if(bob64_page_table_init(&table,info->PhysicalAddressBits,
bob64_bootstrap_space.PageTable.Allocate,
bob64_bootstrap_space.PageTable.Access,
bob64_bootstrap_space.PageTable.Context))return -1;
for(u64 i=0;i<140;i++) {
u64 virtual_address=0x0000001000000000ULL+i*(2ULL<<20);
u64 physical_address=0x0000000010000000ULL+i*BOB64_PAGE_SIZE;
if(bob64_page_map(&table,virtual_address,physical_address,
BOB64_PAGE_WRITE|BOB64_PAGE_NX)) {
bob64_bootstrap_space.TablePoolUsed=checkpoint;
return -1;
}
}
int valid=bob64_bootstrap_space.TablePoolPages>initial_pages&&
bob64_bootstrap_space.TablePoolUsed>checkpoint&&
bob64_page_translate(&table,0x0000001000000000ULL+139*(2ULL<<20),
&physical,0)==1&&
physical==0x0000000010000000ULL+139*BOB64_PAGE_SIZE;
bob64_bootstrap_space.TablePoolUsed=checkpoint;
return valid?0:-1;
}
static int kernel_privilege_stack_init(void) {
kernel_privilege_stack=bob64_heap_alloc(&kernel_heap,
(usize)BOB64_KERNEL_PRIVILEGE_STACK_SIZE);
if(!kernel_privilege_stack)return -1;
for(usize i=0;i<(usize)BOB64_KERNEL_PRIVILEGE_STACK_SIZE;i++)
((u8 *)kernel_privilege_stack)[i]=0;
return bob64_tss_set_rsp0(&bob64_kernel_tss,
(u64)(uintptr_t)kernel_privilege_stack+
BOB64_KERNEL_PRIVILEGE_STACK_SIZE);
}
static void shell_write(void *context,const char *text) {
(void)context;
kernel_write(text);
}
static void shell_clear(void *context) {
(void)context;
bob64_framebuffer_clear();
bob64_early_console_write("\x1b[2J\x1b[H");
}
static EFI_RUNTIME_SERVICES *kernel_runtime_services(
const BOB64_KERNEL_BOOT_INFO *info) {
return info&&info->RuntimeServices?
(EFI_RUNTIME_SERVICES *)(uintptr_t)info->RuntimeServices:0;
}
static int kernel_snapshot_save(void *context,const void *snapshot,
usize snapshot_size) {
const BOB64_KERNEL_BOOT_INFO *info=(const BOB64_KERNEL_BOOT_INFO *)context;
EFI_STATUS firmware_status=EFI_SUCCESS;
UINTN maximum_variable_size=0,remaining_storage_size=0;
int result=bob64_firmware_snapshot_save(kernel_runtime_services(info),
&kernel_heap,snapshot,snapshot_size,&firmware_status,
&maximum_variable_size,&remaining_storage_size);
if(result) {
kernel_write("firmware snapshot save failed (EFI 0x");
kernel_hex64(firmware_status);kernel_write(" max=0x");
kernel_hex64(maximum_variable_size);kernel_write(" free=0x");
kernel_hex64(remaining_storage_size);kernel_write(")\r\n");
}
for(u32 i=0;i<kernel_xhci_storage_device_count;i++) {
KERNEL_XHCI_STORAGE_DEVICE *device=&kernel_xhci_storage_devices[i];
if(!device->HasDataPartition||device->BlockDevice.ReadOnly)continue;
int disk_result=bob64_disk_store_save(&device->BlockDevice,
&device->DataPartition,&kernel_filesystem,&kernel_heap);
if(!disk_result) {
kernel_write("bob64 kernel: disk B64S snapshot saved\r\n");
return 0;
}
kernel_write("bob64 kernel: disk B64S snapshot save failed result=");
kernel_hex64((u64)(s64)disk_result);kernel_write("\r\n");
return result?disk_result:0;
}
return result;
}
static int kernel_snapshot_restore(void *context,BOB64_FILESYSTEM *filesystem) {
const BOB64_KERNEL_BOOT_INFO *info=(const BOB64_KERNEL_BOOT_INFO *)context;
EFI_RUNTIME_SERVICES *services=kernel_runtime_services(info);
int disk_error=0;
for(u32 i=0;i<kernel_xhci_storage_device_count;i++) {
KERNEL_XHCI_STORAGE_DEVICE *device=&kernel_xhci_storage_devices[i];
if(!device->HasDataPartition)continue;
int disk_result=bob64_disk_store_restore(&device->BlockDevice,
&device->DataPartition,filesystem,&kernel_heap);
if(!disk_result) {
kernel_write("bob64 kernel: disk B64S snapshot restored\r\n");
return 0;
}
if(disk_result<0)disk_error=disk_result;
}
int firmware_result=services?bob64_firmware_snapshot_restore(services,
filesystem):BOB64_FIRMWARE_SNAPSHOT_NOT_FOUND;
if(!firmware_result)return 0;
return disk_error?disk_error:firmware_result;
}
__attribute__((noreturn)) void bob64_kernel_main(const BOB64_KERNEL_BOOT_INFO *info) {
bob64_early_console_init();
if(info&&info->HasFramebuffer&&bob64_framebuffer_init(info->FramebufferBase,
info->FramebufferSize,info->FramebufferWidth,info->FramebufferHeight,
info->FramebufferPixelsPerScanLine,info->FramebufferPixelFormat))
bob64_early_console_write("bob64 kernel: framebuffer rejected\r\n");
kernel_write("bob64!\r\nbob64 kernel: long mode entry\r\n");
if(!info||sizeof(void *)!=8||read_cr3()!=info->PageTableRoot||
!info->MemoryMapAddress||!info->MemoryMapSize||
info->MemoryDescriptorSize<40||!info->StackBase||!info->StackSize||
info->PhysicalAddressBits<36||info->PhysicalAddressBits>52||
info->NxSupported>1||info->HasFramebuffer>1) {
kernel_write("bob64 kernel: invalid boot information\r\n");
kernel_halt();
}
if(kernel_heap_smoke_test(info)) {
kernel_write("bob64 kernel: heap initialization failed\r\n");
kernel_halt();
}
if(kernel_table_pool_smoke_test(info)) {
kernel_write("bob64 kernel: page-table pool growth failed\r\n");
kernel_halt();
}
if(bob64_interrupts_enable_timer()) {
kernel_write("bob64 kernel: timer IRQ unavailable\r\n");
kernel_halt();
}
kernel_write("bob64 kernel: timer IRQ enabled\r\n");
u64 initial_timer_ticks=bob64_timer_ticks();
for(volatile u32 timer_wait=0;
timer_wait<100000000u&&bob64_timer_ticks()==initial_timer_ticks;
timer_wait++)
__asm__ volatile("pause");
if(bob64_timer_ticks()==initial_timer_ticks) {
kernel_write("bob64 kernel: timer tick test failed\r\n");
kernel_halt();
}
kernel_write("bob64 kernel: timer tick passed\r\n");
u32 pci_device_count=0;
if(!bob64_pci_enumerate(kernel_pci_read_config,0,
kernel_pci_report_usb,0,&pci_device_count)) {
kernel_write("bob64 kernel: PCI enumeration passed, devices=");
kernel_hex64(pci_device_count);kernel_write("\r\n");
if(kernel_xhci_found) {
int xhci_result=kernel_xhci_probe_capabilities(info);
if(xhci_result) {
kernel_write("bob64 kernel: xHCI capability probe failed, result=0x");
kernel_hex64((u64)(s64)xhci_result);kernel_write(" phys-bits=");
kernel_hex64(info->PhysicalAddressBits);kernel_write("\r\n");
}
}
} else kernel_write("bob64 kernel: PCI enumeration unavailable\r\n");
kernel_write("bob64 kernel: dynamic page-table pool passed\r\n");
if(kernel_privilege_stack_init()) {
kernel_write("bob64 kernel: privilege stack initialization failed\r\n");
kernel_halt();
}
int process_test=kernel_process_smoke_test(info);
if(process_test<0) {
kernel_write("bob64 kernel: application loader smoke test failed\r\n");
kernel_halt();
}
if(!process_test)kernel_write("bob64 kernel: ring-3 bob! app passed\r\n");
else kernel_write("bob64 kernel: app test skipped (NX unavailable)\r\n");
if(bob64_fs_init(&kernel_filesystem,&kernel_heap)) {
kernel_write("bob64 kernel: filesystem initialization failed\r\n");
kernel_halt();
}
if(info->NxSupported&&kernel_install_bob_application(&kernel_filesystem))
kernel_write("bob64 kernel: demo app install failed\r\n");
if(info->NxSupported&&kernel_install_gui_application(&kernel_filesystem))
kernel_write("bob64 kernel: desktop app install failed\r\n");
if(info->NxSupported&&kernel_install_nested_application(&kernel_filesystem))
kernel_write("bob64 kernel: nested smoke app install failed\r\n");
if(info->NxSupported&&kernel_install_echo_application(&kernel_filesystem))
kernel_write("bob64 kernel: echo app install failed\r\n");
if(kernel_install_bob_source(&kernel_filesystem))
kernel_write("bob64 kernel: C demo source install failed\r\n");
if(info->NxSupported&&kernel_install_cat_application(&kernel_filesystem))
kernel_write("bob64 kernel: cat app install failed\r\n");
if(info->NxSupported&&kernel_install_ls_application(&kernel_filesystem))
kernel_write("bob64 kernel: ls app install failed\r\n");
if(info->NxSupported&&kernel_install_notes_application(&kernel_filesystem))
kernel_write("bob64 kernel: notes app install failed\r\n");
if(info->NxSupported&&kernel_install_info_application(&kernel_filesystem))
kernel_write("bob64 kernel: info app install failed\r\n");
if(info->NxSupported&&kernel_install_mouse_smoke_application(&kernel_filesystem))
kernel_write("bob64 kernel: mouse smoke app install failed\r\n");
if(info->NxSupported&&kernel_install_launcher_application(&kernel_filesystem))
kernel_write("bob64 kernel: launcher app install failed\r\n");
if(info->NxSupported&&kernel_install_display_application(&kernel_filesystem))
kernel_write("bob64 kernel: graphics demo install failed\r\n");
kernel_app_context.Filesystem=&kernel_filesystem;
kernel_app_context.NxSupported=info->NxSupported;
kernel_app_context.PhysicalAddressBits=info->PhysicalAddressBits;
kernel_write("CR3=0x");kernel_hex64(info->PageTableRoot);
kernel_write(" memory-map=0x");kernel_hex64(info->MemoryMapAddress);
kernel_write(" bytes=0x");kernel_hex64(info->MemoryMapSize);
kernel_write(" heap=0x");kernel_hex64(BOB64_KERNEL_HEAP_BASE);
kernel_write(" mapped=0x");kernel_hex64((u64)bob64_heap_mapped_bytes(&kernel_heap));
kernel_write("\r\n");
kernel_write("bob64 kernel: memory checks passed\r\n");
if(!bob64_interrupts_enable_serial())
kernel_write("bob64 kernel: serial RX IRQ enabled\r\n");
else
kernel_write("bob64 kernel: serial RX polling fallback\r\n");
bob64_keyboard_reset();
int keyboard_irq_enabled=bob64_interrupts_enable_keyboard()==0;
if(!keyboard_irq_enabled)
kernel_write("bob64 kernel: keyboard IRQ unavailable; using polling\r\n");
else kernel_write("bob64 kernel: keyboard IRQ enabled\r\n");
if(info->NxSupported) {
int started=0;
s64 status=kernel_run_application(&kernel_app_context,"bob.b64e",0,0,&started);
if(!started||status||(keyboard_irq_enabled&&!kernel_interrupts_enabled())) {
kernel_write("bob64 kernel: native streaming app test failed\r\n");
kernel_halt();
}
kernel_write("bob64 kernel: native streaming app passed\r\n");
started=0;
status=kernel_run_application(&kernel_app_context,"nested.b64e",0,0,
&started);
if(!started||status||(keyboard_irq_enabled&&!kernel_interrupts_enabled())) {
kernel_write("bob64 kernel: nested app return test failed\r\n");
kernel_halt();
}
kernel_write("bob64 kernel: nested app return passed\r\n");
static const char *echo_arguments[]={"echo.b64e","bob!"};
started=0;
status=kernel_run_application(&kernel_app_context,"echo.b64e",2,
echo_arguments,&started);
if(!started||status||kernel_user_output_length!=5||
!kernel_bytes_equal(kernel_user_output,"bob!\n")||
(keyboard_irq_enabled&&!kernel_interrupts_enabled())) {
kernel_write("bob64 kernel: echo argument test failed\r\n");
kernel_halt();
}
kernel_write("bob64 kernel: native echo arguments passed\r\n");
static const char *cat_arguments[]={"cat.b64e","cat-smoke.txt"};
static const char cat_fixture[]="bob!";
if(bob64_fs_write(&kernel_filesystem,"cat-smoke.txt",cat_fixture,
sizeof(cat_fixture)-1)) {
kernel_write("bob64 kernel: cat fixture setup failed\r\n");
kernel_halt();
}
started=0;
status=kernel_run_application(&kernel_app_context,"cat.b64e",2,
cat_arguments,&started);
int cat_output_ok=kernel_user_output_length==5&&
kernel_bytes_equal(kernel_user_output,"bob!\n");
if(bob64_fs_delete(&kernel_filesystem,"cat-smoke.txt")) {
kernel_write("bob64 kernel: cat fixture cleanup failed\r\n");
kernel_halt();
}
if(!started||status||!cat_output_ok||
(keyboard_irq_enabled&&!kernel_interrupts_enabled())) {
kernel_write("bob64 kernel: native file-read app test failed\r\n");
kernel_halt();
}
kernel_write("bob64 kernel: native file-read app passed\r\n");
started=0;
status=kernel_run_application(&kernel_app_context,"ls.b64e",0,0,
&started);
if(!started||status||kernel_user_output_truncated||
!kernel_text_contains(kernel_user_output,"bob.b64e ")||
(keyboard_irq_enabled&&!kernel_interrupts_enabled())) {
kernel_write("bob64 kernel: native file-list app test failed\r\n");
kernel_halt();
}
kernel_write("bob64 kernel: native file-list app passed\r\n");
static const char *notes_put_arguments[]={"notes.b64e","put",
"notes-smoke.txt","bob!"};
static const char *notes_show_arguments[]={"notes.b64e","show",
"notes-smoke.txt"};
static const char *notes_delete_arguments[]={"notes.b64e","delete",
"notes-smoke.txt"};
static const char *notes_copy_arguments[]={"notes.b64e","copy",
"notes-large-source.txt","notes-large-copy.txt"};
static u8 note_fixture_check[4];
started=0;
status=kernel_run_application(&kernel_app_context,"notes.b64e",4,
notes_put_arguments,&started);
s64 note_read_result=kernel_app_read_file(&kernel_app_context,
"notes-smoke.txt",note_fixture_check,sizeof(note_fixture_check));
if(!started||status||note_read_result!=4||
note_fixture_check[0]!='b'||note_fixture_check[1]!='o'||
note_fixture_check[2]!='b'||note_fixture_check[3]!='!'||
(keyboard_irq_enabled&&!kernel_interrupts_enabled())) {
kernel_write("bob64 kernel: native notes write test failed\r\n");
kernel_halt();
}
started=0;
status=kernel_run_application(&kernel_app_context,"notes.b64e",3,
notes_show_arguments,&started);
if(!started||status||kernel_user_output_length!=5||
!kernel_bytes_equal(kernel_user_output,"bob!\n")||
(keyboard_irq_enabled&&!kernel_interrupts_enabled())) {
kernel_write("bob64 kernel: native notes show test failed\r\n");
kernel_halt();
}
started=0;
status=kernel_run_application(&kernel_app_context,"notes.b64e",3,
notes_delete_arguments,&started);
if(!started||status||kernel_app_read_file(&kernel_app_context,
"notes-smoke.txt",note_fixture_check,sizeof(note_fixture_check))!=-2||
(keyboard_irq_enabled&&!kernel_interrupts_enabled())) {
kernel_write("bob64 kernel: native notes delete test failed\r\n");
kernel_halt();
}
static char large_note_fixture[12288];
const char *large_note_copy=0;
usize large_note_copy_length=0;
for(usize i=0;i<sizeof(large_note_fixture);i++)
large_note_fixture[i]=(i%80==79)?'\n':(char)('a'+i%26);
if(bob64_fs_write(&kernel_filesystem,"notes-large-source.txt",
large_note_fixture,sizeof(large_note_fixture))) {
kernel_write("bob64 kernel: large notes fixture setup failed\r\n");
kernel_halt();
}
started=0;
status=kernel_run_application(&kernel_app_context,"notes.b64e",4,
notes_copy_arguments,&started);
int large_note_match=!bob64_fs_read(&kernel_filesystem,
"notes-large-copy.txt",&large_note_copy,&large_note_copy_length)&&
large_note_copy_length==sizeof(large_note_fixture);
for(usize i=0;large_note_match&&i<sizeof(large_note_fixture);i++)
if(large_note_copy[i]!=large_note_fixture[i])large_note_match=0;
if(!started||status||kernel_user_output_length!=8||
!kernel_bytes_equal(kernel_user_output,"Copied.\n")||
kernel_user_output_truncated||!large_note_match||
(keyboard_irq_enabled&&!kernel_interrupts_enabled())) {
kernel_write("bob64 kernel: large notes streaming test failed\r\n");
kernel_halt();
}
kernel_write("bob64 kernel: large notes streaming copy passed\r\n");
if(bob64_fs_delete(&kernel_filesystem,"notes-large-source.txt")||
bob64_fs_delete(&kernel_filesystem,"notes-large-copy.txt")) {
kernel_write("bob64 kernel: large notes fixture cleanup failed\r\n");
kernel_halt();
}
kernel_write("bob64 kernel: native notes app passed\r\n");
started=0;
status=kernel_run_application(&kernel_app_context,"info.b64e",0,0,
&started);
if(!started||status||kernel_user_output_truncated||
!kernel_text_contains(kernel_user_output,
"bob64 system information\n")||
!kernel_text_contains(kernel_user_output,"System call ABI: 12\n")||
!kernel_text_contains(kernel_user_output,"Uptime: ")||
!kernel_text_contains(kernel_user_output,"RAM files: ")||
!kernel_text_contains(kernel_user_output,"RAM file data: ")||
kernel_user_output_length<5||
!kernel_bytes_equal(kernel_user_output+
kernel_user_output_length-5,"bob!\n")||
(keyboard_irq_enabled&&!kernel_interrupts_enabled())) {
kernel_write("bob64 kernel: native info app test failed\r\n");
kernel_halt();
}
kernel_write("bob64 kernel: native info app passed\r\n");
}
int snapshot_restore=kernel_snapshot_restore((void *)info,
&kernel_filesystem);
if(!snapshot_restore) {
kernel_write("bob64 kernel: persistent B64S snapshot restored\r\n");
const char *persisted_contents;
usize persisted_length;
if(!bob64_fs_read(&kernel_filesystem,"persist.txt",&persisted_contents,
&persisted_length)&&persisted_length==8&&
kernel_bytes_equal(persisted_contents,"survived"))
kernel_write("bob64 kernel: persistent B64S smoke file passed\r\n");
} else if(snapshot_restore<0)
kernel_write("bob64 kernel: persistent B64S snapshot rejected\r\n");
u32 display_width,display_height;
if(!bob64_framebuffer_resolution(&display_width,&display_height)) {
if(bob64_mouse_init(display_width,display_height))
kernel_write("bob64 kernel: PS/2 mouse unavailable\r\n");
else if(bob64_interrupts_enable_mouse())
kernel_write("bob64 kernel: mouse IRQ unavailable; using polling\r\n");
else kernel_write("bob64 kernel: PS/2 mouse IRQ enabled\r\n");
}
if(bob64_shell_init(&kernel_shell,&bob64_boot_page_allocator,&kernel_heap,
&kernel_filesystem,
shell_write,shell_clear,&kernel_app_context)) {
kernel_write("bob64 kernel: shell initialization failed\r\n");
kernel_halt();
}
bob64_shell_set_runner(&kernel_shell,kernel_run_application);
bob64_shell_set_compiler(&kernel_shell,kernel_compile_application);
bob64_shell_set_snapshot_storage(&kernel_shell,kernel_snapshot_save,
kernel_snapshot_restore,(void *)info);
for(;;) {
kernel_xhci_usb_hid_poll();
int character=bob64_keyboard_poll();
if(character<0)character=bob64_early_console_try_read();
if(character>=0)bob64_shell_input(&kernel_shell,character);
else __asm__ volatile("pause");
}
}