基于 ESP32-S3 + VB6824 的四博 AI 双目交互终端设计：从双目动画到多模态事件系统

基于 ESP32-S3 VB6824 的四博 AI 双目交互终端设计从双目动画到多模态事件系统1. 项目背景AI 硬件如果只停留在“语音问答”用户体验会比较单薄。真正有产品感的 AI 终端需要具备完整的多模态交互能力语音输入唤醒、录音、打断 视觉输出双目动画、状态表情、情绪反馈 触摸输入顶部、左右、背部多点触控 姿态输入摇晃、翻转、倾斜、拿起 触觉输出震动确认、错误提醒、OTA节奏反馈 云端能力ASR / LLM / TTS / 知识库 / MCP 小程序能力配网、声音克隆、知识库、素材与固件更新四博 AI-S3 双目方案采用 ESP32-S3 主控配合 VB6824 语音前端和双目屏可以对接小智、豆包、ChatGPT 等主流大模型资料中也提到 AI-S3 双目双屏方案主要配件包括 1.28 寸 LCD 显示屏 x2、主板、麦克风、喇叭和线材选用 ESP32S3R8 16M Flash VB6824并支持二次开发。本文从固件工程角度设计一套适合 AI 双目音箱、AI 桌宠、AI 陪伴机器人、儿童学习终端和 IP 潮玩设备的嵌入式软件架构。2. 硬件架构四博 AI 双目可以抽象为 6 个硬件子系统1. 主控系统ESP32-S3R8 / ESP32-S3R2 2. 语音系统VB6824 Mic Speaker AMP 3. 显示系统0.71 / 1.28 寸双目屏 4. 交互系统四路触摸 / 按键 / 震动马达 5. 姿态系统三轴加速度计 / 六轴 IMU 6. 平台系统四博小助手 云端 AI MCP OTA资料中双目主板支持 0.71 寸和 1.28 寸屏V1/V2 版本对屏幕类型和调光能力有不同支持QA 中也给出 1.28 寸屏为 240×2400.71 寸屏为 160×160。推荐配置如下模块推荐配置主控ESP32-S3R8Flash16MBPSRAM推荐使用便于动画与音频缓存语音前端VB6824显示0.71 / 1.28 寸双目屏触摸4 路触摸节点姿态三轴加速度计反馈震动马达联网Wi-Fi BLE可扩展 4G平台四博小助手小程序AI 能力声音克隆、知识库、MCP、OTA3. 软件总体架构AI 双目设备不建议采用单一while(1)逻辑而应采用FreeRTOS 多任务 事件队列 状态机的方式。推荐任务划分app_main ├── wifi_task // Wi-Fi连接、断线重连 ├── blufi_task // 四博小助手 BLE 配网 ├── vb6824_task // 唤醒、打断、录音事件 ├── ai_ws_task // WebSocket / MQTT AI通信 ├── eye_render_task // 双目动画刷新 ├── touch_scan_task // 四路触摸扫描 ├── imu_detect_task // 姿态检测 ├── haptic_task // 震动马达控制 ├── mcp_dispatch_task // AI工具调用 ├── ota_task // 固件和素材升级 └── app_dispatch_task // 全局事件分发工程目录建议sibo_ai_eye/ ├── main/ │ ├── app_main.c │ ├── app_event.c │ ├── app_state.c │ ├── app_dispatch.c │ └── board_config.h ├── components/ │ ├── eye_display/ │ ├── touch_input/ │ ├── imu_sensor/ │ ├── haptic_motor/ │ ├── vb6824/ │ ├── ai_client/ │ ├── mcp_tools/ │ ├── ota_manager/ │ ├── asset_manager/ │ └── factory_test/ ├── partitions.csv └── sdkconfig.defaults4. 板级配置抽象因为 AI 双目可能有 0.71 寸、1.28 寸、1.8 寸等不同版本建议用board_config.h统一管理。#pragma once #define BOARD_NAME SIBO_AI_EYE_PRO #define BOARD_VERSION V1.0 #define CONFIG_USE_ESP32S3 1 #define CONFIG_USE_VB6824 1 #define CONFIG_EYE_SCREEN_128 1 #define CONFIG_EYE_SCREEN_071 0 #define CONFIG_USE_TOUCH_4CH 1 #define CONFIG_USE_IMU 1 #define CONFIG_USE_HAPTIC 1 #define CONFIG_USE_BATTERY 1 #define MOTOR_GPIO 15 #define IMU_I2C_PORT I2C_NUM_0 #define IMU_SDA_GPIO 8 #define IMU_SCL_GPIO 9 #define VB6824_UART UART_NUM_1 #define VB6824_TX_GPIO 17 #define VB6824_RX_GPIO 18 #define VB6824_BAUD 115200屏幕参数抽象typedef enum { EYE_SCREEN_TYPE_071 0, EYE_SCREEN_TYPE_128, EYE_SCREEN_TYPE_180, } eye_screen_type_t; typedef struct { eye_screen_type_t type; int width; int height; int fps; const char *asset_path; } eye_screen_config_t; eye_screen_config_t board_get_eye_config(void) { #if CONFIG_EYE_SCREEN_128 return (eye_screen_config_t) { .type EYE_SCREEN_TYPE_128, .width 240, .height 240, .fps 30, .asset_path /spiffs/eye_128, }; #else return (eye_screen_config_t) { .type EYE_SCREEN_TYPE_071, .width 160, .height 160, .fps 24, .asset_path /spiffs/eye_071, }; #endif }这样同一套固件工程可以适配不同屏幕 SKU。5. 全局事件系统设计所有输入都转化为事件由统一状态机分发typedef enum { APP_EVT_NONE 0, APP_EVT_WIFI_CONNECTED, APP_EVT_WIFI_DISCONNECTED, APP_EVT_BLUFI_START, APP_EVT_BLUFI_DONE, APP_EVT_WAKE_WORD, APP_EVT_VOICE_INTERRUPT, APP_EVT_RECORD_START, APP_EVT_RECORD_STOP, APP_EVT_TOUCH_TOP, APP_EVT_TOUCH_LEFT, APP_EVT_TOUCH_RIGHT, APP_EVT_TOUCH_BACK, APP_EVT_IMU_PICKUP, APP_EVT_IMU_SHAKE, APP_EVT_IMU_FLIP, APP_EVT_IMU_TILT_LEFT, APP_EVT_IMU_TILT_RIGHT, APP_EVT_AI_LISTENING, APP_EVT_AI_THINKING, APP_EVT_AI_SPEAKING, APP_EVT_AI_FINISHED, APP_EVT_AI_ERROR, APP_EVT_MCP_TOOL_CALL, APP_EVT_LOW_BATTERY, APP_EVT_OTA_START, APP_EVT_OTA_DONE, } app_event_id_t; typedef struct { app_event_id_t id; int param1; int param2; char payload[256]; } app_event_t; static QueueHandle_t g_app_event_queue;事件投递函数void app_post_event(app_event_id_t id, int p1, int p2, const char *payload) { if (!g_app_event_queue) { return; } app_event_t evt { .id id, .param1 p1, .param2 p2, }; if (payload) { strncpy(evt.payload, payload, sizeof(evt.payload) - 1); } xQueueSend(g_app_event_queue, evt, 0); }主程序入口void app_main(void) { ESP_ERROR_CHECK(nvs_flash_init()); g_app_event_queue xQueueCreate(32, sizeof(app_event_t)); if (!g_app_event_queue) { ESP_LOGE(APP, 事件队列创建失败); return; } eye_app_start(); haptic_app_start(); touch_app_start(); imu_app_start(); wifi_app_start(); blufi_app_start(); vb6824_app_start(); ai_client_start(); mcp_app_start(); ota_app_start(); app_dispatch_start(); ESP_LOGI(APP, 四博 AI 双目系统启动完成); }6. AI 状态机设计状态机负责将双目动画、音频、震动和联网状态统一起来。typedef enum { AI_STATE_BOOT 0, AI_STATE_IDLE, AI_STATE_WAKEUP, AI_STATE_LISTENING, AI_STATE_THINKING, AI_STATE_SPEAKING, AI_STATE_TOUCH_FEEDBACK, AI_STATE_SHAKE_FEEDBACK, AI_STATE_SLEEP, AI_STATE_NETWORK_ERROR, AI_STATE_LOW_BATTERY, AI_STATE_OTA, } ai_state_t; static ai_state_t g_ai_state AI_STATE_BOOT;状态切换void ai_set_state(ai_state_t state) { g_ai_state state; switch (state) { case AI_STATE_BOOT: eye_play_anim(boot); audio_play_prompt(boot.wav); break; case AI_STATE_IDLE: eye_play_anim(idle_blink); break; case AI_STATE_WAKEUP: eye_play_anim(wake); haptic_double(); audio_play_prompt(ding.wav); break; case AI_STATE_LISTENING: eye_play_anim(listening); audio_start_record(); break; case AI_STATE_THINKING: eye_play_anim(thinking); break; case AI_STATE_SPEAKING: eye_play_anim(speaking); break; case AI_STATE_TOUCH_FEEDBACK: eye_play_anim(happy); haptic_short(); break; case AI_STATE_SHAKE_FEEDBACK: eye_play_anim(surprised); haptic_double(); break; case AI_STATE_SLEEP: eye_play_anim(sleep); audio_stop_tts(); break; case AI_STATE_NETWORK_ERROR: eye_play_anim(net_error); haptic_long(); audio_play_prompt(network_error.wav); break; case AI_STATE_LOW_BATTERY: eye_play_anim(low_battery); haptic_long(); audio_play_prompt(low_battery.wav); break; case AI_STATE_OTA: eye_play_anim(ota); haptic_rhythm_ota(); break; default: eye_play_anim(idle_blink); break; } }7. 四路触摸输入四路触摸建议映射为触摸区域事件功能顶部APP_EVT_TOUCH_TOP唤醒 / 暂停 / 继续左侧APP_EVT_TOUCH_LEFT上一个角色 / 上一首右侧APP_EVT_TOUCH_RIGHT下一个角色 / 下一首背部APP_EVT_TOUCH_BACK配网 / 静音 / 恢复出厂触摸检测#include driver/touch_sensor.h #define TOUCH_THRESHOLD_PERCENT 70 typedef struct { touch_pad_t pad; uint32_t baseline; app_event_id_t evt; const char *name; } touch_key_t; static touch_key_t s_touch_keys[] { {TOUCH_PAD_NUM1, 0, APP_EVT_TOUCH_TOP, TOP}, {TOUCH_PAD_NUM2, 0, APP_EVT_TOUCH_LEFT, LEFT}, {TOUCH_PAD_NUM3, 0, APP_EVT_TOUCH_RIGHT, RIGHT}, {TOUCH_PAD_NUM4, 0, APP_EVT_TOUCH_BACK, BACK}, };触摸基线校准static void touch_calibrate(void) { for (int i 0; i 4; i) { uint32_t sum 0; for (int j 0; j 20; j) { uint32_t raw 0; touch_pad_read_raw_data(s_touch_keys[i].pad, raw); sum raw; vTaskDelay(pdMS_TO_TICKS(10)); } s_touch_keys[i].baseline sum / 20; ESP_LOGI(TOUCH, %s baseline%lu, s_touch_keys[i].name, s_touch_keys[i].baseline); } }触摸扫描任务static void touch_task(void *arg) { while (1) { for (int i 0; i 4; i) { uint32_t raw 0; touch_pad_read_raw_data(s_touch_keys[i].pad, raw); uint32_t threshold s_touch_keys[i].baseline * TOUCH_THRESHOLD_PERCENT / 100; if (raw threshold) { ESP_LOGI(TOUCH, %s touched raw%lu, s_touch_keys[i].name, raw); app_post_event(s_touch_keys[i].evt, raw, 0, NULL); vTaskDelay(pdMS_TO_TICKS(250)); } } vTaskDelay(pdMS_TO_TICKS(30)); } } void touch_app_start(void) { ESP_ERROR_CHECK(touch_pad_init()); for (int i 0; i 4; i) { ESP_ERROR_CHECK(touch_pad_config(s_touch_keys[i].pad)); } vTaskDelay(pdMS_TO_TICKS(300)); touch_calibrate(); xTaskCreate(touch_task, touch_task, 4096, NULL, 5, NULL); }8. 三轴姿态识别三轴传感器可实现摇晃、翻转、倾斜和拿起唤醒。typedef struct { int16_t x; int16_t y; int16_t z; } accel_data_t;I2C 读取#define I2C_PORT I2C_NUM_0 #define I2C_SDA_GPIO 8 #define I2C_SCL_GPIO 9 #define I2C_FREQ_HZ 400000 #define IMU_ADDR 0x68 #define IMU_REG_ACCEL_X 0x3B static esp_err_t imu_read_accel(accel_data_t *acc) { uint8_t reg IMU_REG_ACCEL_X; uint8_t buf[6] {0}; esp_err_t ret i2c_master_write_read_device( I2C_PORT, IMU_ADDR, reg, 1, buf, sizeof(buf), pdMS_TO_TICKS(50) ); if (ret ! ESP_OK) { return ret; } acc-x (int16_t)((buf[0] 8) | buf[1]); acc-y (int16_t)((buf[2] 8) | buf[3]); acc-z (int16_t)((buf[4] 8) | buf[5]); return ESP_OK; }动作判断static bool imu_detect_shake(accel_data_t now, accel_data_t last) { int dx abs(now.x - last.x); int dy abs(now.y - last.y); int dz abs(now.z - last.z); return (dx dy dz) 18000; } static bool imu_detect_flip(accel_data_t now) { return now.z -12000; } static bool imu_detect_tilt_left(accel_data_t now) { return now.x -10000; } static bool imu_detect_tilt_right(accel_data_t now) { return now.x 10000; }IMU 任务static void imu_task(void *arg) { accel_data_t last {0}; while (1) { accel_data_t now; if (imu_read_accel(now) ESP_OK) { if (imu_detect_shake(now, last)) { app_post_event(APP_EVT_IMU_SHAKE, now.x, now.y, NULL); } if (imu_detect_flip(now)) { app_post_event(APP_EVT_IMU_FLIP, now.x, now.z, NULL); } if (imu_detect_tilt_left(now)) { app_post_event(APP_EVT_IMU_TILT_LEFT, now.x, 0, NULL); } if (imu_detect_tilt_right(now)) { app_post_event(APP_EVT_IMU_TILT_RIGHT, now.x, 0, NULL); } last now; } vTaskDelay(pdMS_TO_TICKS(80)); } }9. VB6824 语音事件解析资料中 ESP32-C2/C3/S3 VB6824 方案由 VB6824 完成音频编解码、AEC、语音唤醒和改唤醒词等功能让主控专注于通信和 UI该方案已经用于电子吧唧、S3 双目、S3 拍学机、地球仪、拍拍灯等产品。建议 VB6824 与 ESP32-S3 之间通过 UART 上报事件0xA5 0x01 0x00 0x5A 唤醒成功 0xA5 0x02 0x00 0x5A 用户打断 0xA5 0x03 0x00 0x5A 开始录音 0xA5 0x04 0x00 0x5A 停止录音代码#define VB_UART_NUM UART_NUM_1 #define VB_UART_TX 17 #define VB_UART_RX 18 #define VB_UART_BAUD 115200 #define VB_HEAD 0xA5 #define VB_TAIL 0x5A #define VB_CMD_WAKEUP 0x01 #define VB_CMD_INTERRUPT 0x02 #define VB_CMD_REC_START 0x03 #define VB_CMD_REC_STOP 0x04 static void vb6824_parse_frame(uint8_t *buf, int len) { if (len 4) { return; } if (buf[0] ! VB_HEAD || buf[3] ! VB_TAIL) { return; } switch (buf[1]) { case VB_CMD_WAKEUP: app_post_event(APP_EVT_WAKE_WORD, 0, 0, NULL); break; case VB_CMD_INTERRUPT: app_post_event(APP_EVT_VOICE_INTERRUPT, 0, 0, NULL); break; case VB_CMD_REC_START: app_post_event(APP_EVT_RECORD_START, 0, 0, NULL); break; case VB_CMD_REC_STOP: app_post_event(APP_EVT_RECORD_STOP, 0, 0, NULL); break; default: break; } }10. 统一事件分发static void app_dispatch_task(void *arg) { app_event_t evt; while (1) { if (xQueueReceive(g_app_event_queue, evt, portMAX_DELAY)) { switch (evt.id) { case APP_EVT_WIFI_CONNECTED: ai_set_state(AI_STATE_IDLE); break; case APP_EVT_WIFI_DISCONNECTED: ai_set_state(AI_STATE_NETWORK_ERROR); break; case APP_EVT_WAKE_WORD: ai_set_state(AI_STATE_WAKEUP); ai_set_state(AI_STATE_LISTENING); break; case APP_EVT_VOICE_INTERRUPT: audio_stop_tts(); ai_set_state(AI_STATE_LISTENING); break; case APP_EVT_RECORD_START: ai_set_state(AI_STATE_LISTENING); break; case APP_EVT_RECORD_STOP: audio_stop_record(); ai_set_state(AI_STATE_THINKING); break; case APP_EVT_TOUCH_TOP: ai_set_state(AI_STATE_TOUCH_FEEDBACK); app_post_event(APP_EVT_WAKE_WORD, 0, 0, NULL); break; case APP_EVT_TOUCH_LEFT: role_switch_prev(); eye_play_anim(prev_role); haptic_short(); break; case APP_EVT_TOUCH_RIGHT: role_switch_next(); eye_play_anim(next_role); haptic_short(); break; case APP_EVT_TOUCH_BACK: eye_play_anim(config); haptic_double(); blufi_start_config(); break; case APP_EVT_IMU_SHAKE: ai_set_state(AI_STATE_SHAKE_FEEDBACK); role_switch_random(); break; case APP_EVT_IMU_FLIP: ai_set_state(AI_STATE_SLEEP); audio_stop_tts(); break; case APP_EVT_IMU_TILT_LEFT: eye_set_gaze(-30, 0); break; case APP_EVT_IMU_TILT_RIGHT: eye_set_gaze(30, 0); break; case APP_EVT_AI_THINKING: ai_set_state(AI_STATE_THINKING); break; case APP_EVT_AI_SPEAKING: ai_set_state(AI_STATE_SPEAKING); break; case APP_EVT_AI_ERROR: ai_set_state(AI_STATE_NETWORK_ERROR); break; case APP_EVT_OTA_START: ai_set_state(AI_STATE_OTA); break; default: break; } } } }11. 眼睛素材与 OTA 设计资料中提到0.71/1.28 寸双目固件可通过修改screen_type指向屏幕类型并替换defaultEye_1.28.h或defaultEye_0.71.h素材数组来定制眼睛样式。建议不要把所有眼睛动画写死到固件里而是放到素材分区/spiffs/ ├── eye_128/ │ ├── idle.bin │ ├── wake.bin │ ├── thinking.bin │ ├── speaking.bin │ └── happy.bin ├── eye_071/ │ ├── idle.bin │ └── happy.bin └── prompt/ ├── boot.wav ├── ding.wav └── net_error.wav动画加载typedef struct { char name[32]; int frame_count; int width; int height; int fps; FILE *fp; } eye_anim_t; static eye_anim_t s_current_anim; esp_err_t eye_anim_open(const char *name) { char path[96]; eye_screen_config_t cfg board_get_eye_config(); snprintf(path, sizeof(path), %s/%s.bin, cfg.asset_path, name); FILE *fp fopen(path, rb); if (!fp) { ESP_LOGW(EYE, 动画文件不存在: %s, path); return ESP_FAIL; } memset(s_current_anim, 0, sizeof(s_current_anim)); strncpy(s_current_anim.name, name, sizeof(s_current_anim.name) - 1); s_current_anim.fp fp; fread(s_current_anim.frame_count, sizeof(int), 1, fp); fread(s_current_anim.width, sizeof(int), 1, fp); fread(s_current_anim.height, sizeof(int), 1, fp); fread(s_current_anim.fps, sizeof(int), 1, fp); return ESP_OK; } void eye_play_anim(const char *name) { if (s_current_anim.fp) { fclose(s_current_anim.fp); s_current_anim.fp NULL; } if (eye_anim_open(name) ! ESP_OK) { eye_anim_open(idle); } }推荐分区表# Name, Type, SubType, Offset, Size nvs, data, nvs, 0x9000, 0x6000 otadata, data, ota, 0xf000, 0x2000 phy_init, data, phy, 0x11000, 0x1000 factory, app, factory, 0x20000, 2M ota_0, app, ota_0, 0x220000, 2M ota_1, app, ota_1, 0x420000, 2M assets, data, spiffs, 0x620000, 4M storage, data, fat, 0xA20000, 4M12. MCP 工具扩展资料中 MCP 管理指令支持ATADDMCP可配置不返回 AI 参数或返回 AI 参数的工具调用示例中包括设置屏幕主题、设置 RGB 灯光颜色并规定了对应串口返回协议。定义一个双目表情工具{ name: self.eye.set_expression, description: 设置四博AI双目的表情状态, parameters: { type: object, properties: { expression: { type: string, enum: [开心, 思考, 惊讶, 困惑, 睡觉, 待机] } }, required: [expression] } }设备侧处理void mcp_set_eye_expression(const char *expression) { if (strcmp(expression, 开心) 0) { eye_play_anim(happy); haptic_short(); } else if (strcmp(expression, 思考) 0) { eye_play_anim(thinking); } else if (strcmp(expression, 惊讶) 0) { eye_play_anim(surprised); haptic_double(); } else if (strcmp(expression, 困惑) 0) { eye_play_anim(confused); } else if (strcmp(expression, 睡觉) 0) { eye_play_anim(sleep); } else { eye_play_anim(idle_blink); } }工具分发void mcp_dispatch_tool(const char *tool_name, cJSON *args) { if (strcmp(tool_name, self.eye.set_expression) 0) { cJSON *exp cJSON_GetObjectItem(args, expression); if (cJSON_IsString(exp)) { mcp_set_eye_expression(exp-valuestring); } return; } if (strcmp(tool_name, self.device.enter_config) 0) { blufi_start_config(); eye_play_anim(config); haptic_double(); return; } if (strcmp(tool_name, self.audio.stop) 0) { audio_stop_tts(); ai_set_state(AI_STATE_IDLE); return; } }13. 产测设计AI 双目量产建议准备独立 factory 固件覆盖screen 双目屏 RGB、坏点、左右屏同步 touch 四路触摸原始值、阈值 motor 短震、双震、长震 imu X/Y/Z 方向、摇晃、翻转 mic 录音电平、底噪 speaker 提示音播放 vb6824 唤醒、打断、升级模式 wifi RSSI、连接、重连 flash NVS、OTA、素材分区产测代码void factory_reply(const char *item, const char *result) { printf({\item\:\%s\,\result\:\%s\}\n, item, result); } void factory_test_screen(void) { lcd_fill_color(0xF800); vTaskDelay(pdMS_TO_TICKS(300)); lcd_fill_color(0x07E0); vTaskDelay(pdMS_TO_TICKS(300)); lcd_fill_color(0x001F); vTaskDelay(pdMS_TO_TICKS(300)); lcd_fill_color(0x0000); factory_reply(screen, ok); } void factory_test_touch(void) { int top touch_get_raw(0); int left touch_get_raw(1); int right touch_get_raw(2); int back touch_get_raw(3); printf({\item\:\touch\,\result\:\ok\, \top\:%d,\left\:%d,\right\:%d,\back\:%d}\n, top, left, right, back); } void factory_test_imu(void) { accel_data_t acc; if (imu_read_accel(acc) ESP_OK) { printf({\item\:\imu\,\result\:\ok\, \x\:%d,\y\:%d,\z\:%d}\n, acc.x, acc.y, acc.z); } else { factory_reply(imu, fail); } }14. 总结四博 AI 双目方案可以理解为一个多模态 AI 终端底座ESP32-S3负责联网、UI、状态机、AI通信 VB6824负责唤醒、AEC、打断、语音前端 双目屏负责表情显示和状态可视化 触摸节点负责本地输入 IMU负责姿态感知 震动马达负责触觉反馈 四博小助手负责配网、声音克隆、知识库、MCP、OTA它不仅适合 AI 音箱也适合 AI 桌宠、儿童陪伴机、学习终端、IP 潮玩和品牌定制设备。真正的 AI 硬件不只是“会回答问题”而是要在声音、表情、触摸、姿态、触觉和云端智能之间形成完整闭环。