Teams集成-会议侧边栏应用开发-实时转写

Teams虽然提供了转写的接口，但是不是实时的，即便使用订阅事件也不是实时的，为了达到实时转写的效果，使用recall.ai的转录和assembly_ai的转写实现。

前提：除Teams会议侧边栏应用开发-会议转写-CSDN博客的基本要求外，还需要修改用户的安全设置及设置Teams 工作账号，参考：Setup Guide (recall.ai)

一、服务端需要实现4个服务端点：

1）开始录音（创建机器人）

/** Send's a Recall Bot to start recording the call*/
server.post('/start-recording', async (req, res) => {const meeting_url = req.body.meetingUrl;try {if (!meeting_url) {return res.status(400).json({ error: 'Missing meetingUrl' });}console.log('recall bot start recording', meeting_url);const url = 'https://us-west-2.recall.ai/api/v1/bot/';const options = {method: 'POST',headers: {accept: 'application/json','content-type': 'application/json',Authorization: `Token ${RECALL_API_KEY}`},body: JSON.stringify({bot_name: 'teams bot',real_time_transcription: {destination_url: 'https://shortly-adapted-akita.ngrok-free.app/transcription?secret=' + WEBHOOK_SECRET,partial_results: false},transcription_options: {provider: 'assembly_ai'},meeting_url: meeting_url})};const response = await fetch(url, options);const bot = await response.json();local_botId = bot.idconsole.log('botId:', local_botId);res.send(200, JSON.stringify({botId: local_botId}));} catch (error) {console.error("start-recoding error:", error);}
});

2）停止录音

/*
* Tells the Recall Bot to stop recording the call
*/
server.post('/stop-recording', async (req, res) => {try {const botId = local_botId;if (!botId) {res.send(400, JSON.stringify({ error: 'Missing botId' }));}await fetch(`https://us-west-2.recall.ai/api/v1/bot/${botId}/leave_call`, {method: 'POST',headers: {'Content-Type': 'application/json',Accept: 'application/json',Authorization: `Token ${RECALL_API_KEY}`},});console.log('recall bot stopped');res.send(200, {})} catch (error) {console.error("stop-recoding error:", error);}
});

3）轮询机器人状态

/*
* Gets the current state of the Recall Bot
*/
server.get('/recording-state', async (req, res) => {try {const botId = local_botId;if (!botId) {res.send(400, JSON.stringify({ error: 'Missing botId' }));}const response = await fetch(`https://us-west-2.recall.ai/api/v1/bot/${botId}`, {method: 'GET',headers: {'Content-Type': 'application/json',Accept: 'application/json',Authorization: `Token ${RECALL_API_KEY}`},});const bot = await response.json();const latestStatus = bot.status_changes.slice(-1)[0].code;console.log('state:', latestStatus);res.send(200, JSON.stringify({state: latestStatus,transcript: db.transcripts[botId] || [],}));} catch (error) {console.error("recoding-state error:", error);}
});

4）接收转写存储在db中（本例使用的是内存)

/** Receives transcription webhooks from the Recall Bot*/
server.post('/transcription', async (req, res) => {try {console.log('transcription webhook received: ', req.body);const { bot_id, transcript } = req.body.data;if (!db.transcripts[bot_id]) {db.transcripts[bot_id] = [];}if (transcript){db.transcripts[bot_id].push(transcript);}res.send(200, JSON.stringify({ success: true }));} catch (error) {console.error("transcription error:", error);}
});

完整的服务端代码：

import restify from "restify";
import send from "send";
import fs from "fs";
import fetch from "node-fetch";
import path from 'path';
import { fileURLToPath } from 'url';
import { storeToken, getToken } from './redisClient.js';
import { WebSocketServer, WebSocket } from 'ws';const __filename = fileURLToPath(import.meta.url);
console.log('__filename: ', __filename);const __dirname = path.dirname(__filename);
console.log('__dirname: ', __dirname);// Create HTTP server.
const server = restify.createServer({key: process.env.SSL_KEY_FILE ? fs.readFileSync(process.env.SSL_KEY_FILE) : undefined,certificate: process.env.SSL_CRT_FILE ? fs.readFileSync(process.env.SSL_CRT_FILE) : undefined,formatters: {"text/html": function (req, res, body) {return body;},},
});server.use(restify.plugins.bodyParser());
server.use(restify.plugins.queryParser());server.get("/static/*",restify.plugins.serveStatic({directory: __dirname,})
);server.listen(process.env.port || process.env.PORT || 3000, function () {console.log(`\n${server.name} listening to ${server.url}`);
});// Adding tabs to our app. This will setup routes to various views
// Setup home page
server.get("/config", (req, res, next) => {send(req, __dirname + "/config/config.html").pipe(res);
});// Setup the static tab
server.get("/meetingTab", (req, res, next) => {send(req, __dirname + "/panel/panel.html").pipe(res);
});//获得用户token
server.get('/auth', (req, res, next) => {res.status(200);res.send(`
<!DOCTYPE html>
<html>
<head><script>// Function to handle the token storageasync function handleToken() {const hash = window.location.hash.substring(1);const hashParams = new URLSearchParams(hash);const access_token = hashParams.get('access_token');console.log('Received hash parameters:', hashParams);if (access_token) {console.log('Access token found:', access_token);localStorage.setItem("access_token", access_token);console.log('Access token stored in localStorage');try {const response = await fetch('https://shortly-adapted-akita.ngrok-free.app/store_user_token', {method: 'POST',headers: {'Content-Type': 'application/json'},body: JSON.stringify({ "user_token" : access_token })});if (response.ok) {console.log('Token stored successfully');} else {console.error('Failed to store token:', response.statusText);}} catch (error) {console.error('Error storing token:', error);}} else {console.log('No access token found');}window.close();}// Call the function to handle the tokenhandleToken();</script>
</head>
<body></body>
</html>`);next();
});// 存储 user_token
server.post('/store_user_token', async (req, res) => {const user_token = req.body.user_token;if (!user_token) {res.status(400);res.send('user_token are required');}try {// Store user tokenawait storeToken('user_token', user_token);console.log('user_token stored in Redis');} catch (err) {console.error('user_token store Error:', err);}res.status(200);   res.send('Token stored successfully');
});// 获取 user_token
server.get('/get_user_token', async (req, res) => {try {// Store user tokenconst user_token = await getToken('user_token');console.log('user_token get in Redis');res.send({"user_token": user_token});} catch (err) {console.error('user_token get Error:', err);}
});//应用token
let app_token = '';
const app_token_refresh_interval = 3000 * 1000; // 3000秒const getAppToken = async () => {try {// 构建请求体const requestBody = new URLSearchParams({"grant_type": "client_credentials","client_id": "Azure注册应用ID","client_secret": "Azure注册应用密钥","scope": "https://graph.microsoft.com/.default",}).toString();// 获取app令牌const tokenUrl = `https://login.microsoftonline.com/864168b4-813c-411a-827a-af408f70c665/oauth2/v2.0/token`;const tokenResponse = await fetch(tokenUrl, {method: 'POST',headers: {'Content-Type': 'application/x-www-form-urlencoded',},body: requestBody,});if (!tokenResponse.ok) {const errorData = await tokenResponse.json();throw new Error(errorData.error_description);}const tokenData = await tokenResponse.json();app_token = tokenData.access_token;console.log("app_token received!");} catch (error) {console.error('Error getting app token:', error);}
};// 定期刷新 app_token
setInterval(getAppToken, app_token_refresh_interval);// 确保在服务器启动时获取 app_token
getAppToken();//存储机器人转写信息
const db = {transcripts: {// [bot id]: [transcript]},
};const RECALL_API_KEY = '你的recall.ai的API KEY';
const WEBHOOK_SECRET = '在recall.ai配置webhook端点时的密钥';let local_botId = null;
/** Send's a Recall Bot to start recording the call*/
server.post('/start-recording', async (req, res) => {const meeting_url = req.body.meetingUrl;try {if (!meeting_url) {return res.status(400).json({ error: 'Missing meetingUrl' });}console.log('recall bot start recording', meeting_url);const url = 'https://us-west-2.recall.ai/api/v1/bot/';const options = {method: 'POST',headers: {accept: 'application/json','content-type': 'application/json',Authorization: `Token ${RECALL_API_KEY}`},body: JSON.stringify({bot_name: 'teams bot',real_time_transcription: {destination_url: 'https://shortly-adapted-akita.ngrok-free.app/transcription?secret=' + WEBHOOK_SECRET,partial_results: false},transcription_options: {provider: 'assembly_ai'},meeting_url: meeting_url})};const response = await fetch(url, options);const bot = await response.json();local_botId = bot.idconsole.log('botId:', local_botId);res.send(200, JSON.stringify({botId: local_botId}));} catch (error) {console.error("start-recoding error:", error);}
});/*
* Tells the Recall Bot to stop recording the call
*/
server.post('/stop-recording', async (req, res) => {try {const botId = local_botId;if (!botId) {res.send(400, JSON.stringify({ error: 'Missing botId' }));}await fetch(`https://us-west-2.recall.ai/api/v1/bot/${botId}/leave_call`, {method: 'POST',headers: {'Content-Type': 'application/json',Accept: 'application/json',Authorization: `Token ${RECALL_API_KEY}`},});console.log('recall bot stopped');res.send(200, {})} catch (error) {console.error("stop-recoding error:", error);}
});/*
* Gets the current state of the Recall Bot
*/
server.get('/recording-state', async (req, res) => {try {const botId = local_botId;if (!botId) {res.send(400, JSON.stringify({ error: 'Missing botId' }));}const response = await fetch(`https://us-west-2.recall.ai/api/v1/bot/${botId}`, {method: 'GET',headers: {'Content-Type': 'application/json',Accept: 'application/json',Authorization: `Token ${RECALL_API_KEY}`},});const bot = await response.json();const latestStatus = bot.status_changes.slice(-1)[0].code;console.log('state:', latestStatus);res.send(200, JSON.stringify({state: latestStatus,transcript: db.transcripts[botId] || [],}));} catch (error) {console.error("recoding-state error:", error);}
});
/** Receives transcription webhooks from the Recall Bot*/
server.post('/transcription', async (req, res) => {try {console.log('transcription webhook received: ', req.body);const { bot_id, transcript } = req.body.data;if (!db.transcripts[bot_id]) {db.transcripts[bot_id] = [];}if (transcript){db.transcripts[bot_id].push(transcript);}res.send(200, JSON.stringify({ success: true }));} catch (error) {console.error("transcription error:", error);}
});

二、页面需要实现开始录音和停止录音按钮及转写显示。

完整的页面代码：

<!DOCTYPE html>
<html lang="en">
<head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Meeting Transcripts</title><script src="https://res.cdn.office.net/teams-js/2.0.0/js/MicrosoftTeams.min.js"></script><script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.0.1/socket.io.js"></script><style>.subtitle {display: flex;align-items: center;margin-bottom: 10px;}.speaker-photo {width: 20px;height: 20px;border-radius: 50%;margin-right: 10px;}button {padding: 5px 10px; /* 调整按钮的 padding 以减小高度 */font-size: 14px; /* 调整按钮的字体大小 */margin-right: 10px;}#transcript {margin-top: 20px;padding: 10px;border: 1px solid #ccc;min-height: 100px;width: 100%;}</style>
</head>
<body><h2>Meeting Transcripts</h2><button id="startRecording">Start Recording</button><button id="stopRecording" disabled>Stop Recording</button><div id="transcripts"></div><script>const clientId = 'Azure注册应用ID';const tenantId = 'Azure注册应用租户ID';const authUrl = `https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/authorize`;const redirectUri = 'https://shortly-adapted-akita.ngrok-free.app/auth'; // 确保与服务器端一致const scope = 'user.read';let user_token = null;let meetingOrganizerUserId = null;let participants = {}; // 用于存储参会者的信息let userPhotoCache = {}; // 用于缓存用户头像let tokenFetched = false; // 标志变量，用于跟踪是否已经获取了 user_tokenlet displayedTranscriptIds = new Set(); // 用于存储已经显示的转录片段的 IDconst getUserInfo = async (userId, accessToken) => {const graphUrl = `https://graph.microsoft.com/v1.0/users/${userId}`;const response = await fetch(graphUrl, {headers: {'Authorization': `Bearer ${accessToken}`}});if (response.status === 401) {// 如果 token 超期，重新触发 initAuthenticationinitAuthentication();return null;}const userInfo = await response.json();return userInfo;};const getUserPhoto = async (userId, accessToken) => {if (userPhotoCache[userId]) {return userPhotoCache[userId];}const graphUrl = `https://graph.microsoft.com/v1.0/users/${userId}/photo/$value`;const response = await fetch(graphUrl, {headers: {'Authorization': `Bearer ${accessToken}`}});if (!response.ok) {const errorData = await response.json();console.error('Error fetching user photo:', errorData);return null;}const photoBlob = await response.blob();const photoUrl = URL.createObjectURL(photoBlob);userPhotoCache[userId] = photoUrl; // 缓存头像 URLreturn photoUrl;};const getMeetingDetails = async (user_token, joinMeetingId) => {const apiUrl = `https://graph.microsoft.com/v1.0/me/onlineMeetings?$filter=joinMeetingIdSettings/joinMeetingId eq '${joinMeetingId}'`;const response = await fetch(apiUrl, {method: 'GET',headers: {'Authorization': `Bearer ${user_token}`,'Content-Type': 'application/json'}});if (!response.ok) {const errorData = await response.json();throw new Error(`getMeetingDetails status: ${response.status}, message: ${errorData.error}`);}const data = await response.json();return data.value[0];};const getTranscriptContent = async (transcripts) => {const subtitles = [];try {transcripts.forEach(transcript => {const startTime = transcript.words[0].start_time;const endTime = transcript.words[transcript.words.length - 1].end_time;const speaker = transcript.speaker;const content = transcript.words.map(word => word.text).join(' ');subtitles.push({ startTime, endTime, speaker, content, id: transcript.original_transcript_id });});return subtitles;} catch (error) {console.error('getTranscriptContent error:', error);return subtitles;}};const displaySubtitle = async (subtitle, transcriptElement, accessToken) => {const subtitleElement = document.createElement('div');subtitleElement.classList.add('subtitle');// 获取说话者的头像const speakerUserId = participants[subtitle.speaker];const speakerPhotoUrl = speakerUserId ? await getUserPhoto(speakerUserId, accessToken) : 'default-avatar.png';// 创建头像元素const speakerPhotoElement = document.createElement('img');speakerPhotoElement.src = speakerPhotoUrl;speakerPhotoElement.alt = subtitle.speaker;speakerPhotoElement.classList.add('speaker-photo');// 创建输出字符串const output = `${subtitle.startTime} - ${subtitle.endTime}\n${subtitle.content}`;subtitleElement.appendChild(speakerPhotoElement);subtitleElement.appendChild(document.createTextNode(output));transcriptElement.appendChild(subtitleElement);};const init = async () => {try {if (!tokenFetched) {const response = await fetch('https://shortly-adapted-akita.ngrok-free.app/get_user_token');const data = await response.json();if (response.ok) {user_token = data.user_token;console.log('user token retrieved:', user_token);tokenFetched = true;} else {console.error('Failed to get token:', response.statusText);return;}}console.log('User Token:', user_token);const joinMeetingId = '45756456529'; // 替换为你要查询的 joinMeetingIdtry {const meetingDetails = await getMeetingDetails(user_token, joinMeetingId);console.log('Meeting Details:', meetingDetails);meetingOrganizerUserId = meetingDetails.participants.organizer.identity.user.id;const meetingId = meetingDetails.id; // 获取会议 IDconsole.log('Organizer User ID:', meetingOrganizerUserId);console.log('Meeting ID:', meetingId);// 获取主持人信息const organizerInfo = await getUserInfo(meetingOrganizerUserId, user_token);const organizerDisplayName = organizerInfo.displayName;participants[organizerDisplayName] = meetingOrganizerUserId;// 获取参会者信息const attendeesPromises = meetingDetails.participants.attendees.map(async attendee => {const userId = attendee.identity.user.id;const userInfo = await getUserInfo(userId, user_token);const displayName = userInfo.displayName;participants[displayName] = userId;});await Promise.all(attendeesPromises);} catch (error) {console.error('Error fetching meeting details:', error);}} catch (error) {console.error('Error getting token:', error);}};const initAuthentication = () => {microsoftTeams.app.initialize();microsoftTeams.authentication.authenticate({url: `${authUrl}?client_id=${clientId}&response_type=token&redirect_uri=${encodeURIComponent(redirectUri)}&scope=${encodeURIComponent(scope)}`,width: 600,height: 535,successCallback: async (result) => {console.log('Authentication success:', result);},failureCallback: (error) => {console.error('Authentication failed:', error);}});};// 设置较长的轮询时间来防止 user_token 的超期setInterval(initAuthentication, 3000000); // 每3000秒（50分钟）轮询一次initAuthentication();init();// 录音控制功能const startRecordingButton = document.getElementById('startRecording');const stopRecordingButton = document.getElementById('stopRecording');const transcriptDiv = document.getElementById('transcript');let recordingInterval;// Function to start recordingasync function startRecording() {const meetingUrl = await getMeetingUrl();if (!meetingUrl) return;try {const response = await fetch('/start-recording', {method: 'POST',headers: {'Content-Type': 'application/json',},body: JSON.stringify({ meetingUrl }),});if (response.ok) {const data = await response.json();console.log('Bot started:', data);startRecordingButton.disabled = true;stopRecordingButton.disabled = false;startPolling();} else {console.error('Failed to start recording:', response.statusText);}} catch (error) {console.error('Error starting recording:', error);}}// Function to stop recordingasync function stopRecording() {try {const response = await fetch('/stop-recording', {method: 'POST',});if (response.ok) {console.log('Bot stopped');startRecordingButton.disabled = false;stopRecordingButton.disabled = true;clearInterval(recordingInterval);} else {console.error('Failed to stop recording:', response.statusText);}} catch (error) {console.error('Error stopping recording:', error);}}// Function to poll the recording stateasync function pollRecordingState() {try {const response = await fetch('/recording-state');if (response.ok) {const data = await response.json();updateUI(data);} else {console.error('Failed to get recording state:', response.statusText);}} catch (error) {console.error('Error polling recording state:', error);}}// Function to update the UI based on the recording statefunction updateUI(data) {const { state, transcript } = data;console.log(state, transcript);// Update the transcript displayconst transcriptsContainer = document.getElementById('transcripts');const transcriptElement = document.createDocumentFragment(); // 使用 DocumentFragment 优化 DOM 操作if (transcript.length > 0) {getTranscriptContent(transcript).then(subtitles => {subtitles.forEach(subtitle => {if (!displayedTranscriptIds.has(subtitle.id)) {displaySubtitle(subtitle, transcriptElement, user_token);displayedTranscriptIds.add(subtitle.id); // 添加到已显示的转录片段 ID 集合中}});}).catch(error => {const errorElement = document.createElement('div');errorElement.innerHTML = `<strong>${error}</strong>`;transcriptElement.appendChild(errorElement);}).finally(() => {transcriptsContainer.appendChild(transcriptElement); // 一次性插入 DOM});}// Update button states based on the recording stateif (state === 'recording') {startRecordingButton.disabled = true;stopRecordingButton.disabled = false;} else if (state === 'stopped') {startRecordingButton.disabled = false;stopRecordingButton.disabled = true;}}// Function to start polling the recording state every 2 secondsfunction startPolling() {recordingInterval = setInterval(pollRecordingState, 2000);}// Event listeners for buttonsstartRecordingButton.addEventListener('click', startRecording);stopRecordingButton.addEventListener('click', stopRecording);// Function to get the meeting URL from the meeting detailsasync function getMeetingUrl() {const joinMeetingId = '45756456529'; // 替换为你要查询的 joinMeetingIdtry {const meetingDetails = await getMeetingDetails(user_token, joinMeetingId);return meetingDetails.joinWebUrl;} catch (error) {console.error('Error fetching meeting URL:', error);return null;}}</script>
</body>
</html>

最终效果：