transformers.BertTokenizer入门使用

教程link

示例代码

from transformers import OpenAIGPTLMHeadModel, GPT2LMHeadModel, BertTokenizer
import torch
tokenizer = BertTokenizer.from_pretrained("thu-coai/CDial-GPT_LCCC-large")
model = OpenAIGPTLMHeadModel.from_pretrained("thu-coai/CDial-GPT_LCCC-large")sents = ['人工智能是计算机科学的一个分支。','它企图了解智能的实质。','人工智能是一门极富挑战性的科学。',
]token = tokenizer.tokenize(sents[0])
print(token)ids = tokenizer.convert_tokens_to_ids(token)
print(ids)