编译原理实验1——词法分析（python实现）

实验目的

输入一个C语言代码串，输出单词流，识别对象包含关键字、标识符、整型浮点型字符串型常数、科学计数法、操作符和标点、注释等等。

实现

定义单词对应的种别码

自行定义相关单词的种别码

定义输出形式：三元式

# 三元式
class ThreeFml:  # 三元式def __init__(self, syn, inPoint, value):self.syn = syn  # 种别码self.inPoint = inPoint  # 内码值self.value = value  # 自身值def __eq__(self, other):  # 重载 判别对象相等的依据return self.syn == other.syn and self.value == other.valuedef __lt__(self, other):  # 重载 比较对象大小关系的依据if self.syn == other.syn:return self.inPoint < other.inPointelse:return self.syn < other.syn

每个三元组用一个自定义类表示：
类属性：种别码syn、内码值inPoint、自身值value
类方法：

• 方法1：判断两个三元组相等的方法：种别码syn和自身值value相等
• 方法2：确定展示时的先后顺序的方法：先比较种别码syn，再比较内码值inPoint

例如：

• 输入：double a; int a;
• 输出：
• 分析：有两个标识符a，根据类方法1，判断前后两个a为同一个三元组，因此不重复输a。参见种别码表，double为6，int为2，则根据类方法2，进行三元组的展示排序。

python代码实现

import re# 三元式
class ThreeFml:  # 三元式def __init__(self, syn, inPoint, value):self.syn = syn  # 种别码self.inPoint = inPoint  # 内码值self.value = value  # 自身值def __eq__(self, other):  # 重载 判别对象相等的依据return self.syn == other.syn and self.value == other.valuedef __lt__(self, other):  # 重载 比较对象大小关系的依据if self.syn == other.syn:return self.inPoint < other.inPointelse:return self.syn < other.syn# 词法识别
class WordAnalysis:def __init__(self, input_code_str):self.input_code_str = input_code_str  # 源程序字符串self.code_char_list = []  # 源程序字符列表self.code_len = 0  # 源程序字符列表长度self.cp = 0  # 源程序字符列表指针，方便遍历字符串中的字符self.cur = ''  # 当前源程序字符列表的某个字符self.val = []  # 单词自身的值self.syn = 0  # 单词种别码self.errInfo = ""  # 错误信息self.keyWords = ["main", "int", "short", "long", "float","double", "char", "string", "const", "void","struct", "if", "else", "switch", "case","default", "do", "while", "for", "continue","break", "cout", "cin", "endl", "scanf","printf", "return", 'catch', 'class', 'delete','enum', 'export', 'extern', 'false', 'friend','goto', 'inline', 'namespace', 'new', 'not','public', 'static', 'template', 'this', 'true','try', 'typedef', 'union', 'using', 'virtual','std', 'include', 'iostream']  # 关键字self.TFs = []  # 存储三元式def nextChar(self):  # 封装cp++，简化函数scanWord中的代码self.cp += 1self.cur = self.code_char_list[self.cp]def error(self, info):  # errInfo错误信息line = 1for i in range(0, self.cp + 1):if self.code_char_list[i] == '\n':line += 1self.errInfo = "第" + str(line) + "行报错：" + infodef bracket_match(self):pattern = r'(\/\/.*?$|\/\*(.|\n)*?\*\/)'  # 匹配单行或多行注释comments = re.findall(pattern, self.input_code_str, flags=re.MULTILINE | re.DOTALL)comments = [comment[0].strip() for comment in comments]  # 处理结果，去除多余的空格i = 0code_sub_com = []  # 去除注释print(f"comment: {comments}")while i < len(self.input_code_str):ch = self.input_code_str[i]if ch == "/" and comments != []:i += len(comments[0])comments.pop(0)continuecode_sub_com.append((i, ch))i += 1pattern2 = r'"([^"]*)"'  # 匹配双引号包裹的字符串strings = re.findall(pattern2, self.input_code_str)code_sub_com_str = []  # 去除字符串变量i = 0while i < len(code_sub_com):item = code_sub_com[i]ch = item[1]if ch == "\"" and comments != []:i += len(strings[0]) + 2strings.pop(0)continuecode_sub_com_str.append(item)i += 1s = []stack = []mapping = {")": "(", "}": "{", "]": "["}for idx, char in code_sub_com_str:if char in mapping.keys() or char in mapping.values():s.append((idx, char))if not s:return "ok"for item in s:idx = item[0]char = item[1]if char in mapping.values():  # 左括号stack.append(item)elif char in mapping.keys():  # 右括号if not stack:  # 栈为空，当前右括号匹配不到return idxtopitem = stack[-1]topidx = topitem[0]topch = topitem[1]if mapping[char] != topch:  # 当前右括号匹配失败return topidxelse:stack.pop()if not stack:  # 栈为空，匹配完毕return "ok"else:  # 栈不为空，只剩下左括号item = stack[0]idx = item[0]return idxdef scanWord(self):  # 词法分析# 初始化valueself.val = []self.syn = 0# ******获取当前有效字符（去除空白，直至扫描到第一个有效字符）******self.cur = self.code_char_list[self.cp]# print(f"==={self.cp}  {self.code_len-1}===")while self.cur == ' ' or self.cur == '\n' or self.cur == '\t':self.cp += 1if self.cp >= self.code_len - 1:print(f"越界{self.cp}")return  # 越界直接返回self.cur = self.code_char_list[self.cp]# ********************首字符为数字*****************if self.cur.isdigit():# ====首先默认为整数 ====i_value = 0while self.cur.isdigit():  # string数转inti_value = i_value * 10 + int(self.cur)self.nextChar()six_flag = Falseif (self.cur == 'x' or self.cur == 'X') \and self.code_char_list[self.cp - 1] == '0':  # 十六进制整数 0x?????self.nextChar()six_flag = Trues = ""while self.cur.isdigit() or self.cur.isalpha():if self.cur.isalpha():if not (('a' <= self.cur <= 'f') or ('A' <= self.cur <= 'F')):self.syn = -999self.error("十六进制中的字母不为：a~f 或 A~F")returns += self.curself.nextChar()i_value = int(s, 16)  # 将16进制数转为整数self.syn = 201self.val = str(i_value)  # int转strif six_flag:return# ====有小数点或e，则为浮点数====d_value = i_value * 1.0if self.cur == '.':fraction = 0.1self.nextChar()while self.cur.isdigit():  # 计算小数位上的数 形如 123.45d_value += fraction * int(self.cur)fraction = fraction * 0.1self.nextChar()if self.cur == 'E' or self.cur == 'e':  # 形如 123.4E?? 或 123.E??self.nextChar()powNum = 0if self.cur == '+':  # 形如 123.4E+5self.nextChar()while self.cur.isdigit():powNum = powNum * 10 + int(self.cur)self.nextChar()d_value *= 10 ** powNumelif self.cur == '-':  # 形如 123.4E-5self.nextChar()while self.cur.isdigit():powNum = powNum * 10 + int(self.cur)self.nextChar()d_value /= 10 ** powNumelif self.cur.isdigit():  # 形如 123.4E5while self.cur.isdigit():powNum = powNum * 10 + int(self.cur)self.nextChar()d_value *= 10 ** powNumif self.cur.isalpha():self.syn = -999self.error(f"科学计数法后含有多余字母{self.cur}")returnself.syn = 202self.val = str(d_value)  # double转strelif self.cur == 'E' or self.cur == 'e':  # 形如 123E??self.nextChar()powNum = 0if self.cur == '+':  # 形如 123E+4self.nextChar()while self.cur.isdigit():powNum = powNum * 10 + int(self.cur)self.nextChar()d_value *= 10 ** powNumelif self.cur == '-':  # 形如 123E-4self.nextChar()while self.cur.isdigit():powNum = powNum * 10 + int(self.cur)self.nextChar()d_value /= 10 ** powNumelif self.cur.isdigit():  # 形如 123E4while self.cur.isdigit():powNum = powNum * 10 + int(self.cur)self.nextChar()d_value *= 10 ** powNumif self.cur.isalpha():self.syn = -999self.error(f"科学计数法后含有多余字母{self.cur}")returnself.syn = 202self.val = str(d_value)# ********************首字符为字母*****************elif self.cur.isalpha():# ====标识符====while self.cur.isdigit() or self.cur.isalpha() or self.cur == '_':self.val.append(self.cur)self.nextChar()self.syn = 222# ====判断是否为关键字====for i, keyword in enumerate(self.keyWords):if ''.join(self.val) == keyword:self.syn = i + 1break# ********************首字符为标点*****************else:if self.cur == '+':self.syn = 101self.val.append(self.cur)self.nextChar()if self.cur == '+':self.syn = 131self.val.append(self.cur)self.nextChar()elif self.cur == '=':self.syn = 136self.val.append(self.cur)self.nextChar()elif self.cur == '-':self.syn = 102self.val.append(self.cur)self.nextChar()if self.cur == '-':self.syn = 132self.val.append(self.cur)self.nextChar()elif self.cur == '=':self.syn = 137self.val.append(self.cur)self.nextChar()elif self.cur == '*':self.syn = 103self.val.append(self.cur)self.nextChar()if self.cur == '=':self.syn = 138self.val.append(self.cur)self.nextChar()elif self.cur == '/':self.syn = 104self.val.append(self.cur)self.nextChar()if self.cur == '=':self.syn = 139self.val.append(self.cur)self.nextChar()# 单行注释elif self.cur == '/':self.nextChar()while self.cur != '\n':self.nextChar()self.syn = 0# 多行注释elif self.cur == '*':self.cp += 1haveEnd = Falseflag = 0for i in range(self.cp + 1, self.code_len):# print(self.code_char_list[i])if self.code_char_list[i - 1] == '*' and self.code_char_list[i] == '/':haveEnd = Trueflag = ibreakif haveEnd:self.syn = 0self.cp = flag + 1else:self.syn = -999self.error(" 多行注释没有结尾*/ ")elif self.cur == '%':self.syn = 105self.val.append(self.cur)self.nextChar()if self.cur == '=':self.syn = 140self.val.append(self.cur)self.nextChar()elif self.cur == '=':self.syn = 106self.val.append(self.cur)self.nextChar()if self.cur == '=':self.syn = 118self.val.append(self.cur)self.nextChar()elif self.cur == '(':self.syn = 107self.val.append(self.cur)self.nextChar()elif self.cur == ')':self.syn = 108self.val.append(self.cur)self.nextChar()elif self.cur == '[':self.syn = 109self.val.append(self.cur)self.nextChar()elif self.cur == ']':self.syn = 110self.val.append(self.cur)self.nextChar()elif self.cur == '{':self.syn = 111self.val.append(self.cur)self.nextChar()elif self.cur == '}':self.syn = 112self.val.append(self.cur)self.nextChar()elif self.cur == ';':self.syn = 113self.val.append(self.cur)self.nextChar()elif self.cur == '>':self.syn = 114self.val.append(self.cur)self.nextChar()if self.cur == '=':self.syn = 116self.val.append(self.cur)self.nextChar()elif self.cur == '>':self.syn = 119self.val.append(self.cur)self.nextChar()if self.cur == '=':self.syn = 141self.val.append(self.cur)self.nextChar()elif self.cur == '<':self.syn = 115self.val.append(self.cur)self.nextChar()if self.cur == '=':self.syn = 117self.val.append(self.cur)self.nextChar()elif self.cur == '<':self.syn = 120self.val.append(self.cur)self.nextChar()if self.cur == '=':self.syn = 142self.val.append(self.cur)self.nextChar()elif self.cur == '!':self.syn = 121self.val.append(self.cur)self.nextChar()if self.cur == '=':self.syn = 122self.val.append(self.cur)self.nextChar()elif self.cur == '&':self.syn = 123self.val.append(self.cur)self.nextChar()if self.cur == '&':self.syn = 124self.val.append(self.cur)self.nextChar()elif self.cur == '=':self.syn = 143self.val.append(self.cur)self.nextChar()elif self.cur == '|':self.syn = 125self.val.append(self.cur)self.nextChar()if self.cur == '|':self.syn = 126self.val.append(self.cur)self.nextChar()elif self.cur == '=':self.syn = 144self.val.append(self.cur)self.nextChar()elif self.cur == '\\':  # \self.syn = 127self.val.append(self.cur)self.nextChar()elif self.cur == '\'':  # ‘self.syn = 128self.val.append(self.cur)self.nextChar()elif self.cur == '\"':  # ”self.nextChar()haveEnd = Falseflag = 0for i in range(self.cp, self.code_len):if self.code_char_list[i] == '"':haveEnd = Trueflag = ibreakif haveEnd:for j in range(self.cp, flag):self.val.append(self.code_char_list[j])self.cp = flag + 1self.cur = self.code_char_list[self.cp]self.syn = 203else:self.syn = -999self.error(" string常量没有闭合的\" ")elif self.cur == ':':self.syn = 130self.val.append(self.cur)self.nextChar()if self.cur == ':':self.syn = 134self.val.append(self.cur)self.nextChar()elif self.cur == ',':self.syn = 133self.val.append(self.cur)self.nextChar()elif self.cur == '^':  # 按位异或self.syn = 146self.val.append(self.cur)self.nextChar()if self.cur == '=':self.syn = 145self.val.append(self.cur)self.nextChar()elif self.cur == '#':self.syn = 147self.val.append(self.cur)self.nextChar()else:self.syn = -999self.error(f" 无效字符: {self.cur}")def solve(self):print("\n================scan-main  begin================")self.code_char_list = list(self.input_code_str.strip())  # 去除头尾的空格self.code_char_list.append('\n')  # 末尾补充一个\n， 可在一些while判断中 防止越界self.code_len = len(self.code_char_list)if self.bracket_match() != "ok":  # 检测括号匹配self.cp = self.bracket_match()self.error(f"{self.code_char_list[self.cp]}匹配缺失！")intCnt, doubleCnt, stringCnt, idCnt = 0, 0, 0, 0  # 内码值while True:  # 至少执行一次，如同do whileself.scanWord()  # 进入词法分析value = ''.join(self.val)  # char列表 ===> Stringnew_tf = ThreeFml(self.syn, -1, value)  # 创建三元式对象if self.syn == 201:  # 整型常数# print(f"整型常数: {value}")if not any(tf == new_tf for tf in self.TFs):  # append前先判断是否有重复intCnt += 1new_tf.inPoint = intCntself.TFs.append(new_tf)elif self.syn == 202:  # 浮点型常数# print(f"浮点型常数: {value}")if not any(tf == new_tf for tf in self.TFs):doubleCnt += 1new_tf.inPoint = doubleCntself.TFs.append(new_tf)elif self.syn == 203:  # 字符串常数# print(f"字符串常数: {value}")if not any(tf == new_tf for tf in self.TFs):stringCnt += 1new_tf.inPoint = stringCntself.TFs.append(new_tf)elif self.syn == 222:  # 标识符# print(f"标识符: {value}")if not any(tf == new_tf for tf in self.TFs):idCnt += 1new_tf.inPoint = idCntself.TFs.append(new_tf)elif 1 <= self.syn <= 100:  # 关键字# print(f"关键字: {value}")if not any(tf == new_tf for tf in self.TFs):new_tf.inPoint = 1self.TFs.append(new_tf)elif self.syn == 0:  # 注释内容、或者最后的\n# print("注释 or 结束")passelif self.syn == -999:  # 报错# print(f"error: {self.errInfo}")breakelse:  # 符号：标点符、算符# print(f"符号: {value}")if not any(tf == new_tf for tf in self.TFs):new_tf.inPoint = 1self.TFs.append(new_tf)if self.cp >= (self.code_len - 1):  # 最后一个元素 为自主添加的\n，代表结束# print(f"{cp}  跳出")breakif self.errInfo:  # 检查是否有报错print(self.errInfo)returnself.TFs.sort()  # 给三元式列表TFs排序（按种别码、内码）for tf in self.TFs:  # 打印print(f"({tf.syn}, {tf.inPoint}, {tf.value})")print("================scan-main  end================")if __name__ == '__main__':filepath = "./code.txt"with open(filepath, "r") as file:code = file.read()word_analysis = WordAnalysis(code)word_analysis.solve()

运行结果

输入：通过读取txt文件输入要分析的源程序串

输出：

第一行表示注释内容
后面为三元式（种别码，内码值，自身值）
从结果可以看到：

• 输入代码串中有两个int关键字，并没有重复输出，只保留1个
• 输入代码串中识别到多个标识符（我设定的种别码为222），由于它们的值不同，所以在种别码相同的情况下，给出不同的内码值。

检错处理

括号匹配

字符串常量未闭合

多行注释未闭合

十六进制数不规范

科学计数法不规范

总结

体会
在本次的实验中，通过对词法分析器的编写，在理论的基础上加深了对词法分析的理解和实践，所编写的词法分析器在多次的测试中均得到了正确的结果。
此外，我是先用c++编写的代码，确认大部分功能无误后，再改用python编写。在改语言的过程中，明显感受到python 的便利之处，就比如一个简单的判断字符是否为字母，在c++里需要自定义一个函数来判断(if ch>=’a’ and if ch<=’z’)，而python则直接使用系统自带的isalpha函数即可，大大简化了代码量。

问题
编写的程序中，虽然已完成绝大部分单词分析功能，但对一些小细节就没有进行直接的编写。例如在识别用科学表示法表示的浮点型常量时，并没有考虑是否会溢出C++语言中的double类型，当然，这也可以认为是语义分析的任务，而非词法分析的任务，但这可以是程序改善的一处地方。

附思路流程图
总体逻辑：

主函数逻辑