一、实验目标
从左至右逐个字符地对源程序进行扫描，产生一个个的单词符号，把作为字符串的源程序改造成为单词符号串的中间程序。词法分析器的功能是输入源程序，输出单词符号，并保存token的内容。程序语言的单词符号分为以下六种：iT标识符、cT字符、sT字符串、CT常数、KT关键字、PT界符。
二、实验内容
2.1概要设计
词法分析器，实现固定语法的识别，就要明白什么是词法分析器，它的功能是什么。词法分析是编译程序进行编译时第一个要进行的任务，主要是对源程序进行编译预处理（去除注释、无用的回车换行找到包含的文件等）之后，对整个源程序进行分解，分解成一个个单词，这些单词有且只有6类，分别是标识符、保留字、常数、字符、字符串、界符。以便为下面的语法分析和语义分析做准备。词法分析是所有后续工作的基础，如果这一步出错，比如明明是‘<=’却被拆分成‘<’和‘=’就会对下文造成不可挽回的影响。因此，在进行词法分析的时候一定要定义好这6种符号的集合。
词法分析器的设计包括：读取文件里的内容、有限自动机的设计、将读取的token序列存入字典之后按照文件里单词的顺序输出。其中有限自动机的设计为词法分析器的核心，实现了对预处理之后的文件中的token的识别。有限自动机通过分析当前读入的字符，跳转到下一状态，直到进入终止状态。并且根据当前token的终止状态，判断出token所属的类型码，存入相应的符号类型表。
2.2数据结构
（1）Dict为token序列建立字典，键是分割出的单词，值是单词所属的类型码
（2）词法分析类Scanner:
表 3 Scanner的数据成员
| 数据成员 |
|
| countt, IDentifierTbl[1000][20] |
i标识符表和其对应的计数器 |
| countct, SingleChar[1000] |
c字符表和其对应的计数器 |
| counts, StringChar[1000][20] |
S字符串表和其对应的计数器 |
| countc, ConstantTbl[1000][20] |
CT常数表和其对应的计数器 |
| operatorOrDelimiter[36][10] |
P界符表 |
| reserveWord[32][20] |
K关键字表 |
| resourceProject[10000] |
输入源程序的存放处 |
| Token[] |
每次分析出来的单词 |
表 4 关键函数
| 成员函数 |
|
| int searchReserve(char reserve[][20],char s[]) |
搜索解析出来的单词在二维字符数组中是否出现 |
| int searchRReserve(char reserve[],char s) |
搜索解析出来的单词在一维字符数组中是否出现 |
| bool IsLetter(char letter) bool IsDigit(char digit) |
判断当前字符是否是数字或字符，返回布尔型 |
| void filterResource(char r[],int pProject) |
过滤掉注释的部分，得到一个纯净的代码 |
| void Scanner(int &syn,char resourceProject[],char token[],int &pProject) |
生成token序列，同时将序列输出到文件里 |
| int main() |
运行主函数，进行文件数据输入 |
初始化标识符表，字符表，字符串表，字符表，界符表，关键字表和定义存储字符数组表称全局变量，所以不需要初始化函数，变量的调用也方便；
searchRReserve(char reserve[],char s)， searchReserve(char reserve[][20],char s[])函数用于避免出现重复，既每出现一个token类别都到原有的类别数组中搜索一下。；输出在Scanner（）函数中直接传输到文件。
2.3 流程图
图 3 token序列识别流程图
三、源程序代码：（加入注释）
#include <iostream>
#include<stdio.h>
#include<string.h>
#include<stdlib.h>
using namespace std;
//保留字表
static char reserveWord[32][20] =
{
"auto", "break", "case", "char", "const", "continue",
"default", "do", "double", "else", "enum", "extern",
"float", "for", "goto", "if", "int", "long",
"register", "return", "short", "signed", "sizeof", "static",
"struct", "switch", "typedef", "union", "unsigned", "void",
"volatile", "while"
};
//界运算符
static char operatorOrDelimiter[36][10]=
{
"+","-","*","/","<","<=",">",">=","=","==",
"!=",";","(",")","^",",","\"","\'","#","&",
"&&","|","||","%","~","<<",">>","[","]","{",
"}","\\",".","\?",":","!"
};
static char IDentifierTbl[1000][20]= {""}; //标识符表i
static char SingleChar[1000]= {""}; //单个字符表ct
static char StringChar[1000][20]= {""}; //字符串表S
static char ConstantTbl[1000][20]= {""}; //常数表C
int countct=0;//单个字符表ct
int countc=0;//常数表C
int counts=0; //字符串表S
int countt=0;//标识符表i
char resourceProject[10000];//输入的源程序存放处，最大可以存放10000个字符。
//查找保留字，若成功查找，则返回种别码
//否则返回-1，代表查找不成功，即为标识符
int searchReserve(char reserveWord[ ][20], char s[]);
bool IsLetter(char letter);
bool IsDigit(char digit);
void filterResource(char r[],int pProject);
void Scanner(int &syn,char resourceProject[],char token[],int &pProject);
int searchReserve(char reserve[][20],char s[])
{
for(int i=0; reserve[i][0]!='\0'; i++)
{
if(strcmp(reserve[i],s)==0)
return i+1;
}
return -1;
}
int searchRReserve(char reserve[],char s)
{
for(int i=0; reserve[i]!='\0'; i++)
{
if(reserve[i]==s)
return i+1;
}
return -1;
}
bool IsLetter(char letter)
{
if(letter>='a'&&letter<='z'||letter>='A'&&letter<='Z'||letter=='_')
return true;
else
return false;
}
bool IsDigit(char digit)
{
if(digit>='0'&&digit<='9')
return true;
else
return false;
}
void filterResource(char r[],int pProject)
{
char tempString[10000];
int count=0;
for(int i=0; i<=pProject; i++)
{
if(r[i]=='/'&&r[i+1]=='/')
while(r[i]!='\n')
i++;
if(r[i]=='/'&&r[i+1]=='*')
{
i+=2;
while(r[i]!='*'||r[i+1]!='/')
{
i++;
if(r[i]=='$')
{
cout<<"注释出错"<<endl;
exit(0);
}
}
i+=2;
}
if(r[i]!='\n'&&r[i]!='\t'&&r[i]!='\v'&&r[i]!='\r')
{
tempString[count++]=r[i];
}
}
tempString[count]='\0';
strcpy(r,tempString);
}
void Scanner(int &syn,char resourceProject[],char token[],int &pProject)
{
FILE *fp1;
if((fp1=fopen("E:\\2017.txt","at"))==NULL)
{
cout<<"cam not open";
exit(0);
}
int i,count=0;//count用来做token指示器，收集有用字符
char ch;
ch=resourceProject[pProject];
while(ch==' ')
{
pProject++;
ch=resourceProject[pProject];
}
for(i=0; i<20; i++)
{
token[i]='\0';//收集前先清零
}
if(IsLetter(resourceProject[pProject]))
{
token[count++]=resourceProject[pProject];
pProject++;
while(IsLetter(resourceProject[pProject])||IsDigit(resourceProject[pProject]))
{
token[count++]=resourceProject[pProject];
pProject++;
}
token[count]='\0';
syn=searchReserve(reserveWord,token);
if(syn!=-1)
{
cout<<"{"<<"k"<<","<<syn<<","<<reserveWord[syn-1]<<"}"<<endl;
fprintf(fp1, "{k , %d ,%s }\n", syn,reserveWord[syn-1]);
}
if(syn==-1)
{
syn=searchReserve(IDentifierTbl,token);
if(syn==-1)
{
strcpy(IDentifierTbl[countt++],token);
syn=countt;
}
cout<<"{"<<"i"<<","<<syn<<","<<IDentifierTbl[syn-1]<<"}"<<endl;
fprintf(fp1, "{i , %d ,%s }\n", syn,IDentifierTbl[syn-1]);
}
}
else if(IsDigit(resourceProject[pProject]))
{
while(IsDigit(resourceProject[pProject]))
{
token[count++]=resourceProject[pProject];
pProject++;
}
token[count]='\0';
syn=searchReserve(ConstantTbl,token);
if(syn==-1)
{
strcpy(ConstantTbl[countc++],token);
syn=countc;
}
cout<<"{"<<"c"<<","<<syn<<","<<ConstantTbl[syn-1]<<"}"<<endl;
fprintf(fp1, "{c , %d ,%s }\n", syn,ConstantTbl[syn-1]);
}
else if(resourceProject[pProject]=='\'')
{
pProject++;
if(IsLetter(resourceProject[pProject]))
{
pProject++;
if(resourceProject[pProject]=='\'')
{
pProject--;
syn=searchRReserve(SingleChar,resourceProject[pProject]);
if(syn==-1)
{
SingleChar[countct++]=resourceProject[pProject];
syn=countct;
}
}
}
cout<<"{"<<"CT"<<","<<syn<<","<<resourceProject[syn-1]<<"}"<<endl;
fprintf(fp1, "{CT , %d ,%c }\n", syn,resourceProject[syn-1]);
pProject+=2;
}
else if(resourceProject[pProject]=='"')
{
pProject++;
while(resourceProject[pProject]!='"')
{
token[count++]=resourceProject[pProject];
pProject++;
}
token[count]='\0';
syn=searchReserve(StringChar,token);
if(syn==-1)
{
strcpy(StringChar[counts++],token);
syn=counts;
}
cout<<"{"<<"S"<<","<<syn<<","<<StringChar[syn-1]<<"}"<<endl;
fprintf(fp1, "{S , %d ,%s }\n", syn,StringChar[syn-1]);
pProject++;
}
else if (ch == '+' || ch == '-' || ch == '*' || ch == '/' || ch == ';' || ch == '(' || ch == ')' || ch == '^'
|| ch == ',' || ch == '\"' || ch == '\'' || ch == '~' || ch == '#' || ch == '%' || ch == '['
|| ch == ']' || ch == '{' || ch == '}' || ch == '\\' || ch == '.' || ch == '\?' || ch == ':')
{
//若为运算符或者界符，查表得到结果
token[0] = resourceProject[pProject];
token[1] = '\0';//形成单字符串
for (i = 0; i<36; i++)
{
//查运算符界符表
if (strcmp(token, operatorOrDelimiter[i]) == 0)
{
syn = i+1;//获得种别码，使用了一点技巧，使之呈线性映射
break;//查到即推出
}
}
cout<<"{"<<"p"<<","<<syn<<","<<operatorOrDelimiter[syn-1]<<"}"<<endl;
fprintf(fp1, "{p , %d ,%s }\n", syn,operatorOrDelimiter[syn-1]);
pProject++;//指针下移，为下一扫描做准备
}
else if(resourceProject[pProject]=='<')
{
//<,<=,<<
pProject++;
if(resourceProject[pProject]=='=') syn=6;
else if(resourceProject[pProject]=='<')
syn=26;
else
{
pProject--;
syn=5;
}
cout<<"{"<<"p"<<","<<syn<<","<<operatorOrDelimiter[syn-1]<<"}"<<endl;
fprintf(fp1, "{p , %d ,%s }\n", syn,operatorOrDelimiter[syn-1]);
pProject++;
}
else if(resourceProject[pProject]=='>')
{
//>,>=,>>
pProject++;
if(resourceProject[pProject]=='=') syn=8;
else if(resourceProject[pProject]=='>') syn=27;
else
{
pProject--;
syn=7;
}
cout<<"{"<<"p"<<","<<syn<<","<<operatorOrDelimiter[syn-1]<<"}"<<endl;
fprintf(fp1, "{p , %d ,%s }\n", syn,operatorOrDelimiter[syn-1]);
pProject++;
}
else if(resourceProject[pProject]=='=')
{
//=,==
pProject++;
if(resourceProject[pProject]=='=') syn=10;
else
{
pProject--;
syn=9;
}
cout<<"{"<<"p"<<","<<syn<<","<<operatorOrDelimiter[syn-1]<<"}"<<endl;
fprintf(fp1, "{p , %d ,%s }\n", syn,operatorOrDelimiter[syn-1]);
pProject++;
}
else if(resourceProject[pProject]=='!')
{
// !,!=
pProject++;
if(resourceProject[pProject]=='=') syn=11;
else
{
pProject--;
syn=36;
}
cout<<"{"<<"p"<<","<<syn<<","<<operatorOrDelimiter[syn-1]<<"}"<<endl;
fprintf(fp1, "{p , %d ,%s }\n", syn,operatorOrDelimiter[syn-1]);
pProject++;
}
else if(resourceProject[pProject]=='&')
{
//&,&&
pProject++;
if(resourceProject[pProject]=='&') syn=21;
else
{
pProject--;
syn=20;
}
cout<<"{"<<"p"<<","<<syn<<","<<operatorOrDelimiter[syn-1]<<"}"<<endl;
fprintf(fp1, "{p , %d ,%s }\n", syn,operatorOrDelimiter[syn-1]);
pProject++;
}
else if(resourceProject[pProject]=='|')
{
//|,||
pProject++;
if(resourceProject[pProject]=='|') syn=23;
else
{
pProject--;
syn=22;
}
cout<<"{"<<"p"<<","<<syn<<","<<operatorOrDelimiter[syn-1]<<"}"<<endl;
fprintf(fp1, "{p , %d ,%s }\n", syn,operatorOrDelimiter[syn-1]);
pProject++;
}
else if(resourceProject[pProject]=='$')
{
//$
syn=0;
}
else
{
cout<<"error: no exist "<<resourceProject[pProject]<<endl;
exit(0);
}
fclose(fp1);
}
int main()
{
char token[20]= {0}; //每次扫描的时候存储已经扫描的结果。
int syn=-1;//syn即为种别码，约定‘$’的种别码为0，为整个源程序的结束符号一旦扫描到这个字符代表扫描结束
int pProject = 0;//源程序指针，始终指向当前源程序待扫描位置。
FILE *fp;
if((fp=fopen("E:\\2018.txt","r"))==NULL)
{
cout<<"cam not open";
exit(0);
}
resourceProject[pProject]=fgetc(fp);
while(resourceProject[pProject]!='$')
{
pProject++;
resourceProject[pProject]=fgetc(fp);
}
resourceProject[++pProject]='\0';
fclose(fp);
cout<<endl<<"源程序为"<<endl;
cout<<resourceProject<<endl;
//过滤
filterResource(resourceProject, pProject);
cout<<endl<<"过滤之后"<<endl;
cout<<resourceProject<<endl;
pProject=0;
while(syn!=0)
{
Scanner(syn,resourceProject,token,pProject);
}
return 0;
}
四、程序运行结果：（截屏）
五：画蛇添足
有什么问题欢迎大家互相学习讨论~~
