2021SC@SDUSC
总览
本篇主要对PigMacro类进行分析，了解在解析器中对
宏文件（macro）的一些处理。
代码分析
substituteParams()方法
首先判断输入以及参数是否为空以及输入的长度与参数的数目是否匹配,接下来判断预计返回数以及实际返回数是否匹配，插入数组args[]input和output的数据，替换值分别对应参数以及返回值，如果键重叠宏参数应优先于命令行参数。
private String substituteParams(String[] inputs, String[] outputs,
int line, String file) throws ParserException {
if ((inputs == null && !params.isEmpty())
|| (inputs != null && inputs.length != params.size())) {
String msg = getErrorMessage(file, line,
"Failed to expand macro '" + name + "'",
"Expected number of parameters: " + params.size()
+ " actual number of inputs: "
+ ((inputs == null) ? 0 : inputs.length));
throw new ParserException(msg);
}
boolean isVoidReturn = false;
if (rets.isEmpty()) {
if (outputs != null && outputs.length > 0) {
String msg = getErrorMessage(file, line, "Cannot expand macro '"
+ name + "'",
"Expected number of return aliases: 0"
+ " actual number of return values: "
+ outputs.length);
throw new ParserException(msg);
}
isVoidReturn = true;
}
if (!isVoidReturn && ((outputs == null && !rets.isEmpty())
|| (outputs != null && outputs.length != rets.size()))) {
String msg = getErrorMessage(file, line, "Failed to expand macro '"
+ name + "'",
"Expected number of return aliases: " + rets.size()
+ " actual number of return values: "
+ ((outputs == null) ? 0 : outputs.length));
throw new ParserException(msg);
}
String[] args = new String[params.size()];
for (int i=0; i<params.size(); i++) {
if (inputs[i].startsWith("$"))
inputs[i]="\\\\"+inputs[i];
args[i] = params.get(i) + "=" + inputs[i];
}
if (!isVoidReturn) {
String[] args1 = new String[params.size() + rets.size()];
System.arraycopy(args, 0, args1, 0, params.size());
args = args1;
for (int i=0; i<rets.size(); i++) {
args[params.size() + i] = rets.get(i) + "=" + outputs[i];
}
}
StringWriter writer = new StringWriter();
BufferedReader in = new BufferedReader(new StringReader(body));
try {
PreprocessorContext pc = new PreprocessorContext(50);
pc.loadParamVal(Arrays.asList(args), null);
Map<String, String> paramVal = pc.getParamVal();
for (Map.Entry<String, String> e : pigContext.getParamVal().entrySet()) {
// overwrite=false since macro parameters should have precedence
// over commandline parameters (if keys overlap)
pc.processOrdLine(e.getKey(), e.getValue(), false);
}
ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(pc);
psp.genSubstitutedFile(in, writer);
} catch (Exception e) {
// catch both ParserException and RuntimeException
String msg = getErrorMessage(file, line,
"Macro inline failed for macro '" + name + "'",
e.getMessage() + "\n Macro content: " + body);
throw new ParserException(msg);
}
LOG.debug("--- after substition:\n" + writer.toString());
return writer.toString();
}
maskAlias()方法
掩码别名方法，输入的树的0号节点为宏的内联节点，实际行号位于“宏名称”节点中，然后开始解析宏，将它解析为语法树的形式，通过词法生成器生成词法文件，然后通过转换，将词法文件转换为该条语句对应的逻辑执行子计划。将逻辑执行子计划转化为一般树，检查该树，然后将宏调用点添加到展开的宏树中，递归地展开内联宏，在内联宏中屏蔽别名，返回屏蔽宏的树。
private CommonTree maskAlias(String in, Set<String> masks, CommonTree tree,
String file) throws ParserException {
// this is the MACRO_INLINE node. the real line number is in the
// macro name node
int line = tree.getChild(0).getLine();
CharStream input = null;
try {
// parse macro body into ast
input = new QueryParserStringStream(in, file);
} catch (IOException e) {
String msg = getErrorMessage(file, line, "Failed to inline macro '"
+ name + "'", e.getMessage() + "\nmacro content: " + in);
throw new ParserException(msg);
}
QueryLexer lex = new QueryLexer(input);
CommonTokenStream tokens = new CommonTokenStream(lex);
QueryParser.query_return result = null;
QueryParser parser = QueryParserUtils.createParser(tokens, startLine-1);
try {
result = parser.query();
} catch (RecognitionException e) {
e.line += startLine -1;
String msg = (fileName == null) ? parser.getErrorHeader(e)
: QueryParserUtils.generateErrorHeader(e, fileName);
msg += " " + parser.getErrorMessage(e, parser.getTokenNames());
String msg2 = getErrorMessage(file, line, "Failed to parse macro '"
+ name + "'", msg + "\nmacro content: " + in);
throw new ParserException(msg2);
}
CommonTree ast = (CommonTree)result.getTree();
LOG.debug("AST for macro '" + name + "':\n" + ast.toStringTree());
List<CommonTree> macroDefNodes = new ArrayList<CommonTree>();
traverseMacro(ast, macroDefNodes, "MACRO_DEF");
if (!macroDefNodes.isEmpty()) {
String fname = ((PigParserNode)ast).getFileName();
String msg = getErrorMessage(fname, ast.getLine(),
"Invalid macro definition", "macro '" + name
+ "' contains macro definition.\nmacro content: "
+ body);
throw new ParserException(msg);
}
// add macro invocation points to the expanded macro tree
PigParserNode pnode = (PigParserNode)tree;
List<InvocationPoint> invStack = pnode.getInvocationStack();
List<InvocationPoint> newInvStack = (invStack == null) ? new ArrayList<InvocationPoint>()
: new ArrayList<InvocationPoint>(invStack);
InvocationPoint pt = new InvocationPoint(line, file, name);
newInvStack.add(pt);
setInvocationStack(ast, newInvStack);
// recursively expand the inline macros
List<CommonTree> inlineNodes = new ArrayList<CommonTree>();
traverseMacro(ast, inlineNodes, "MACRO_INLINE");
for (CommonTree t : inlineNodes) {
CommonTree newTree = macroInline(t,
new ArrayList<PigMacro>(seen.values()), macroStack, pigContext);
QueryParserUtils.replaceNodeWithNodeList(t, newTree, null);
}
// mask the aliases in the inlined macro
CommonTreeNodeStream nodes = new CommonTreeNodeStream(ast);
AliasMasker walker = new AliasMasker(nodes);
walker.setParams(masks, name, idx++);
AliasMasker.query_return result2 = null;
CommonTree commonTree = null;
try {
result2 = walker.query();
} catch (RecognitionException e) {
e.line += startLine - 1;
String msg = walker.getErrorHeader(e) + " "
+ walker.getErrorMessage(e, walker.getTokenNames());
String msg2 = getErrorMessage(file, line, "Failed to mask macro '"
+ name + "'", msg + "\nmacro content: " + in);
throw new ParserException(msg2);
}
commonTree = result2.tree;
LOG.debug("AST for masked macro '" + name + "':\n"
+ commonTree.toStringTree());
return commonTree;
}
macroInline()方法
首先获取宏的名字以及声明，然后判断宏是否已经被声明或者是否在宏堆栈中，设置嵌套宏调用堆栈，使宏了解到上下文，因此，它可以替换主脚本中的参数，最后获取参数。
static CommonTree macroInline(CommonTree t,
List<PigMacro> macroDefs, Set<String> macroStack,
PigContext pigContext)
throws ParserException {
// get name
String mn = t.getChild(0).getText();
// get macroDef
PigMacro macro = null;
for (PigMacro pm : macroDefs) {
if (pm.getName().equals(mn)) {
macro = pm;
break;
}
}
String file = ((PigParserNode)t).getFileName();
if (macro == null) {
String msg = getErrorMessage(file, t.getLine(),
"Cannot expand macro '" + mn + "'",
"Macro must be defined before expansion.");
throw new ParserException(msg);
}
if (macroStack.contains(macro.name)) {
String msg = getErrorMessage(file, t.getLine(),
"Cannot expand macro '" + mn + "'",
"Macro can't be defined circularly.");
throw new ParserException(msg);
}
// set nested macro call stack
Set<String> newStack = new HashSet<String>(macroStack);
newStack.add(macro.name);
macro.setStack(newStack);
// inform the macro of the PigContext
// so it can substitute parameters from the main pigscript
macro.setPigContext(pigContext);
// get return values
int n = t.getChild(1).getChildCount();
String[] rets = new String[n];
for (int i = 0; i < n; i++) {
rets[i] = t.getChild(1).getChild(i).getText();
}
// get parameters
int m = t.getChild(2).getChildCount();
String[] params = new String[m];
for (int i = 0; i < m; i++) {
params[i] = t.getChild(2).getChild(i).getText();
}
return macro.inline(params, rets, t, file);
}
总结
本次分析主要针对的是在生成抽象语法树的过程中对宏的一些处理方法，例如替换参数，屏蔽别名等操作。