protobuf+netty自定义编码解码

protobuf+netty自定义编

项目背景

protobuf+netty自定义编码解码

比如心跳协议，客户端请求的协议是10001，在java端如何解码，心跳返回协议如何编码，将协议号带过去

// 心跳包
//10001
message c2s_heartbeat {
}

//10002
message s2c_heartbeat {
  int64 timestamp = 1;    // 时间戳 ms
}

解决方案

1.每个协议id和生成的class类名关联起来，使用的时候使用读取文件

比如 proto.txt

com.git.LoginProto$c2s_heartbeat=10001

2.使用jprotobuf 把注释上面的协议id带入到生成文件里面

使用protoc生成java文件的时候带上自定注解

<dependency>
			<groupId>com.baidu</groupId>
			<artifactId>jprotobuf</artifactId>
			<version>2.4.15</version>
		</dependency>

重写根据proto文件生成java代码的方法百度版本的核心文件在ProtobufIDLProxy类

重写核心方法 createCodeByType 生成代码的核心方法

	private static CodeDependent createCodeByType(ProtoFile protoFile, MessageElement type, Set<String> enumNames,
                                                  boolean topLevelClass, List<TypeElement> parentNestedTypes, List<CodeDependent> cds, Set<String> packages,
                                                  Map<String, String> mappedUniName, boolean isUniName) {
    
    
        //...省略

if (topLevelClass) {
    
    
            // define package
            if (!StringUtils.isEmpty(packageName)) {
    
    
                code.append("package ").append(packageName).append(CODE_END);
                code.append("\n");
            }
            // add import;
            code.append("import com.baidu.bjf.remoting.protobuf.FieldType;\n");
            code.append("import com.baidu.bjf.remoting.protobuf.EnumReadable;\n");
            code.append("import com.baidu.bjf.remoting.protobuf.annotation.Protobuf;\n");

        }
		//添加自定义操作
        generateCommentsForClass(code,type,protoFile);

        // define class
        String clsName;
        if (topLevelClass) {
    
    
            clsName = "public class ";
        } else {
    
    
            clsName = "public static class ";
        }

/**
     * 生成class注释
     * @param code 当前代码
     * @param type 当前类型
     * @param protoFile 所有类型
     * @return 是否返回协议码
     */
    private static void generateCommentsForClass(StringBuilder code, MessageElement type, ProtoFile protoFile) {
    
    
        TypeElement typeElement = protoFile.typeElements().stream().filter(i -> i.name().equals(type.name())).findFirst().orElse(null);
        if(typeElement==null){
    
    
            return;
        }
        String documentation = typeElement.documentation();
        if(StringUtils.isEmpty(documentation)){
    
    
            documentation = "";
        }else {
    
    
            documentation = documentation.trim();
        }
        String[] split = documentation.split("\n");
        Integer protoId = null;
        try{
    
    
            protoId = Integer.parseInt(split[split.length-1]);
            String collect = Arrays.stream(split).collect(Collectors.toList()).subList(0, split.length - 1).stream().collect(Collectors.joining());

            //code.append("import com.baidu.bjf.remoting.protobuf.annotation.ProtobufClass;\n");

            String comment = """
                
                /**
                 * %d
                 * %s 
                 * @author authorZhao
                 * @since %s
                 */
                """;

            comment = String.format(comment,protoId,collect,DATE);
            code.append(comment);
            code.append("@com.git.ProtoId("+protoId+")";
        }catch (Exception e){
    
    
            String comment = """
                
                /**
                 * %s
                 * @author authorZhao
                 * @since %s
                 */
                """;
            comment = String.format(comment,documentation,DATE);
            code.append(comment);
        }



        /*code.append("    /**").append(ClassCode.LINE_BREAK);
        code.append("     * ").append(documentation).append(ClassCode.LINE_BREAK);
        code.append("     * ").append(ClassCode.LINE_BREAK);*/
        //code.append("     */").append(ClassCode.LINE_BREAK);
    }

用法

public static void main(String[] args) {
    
    
        File javaOutPath = new File("E:\\java\\workspace\\proto\\src\\main\\java");
        javaOutPath = new File("C:\\Users\\Admin\\Desktop\\工作文档\\worknote\\java");
        File protoDir = new File("E:\\project\\git\\test_proto");
        //protoDir = copy(protoDir);
        //filterFile(protoDir);

        File protoFile = new File(protoDir.getAbsolutePath()+"/activity.proto");
        MyProtobufIDLProxy.setFormatJavaField(true);
        try {
    
    
            //这里改写之后可以根据一个proto文件生成所有的文件
            MyProtobufIDLProxy.createAll(protoFile,protoDir, javaOutPath);
            System.out.println("create success. input file="+protoFile.getName()+"\toutput path=" + javaOutPath.getAbsolutePath());
        } catch (IOException var5) {
    
    
            System.out.println("create failed: " + var5.getMessage());
        }
        System.exit(0);
    }

3.重写protobuf的核心文件protoc

以windows为例

git clone https://github.com/protocolbuffers/protobuf.git
cd protobuf
git submodule update --init --recursive

本文使用clion开发环境，找到核心代码

SourceLocation location;
  if (descriptor->GetSourceLocation(&location)) {
    
    
    WriteDocCommentBodyForLocation(printer, location, kdoc);
  }
std::string comments = location.leading_comments.empty()
                             ? location.trailing_comments
                             : location.leading_comments;
  if (!comments.empty()) {
    
    
    if (kdoc) {
    
    
      comments = EscapeKdoc(comments);
    } else {
    
    
      comments = EscapeJavadoc(comments);
    }

    std::vector<std::string> lines = absl::StrSplit(comments, "\n");
    while (!lines.empty() && lines.back().empty()) {
    
    
      lines.pop_back();
    }

    if (kdoc) {
    
    
      printer->Print(" * ```\n");
    } else {
    
    
      printer->Print(" * <pre>\n");
    }

    for (int i = 0; i < lines.size(); i++) {
    
    
      // Most lines should start with a space.  Watch out for lines that start
      // with a /, since putting that right after the leading asterisk will
      // close the comment.
      if (!lines[i].empty() && lines[i][0] == '/') {
    
    
        printer->Print(" * $line$\n", "line", lines[i]);
      } else {
    
    
        printer->Print(" *$line$\n", "line", lines[i]);
      }
    }

    if (kdoc) {
    
    
      printer->Print(" * ```\n");
    } else {
    
    
      printer->Print(" * </pre>\n");
    }
    printer->Print(" *\n");
  }

重写方法 WriteMessageDocComment 把注释的最后一行协议号提取出来增加一个协议id

void WriteMessageDocComment(io::Printer* printer, const Descriptor* message,
                            const bool kdoc) {
    
    
  printer->Print("/**\n");
  WriteDocCommentBody(printer, message, kdoc);
  if (kdoc) {
    
    
    printer->Print(
        " * Protobuf type `$fullname$`\n"
        " */\n",
        "fullname", EscapeKdoc(message->full_name()));
  } else {
    
    
    printer->Print(
        " * Protobuf type {@code $fullname$}\n"
        " */\n",
        "fullname", EscapeJavadoc(message->full_name()));
  }
}

简单改写一下

       //网上抄袭的
    bool isNum(const std::string& str){
    
    
        std::stringstream sin(str);
        double t;
        char p;
        if(!(sin >> t))
            /*解释：
                sin>>t表示把sin转换成double的变量（其实对于int和float型的都会接收），如果转换成功，则值为非0，如果转换不成功就返回为0
            */
            return false;
        if(sin >> p)
            /*解释：此部分用于检测错误输入中，数字加字符串的输入形式（例如：34.f），在上面的的部分（sin>>t）已经接收并转换了输入的数字部分，在stringstream中相应也会把那一部分给清除，如果此时传入字符串是数字加字符串的输入形式，则此部分可以识别并接收字符部分，例如上面所说的，接收的是.f这部分，所以条件成立，返回false;如果剩下的部分不是字符，那么则sin>>p就为0,则进行到下一步else里面
              */
            return false;
        else
            return true;
    }

    /**
    * 生成自定义代码
    * @param printer
    * @param message
    * @param kdoc
    * */
    void writeWithProtoId(io::Printer *printer, const Descriptor *message) {
    
    
        SourceLocation location;

        bool hasComments = message->GetSourceLocation(&location);
        if (!hasComments) {
    
    
            return;
        }

        std::string comments = location.leading_comments.empty()? location.trailing_comments: location.leading_comments;
        if (comments.empty()) {
    
    
            return;
        }

        //这里当做非kdoc
        comments = EscapeJavadoc(comments);

        //根据换行分割
        std::vector<std::string> lines = absl::StrSplit(comments, "\n");
        while (!lines.empty() && lines.back().empty()) {
    
    
            lines.pop_back();
        }
        if(lines.empty()){
    
    
            return;
        }
        std::string protoId = lines[lines.size()-1];
        if(!isNum(protoId)){
    
    
            return;
        }
        printer->Print("@com.git.protoId($line$)\n","line",protoId);
    }


void WriteMessageDocComment(io::Printer* printer, const Descriptor* message,
                            const bool kdoc) {
    
    
  printer->Print("/**\n");
  WriteDocCommentBody(printer, message, kdoc);
  if (kdoc) {
    
    
    printer->Print(
        " * Protobuf type `$fullname$`\n"
        " */\n",
        "fullname", EscapeKdoc(message->full_name()));
  } else {
    
    
      printer->Print(
              " * Protobuf type {@code $fullname$}\n"
              " */\n",
              "fullname", EscapeJavadoc(message->full_name()));
      writeWithProtoId(printer,message);
  }
}

protoc.exe --plugin=protoc-gen-grpc-java=./protoc-gen-grpc-java-1.57.1-windows-x86_64.exe --proto_path=./proto ./proto*.proto --java_out=./test --grpc-java_out=./test

最后生成的代码

 /**
   * <pre>
   *身份验证c2s
   *10007
   * </pre>
   *
   * Protobuf type {@code login.c2s_auth}
   */
  @com.git.protoId(10007)
  public static final class c2s_auth extends
      com.google.protobuf.GeneratedMessageV3 implements
      // @@protoc_insertion_point(message_implements:login.c2s_auth)
      c2s_authOrBuilder {
    
    

使用方式

本文结合spring扫描，

/**
 * 这个类并不注册什么bean，仅仅扫描protoBuf
 * ProtoScan类似于mybatis的scan，表示proto生成的java文件所在目录
 * 扫描处理protoId
 */
@Slf4j
public class BeanMapperSelector implements ImportBeanDefinitionRegistrar {
    
    

    /**
     * 扫描的包路径
     */
    private String[] basePackage;
    /**
     * 需要扫描的类
     */
    private Class[] classes;

    @Override
    public void registerBeanDefinitions(AnnotationMetadata importingClassMetadata, BeanDefinitionRegistry registry) {
    
    
        Map<String, Object> annotationAttributes = importingClassMetadata.getAnnotationAttributes(ProtoScan.class.getName());
        this.basePackage = (String[])annotationAttributes.get("basePackages");

        this.classes = (Class[])annotationAttributes.get("classes");

        List<Class> classList = new ArrayList<>();
        for (Class aClass : classes) {
    
    
            if(aClass.isAnnotationPresent(ProtoId.class) && com.google.protobuf.GeneratedMessageV3.class.isAssignableFrom(aClass)){
    
    
                classList.add(aClass);
            }
        }
        if(basePackage.length>0){
    
    
            List<String> list = List.of(basePackage).stream().map(this::resolveBasePackage).toList();
            List<Class> classes1 = ClassScanUtil.scanPackageClass(list, null, clazz -> clazz.isAnnotationPresent(ProtoId.class) && com.google.protobuf.GeneratedMessageV3.class.isAssignableFrom(clazz));
            classList.addAll(classes1);
        }
        for (Class aClass : classList) {
    
    
            try {
    
    
                ProtoId protoId = AnnotationUtils.getAnnotation(aClass, ProtoId.class);
                if(aClass.getSimpleName().startsWith("c2s")){
    
    
                    //将byte[]转化为对象的方法缓存
                    //com.google.protobuf.GeneratedMessageV3 protoObject = (com.google.protobuf.GeneratedMessageV3) method
                    .invoke(null, bytes);
                    Method m = aClass.getMethod("parseFrom", byte[].class);
                    AppProtocolManager.putProtoIdC2SMethod(protoId.value(),m);
                }else {
    
    
                    //class->protoId映射缓存
                    AppProtocolManager.putOldProtoIdByClass(protoId.value(),aClass);
                }
            }catch (Exception e){
    
    
                log.error("protoId 注册失败",e);
            }
        }
        //
        AppProtocolManager.info();
    }


    protected String resolveBasePackage(String basePackage) {
    
    
        String replace = basePackage.replace(".", "/");
        return ResourcePatternResolver.CLASSPATH_ALL_URL_PREFIX+replace+"/*.class";
    }

}

本文原创，转载请申明