某项目需要通过https请求服务器某接口，该项目使用 golang 编写。由于服务端接口实现较早，其使用的https证书无法被新版本golang识别。因为这个问题，一直未启动https。经考虑，决定在 golang 调用 libcur 实现 https的请求。
概述
在去年时已经发现这个问题，由于时间急，且服务端保留http接口，所以就一直用着。之前在原https证书上添加额外信息，golang中能正常连接。但因为涉及到服务端停机切换的事，影响范围很大，上峰慎之又慎，接着有其它的事务，这个事耽搁下来了。但一直没做也不是办法，于是最近持续抽出点时间，着手解决。
技术小结
- golang 内置有相应的库可以连接https服务器，但因为所用证书缺少信息，无法正常连接。
- 使用curl命令能正常连接，考虑用之，找到 go-curl 库，但调试后无法达到目的。可能是实际项目工程中设置的一些选项没有支持到位。
- 考虑自己实现接口，通过cgo调用curl库函数，将一次请求的全部过程都封装在C代码接口中，最终成功，但运行时系统上必须有libcurl等动态库库。
使用 go-curl
github仓库在此，下载：
$ go get -u github.com/andelf/go-curl
测试代码片段：
package mypostservice
import (
"fmt"
curl "go-curl"
"strings"
)
type CCurlDemo struct {
url string
cafile string
clientfile string
keyfile string
timeout int
}
func NewCCurlDemo() *CCurlDemo {
return &CCurlDemo{}
}
func (a *CCurlDemo) Init(url, cafile, clientfile, keyfile string, timeout int) error {
if url[0:5] == "https" {
if !com.IsExist(cafile) ||
!com.IsExist(clientfile) ||
!com.IsExist(keyfile) {
return fmt.Errorf("配置了https，但证书不存在")
}
}
a.url = url
a.cafile = cafile
a.clientfile = clientfile
a.keyfile = keyfile
a.timeout = timeout
return nil
}
func (a *CCurlDemo) CurlWriteCb(ptr []byte, userdata interface{}) bool {
ptrstr := string(ptr)
// 注：返回的字符串有回车换行
fmt.Printf("debug what? [%v]\n", ptrstr)
if strings.Contains(ptrstr, "HTTP/") {
}
if strings.Contains(ptrstr, "Server") {
}
if strings.Contains(ptrstr, "Date") {
}
if strings.Contains(ptrstr, "Content-Type") {
}
if strings.Contains(ptrstr, "Content-Length") {
}
if strings.Contains(ptrstr, "Connection") {
}
if strings.Contains(ptrstr, "Content-Disposition") {
}
if strings.Contains(ptrstr, "<html>") {
}
if strings.Contains(ptrstr, "{") {
fmt.Println("got json ", ptrstr)
}
return true
}
func (a *CCurlDemo) RunCurlSimple(jsonBytes []byte) (respInfo string, logmsg string, err error) {
easy := curl.EasyInit()
// defer easy.Cleanup()
if easy == nil {
err = fmt.Errorf("curl init error")
return
}
easy.Setopt(curl.OPT_URL, a.url)
easy.Setopt(curl.OPT_POST, true)
easy.Setopt(curl.OPT_VERBOSE, true)
easy.Setopt(curl.OPT_SSLVERSION, 4)
// 判断是否需要证书
if a.url[0:5] == "https" {
fmt.Println("sending https....")
easy.Setopt(curl.OPT_SSL_VERIFYHOST, false)
easy.Setopt(curl.OPT_SSL_VERIFYPEER, false)
easy.Setopt(curl.OPT_CAINFO, a.cafile)
easy.Setopt(curl.OPT_SSLCERT, a.clientfile)
easy.Setopt(curl.OPT_SSLCERTPASSWD, "123456")
easy.Setopt(curl.OPT_SSLCERTTYPE, "PEM")
easy.Setopt(curl.OPT_SSLKEY, a.keyfile)
easy.Setopt(curl.OPT_SSLKEYPASSWD, "123456")
easy.Setopt(curl.OPT_SSLKEYTYPE, "PEM")
}
// contentType := "application/json"
// contentType = "multipart/form-data"
form := curl.NewForm()
form.AddWithFileType("file", jsonBytes)
easy.Setopt(curl.OPT_HTTPPOST, form) // TOCHECK
easy.Setopt(curl.OPT_HEADER, 1) // 下载数据包括HTTP头部
easy.Setopt(curl.OPT_CONNECTTIMEOUT_MS, a.timeout) // 超时
easy.Setopt(curl.OPT_TIMEOUT_MS, a.timeout) // 超时
// 接收回调
easy.Setopt(curl.OPT_WRITEFUNCTION, a.CurlWriteCb)
var curlback string
easy.Setopt(curl.OPT_WRITEDATA, curlback) // 接收回调函数第4个参数
fmt.Printf("oustr [%v]\n", curlback)
if err = easy.Perform(); err != nil {
println("easy perform: ", err.Error())
}
return
}
注：使用上面代码时，达不到预期结果，且考虑性能，不再使用，代码片段不能直接运行，仅作备份。
结果：
[latelee@master test]$ go test -v -run TestOtherServer
=== RUN TestOtherServer
online fee test....
version: libcurl/7.29.0 OpenSSL/1.0.2k zlib/1.2.7 libssh2/1.8.0
malloc 2774 0x133a370
libcurl debug 0x133a370 0x1339890
name: foo.json
* About to connect() to 172.18.18.10 port 86 (#0)
* Trying 172.18.18.10...
* Connected to 172.18.18.10 (172.18.18.10) port 86 (#0)
* successfully set certificate verify locations:
* CAfile: ../../../cert/all.pem
CApath: none
* SSL connection using ECDHE-RSA-AES256-GCM-SHA384
* Server certificate:
* subject: CN=172.18.18.10
* start date: 2023-02-16 08:19:00 GMT
* expire date: 2024-02-16 08:19:00 GMT
* Connection #0 to host 172.18.18.10 left intact
my.... errno 27
easy perform: curl: Out of memory ## !!!!! 此处出错
oustr []
free 0x133a370
Cal failed curl: Out of memory
--- PASS: TestOtherServer (0.03s)
PASS
ok curl-project/test 0.053s
封装调用libcurl
思路：
go-curl库经常性用调用cgo，比如调用一次easy.Setopt(xxx)就是一次cgo，毕竟cgo性能损失摆在那里，于是决定将一次请求都封装在同一个C函数中，而像证书、超时时间等参数，一旦确定不会修改。因此实际上一共封装2个函数。由于是使用 libcurl，系统上必须存在libcurl运行库及其依赖库，如libssh2等。
代码：
/*
使用 curl 库封装的请求接口
为减少cgo开销，在 C 中实现完整的初始化、请求过程，使用静态变量减少内存碎片
编译、运行的系统必须有libcurl、libssh2等库
*/
package mypostservice
/*
#cgo linux pkg-config: libcurl
#cgo darwin LDFLAGS: -lcurl
#cgo windows LDFLAGS: -lcurl
#include <stdlib.h>
#include <string.h>
#include <sys/time.h>
#include <curl/curl.h>
static char g_url[128] = {0};
static char g_cafile[128] = {0};
static char g_clifile[128] = {0};
static char g_keyfile[128] = {0};
static char g_filename[128] = {0};
static char g_backjson[2*1024*1024] = {0}; // 2MBit 应该足够
int g_timeout = 20000;
static void setOpt(char* url, char* cafile, char* clientfile, char* keyfile, int timeout) {
strncpy(g_url, url, sizeof(g_url));
strncpy(g_cafile, cafile, sizeof(g_cafile));
strncpy(g_clifile, clientfile, sizeof(g_clifile));
strncpy(g_keyfile, keyfile, sizeof(g_keyfile));
g_timeout = timeout;
}
typedef struct curl_https_reply
{
char head[64];
char filename[64];
char *pdata;
int len;
}TCurlHttpsReply;
size_t curl_write_cb(void *buffer, size_t size, size_t nmemb, void *stream)
{
int len = size * nmemb;
struct curl_https_reply *args = (struct curl_https_reply *)stream;
if(NULL == stream)
return 0;
// printf("%s() debug (%s)\n", __func__, (const char *)buffer);
// 获取返回的文件名称
if(strncmp((const char *)buffer, "Content-Disposition", 19) == 0) {
char *pos = strstr(buffer, "LL_FOO");
if (pos != NULL) {
// printf("got filename ... \n");
snprintf(args->filename, sizeof(args->filename), "%s", (const char *)pos);
args->filename[strlen(args->filename)-2] = '\0'; // 去掉最后的\r\n
}
}
// 有数据，且是json的
if(strncmp((const char *)buffer, "{", 1) == 0 && args->len == 0)
{
args->pdata = (char*)malloc(len + 1);
if (NULL == args->pdata)
return 0;
memset((char*)args->pdata , 0, len + 1);
memcpy((char*)args->pdata, buffer, len);
args->len = len;
return len;
}
if(args->len > 0)
{
args->pdata = (char*)realloc(args->pdata, args->len + len + 1);
if (NULL == args->pdata)
return 0;
memset((char*)args->pdata + args->len, 0x00, len + 1);
memcpy((char*)args->pdata + args->len, buffer, len);
args->len += len;
}
return len;
}
static char* postdata(char* jsonStr, int len) {
CURL *curl;
CURLcode ret;
curl_global_init(CURL_GLOBAL_SSL);
curl = curl_easy_init();
if (!curl) {
return NULL;
}
if (strncmp(g_url, "https://", 8) == 0) {
// 添加验证，暂不用
if(0 == 1)
{
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 1L);
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1L);
}
else
{
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L);
curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
}
curl_easy_setopt(curl, CURLOPT_CAINFO, g_cafile);
curl_easy_setopt(curl, CURLOPT_SSLCERT, g_clifile);
curl_easy_setopt(curl, CURLOPT_SSLCERTPASSWD, "123456");
curl_easy_setopt(curl, CURLOPT_SSLCERTTYPE, "PEM");
curl_easy_setopt(curl, CURLOPT_SSLKEY, g_keyfile);
curl_easy_setopt(curl, CURLOPT_SSLKEYPASSWD, "123456");
curl_easy_setopt(curl, CURLOPT_SSLKEYTYPE, "PEM");
}
// curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
curl_easy_setopt(curl, CURLOPT_SSLVERSION, 4);
struct curl_httppost *formpost=NULL;
struct curl_httppost *lastptr=NULL;
char time_str[32] = {0};
GetDateTimeStr(time_str, sizeof(time_str));
snprintf(g_filename, sizeof(g_filename), "LL_BAR_%s.json", time_str);
curl_formadd(&formpost, &lastptr,
CURLFORM_COPYNAME, "file",
CURLFORM_BUFFER, g_filename,
CURLFORM_BUFFERPTR, jsonStr,
CURLFORM_BUFFERLENGTH, len,
CURLFORM_CONTENTTYPE, "application/json",
CURLFORM_END);
TCurlHttpsReply *pOutStr = (TCurlHttpsReply *)malloc(sizeof(TCurlHttpsReply));
if (pOutStr == NULL) {
strcpy(g_backjson, "malloc out buffer NULL");
return g_backjson;
}
curl_easy_setopt(curl, CURLOPT_HTTPPOST, formpost);
curl_easy_setopt(curl, CURLOPT_HEADER, 1L);
curl_easy_setopt(curl, CURLOPT_URL, g_url);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_cb); // 回调处理
curl_easy_setopt(curl, CURLOPT_WRITEDATA, pOutStr); // 回调函数的第4 个参数
curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS, g_timeout);
curl_easy_setopt(curl,CURLOPT_TIMEOUT_MS, g_timeout);
curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
// curl_easy_setopt(curl,CURLOPT_POST, 1); // 似乎不调用亦可
// printf("%s() debug %s %s\n%s\n", __func__, g_url, g_filename, jsonStr);
ret = curl_easy_perform(curl);
if (CURLE_OK != ret)
{
sprintf(g_backjson, "curl perform error: %d", ret);
return g_backjson;
}
if(pOutStr->pdata != NULL)
{
printf("got data [%s] %d\n", pOutStr->pdata, pOutStr->len);
strncpy(g_backjson, pOutStr->pdata, pOutStr->len);
free(pOutStr->pdata);
}
free(pOutStr);
curl_formfree(formpost);
curl_easy_cleanup(curl);
curl_global_cleanup();
return g_backjson;
}
*/
import "C"
import "unsafe"
type MyCURL struct {
}
func NewCurl() *MyCURL {
return &MyCURL{}
}
func (this *MyCURL) SetOpt(url, cafile, clientfile, keyfile string, timeout int) {
var (
c_url *C.char
c_cafile *C.char
c_clientfile *C.char
c_keyfile *C.char
c_timeout C.int
)
c_url = C.CString(url)
c_cafile = C.CString(cafile)
c_clientfile = C.CString(clientfile)
c_keyfile = C.CString(keyfile)
c_timeout = C.int(timeout)
defer C.free(unsafe.Pointer(c_url))
defer C.free(unsafe.Pointer(c_cafile))
defer C.free(unsafe.Pointer(c_clientfile))
defer C.free(unsafe.Pointer(c_keyfile))
C.setOpt(c_url, c_cafile, c_clientfile, c_keyfile, c_timeout)
}
func (this *MyCURL) PostFiledata(jsonStr []byte) (outJson string) {
var (
c_jsonStr *C.char
length C.int
)
c_jsonStr = C.CString(string(jsonStr))
length = C.int(len(jsonStr))
defer C.free(unsafe.Pointer(c_jsonStr))
backjson := C.postdata(c_jsonStr, length)
outJson = C.GoString(backjson)
return
}
结果：
[latelee@master test]$ go test -v -run TestOtherServer
=== RUN TestOtherServer
online fee test....
version: libcurl/7.29.0 OpenSSL/1.0.2k zlib/1.2.7 libssh2/1.8.0
libcurl debug (nil) 0x236d620
name: foo.json
* About to connect() to 172.18.18.10 port 86 (#0)
* Trying 172.18.18.10...
* Connected to 172.18.18.10 (172.18.18.10) port 86 (#0)
* successfully set certificate verify locations:
* CAfile: ../../../cert/all.pem
CApath: none
* SSL connection using ECDHE-RSA-AES256-GCM-SHA384
* Server certificate:
* subject: CN=172.18.18.10
* start date: 2023-02-16 08:19:00 GMT
* expire date: 2024-02-16 08:19:00 GMT
> POST /mypost/foobar HTTP/1.1
Host: 172.18.18.10:86
Accept: */*
Content-Length: 2999
Expect: 100-continue
Content-Type: multipart/form-data; boundary=----------------------------b6f2fa93226e
< HTTP/1.1 100 Continue
debug what? [HTTP/1.1 100 Continue
]
debug what? [
]
< HTTP/1.1 200 OK
debug what? [HTTP/1.1 200 OK
]
< Server: nginx/1.16.1
debug what? [Server: nginx/1.16.1
]
< Date: Fri, 05 May 2023 00:36:40 GMT
debug what? [Date: Fri, 05 May 2023 00:36:40 GMT
]
< Content-Type: application/json
debug what? [Content-Type: application/json
]
< Content-Length: 690
debug what? [Content-Length: 690
]
< Connection: keep-alive
debug what? [Connection: keep-alive
]
< Content-Disposition: form-data;filename=bar.json
debug what? [Content-Disposition: form-data;filename=bar.json
]
debug what? [
]
<
debug what?
...
* Connection #0 to host 172.18.18.10 left intact
oustr1
...
logmsg :
test resp:
...
--- PASS: TestOtherServer (0.41s)
PASS
ok curl-project/test 0.428s
小结
上述代码目前只在测试环境测试，后续择机在生产环境中使用。就测试结果看，应该是没有大问题的。