import os
import subprocess
from elasticsearch import Elasticsearch
from elasticsearch.client.indices import IndicesClient
def get_all_index(es_host):
client = Elasticsearch(es_host)
schema = IndicesClient(client).get_mapping()
indices_full_list = schema.keys()
return [index for index in indices_full_list if not index.startswith(".")]
ALL_INDEX_LIST = get_all_index("http://127.0.0.1:9200") # 所有的index字符串列表
# 使用 npm install elasticdump
# elasticdump 所在的路径
ES_DUMP_CMD = os.path.join(os.path.dirname(__file__), "../node_modules/.bin/elasticdump")
ES_QUERY_JSON_DIR = os.path.expanduser("~/Desktop/es_query_json_dir")
def subprocess_popen(statement):
# 可以打印出命令的输出
file_out = subprocess.Popen(statement, shell=True, stdout=subprocess.PIPE)
break_count = 0
while True:
line = file_out.stdout.readline()
print(line.decode('utf-8').strip('
'))
if subprocess.Popen.poll(file_out) == 0:
break
if not line.decode('utf-8').strip('
'):
break_count += 1
else:
break_count = 0
if break_count >= 10:
break
def export_es_data():
source_es_host = "http://127.0.0.1:9200" # http://username:[email protected]:9200"
# 可以根据search_body仅导出查询出来的数据
search_body = '{"query":{"match":{"xx" : xx}}}'
limit = 10000 # 每次导出的数据条数,听说最大为10000,本人没有测试,根据自身机器的内存大小进行调节
index_count = 0
for index_str in ALL_INDEX_LIST[index_count:]:
es_export_cmd = f"{ES_DUMP_CMD} --limit={limit} --input={source_es_host}/{index_str} --output={ES_QUERY_JSON_DIR}{index_str}.json --searchBody '{search_body}'"
print(f"-------------export {index_str} ------------- index of ALL_INDEX_LIST: {index_count}")
print(es_export_cmd)
subprocess_popen(es_export_cmd)
index_count += 1
print("")
def import_es_data():
target_es_host = "http://127.0.0.1:9200" # http://username:[email protected]:9200"
index_count = 0
limit = 10000
for index_str in ALL_INDEX_LIST[index_count:]:
es_import_cmd = f"{ES_DUMP_CMD} --limit={limit} --input={ES_QUERY_JSON_DIR}{index_str}.json --output={target_es_host}"
print(f"-------------import {index_str} ------------- index of ALL_INDEX_LIST: {index_count}")
print(es_import_cmd)
subprocess_popen(es_import_cmd)
index_count += 1
print("")
if __name__ == '__main__':
export_es_data() # 导出的方法
import_es_data() # 导入的方法
python elasticsearch 导出数据到json文件导入到另一个es中
猜你喜欢
转载自blog.csdn.net/m0_67393157/article/details/124325780
今日推荐
周排行