python使用dataframe统计的一个小脚本：

 此脚本是我读取JSON文件，解析为词典，然后读取为dataframe，通过列名进行统计，最后统计输出到结果文件

_metaclass_=type
import ConfigParser
import os
import pandas as pd
import json
from __future__ import division

#读取配置文件获取输入路径，输出路径
cf = ConfigParser.ConfigParser()
cf.read("config.conf")#配置文件路径
inpath = cf.get("config","inpath")
outpath = cf.get("config","outpath")
print inpath
#遍历dirname，获取所有文件路径
result = []
for maindir, subdir, file_name_list in os.walk(inpath):
    for filename in file_name_list:
        apath = os.path.join(maindir, filename)
        result.append(apath)
print result
#遍历文件路径，获取每个文件的质控结果
results = pd.DataFrame([['表名称','字段名称','总行数','空值行数','空值占比','去重后行数','样例数据','样例数据对应条数']],columns=['tablename','columnName','总行数','无效记录数','无效占比','去重行数','样例数据','样例数据对应条数'])
df = pd.DataFrame()
for inpath in result:
    if "_SUCCESS" not in inpath:#测试过滤出目标文件并统计
        f = open(inpath)
        lines = f.readlines()
        for line in lines:
            dic = json.loads(line)
            ps = pd.DataFrame(dic,index=['1'])
            df = df.append(ps)
names = df.columns
tablename = inpath.split('\\')[len(inpath.split('\\'))-1]
sumcounts = len(df)
print sumcounts
for name in names:
    ylsj = df[name].value_counts()[0:5].reset_index()['index'].tolist()
    ylsjcount = df[name].value_counts()[0:5].reset_index()[name].tolist()
    discount = len(df.drop_duplicates([name]))
    nullcounts = len(df[df[name].isin(['','NULL'])])
    ratio = nullcounts/sumcounts*100
    result = pd.DataFrame([[tablename,name,sumcounts,nullcounts,ratio,discount,ylsj,ylsjcount]],columns=['tablename','columnName','总行数','无效记录数','无效占比','去重行数','样例数据','样例数据对应条数'])
    print result
    results = results.append(result,ignore_index=True)
results.to_csv(outpath, index=False,mode='a', header=False )

 版权所有！