一些常用的python代码合集，方便检索引用

模块1：读写excel文件

from datetime import datetimeimport odpsimport xlwtimport osfrom odps import DataFrameimport pandas as pdimport xlrdimport numpy as npfrom collections import defaultdictfrom collections import Counter# 写入工作簿def write_imf(fl_save_path, data):    wb = xlwt.Workbook(encoding='utf-8')  # 不写encoding会出现编码错误    sh = wb.add_sheet(u'data', cell_overwrite_ok=True)    # 表头部分,单独写    colnames = data.columns.values    for i in range(0, data.shape[1]):        sh.write(0, i, colnames[i])    # 表内容,循环写入,好像没简便的方法    for i in range(1, len(data) + 1):        for j in range(0, data.shape[1]):            value = data.iloc[i - 1, j]            # print(value)            # 这里的坑特别多！！！数据读进来之后就成numpy.xxx64的类型了，在dataframe的时候就需要统一干掉！            try:                value.dtype                if value.dtype == 'int64':                    value = int(value)                # print('value is:%d,type is:%s'%(value,type(value)))                if value.dtype == 'float64':                    value = float(value)                    # print('value is:%d,type is:%s' % (value, type(value)))            except(RuntimeError, TypeError, NameError, ValueError, AttributeError):                pass            sh.write(i, j, value)    wb.save(fl_save_path)    print('congratulation save successful!')def save_pd_to_csv(fl_save_path, data):    try:        # 直接转csv不加encoding，中文会乱码        data.to_csv(fl_save_path, encoding="utf_8_sig", header=True, index=False)  # 存储        return True    except:        return Falsedef get_excel_content(file_path):    # 获取excel内的SQL语句，需要通过xlrd获取workbook中的SQL内容，或者读txt，后续改为配置文件    wb = xlrd.open_workbook(file_path, encoding_override='utf-8')    sht = wb.sheet_by_index(0)  # 默认第一个工作表    # print(sht.name)    wb_cont_imf = []    nrows = sht.nrows  # 行数    wb_cont_imf = [sht.row_values(i) for i in range(0, nrows)]  # 第一个工作表内容按行循环写入    df = pd.DataFrame(wb_cont_imf[1:], columns=wb_cont_imf[0])    return df

# 获取年月第一天最后一天def getMonthFirstDayAndLastDay(year=None, month=None):    """    :param year: 年份，默认是本年，可传int或str类型    :param month: 月份，默认是本月，可传int或str类型    :return: firstDay: 当月的第一天，datetime.date类型              lastDay: 当月的最后一天，datetime.date类型    """    if year:        year = int(year)    else:        year = datetime.date.today().year    if month:        month = int(month)    else:        month = datetime.date.today().month    # 获取当月第一天的星期和当月的总天数    firstDayWeekDay, monthRange = calendar.monthrange(year, month)    # 获取当月的第一天    firstDay = datetime.date(year=year, month=month, day=1)    lastDay = datetime.date(year=year, month=month, day=monthRange)    # return firstDay, lastDay    return lastDay

# dataframe2pngdef render_mpl_table(data, col_width=5.0, row_height=0.625, font_size=1,                     header_color='#40466e', row_colors=['#f1f1f2', 'w'], edge_color='w',                     bbox=[0, 0, 1, 1], header_columns=0,                     ax=None,**kwargs):    if ax is None:        # size = (np.array(data.shape[::-1]) + np.array([0, 1])) * np.array([col_width, row_height])        # fig, ax = plt.subplots(figsize=size)        fig, ax = plt.subplots() # 创建一个空的绘图区        # 衍生知识点，服务器上安装中文字体        plt.rcParams['font.sans-serif'] = ['SimHei']  # 用来正常显示中文标签        # plt.rcParams['font.sans-serif'] = ['WenQuanYi Zen Hei Mono']        plt.rcParams['axes.unicode_minus'] = False  # 用来正常显示负号        plt.style.use('ggplot')        ax.axis('off')    mpl_table = ax.table(cellText=data.values, bbox=bbox, colLabels=data.columns, **kwargs)    mpl_table.auto_set_font_size(False)    mpl_table.set_fontsize(font_size)    for k, cell in six.iteritems(mpl_table._cells):        cell.set_edgecolor(edge_color)        nrow = k[0]        ncol = k[1]        # 设置表格底色        if nrow == 0 or ncol < header_columns:            cell.set_text_props(weight='bold', color='w')            cell.set_facecolor(header_color)        else:            cell.set_facecolor(row_colors[k[0] % len(row_colors)])    # # 对当日异常数据为0的部分，着重体现    # row_num = []    # for k, cell in mpl_table._cells.items():    #     nrow = k[0]    #     ncol = k[1]    #     val = cell.get_text().get_text()    #     if nrow > 0 and ncol == 2 and val != '0':    #         row_num.append(nrow)    # for k, cell in six.iteritems(mpl_table._cells):    #     nrow = k[0]    #     # 设置表格底色    #     if nrow in row_num:    #         cell.set_facecolor('gold')    # 保留原图的设置    # fig.set_size_inches(width/100.0,height/100.0)#输出width*height像素    plt.gca().xaxis.set_major_locator(plt.NullLocator())    plt.gca().yaxis.set_major_locator(plt.NullLocator())    plt.subplots_adjust(top=1, bottom=0, left=0, right=1, hspace=0, wspace=0)    plt.margins(0, 0)    return ax

#!/user/bin/python# -*- coding:utf-8 -*-_author_ = 'xisuo'import datetimeimport calendarimport xlwtimport osimport pandas as pdimport xlrdimport openpyxlimport numpy as npfrom collections import defaultdictimport platformfrom wordcloud import WordCloud,STOPWORDSimport matplotlib.pyplot as pltfrom PIL import Imagedef create_wordcloud(docs=None,imgs=None,filename=None):    '''    :param docs:读入词汇txt，尽量不重复    :param imgs: 读入想要生成的图形，网上随便找    :param filename: 保存图片文件名    :return:    '''    # Read the whole text.    text = open(os.path.join(current_file, docs)).read()    alice_mask = np.array(Image.open(os.path.join(current_file, imgs)))    print(font_path)    wc = WordCloud(background_color="white",                   max_words=2000,                   font_path=font_path,  # 设置字体格式，如不设置显示不了中文                   mask=alice_mask,                   stopwords=STOPWORDS.add("said")                   )    # generate word cloud    wc.generate(text)    # store to file    if filename is None:filename="词云结果.png"    wc.to_file(os.path.join(current_file, filename))def main():    docs='demo.txt'    #读入的文本    imgs="eg.jpg"     #需要绘制的图像    filename='res_eg.png'     #保存图片文件名    create_wordcloud(docs=docs,imgs=imgs,filename=filename)    print('create wordcloud successful')if __name__ == '__main__':    start_time = datetime.datetime.now()    print('start running program at:%s' % start_time)    systemp_type = platform.system()    if (systemp_type == 'Windows'):        plt.rcParams['font.sans-serif'] = ['SimHei']  # 用来正常显示中文标签        plt.rcParams['axes.unicode_minus'] = False  # 用来正常显示负号        font_path='simfang.ttf'        try:            current_path = os.getcwd()        except:            current_path = os.path.dirname(__file__)        current_file = os.path.join(current_path, 'docs')        current_file = current_path    elif (systemp_type == 'Linux'):        font_path = 'Arial Unicode MS.ttf'        plt.rcParams['font.family'] = ['Arial Unicode MS']  # 用来正常显示中文标签        plt.rcParams['axes.unicode_minus'] = False  # 用来正常显示负号        current_file = '/home/xisuo/mhc_work/docs/'  # 服务器上的路径    else:        quit()    if not os.path.exists(current_file):        os.mkdir(current_file)        print('目录中部存在docs文件夹，完成新文件夹创建过程。')    print('当前操作系统：%s，文件存储路径为：%s' % (systemp_type, current_file))    main()    end_time = datetime.datetime.now()    tt = end_time - start_timepython    print('ending time:%s', end_time)    print('this analysis total spend time:%s' % tt.seconds)

#!/user/bin/python#-*- coding:utf-8 -*-_author_ = 'xisuo'import urllib.requestimport requestsfrom bs4 import BeautifulSoupfrom lxml import etreeimport osurl=';response=requests.get(url).text# soup=BeautifulSoup(response,'lxml')# cont=soup.find('article', class_='article-content')html=etree.HTML(response)src_list=html.xpath('//div/article/p/img/@src')current_path=os.path.dirname(__file__)save_path=os.path.join(current_path,'ppt_img')if os.path.exists(save_path):    passelse:    os.mkdir(save_path)    print('img folder create successful')i=1for src in src_list:    save_img_path=os.path.join(save_path,'%d.jpg'%i)    try:        with open(save_img_path,'wb') as f:            f.write(urllib.request.urlopen(src).read())            f.close()        i=i+1        print('save true')    except Exception as e:        print('save img fail')

rom sklearn import joblibfrom sklearn import svmfrom sklearn2pmml import PMMLPipeline, sklearn2pmmlimport pickledef save_model(train_X,train_y):    ''''    save model    :return:    '''    clf = svm.SVC()    clf.fit(X, y)    joblib.dump(clf, "train_model.m")    sklearn2pmml(clf, "train_model.pmml")    with open('train_model.pickle', 'wb') as f:        pickle.dump(clf, f)    return Truedef load_model():    '''    laod model    :return:    '''    clf_joblib=joblib.load('train_model.m')    clf_pickle== pickle.load(open('linearregression.pickle','rb'))    return clf_joblib,clf_pickle

import timeimport pandas as pdimport numpy as npfrom sklearn.feature_extraction.text import CountVectorizerfrom sklearn.feature_extraction.text import TfidfVectorizer# 读取数据 - 性能不好待优化print('开始读取KeyTag标签...')read_data_path = 'D:/untitled/incomelevel_kwtag_20190801.txt'load_data = pd.read_csv(read_data_path, sep='\t',encoding='utf-8')data = pd.DataFrame(load_data,columns = ['income_level','kw_tag'])print('...读取KeyTag标签完成')# 将数据分组处理print('开始分组处理KeyTag标签...')# 高收入incomelevel_top = data[data['income_level'] == '高']incomelevel_top = incomelevel_top.head() #testkw_tag_top = ' '.join(incomelevel_top['kw_tag'])print('kw_tag_top : \n',kw_tag_top)# 中收入incomelevel_mid = data[data['income_level'] == '中']incomelevel_mid = incomelevel_mid.head()  #testkw_tag_mid = ' '.join(incomelevel_mid['kw_tag'])print('kw_tag_mid : \n',kw_tag_mid)# 低收入incomelevel_low = data[data['income_level'] == '低']incomelevel_low = incomelevel_low.head()  #testkw_tag_low = ' '.join(incomelevel_low['kw_tag'])print('kw_tag_low : \n',kw_tag_low)print('...分组处理KeyTag标签完成')# 开始加载TF-IDFvectorizer = CountVectorizer()result = vectorizer.fit_transform([kw_tag_top, kw_tag_mid, kw_tag_low])transformer = TfidfVectorizer()kw_tag_score = transformer.fit_transform([kw_tag_top, kw_tag_mid, kw_tag_low])print('...KeyTag分词结束')# 获取全量标签kw_tag_value = transformer.get_feature_names()result_target = pd.DataFrame(kw_tag_value,columns = ['kw_tag'])print('result_target : \n',result_target)# 分词得分处理tf_score = kw_tag_score.toarray()print('tf_score : \n',tf_score)kw_tag_score_top = pd.DataFrame(tf_score[0],columns = ['kw_tag_score_top']) # 217kw_tag_score_mid = pd.DataFrame(tf_score[1],columns = ['kw_tag_score_mid'])kw_tag_score_low = pd.DataFrame(tf_score[2],columns = ['kw_tag_score_low'])print(len(kw_tag_score_top))

import timeimport pandas as pdimport xlrdimport reimport matplotlib.pyplot as pltimport siximport numpy as np# 载入ppt和pyecharts相关的包from pyecharts.render import make_snapshotfrom snapshot_phantomjs import snapshotfrom pyecharts import options as optsfrom collections import defaultdictfrom pyecharts.charts import Bar, Geo, Map, Line,Funnel,Pageimport osfrom example.commons import Fakerdef create_zjs_map():    folder_path = os.getcwd()    file_name = "白皮书数据地图.xlsx"    file_path = os.path.join(folder_path, file_name)    dat = get_excel_content(file_path, sheet_name="省份地图")    df = dat[['城市', '渗透率']]    df.columns = ['city', 'penarate']    print(df)    # df['city'] = df['city'].apply(lambda x: reg.sub('', x))    citys = df['city'].values.tolist()    values = df['penarate'].values.tolist()    print(citys)    print('{:.0f}%'.format(max(values)*100),'{:.0f}%'.format(min(values)*100))    city_name='浙江'    penetration_map = (        Map(init_opts=opts.InitOpts(width='1200px', height='1000px', bg_color='white'))            .add("{}透率分布".format(city_name), [list(z) for z in zip(citys, values)], city_name)            .set_series_opts(                label_opts=opts.LabelOpts(                    is_show=True,                    font_size=15            )        )            .set_global_opts(                visualmap_opts=opts.VisualMapOpts(                    is_show=True,                    max_=max(values),                    min_=min(values),                    is_calculable=False,                    orient='horizontal',                    split_number=3,                    range_color=['#C2D5F8', '#88B0FB', '#4D8AFD'],                    range_text=['{:.0f}%'.format(max(values)*100),'{:.0f}%'.format(min(values)*100)],                    pos_left='10%',                    pos_bottom='15%'                ),                legend_opts=opts.LegendOpts(is_show=False)        )    )    # penetration_map.render()    make_snapshot(snapshot, penetration_map.render(), "zj_map.png")    print('保存 zj_map.png')    return penetration_mapdef create_county_map(city_name):    folder_path = os.getcwd()    file_name = "白皮书数据地图.xlsx"    file_path = os.path.join(folder_path, file_name)    dat = get_excel_content(file_path, sheet_name="城市地图")    df = dat[['city', 'county', 'penarate']][dat.city == city_name]    citys = df['county'].values.tolist()    values = df['penarate'].values.tolist()    max_insurance = max(values)    print(citys)    province_penetration_map = (        Map(init_opts=opts.InitOpts(width='1200px', height='1000px', bg_color='white'))            .add("{}透率分布".format(city_name), [list(z) for z in zip(citys, values)], reg.sub('',city_name))            .set_series_opts(                label_opts=opts.LabelOpts(                is_show=True,                font_size=15            )        )            .set_global_opts(            visualmap_opts=opts.VisualMapOpts(                is_show=True,                max_=max(values),                min_=min(values),                is_calculable=False,                orient='horizontal',                split_number=3,                range_color=['#C2D5F8', '#88B0FB', '#4D8AFD'],                range_text=['{:.0f}%'.format(max(values) * 100), '{:.0f}%'.format(min(values) * 100)],                pos_left='10%',                pos_bottom='5%'            ),            legend_opts=opts.LegendOpts(is_show=False)        )    )    # insurance_map.render()    make_snapshot(snapshot, province_penetration_map.render(), "city_map_{}.png".format(city_name))    print('保存 city_map_{}.png'.format(city_name))    return province_penetration_mapdef create_funnel_label():    folder_path=os.getcwd()    file_name = "白皮书数据地图.xlsx"    file_path = os.path.join(folder_path, file_name)    dat = get_excel_content(file_path, sheet_name="漏斗图")    df = dat[['category', 'cnt']]    print(df)    category = df['category'].values.tolist()    values = df['cnt'].values.tolist()    funnel_map = (        Funnel(init_opts=opts.InitOpts(width='1200px', height='1000px', bg_color='white'))            .add("漏斗图", [list(z) for z in zip(category, values)])            .set_series_opts(                label_opts=opts.LabelOpts(                    position='inside',                    font_size=16,                )            )            .set_global_opts(                legend_opts=opts.LegendOpts(is_show=False)            )    )    # insurance_map.render()    make_snapshot(snapshot, funnel_map.render(), "funnel.png")    print('保存 funnel.png')    return funnel_mapcity_list=['温州市','杭州市','绍兴市','嘉兴市','湖州市','宁波市','金华市','台州市','衢州市','丽水市','舟山市']for city_name in city_list:    create_county_map(city_name)