对100M左右的数据进行保存和简单分析

老师给了一个100M左右的数据（.csv 格式），然后让我们用python 和excel 进行分析。

具体要求和代码：点击打开链接（数据太大，无法上传，谁需要可以私我）

t2.py

import sqlite3
import pandas as pd####   main       ##################################################
##分为两个步骤，先是对数据读取后进行处理
##然后将数据放入数据库
if __name__ == '__main__':##三个文件路径###########  对数据读取后进行处理   ############################################################fileName = ['./data/openData_Order201608_afterClear.csv', './data/openData_Order201609_afterClear.csv','./data/openData_Order201610_afterClear.csv']##对每个文件进行处理for index in range(3):##一行行处理数据，将所需要增加的都放在后面，不影响结果，但是操作更加方便rFile = open(fileName[index], "r")# 用来临时保存数据的文件，openData_Order201608.csvtargetFile = fileName[index][7:27] + '.csv'wFile = open(targetFile, "w")# print(targetFile)lineCount = 0dataLine = rFile.readline()  # 读取一行数据while dataLine:lineCount += 1# 分割数据，用来接下来的处理dataList = dataLine.split(",")# print(dataList)##对第一行，即header进行处理，增加所要求的内容if (lineCount == 1):dataList.append('MONTH')dataList.append('YEAR')dataList.append('AREA_NAME')dataList.append('BRAND_CODE')# print(dataList)else:# 对具体内容进行处理，增加所要求的内容MONTH = (dataList[0][4:6])#YEAR = (dataList[0][0:4])# print(YEAR)AREA_NAME = 'QX' + dataList[1][-3:]# print(AREA_NAME)BRAND_CODE = dataList[3][0:6]# print(BRAND_CODE)dataList.append(MONTH)dataList.append(YEAR)dataList.append(AREA_NAME)dataList.append(BRAND_CODE)# print(dataList)##将内容整理好，放入临时文件lineContent = ""rolCount = 0for col in dataList:rolCount += 1if (rolCount == 1):lineContent += col.strip()else:lineContent += "," + col.strip()# 一行行写入wFile.write(lineContent + "\n")# print(lineContent)dataLine = rFile.readline()print("数据行数=", lineCount)rFile.close()wFile.close()##########       将数据放入数据库             #################################### 连接数据库conn = sqlite3.connect("./db/saleData.db")##采用pandas 的一个函数，可以很方便的直接将数据从文件中写入到数据库df = pd.read_csv(targetFile)df.to_sql(fileName[index][7:27], conn, if_exists='append', index=False)

t3.py

import sqlite3# 连接数据库
conn = sqlite3.connect('./db/saleData.db')
# 查询语句
query_sql = '''
SELECT
openData_Order201608.KIND,
openData_Order201608.AREA_NAME,
Sum(openData_Order201608.PURCH_TAX_AMT)
FROM
openData_Order201608
GROUP BY
openData_Order201608.KIND,
openData_Order201608.AREA_NAME
'''
##进行查询
query = conn.execute(query_sql)
##将查询结果写入
filename = './txt/areaKind.txt'
fd = open(filename, 'w', encoding='utf-8')
print('烟的种类', '县区', '销售利润', file=fd, sep=',')
for kind in query:print(kind[0], kind[1], kind[2], file=fd, sep=',')
fd.close()
# print(query[1][1])

t4.py

import sqlite3# 连接数据库
conn = sqlite3.connect('./db/saleData.db')
###############################################
##对每个月的查询分开处理，先是8月
# 查询语句
query_sql = '''
SELECT
openData_Order201608.CUST_CODE,AVG(PURCH_TAX_AMT)
FROM
openData_Order201608
GROUP BY
openData_Order201608.CUST_CODE
'''
##查询
query = conn.execute(query_sql)
##将查询结果进行写入
filename = './txt/t4.txt'
num = 0
fd = open(filename, 'w', encoding='utf-8')
print('月份', '商家', '销售利润', file=fd, sep=',')
for kind in query:num += 1print('8月', kind[0], kind[1], file=fd, sep=',')
print('8月商户数量：', num)
###################     9月     ######################################
query_sql = '''
SELECT
openData_Order201609.CUST_CODE,AVG(PURCH_TAX_AMT)
FROM
openData_Order201609
GROUP BY
openData_Order201609.CUST_CODE
'''
query = conn.execute(query_sql)filename = './txt/t4.txt'
num = 0
fd = open(filename, 'a', encoding='utf-8')
# print('月份','商家','销售利润',file=fd,sep=',')
for kind in query:num += 1print('9月', kind[0], kind[1], file=fd, sep=',')
print('9月商户数量：', num)
###################       10月    ####################################
query_sql = '''
SELECT
openData_Order201610.CUST_CODE,AVG(PURCH_TAX_AMT)
FROM
openData_Order201610
GROUP BY
openData_Order201610.CUST_CODE
'''
query = conn.execute(query_sql)filename = './txt/t4.txt'
num = 0
fd = open(filename, 'a', encoding='utf-8')
# print('月份','商家','销售利润',file=fd,sep=',')
for kind in query:num += 1print('10月', kind[0], kind[1], file=fd, sep=',')
print('10月商户数量：', num)fd.close()

t5.py

import sqlite3##连接数据库
conn = sqlite3.connect('./db/saleData.db')##对每个月的查询分开处理，先是8月##############################################
query_sql = '''
SELECT
openData_Order201608.ITEM_CODE,
sum(openData_Order201608.PURCH_QTY),
sum(openData_Order201608.PURCH_TAX_AMT)/sum(openData_Order201608.PURCH_QTY)
FROM
openData_Order201608
GROUP BY
openData_Order201608.ITEM_CODE
'''
query = conn.execute(query_sql)filename = './txt/t5.txt'
num = 0
fd = open(filename, 'w', encoding='utf-8')
print('月', '规格', '数量', '单价', file=fd, sep=',')
for kind in query:if (kind[1] == 0):continuenum += 1print('8月', kind[0], kind[1], kind[2], file=fd, sep=',')
print(num)
####################   9月    ############################################
query_sql = '''
SELECT
openData_Order201609.ITEM_CODE,
openData_Order201609.PURCH_QTY,
openData_Order201609.PURCH_TAX_AMT/openData_Order201609.PURCH_QTY
FROM
openData_Order201609
GROUP BY
openData_Order201609.ITEM_CODE
'''
query = conn.execute(query_sql)filename = './txt/t5.txt'
num = 0
fd = open(filename, 'a', encoding='utf-8')
# print('月','规格','数量','单价',file=fd,sep=',')
for kind in query:if (kind[1] == 0):continuenum += 1print('9月', kind[0], kind[1], kind[2], file=fd, sep=',')
print(num)
#######################    10月  ##################################################
query_sql = '''
SELECT
openData_Order201610.ITEM_CODE,
openData_Order201610.PURCH_QTY,
openData_Order201610.PURCH_TAX_AMT/openData_Order201610.PURCH_QTY
FROM
openData_Order201610
GROUP BY
openData_Order201610.ITEM_CODE
'''
query = conn.execute(query_sql)filename = './txt/t5.txt'
num = 0
fd = open(filename, 'a', encoding='utf-8')
# print('月','规格','数量','单价',file=fd,sep=',')
for kind in query:if (kind[1] == 0):continuenum += 1print('10月', kind[0], kind[1], kind[2], file=fd, sep=',')
print(num)
fd.close()

t6.py

import numpy
import sqlite3# 连接数据库
conn = sqlite3.connect('./db/saleData.db')
##由于需要知道每一个商家的销售额占总销售额的比例，所以需要查询两次
# 第一次   每一个商家的销售额
query_sql = '''
SELECT
openData_Order201608.CUST_CODE,
Sum(openData_Order201608.PURCH_TAX_AMT)
FROM
openData_Order201608
GROUP BY
openData_Order201608.CUST_CODE
ORDER BY
Sum(openData_Order201608.PURCH_TAX_AMT) DESC'''
query = conn.execute(query_sql)
# 第二次   总的销售额
query_sql_total = '''
SELECT
Sum(openData_Order201608.PURCH_TAX_AMT)
FROM
openData_Order201608
'''
query_total = conn.execute(query_sql_total)
total_sum = query_total.fetchone()[0]
##      写入     ####################################################
filename = './txt/t6.txt'
num_total = 0  # 总的商家数
fd = open(filename, 'w', encoding='utf-8')
print('商户', '销售金额', '占比', file=fd, sep=',')
zhanbi_rate = 0  # 销售收入占比率
flag_num = 0  # 当超过80时，商家的个数
flag = 0  # 用来标记第一次超过80%
for kind in query:num_total += 1zhanbi = round((kind[1]) / (total_sum) * 100, 5)# 百分比    保留五位小数zhanbi_rate += zhanbiif zhanbi_rate > 80 and flag == 0:flag_num = num_totalprint('当商户的销售占比率超过80%,商户的数量:    ', flag_num)flag = 1print(kind[0], kind[1], zhanbi, file=fd, sep=',')
print('总的商户数量   :', num_total)
print('商户占比率   ', round(flag_num / num_total, 5))

t7.py

import sqlite3
import numpy as np# 连接数据库
conn = sqlite3.connect('./db/saleData.db')
#####   平均值      ##############################
query_sql = '''
SELECT
openData_Order201608.CUST_CODE,
Avg(openData_Order201608.PURCH_TAX_AMT)FROM
openData_Order201608
GROUP BY
openData_Order201608.CUST_CODE
ORDER BY
openData_Order201608.CUST_CODE ASC
'''
query = conn.execute(query_sql)
######      方差              ############################
query_sql_per = '''
SELECT
openData_Order201608.CUST_CODE,
openData_Order201608.PURCH_TAX_AMT
FROM
openData_Order201608
ORDER BY
openData_Order201608.CUST_CODE ASC
'''
query_per = conn.execute(query_sql_per)purch_per = []  # 用来保存每一个商家的销售额
fangcha = {}  # 用来保存每一个商家的销售额的方差
cust_code = 1  # 用来保存每一个商家的cust_code
last_custcode = 0  # 用来保存最后一个商家的cust_code
for i in query_per.fetchall():# 如果当前cust_code不等于之前的cust_code，则说明上一个商家的记录结束，进行处理if i[0] != cust_code:# print(purch_per)# 计算方差cust_fangcha = round(np.std(purch_per, ddof=1), 5)# print(cust_fangcha)# 用字典保存方差fangcha[cust_code] = cust_fangcha# 更新 cust_codecust_code = i[0]##将这个[]清空purch_per = []# 保存销售额purch_per.append(i[1])last_custcode = i[0]
###在之前的操作中，没有计算最后一个商家的方差，这里加上
last_fangcha = round(np.std(purch_per, ddof=1), 5)
fangcha[last_custcode] = last_fangcha
############    写入      ##############################
filename = './txt/t7.txt'
fd = open(filename, 'w', encoding='utf-8')
print('商家', '平均', '方差', file=fd, sep=',')
# fnagchaNum=0
for kind in query:print(kind[0], round(kind[1], 5), fangcha[kind[0]], file=fd, sep=',')# fnagchaNum += 1
fd.close()

t8.py

import sqlite3# 连接数据库
conn = sqlite3.connect('./db/saleData.db')
# 查询语句
query_sql = '''
SELECT
openData_Order201608.CUST_CODE,
Sum(openData_Order201608.PURCH_TAX_AMT)
FROM
openData_Order201608
GROUP BY
openData_Order201608.CUST_CODE
ORDER BY
SUM(PURCH_TAX_AMT) DESC
'''
# 查询
query = conn.execute(query_sql)# 遍历一遍，计算商家数量
num_cust = 0
for kind in query:num_cust += 1
# 一个很有趣的现象，如果查询结果被遍历一边，然后query就为[],所以需要再遍历一遍
query_again = conn.execute(query_sql)
numAgain = 0
# 每个等分中 商家的销售额
sumPer = [0 for i in range(5)]
# print(query.fetchall())
for Per in query_again:index = int(numAgain / (num_cust / 5))# print(index)sumPer[index] += Per[1]# print('-'*16)numAgain += 1
# 保存所要求的倍数
rate = []
for index in range(5):# 计算倍数，写入listrate.append(round(sumPer[0] / sumPer[index], 3))print(rate[index])
# print(sumPer)##写入
filename = './txt/t8.txt'
fd = open(filename, 'w', encoding='utf-8')
print('等分', '比值', file=fd, sep=',')
for index in range(5):print(index, rate[index], file=fd, sep=',')
fd.close()

t9.py

import sqlite3
#在这一道题中，我想要看一下每一类烟的平均单价与销售数量，看看有什么联系
##连接数据库
conn = sqlite3.connect('./db/saleData.db')##对每个月的查询分开处理，先是8月##############################################
query_sql = '''
SELECT
openData_Order201608.BRAND_CODE,
Sum(openData_Order201608.PURCH_TAX_AMT)/Sum(openData_Order201608.PURCH_QTY),
Sum(openData_Order201608.PURCH_QTY)
FROM
openData_Order201608
GROUP BY
openData_Order201608.BRAND_CODE
ORDER BY
Sum(openData_Order201608.PURCH_TAX_AMT)/Sum(openData_Order201608.PURCH_QTY) DESC
'''
query = conn.execute(query_sql)filename = './txt/t9.txt'
num = 0
fd = open(filename, 'w', encoding='utf-8')
print( '品牌', '单价', '数量', file=fd, sep=',')
for kind in query:if (kind[1] == 0):continuenum += 1print( kind[0], kind[1], kind[2], file=fd, sep=',')
print(num)

对100M左右的数据进行保存和简单分析相关推荐

爬取表格类网站数据并保存为excel文件
本文转载自以下网站:50 行代码爬取东方财富网上市公司 10 年近百万行财务报表数据 https://www.makcyun.top/web_scraping_withpython6.html 主要学 ...
python爬取网页表格数据匹配_爬取表格类网站数据并保存为excel文件
本文转载自以下网站:50 行代码爬取东方财富网上市公司 10 年近百万行财务报表数据 https://www.makcyun.top/web_scraping_withpython6.html 主要学 ...
python爬取表格数据匹配_爬取表格类网站数据并保存为excel文件
本文转载自以下网站:50 行代码爬取东方财富网上市公司 10 年近百万行财务报表数据 https://www.makcyun.top/web_scraping_withpython6.html 主要学 ...
[转]Asp.Net 上传大文件专题(3)--从请求流中获取数据并保存为文件[下]
转自:http://www.cnblogs.com/stg609/archive/2008/08/03/1259206.html 接着上一篇讲 3.4 读取剩余的请求前面我们已经提到过R ...
Asp.Net 上传大文件专题(3)--从请求流中获取数据并保存为文件[下]
接着上一篇讲 3.4 读取剩余的请求前面我们已经提到过ReadEntityBody (Byte[] buffer, Int32 size)方法,该方法可以用来读取客户端的请求数据 ...
python爬取贴吧所有帖子-通过python爬取贴吧数据并保存为word
前言 Python是一种跨平台的计算机程序设计语言.是一种面向对象的动态类型语言,最初被设计用于编写自动化脚本(shell),随着版本的不断更新和语言新功能的添加,越来越多被用于独立的.大型项目的开发 ...
Matlab的File菜单功能图解 - 导入数据、保存工作空间、搜索路径、系统参数
File菜单主要是打开/关闭文件.关闭窗口.导入数据.保存工作空间.设置搜索路径.设置系统参数.打印设置: 导入数据: Matlab的toolbox\stats目录下有一些自带的mat文件: 导入对话 ...
DL之AlexNet：利用卷积神经网络类AlexNet实现猫狗分类识别(图片数据增强→保存h5模型)
DL之AlexNet:利用卷积神经网络类AlexNet实现猫狗分类识别(图片数据增强→保存h5模型) 目录利用卷积神经网络类AlexNet实现猫狗分类识别(图片数据增强→保存h5模型) 设计思路处 ...
form 多条数据只保存修改的行_MVC中的新增、修改、删除
新增.修改.删除开发工具与关键技术:MVC作者:盘洪源撰写时间:2019年3月30日星期六新增.删除.修改.查询这都是MVC中最基础的,所以说想要学好MVC必须先把基础打好,查询是比较复杂点,这里 ...

对100M左右的数据进行保存和简单分析

对100M左右的数据进行保存和简单分析相关推荐

最新文章

热门文章