def parse2csv(filepath):
logfile = open(filepath,'r')
source_ip_dict={}
res_url_dict={}
from_url_dict={}
category_dict={}
print('start.....')
for line in logfile:
line=line.strip()
if line!="":
source_ip = line.split('- -')[0].strip()
if re.match('[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}',source_ip):
if source_ip_dict.get(source_ip,'-')=='-':
source_ip_dict[source_ip]=1
else:
source_ip_dict[source_ip]=source_ip_dict[source_ip]+1
reg='"[GETPUOSHADINS]{3,8} /'
url_start = re.compile(reg)
re_result = url_start.findall(line)
if len(re_result)>=1:
res_url = '/'+line.split(re_result[0])[1].split(' ')[0]
category = strip_detail(res_url.split('/'))
if len(category)>=1:
if category[0] in ['article','item','question','topic','people','celebrity','brand','page','category']:
if category_dict.get(category[0],'-')=='-':
category_dict[category[0]]=1
else:
category_dict[category[0]]=category_dict[category[0]]+1
if res_url.endswith('.jpg') or res_url.endswith('.css') or res_url.endswith('.js') or res_url.endswith('.png') or res_url.endswith('.gif'):
pass
else:
if res_url.find(r'.css?')!=-1 or res_url.find(r'.js?')!=-1:
pass
else:
if res_url_dict.get(res_url,'-')=='-':
res_url_dict[res_url]=1
else:
res_url_dict[res_url]=res_url_dict[res_url]+1
from_url_start =' "http://'
if line.find(from_url_start)!=-1:
from_url = 'http://'+line.split(from_url_start)[1].split('" ')[0]
if from_url_dict.get(from_url,'-')=='-':
from_url_dict[from_url]=1
else:
from_url_dict[from_url]=from_url_dict[from_url]+1
logfile.close()
yesterday = datetime.date.today()-datetime.timedelta(days=1)
yesterday_str = str(yesterday).replace('-','')
conn = pymysql.connect(host='127.0.0.1', user='fenxi',passwd='', db='rizhifenxi', port = 3306, charset = 'utf8')
cur = conn.cursor()
cur.execute('insert into categort(yesterday_str, item, question, topic, people, celebrity, brand, page, category, article) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)', (yesterday_str, category['item'], category['question'], category['topic'], category['people'], category['celebrity'], category['brand'], category['page'], category['category'], category['article']))
#temp = "insert into ipList(ipList) values ("+ ip_list[j] + ")"
#print temp
#print ('success connect')
conn.commit()
cur.close()
conn.close()代码如上
报错如下：
root@iZbp1iqn00z9x3jov6bas1Z:/data/wwwlogs/www# python parselog2.py
/data/wwwlogs/www/m.xxx.com-access_20170419.log
start.....
Traceback (most recent call last):
File "parselog2.py", line 145, in <module>
parse2csv(filename)
File "parselog2.py", line 92, in parse2csv
cur.execute('insert into categort(yesterday_str, item, question, topic, people, celebrity, brand, page, category, article) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)', (yesterday_str, category['item'], category['question'], category['topic'], category['people'], category['celebrity'], category['brand'], category['page'], category['category'], category['article']))
TypeError: list indices must be integers or slices, not str
root@iZbp1iqn00z9x3jov6bas1Z:/data/wwwlogs/www#
这是哪里错了呢？我想把category的统计结果这个结果导入到数据库，要怎么修改呢？
category修改成category_dict之后其他取值正常，就是这个取值失败，是什么原因呢？
cur.execute('insert into categort(yesterday_str, item, question, topic, people, celebrity, brand, page, category, article) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)', (yesterday_str, category_dict['item'], category_dict['question'], category_dict['topic'], category_dict['people'], category_dict['celebrity'], category_dict['brand'], 0, category_dict['category'], category_dict['article']))
KeyError: 'category'
