博客
关于我
强烈建议你试试无所不能的chatGPT,快点击我
ELK-Python(二)
阅读量:7094 次
发布时间:2019-06-28

本文共 3026 字,大约阅读时间需要 10 分钟。

不具有通用性,留作纪念。

[root@GXB-CTRLCENTER python]# cat insert_pv.py #!/usr/bin/env python# -*- coding:utf-8 -*-from datetime import *from with_conn_to_db import conn_to_mysqlimport urllib2,jsonimport time###define yestoday 0-24 hours delta part##########today = date.today()yestoday = today - timedelta(days=1)#print today,yestodaya = str(yestoday) + ' ' + '00:00:00'b = str(today) + ' ' + '00:00:00'timeArray1 = time.strptime(a, "%Y-%m-%d %H:%M:%S")timeArray2 = time.strptime(b, "%Y-%m-%d %H:%M:%S")start_time = int(time.mktime(timeArray1)) * 1000end_time = int(time.mktime(timeArray2)) * 1000#####define es index and search part########server = 'http://elk.xkops.com:9200/'#stat_index = 'client-visit-*'index='client-*'#start_time = 1459146210879#stop_time = 1459147110879url = server + index + "/_search?pretty=true"query_date={  "query": {    "filtered": {      "query": {        "query_string": {          "query": "*",          "analyze_wildcard": True        }      },      "filter": {        "bool": {          "must": [            {              "range": {                "@timestamp": {                  "gte": start_time,                  "lte": end_time,                  "format": "epoch_millis"                }              }            }          ],          "must_not": []        }      }    }  },  "size": 0,  "aggs": {    "2": {      "terms": {        "field": "visit_tenant_id",        "size": 1000,        "order": {          "1": "desc"        }      },      "aggs": {        "1": {          "sum": {            "field": "count"          }        },        "3": {          "terms": {            "field": "client_type",            "size": 10000000,            "order": {              "1": "desc"            }          },          "aggs": {            "1": {              "sum": {                "field": "count"              }            }          }        }      }    }  }}query_date = json.dumps(query_date)req = urllib2.Request(url,query_date)response = urllib2.urlopen(req)page = response.read()#print pageresult = json.loads(page)###避免当天多次插入,插入前先删除#######sql = "delete from pv_stat where create_time = '%s'" % (yestoday)with conn_to_mysql('logstash') as db:    db.execute(sql)for s in result['aggregations']['2']['buckets']:    tenant_id =  s['key']    type1 = s["3"]['buckets'][0]['key']    type1_count = s["3"]['buckets'][0]['doc_count']    sql = "insert into pv_stat(tenant_id,create_time,pv_count,client_type) values('%s','%s','%s','%s')" % (tenant_id,yestoday,type1_count,type1)    #print sql    with conn_to_mysql('logstash') as db:         db.execute(sql)    if len(s["3"]['buckets']) > 1:        type2 = s["3"]['buckets'][1]['key']        type2_count = s["3"]['buckets'][1]['doc_count']        sql = "insert into pv_stat(tenant_id,create_time,pv_count,client_type)  values('%s','%s','%s','%s')" % (tenant_id,yestoday,type2_count,type2)        #print sql        with conn_to_mysql('logstash') as db:            db.execute(sql)    else:        continue

 

转载于:https://www.cnblogs.com/xkops/p/5602061.html

你可能感兴趣的文章
什么是大数据?如何成为大数据工程师?
查看>>
Spring Security OAuth 个性化token
查看>>
妙回春堂完成首轮融资,投资方主体为华园科创 ...
查看>>
java开发之使用websocket实现web客户端与服务器之间的实时通讯
查看>>
阿里云Kubernetes容器服务上体验Knative
查看>>
想知道Python的 数据驱动编程框架 Da0tabot 是怎么运行?
查看>>
ASP.NET Core中自定义路由约束
查看>>
全栈看到的技术债务
查看>>
NeurIPS 2018 中的贝叶斯研究
查看>>
Linux虚拟机下安装Oracle 11G教程
查看>>
关于阿里云主机数据丢失问题,是常态还是个例?如何保障数据安全?
查看>>
Thinking in java(1):对象导论
查看>>
Powershell快速入门(一) 安装和使用
查看>>
11.OkHttp之java网络编程
查看>>
微博(APP)榜单爬虫及数据可视化
查看>>
Derek解读Bytom源码-创世区块
查看>>
常用的HTTP状态码
查看>>
快看!这个机器人穿着人类的鞋子在走路
查看>>
数据中心产业的五大现象
查看>>
搭建git私服
查看>>