hive 导入mysql 脚本 python
hive 导入mysql 脚本 python
sunhongxi 发表于4年前
hive 导入mysql 脚本 python
  • 发表于 4年前
  • 阅读 54
  • 收藏 0
  • 点赞 0
  • 评论 0

移动开发云端新模式探索实践 >>>   

摘要: hive 导入mysql 脚本 python

import subprocess

import sys 


print "脚本名:", sys.argv[0]

tableName=sys.argv[1]

startdate=sys.argv[2]

enddate=sys.argv[3]


if startdate=="":

print "please input statdate !"

sys.exit() 


if enddate=="" :

enddate=startdate


com=""

if tableName=="t_netflow_pc_city_cate_pv" :

#t_netflow_pc_city_cate_pv_yyyymm  hql

com="hive -e \"use tmpdb; select statdate , chlname,   pagetype,area1 ,cate1, cate2 ,   pv, uv ,newuv ,  visit,newvisit , avgpage , avgtime , jumpin , jumpout , outrate from ext_pctrack_is where statdate>='"+startdate+"' and statdate<='"+enddate+"'  and cate3='A' and cate4='A' and cate5='A' and area2='A' and area3='A' and area4='A' and source1='A' and source2='A';\" > tmp/t_netflow_pc_city_cate_pv_"+startdate+".txt;"


elif tableName=="t_netflow_pc_source_cate_pv":

com="hive -e \"use tmpdb; select statdate , chlname,  pagetype,source1 ,source2 ,  cate1, cate2,cate3 ,   pv, uv ,newuv ,  visit,newvisit , avgpage , avgtime , jumpin , jumpout , outrate   from ext_pctrack_is  where  statdate>='"+startdate+"' and statdate<='"+enddate+"' and cate4='all' and cate5='all' and area1='all' and area2='all' and area3='all' and area4='all' ;\" > tmp/t_netflow_pc_source_cate_pv_"+enddate+".txt;"


elif tableName=="t_netflow_pc_pv":

com="hive -e \"use tmpdb; select statdate , chlname,pagetype,source1 , area1,cate1,   pv, uv ,newuv , visit,newvisit , avgpage , avgtime , jumpin , jumpout , outrate   from ext_pctrack_is  where  statdate>='"+startdate+"' and statdate<='"+enddate+"' and cate2='all' and cate3='all' and cate4='all' and cate5='all' and area2='all' and area3='all' and area4='all' and  source2='all';\" > tmp/t_netflow_pc_pv_"+startdate+".txt;"


print com


def executeHql(hql):

p = subprocess.Popen(hql, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

res="1"

for line in p.stdout.readlines():

print line

if "Exception" in line :

executeHql(com)

res="0"

else :

res="1"

retval = p.wait()

return res



print "hive select  start ------------------------------------------"

count=1

while 1:

print "count is ----------------------" +str(count)

count=count+1

res=executeHql(com)

if res=="1":

break


print "hive select  end  ------------------------------------------"

print "mysql load  start ------------------------------------------"


mload="mysql -uXXX -pXXXX -hXXXX -PXXX -DXXX -e\"LOAD DATA LOCAL INFILE 'tmp/"+tableName+"_"+startdate+".txt' INTO TABLE "+tableName+"  FIELDS TERMINATED BY '\t' LINES TERMINATED BY '\n'\""


p = subprocess.Popen(mload, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

for line in p.stdout.readlines():

print line 

retval = p.wait()


print "mysql load  end ------------------------------------------"


  • 打赏
  • 点赞
  • 收藏
  • 分享
共有 人打赏支持
粉丝 1
博文 7
码字总数 3845
×
sunhongxi
如果觉得我的文章对您有用,请随意打赏。您的支持将鼓励我继续创作!
* 金额(元)
¥1 ¥5 ¥10 ¥20 其他金额
打赏人
留言
* 支付类型
微信扫码支付
打赏金额:
已支付成功
打赏金额: