文档章节

day6-threading

Bing-
 Bing-
发布于 2016/05/17 10:11
字数 1496
阅读 23
收藏 0

thread instance:

#!/usr/bin/env python
#coding:utf-8
#author:Bing

from threading import Thread
import time

def test(arg):
    for i in range(10):
        print i
        #time.sleep(1)

t1 = Thread(target=test, args=("11",))
#t1.setDaemon(True)  don't waiting for subprogress
t1.start()
print t1.getName()
t1.join(5)
#t1.join()

print "after"
print "after"

class MyThread(Thread):
    def run(self):
        #time.sleep(15)
        print "test"
        Thread.run(self) #call bone excute

def bone():
    print "bone"

t2 = MyThread(target=bone)
t2.start()
print "over"

Product and Consumer mode:

#!/usr/bin/env python
#coding:utf-8
#author:Bing

from threading import Thread
import time,Queue,random

que = Queue.Queue(maxsize=100)

class Product(Thread):
    def __init__(self,name,que):
        super(Product,self).__init__()
        self.name = name
        self.que = que

    def run(self):
        while True:
            if self.que.full():
                time.sleep(1)
            else:
                self.que.put("baozi")
                print "%s:made a baozi...===" % self.name

class Consumer(Thread):
    def __init__(self,name,que):
        super(Consumer,self).__init__()
        self.name = name
        self.que = que

    def run(self):
        while True:
            if self.que.empty():
                time.sleep(1)
            else:
                self.que.get()
                print "%s:got a baozi" % self.name

#three productor
t1 = Product("chef1",que)
t2 = Product("chef1",que)
t3 = Product("chef1",que)
t1.start()
t2.start()
t3.start()

#twenty consumer
for item in range(20):
    name = "customer%d"%(item,)
    temp = Consumer(name,que)
    temp.start()

'''

def Pro(name,que):
    while True:
        if que.qsize() <3:
            que.put("baozi")
            print "%s:made a lion...=====" % name
        else:
            print "%s:we have three lions..." % name
        time.sleep(random.randrange(2))  #control made speed

def Con(name,que):
    while True:
        try:
            que.get_nowait()
            print "%s:got a lion" % name
        except Exception:
            print "%s:don't have a lion" % name
        time.sleep(random.randrange(3))  #control consumer speed

p1 = Thread(target=Pro,args=("chef1", que))
p2 = Thread(target=Pro,args=("chef2", que))
p1.start()
p2.start()

c1 = Thread(target=Con,args=("customer1", que))
c2 = Thread(target=Con,args=("customer2", que))
c1.start()
c2.start()
'''

Threading.Event

#!/usr/bin/env python
#coding:utf-8
#author:Bing

from threading import Thread,Event
import time,Queue,random

que = Queue.Queue(maxsize=100)

def Pro():
    print "chef:waiting for get a baozi of people."
    event.wait()
    event.clear()
    print "chef:a person comming for baozi"
    print "chef:make a baozi for a person"
    time.sleep(3)

    print "chef:your baozi is ok!"
    event.set()
def Con():
    print "I go to buy a baozi."
    event.set()

    time.sleep(2)
    print "waiting for a baozi"
    event.wait()

    print "think you!"

event = Event()
p = Thread(target=Pro)
c = Thread(target=Con)

p.start()
c.start()

for examlpe:

#!/usr/bin/python
# encoding=utf-8
# Filename: port.py
# Author:Bing
import sys,re
reload(sys)
sys.setdefaultencoding('utf-8')
import urllib,time,threading
import urllib2,random,Queue
from bs4 import BeautifulSoup

USER_AGENTS = [
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
    "Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
    "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
    "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
    "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
    "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
    "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
]

class Spiderdomain:
    def __init__(self,target,page,threads_num):
        self.target = target
        self.page = page
        self.lock = threading.Lock()
        self.threads_num = threads_num
        self._que(self.target)
        self.lock = threading.Lock()

    def random_useragent(self):
        global USER_AGENTS
        return random.choice(USER_AGENTS)

    def _que(self,target):
        self.queue = Queue.Queue()
        for sub in target:
            if sub:self.queue.put(sub)

    def baidu(self):
        count = 0
        pages = self.page*10
        urls = []
        reg = r'((\w+\.)+(com|edu|cn|gov|net|org|hk){1,2})'
        thread_id = int( threading.currentThread().getName() )
        while self.queue.qsize() > 0:
            sub = self.queue.get()
            while count <= pages:
                url = "http://www.baidu.com/s?wd=%s&pn=%s" % (urllib.quote(str(sub)),count)
                request = urllib2.Request(url)
                request.add_header('User-Agent', self.random_useragent())
                request.add_header('Referer', request.get_full_url())
                htmlpage = urllib2.urlopen(request).read()
                soup = BeautifulSoup(htmlpage,"html5lib")           #tags_a =soup.findAll(name='a',attrs={'href':re.compile("^https?://")})
                Tags = soup.findAll(name='a',attrs={'class':'c-showurl'})
                for i in Tags:
                    result = re.findall(reg,str(i))
                    try:
                        urls.append(result[1][0])
                    except Exception,e:
                        pass
                count += 10
            result = {}.fromkeys(urls).keys()
            print result

    def run(self):
        threads = []
        # start threads
        for i in range(self.threads_num):
            t = threading.Thread(target=self.baidu, name=str(i))
            t.start()
            threads.append(t)
        # wait for all
        for i in range(self.threads_num):
        	threads[i].join()

d = Spiderdomain(target=["site:"+"sina.com","site:"+"baidu.com","site:"+"qq.com","site:"+"360.com","site:"+"58.com"],page=20,threads_num=20)#,"site:"+"qq.com","site:"+"360.com","site:"+"58.com"
d.run()

#27s

The following is a process instance

#!/usr/bin/python
# encoding=utf-8
# Filename: port.py
# Author:Bing
import sys,re
reload(sys)
sys.setdefaultencoding('utf-8')
import urllib,time,threading
import urllib2,random,Queue
from bs4 import BeautifulSoup
from multiprocessing.dummy import Pool as ThreadPool

USER_AGENTS = [
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
    "Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
    "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
    "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
    "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
    "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
    "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
]

def baidu(target):
    count = 0
    pages = 20*10
    urls = []
    reg = r'((\w+\.)+(com|edu|cn|gov|net|org|hk){1,2})'
    while count <= pages:
        url = "http://www.baidu.com/s?wd=%s&pn=%s" % (urllib.quote(str(target)),count)
        request = urllib2.Request(url)
        request.add_header('User-Agent', USER_AGENTS)
        request.add_header('Referer', request.get_full_url())
        htmlpage = urllib2.urlopen(request).read()
        soup = BeautifulSoup(htmlpage,"html5lib")           #tags_a =soup.findAll(name='a',attrs={'href':re.compile("^https?://")})
        Tags = soup.findAll(name='a',attrs={'class':'c-showurl'})
        for i in Tags:
            result = re.findall(reg,str(i))
            try:
                urls.append(result[1][0])
            except Exception,e:
                pass
        count += 10
    result = {}.fromkeys(urls).keys()
    return result

urls = ["site:"+"sina.com","site:"+"baidu.com","site:"+"qq.com","site:"+"360.com","site:"+"58.com"]
pool = ThreadPool(20) # pool size
res  = map(baidu, urls)
print res
pool.close()
pool.join()

#46s

Pool和threading 进程池和多线程:

'''     
from multiprocessing import Pool
import threading
import traceback
#from gevent.pool import Pool
import time

def t(li,n):
    #li.append(n)
    print "thread number:",n
 
#一个子进程运行5个线程
def p(x):
    info_list = []
    for i in range(5):
        t = threading.Thread(target=t,args=[info_list,i])
        t.start()
    print info_list,"********"
    
#最多一次运行5个子进程
pls = Pool(processes=5)

res_list = []
#生产10个子进程,一次运行最多5个子进程,异步运行(类似于一次并发5个)
for i in range(10):
    res = pls.apply_async(p,[i,])
    print '------------:',i
    res_list.append(res)

for r in res_list:
    print r.get(timeout=1)

pls.close()
pls.join()
'''

© 著作权归作者所有

共有 人打赏支持
Bing-
粉丝 0
博文 25
码字总数 18863
作品 0
大连
HttpServletResponse

一、浏览器与服务器之间的交互过程 当服务器收到一次浏览器的Http(http底层是基于TCP传输协议)请求后在服务器同时会产生一个Request对象和一个Response对象 通过request请求对象获得客户端...

谢思华
2014/01/09
0
0
python中的模块

############################################# ################module####################### ############################################# 在 Python 中,一个.py文件就称之为一个模块......

2017lw
2017/07/26
0
0
【目录导航】JAVA零基础进阶之路

【JAVA零基础入门系列】(已完结)导航目录 Day1 开发环境搭建 Day2 Java集成开发环境IDEA Day3 Java基本数据类型 Day4 变量与常量 Day5 Java中的运算符 Day6 Java字符串 Day7 Java输入与输出...

MFrank
06/21
0
0
python 多线程笔记(3)-- 线程的私有命名空间

线程的私有命名空间实现:   threading_namespace = threading.local() import threadingimport timeimport random threading_namespace = threading.local() # 命名空间 def print_countr......

罗兵
2016/02/02
0
0
Python线程锁

多线程适用于IO密集型,多线程实现方式有两种,详见下方例子 例子: import threading class MyThread(threading.Thread): def init(self, args): #使用super写法,按照父类.方法的方式直接重...

粗粮面包
2017/12/13
0
0

没有更多内容

加载失败,请刷新页面

加载更多

下一页

JS:异步 - 面试惨案

为什么会写这篇文章,很明显不符合我的性格的东西,原因是前段时间参与了一个面试,对于很多程序员来说,面试时候多么的鸦雀无声,事后心里就有多么的千军万马。去掉最开始毕业干了一年的Jav...

xmqywx
今天
0
0
Win10 64位系统,PHP 扩展 curl插件

执行:1. 拷贝php安装目录下,libeay32.dll、ssleay32.dll 、 libssh2.dll 到 C:\windows\system32 目录。2. 拷贝php/ext目录下, php_curl.dll 到 C:\windows\system32 目录; 3. p...

放飞E梦想O
今天
0
0
谈谈神秘的ES6——(五)解构赋值【对象篇】

上一节课我们了解了有关数组的解构赋值相关内容,这节课,我们接着,来讲讲对象的解构赋值。 解构不仅可以用于数组,还可以用于对象。 let { foo, bar } = { foo: "aaa", bar: "bbb" };fo...

JandenMa
今天
1
0
OSChina 周一乱弹 —— 有人要给本汪介绍妹子啦

Osc乱弹歌单(2018)请戳(这里) 【今日歌曲】 @莱布妮子 :分享水木年华的单曲《中学时代》@小小编辑 手机党少年们想听歌,请使劲儿戳(这里) @须臾时光:夏天还在做最后的挣扎,但是晚上...

小小编辑
今天
21
5
centos7安装redis及开机启动

配置编译环境: sudo yum install gcc-c++ 下载源码: wget http://download.redis.io/releases/redis-3.2.8.tar.gz 解压源码: tar -zxvf redis-3.2.8.tar.gz 进入到解压目录: cd redis-3......

hotsmile
今天
2
0

没有更多内容

加载失败,请刷新页面

加载更多

下一页

返回顶部
顶部