文档章节

day6-threading

Bing-
 Bing-
发布于 2016/05/17 10:11
字数 1496
阅读 23
收藏 0
点赞 2
评论 0

thread instance:

#!/usr/bin/env python
#coding:utf-8
#author:Bing

from threading import Thread
import time

def test(arg):
    for i in range(10):
        print i
        #time.sleep(1)

t1 = Thread(target=test, args=("11",))
#t1.setDaemon(True)  don't waiting for subprogress
t1.start()
print t1.getName()
t1.join(5)
#t1.join()

print "after"
print "after"

class MyThread(Thread):
    def run(self):
        #time.sleep(15)
        print "test"
        Thread.run(self) #call bone excute

def bone():
    print "bone"

t2 = MyThread(target=bone)
t2.start()
print "over"

Product and Consumer mode:

#!/usr/bin/env python
#coding:utf-8
#author:Bing

from threading import Thread
import time,Queue,random

que = Queue.Queue(maxsize=100)

class Product(Thread):
    def __init__(self,name,que):
        super(Product,self).__init__()
        self.name = name
        self.que = que

    def run(self):
        while True:
            if self.que.full():
                time.sleep(1)
            else:
                self.que.put("baozi")
                print "%s:made a baozi...===" % self.name

class Consumer(Thread):
    def __init__(self,name,que):
        super(Consumer,self).__init__()
        self.name = name
        self.que = que

    def run(self):
        while True:
            if self.que.empty():
                time.sleep(1)
            else:
                self.que.get()
                print "%s:got a baozi" % self.name

#three productor
t1 = Product("chef1",que)
t2 = Product("chef1",que)
t3 = Product("chef1",que)
t1.start()
t2.start()
t3.start()

#twenty consumer
for item in range(20):
    name = "customer%d"%(item,)
    temp = Consumer(name,que)
    temp.start()

'''

def Pro(name,que):
    while True:
        if que.qsize() <3:
            que.put("baozi")
            print "%s:made a lion...=====" % name
        else:
            print "%s:we have three lions..." % name
        time.sleep(random.randrange(2))  #control made speed

def Con(name,que):
    while True:
        try:
            que.get_nowait()
            print "%s:got a lion" % name
        except Exception:
            print "%s:don't have a lion" % name
        time.sleep(random.randrange(3))  #control consumer speed

p1 = Thread(target=Pro,args=("chef1", que))
p2 = Thread(target=Pro,args=("chef2", que))
p1.start()
p2.start()

c1 = Thread(target=Con,args=("customer1", que))
c2 = Thread(target=Con,args=("customer2", que))
c1.start()
c2.start()
'''

Threading.Event

#!/usr/bin/env python
#coding:utf-8
#author:Bing

from threading import Thread,Event
import time,Queue,random

que = Queue.Queue(maxsize=100)

def Pro():
    print "chef:waiting for get a baozi of people."
    event.wait()
    event.clear()
    print "chef:a person comming for baozi"
    print "chef:make a baozi for a person"
    time.sleep(3)

    print "chef:your baozi is ok!"
    event.set()
def Con():
    print "I go to buy a baozi."
    event.set()

    time.sleep(2)
    print "waiting for a baozi"
    event.wait()

    print "think you!"

event = Event()
p = Thread(target=Pro)
c = Thread(target=Con)

p.start()
c.start()

for examlpe:

#!/usr/bin/python
# encoding=utf-8
# Filename: port.py
# Author:Bing
import sys,re
reload(sys)
sys.setdefaultencoding('utf-8')
import urllib,time,threading
import urllib2,random,Queue
from bs4 import BeautifulSoup

USER_AGENTS = [
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
    "Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
    "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
    "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
    "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
    "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
    "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
]

class Spiderdomain:
    def __init__(self,target,page,threads_num):
        self.target = target
        self.page = page
        self.lock = threading.Lock()
        self.threads_num = threads_num
        self._que(self.target)
        self.lock = threading.Lock()

    def random_useragent(self):
        global USER_AGENTS
        return random.choice(USER_AGENTS)

    def _que(self,target):
        self.queue = Queue.Queue()
        for sub in target:
            if sub:self.queue.put(sub)

    def baidu(self):
        count = 0
        pages = self.page*10
        urls = []
        reg = r'((\w+\.)+(com|edu|cn|gov|net|org|hk){1,2})'
        thread_id = int( threading.currentThread().getName() )
        while self.queue.qsize() > 0:
            sub = self.queue.get()
            while count <= pages:
                url = "http://www.baidu.com/s?wd=%s&pn=%s" % (urllib.quote(str(sub)),count)
                request = urllib2.Request(url)
                request.add_header('User-Agent', self.random_useragent())
                request.add_header('Referer', request.get_full_url())
                htmlpage = urllib2.urlopen(request).read()
                soup = BeautifulSoup(htmlpage,"html5lib")           #tags_a =soup.findAll(name='a',attrs={'href':re.compile("^https?://")})
                Tags = soup.findAll(name='a',attrs={'class':'c-showurl'})
                for i in Tags:
                    result = re.findall(reg,str(i))
                    try:
                        urls.append(result[1][0])
                    except Exception,e:
                        pass
                count += 10
            result = {}.fromkeys(urls).keys()
            print result

    def run(self):
        threads = []
        # start threads
        for i in range(self.threads_num):
            t = threading.Thread(target=self.baidu, name=str(i))
            t.start()
            threads.append(t)
        # wait for all
        for i in range(self.threads_num):
        	threads[i].join()

d = Spiderdomain(target=["site:"+"sina.com","site:"+"baidu.com","site:"+"qq.com","site:"+"360.com","site:"+"58.com"],page=20,threads_num=20)#,"site:"+"qq.com","site:"+"360.com","site:"+"58.com"
d.run()

#27s

The following is a process instance

#!/usr/bin/python
# encoding=utf-8
# Filename: port.py
# Author:Bing
import sys,re
reload(sys)
sys.setdefaultencoding('utf-8')
import urllib,time,threading
import urllib2,random,Queue
from bs4 import BeautifulSoup
from multiprocessing.dummy import Pool as ThreadPool

USER_AGENTS = [
    "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
    "Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
    "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
    "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
    "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
    "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
    "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
    "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
    "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
]

def baidu(target):
    count = 0
    pages = 20*10
    urls = []
    reg = r'((\w+\.)+(com|edu|cn|gov|net|org|hk){1,2})'
    while count <= pages:
        url = "http://www.baidu.com/s?wd=%s&pn=%s" % (urllib.quote(str(target)),count)
        request = urllib2.Request(url)
        request.add_header('User-Agent', USER_AGENTS)
        request.add_header('Referer', request.get_full_url())
        htmlpage = urllib2.urlopen(request).read()
        soup = BeautifulSoup(htmlpage,"html5lib")           #tags_a =soup.findAll(name='a',attrs={'href':re.compile("^https?://")})
        Tags = soup.findAll(name='a',attrs={'class':'c-showurl'})
        for i in Tags:
            result = re.findall(reg,str(i))
            try:
                urls.append(result[1][0])
            except Exception,e:
                pass
        count += 10
    result = {}.fromkeys(urls).keys()
    return result

urls = ["site:"+"sina.com","site:"+"baidu.com","site:"+"qq.com","site:"+"360.com","site:"+"58.com"]
pool = ThreadPool(20) # pool size
res  = map(baidu, urls)
print res
pool.close()
pool.join()

#46s

Pool和threading 进程池和多线程:

'''     
from multiprocessing import Pool
import threading
import traceback
#from gevent.pool import Pool
import time

def t(li,n):
    #li.append(n)
    print "thread number:",n
 
#一个子进程运行5个线程
def p(x):
    info_list = []
    for i in range(5):
        t = threading.Thread(target=t,args=[info_list,i])
        t.start()
    print info_list,"********"
    
#最多一次运行5个子进程
pls = Pool(processes=5)

res_list = []
#生产10个子进程,一次运行最多5个子进程,异步运行(类似于一次并发5个)
for i in range(10):
    res = pls.apply_async(p,[i,])
    print '------------:',i
    res_list.append(res)

for r in res_list:
    print r.get(timeout=1)

pls.close()
pls.join()
'''

© 著作权归作者所有

共有 人打赏支持
Bing-
粉丝 0
博文 25
码字总数 18863
作品 0
大连
HttpServletResponse

一、浏览器与服务器之间的交互过程 当服务器收到一次浏览器的Http(http底层是基于TCP传输协议)请求后在服务器同时会产生一个Request对象和一个Response对象 通过request请求对象获得客户端...

谢思华 ⋅ 2014/01/09 ⋅ 0

python中的模块

############################################# ################module####################### ############################################# 在 Python 中,一个.py文件就称之为一个模块......

2017lw ⋅ 2017/07/26 ⋅ 0

【目录导航】JAVA零基础进阶之路

【JAVA零基础入门系列】(已完结)导航目录 Day1 开发环境搭建 Day2 Java集成开发环境IDEA Day3 Java基本数据类型 Day4 变量与常量 Day5 Java中的运算符 Day6 Java字符串 Day7 Java输入与输出...

MFrank ⋅ 昨天 ⋅ 0

python基本 -- threading多线程模块的使用

python多线程编程,一般使用thread和threading模块。thread模块想对较底层,threading模块对thread模块进行了封装,更便于使用。所有,通常多线程编程使用threading模块。 (一)threading模...

明天以后 ⋅ 2014/08/23 ⋅ 0

python之多线程并发处理模块-threading

thread:多线程的底层支持模块,一般不建议使用; threading:对thread进行了封装,将一些线程的操作对象化,一般采用这种方法实现多线程编程 多线程实现有两种模式: 1.创建线程要执行的函数...

zhpfxl ⋅ 2016/12/27 ⋅ 0

Python: 关于thread模块和threading模块的选择

最近在阅读Python核心编程(第二版)。 作者在多线程编程一章讲到thread和threading这两个模块时, 提到要避免使用thread模块,取而代之地使用threading模块。 之后点明了原因—— "更高级别...

mickelfeng ⋅ 2016/08/05 ⋅ 0

c# 线程 运行一段时间会报错

用c# 做的一个winform 程序,里面开启了两个线程,程序运行一段时间就会出错,这个是根据系统日志获取到的信息,请高手帮忙看下是什么问题 Application: MYSQL数据库还原.exe Framework Vers...

junan ⋅ 2015/12/14 ⋅ 1

python_day6_对象

6.1: 类与对象实例 python是一门面向对象编程语言,其中编程方式分为三种范式: 1、面向过程编程 2、函数式编程 分为两种:数学层次的编程与python函数式编程 3、面向对象编程 对象是由类产...

812374156 ⋅ 2017/12/22 ⋅ 0

Python中的threading

#!/usr/bin/env python # -- coding: utf-8 -- import threading, time #新线程执行的代码: def loop(): print('thread %s is running...' % threading.current_thread().name) n = 0 while ......

io123 ⋅ 2016/04/01 ⋅ 0

Python中的threading

#!/usr/bin/env python # -- coding: utf-8 -- import threading, time #新线程执行的代码: def loop(): print('thread %s is running...' % threading.current_thread().name) n = 0 while ......

io123 ⋅ 2016/04/01 ⋅ 0

没有更多内容

加载失败,请刷新页面

加载更多

下一页

Confluence 6 从其他备份中恢复数据

一般来说,Confluence 数据库可以从 Administration Console 或者 Confluence Setup Wizard 中进行恢复。 如果你在恢复压缩的 XML 备份的时候遇到了问题,你还是可以对整个站点进行恢复的,如...

honeymose ⋅ 11分钟前 ⋅ 0

myeclipse10 快速搭建spring boot开发环境(入门)

1.创建一个maven的web项目 注意上面标红的部分记得选上 2.创建的maven目录结构,有缺失的目录可以自己建立目录补充 补充后 这时候一个maven的web项目创建完成 3.配置pom.xml配置文件 <proje...

小海bug ⋅ 23分钟前 ⋅ 0

nginx.conf

=========================================================================== nginx.conf =========================================================================== user nobody; #......

A__17 ⋅ 26分钟前 ⋅ 0

645. Set Mismatch - LeetCode

Question 645. Set Mismatch Solution 思路: 遍历每个数字,然后将其应该出现的位置上的数字变为其相反数,这样如果我们再变为其相反数之前已经成负数了,说明该数字是重复数,将其将入结果r...

yysue ⋅ 40分钟前 ⋅ 0

Python这么强?红包杀手、消息撤回也可以无视,手机App辅助!

论述 标题也许有点不好理解,其实就是一款利用Python实现的可以监控微信APP内的红包与消息撤回的助手。不得不说,这确实是一款大家钟意的神器。 消息撤回是一件很让人恶心的事,毕竟人都是有...

Python燕大侠 ⋅ 55分钟前 ⋅ 0

压缩打包介绍、gzip压缩工具、bzip2压缩工具、xz压缩工具

压缩打包介绍 压缩的好处不仅能节省磁盘空间而且在传输的时候节省传输时间和网络带宽 windows系统下文件带有 .rar .zip .7z 后缀的就是压缩文件 linux系统下则是 .zip, .gz, .bz2, .xz, ...

黄昏残影 ⋅ 今天 ⋅ 0

观察者模式

1.利用java原生类进行操作 package observer;import java.util.Observable;import java.util.Observer;/** * @author shadow * @Date 2016年8月12日下午7:29:31 * @Fun 观察目标 **/......

Cobbage ⋅ 今天 ⋅ 0

Ubuntu打印服务器配置

参考:https://blog.csdn.net/gsls200808/article/details/50950586 https://blog.csdn.net/jiay2/article/details/80252369 https://wiki.gentoo.org/wiki/HPLIP 由于媳妇儿要大量打印资料,......

大熊猫 ⋅ 今天 ⋅ 0

面试的角度诠释Java工程师(二)

原文出处: locality 续言: 相信每一位简书的作者,都会有我这样的思考:怎么写好一篇文章?或者怎么写好一篇技术类的文章?我就先说说我的感悟吧,写文章其实和写程序是一样的。为什么我会...

颖伙虫 ⋅ 今天 ⋅ 0

github中SSH的Key

https://help.github.com/articles/connecting-to-github-with-ssh/ https://help.github.com/articles/testing-your-ssh-connection/ https://help.github.com/articles/adding-a-new-ssh-k......

whoisliang ⋅ 今天 ⋅ 0

没有更多内容

加载失败,请刷新页面

加载更多

下一页

返回顶部
顶部