关于模块选择

　　Python提供了几个用于多线程编程的模块，包括thread、threading和Queue等。thread和threading模块允许程序员创建和管理线程。thread模块提供了基本的线程和锁的支持，threading提供了更高级别、功能更强的线程管理的功能。Queue模块允许用户创建一个可以用于多个线程之间共享数据的队列数据结构。　　避免使用thread模块，因为更高级别的threading模块更为先进，对线程的支持更为完善，而且使用thread模块里的属性有可能会与threading出现冲突；其次低级别的thread模块的同步原语很少(实际上只有一个)，而threading模块则有很多；再者，thread模块中当主线程结束时，所有的线程都会被强制结束掉，没有警告也不会有正常的清除工作，至少threading模块能确保重要的子线程退出后进程才退出。

　　thread模块不支持守护线程，当主线程退出时，所有的子线程不论它们是否还在工作，都会被强行退出。而threading模块支持守护线程，守护线程一般是一个等待客户请求的服务器，如果没有客户提出请求它就在那等着，如果设定一个线程为守护线程，就表示这个线程是不重要的，在进程退出的时候，不用等待这个线程退出。

threading模块

多线程启动

# 多线程启动import osimport timefrom threading import Threaddef func():    time.sleep(1)    print('hello 线程', os.getpid())t = Thread(target=func)t.start()print(os.getpid())# 结果# 6360# hello 线程 6360

同步开启多线程

# 同步开启多线程import osimport timefrom threading import Threaddef func():    time.sleep(1)    print('hello 线程', os.getpid())thread_l = []for i in range(10):    t = Thread(target=func)    t.start()    thread_l.append(t)for j in thread_l:    j.join()print(os.getpid())

开启线程的另一种方法

# 开启线程的另一种方法import osimport timefrom threading import Threadclass My_thread(Thread):    def run(self):        time.sleep(1)        print('hello 线程', os.getpid())thread_l = []for i in range(10):    t = My_thread()    t.start()    thread_l.append(t)for j in thread_l:    j.join()print(os.getpid())

以类的方式开启进程中的注意汇总

# 以类的方式开启进程中的注意汇总import osimport timefrom threading import Threadclass My_thread(Thread):    count = 0  # 进程中的静态属性是共享的  计算调用线程的次数    def __init__(self, arg):  # 传参方法        super().__init__()  # 继承父类的init方法        self.arg = arg    def run(self):        My_thread.count += 1        time.sleep(1)        print('%s,%s' % (self.arg, os.getpid()))thread_l = []for i in range(10):    t = My_thread(i, )    t.start()    thread_l.append(t)for j in thread_l:    j.join()print(t.count)

self.name获取线程名字

# self.name获取线程名字import osimport timefrom threading import Threadclass My_thread(Thread):    count = 0  # 进程中的静态属性是共享的  计算调用线程的次数    def __init__(self, arg):  # 传参方法        super().__init__()  # 继承父类的init方法        self.arg = arg    def run(self):        My_thread.count += 1        time.sleep(1)        print('%s,%s,%s' % (self.arg, self.name, os.getpid()))thread_l = []for i in range(10):    t = My_thread(i, )    t.start()    thread_l.append(t)for j in thread_l:    j.join()print(t.count)# 结果# 4,Thread-5,6284# 3,Thread-4,6284# 2,Thread-3,6284# 1,Thread-2,6284# 0,Thread-1,6284# 9,Thread-10,6284# 8,Thread-9,6284# 6,Thread-7,6284# 7,Thread-8,6284# 5,Thread-6,6284# 10

线程周边

# 线程周边import timeimport threadingdef func(i):    time.sleep(0.5)    print(i, threading.currentThread().name, threading.currentThread().ident)#     ident  线程idfor i in range(10):    t = threading.Thread(target=func, args=(i, ))    t.start()print(threading.enumerate())  # 返回正在运行着的线程列表print(len(threading.enumerate()))  # 打印线程数量print(threading.activeCount())  # 记录活着的线程数量# 结果# [<_MainThread(MainThread, started 9004)>, <Thread(Thread-1, started 18804)>, <Thread(Thread-2, started 4892)>, <Thread(Thread-3, started 12760)>, <Thread(Thread-4, started 22276)>, <Thread(Thread-5, started 20732)>, <Thread(Thread-6, started 19072)>, <Thread(Thread-7, started 17020)>, <Thread(Thread-8, started 22228)>, <Thread(Thread-9, started 22252)>, <Thread(Thread-10, started 22280)>]# 11# 11# 0 Thread-1 18804# 1 Thread-2 4892# 3 Thread-4 22276# 4 Thread-5 20732# 2 Thread-3 12760# 8 Thread-9 22252# 9 Thread-10 22280# 7 Thread-8 22228# 6 Thread-7 17020# 5 Thread-6 19072

socket启动多线程

# socket启动多线程# client端import socketsk = socket.socket()sk.connect(('127.0.0.1', 8080))ret = sk.recv()print(ret)msg = input('>>>')sk.send(msg.encode('utf-8'))sk.close()# server端import socketfrom threading import Threaddef func(conn):    conn.send(b'hello')    ret = conn.recv(1024)    print(ret)    conn.close()sk = socket.socket()sk.bind(('127.0.0.1', 8080))sk.listen()while True:    conn, addr = sk.accept()    Thread(target=func, args=(conn, )).start()sk.close()

# 守护进程import timefrom threading import Threaddef func():    print('开始执行子线程')    time.sleep(2)    print('子线程执行结束')t = Thread(target=func)t.setDaemon(True)  # 进程设置守护进程 是属性 daemon = Truet.start()t2 = Thread(target=func)t2.start()t2.join()  # 等待t2结束  \\  t2执行完毕 主线程结束# 注意# 守护线程 守护进程 都是等待主进程或主线程中的代码 执行完毕# 结果# 开始执行子线程# 开始执行子线程# 子线程执行结束# 子线程执行结束

锁的使用

# 锁import timefrom threading import Threadfrom threading import Lockdef func():    global n    lock.acquire()    temp = n  # 从进程中获取n    time.sleep(0.01)    n = temp-1  # 得到结果再存储回进程    lock.release()n = 100lock = Lock()t_lis = []for i in range(100):    t = Thread(target=func)    t.start()    t_lis.append(t)[t.join() for t in t_lis]print(n)

PS：GIL 不是锁数据，而是锁线程

在多线程中，特殊情况，仍要加锁，对数据加锁（具体见锁得使用）

互斥锁与join的区别

#不加锁:并发执行,速度快,数据不安全from threading import current_thread,Thread,Lockimport os,timedef task():    global n    print('%s is running' %current_thread().getName())    temp=n    time.sleep(0.5)    n=temp-1if __name__ == '__main__':    n=100    lock=Lock()    threads=[]    start_time=time.time()    for i in range(100):        t=Thread(target=task)        threads.append(t)        t.start()    for t in threads:        t.join()    stop_time=time.time()    print('主:%s n:%s' %(stop_time-start_time,n))'''Thread-1 is runningThread-2 is running......Thread-100 is running主:0.5216062068939209 n:99'''#不加锁:未加锁部分并发执行,加锁部分串行执行,速度慢,数据安全from threading import current_thread,Thread,Lockimport os,timedef task():    #未加锁的代码并发运行    time.sleep(3)    print('%s start to run' %current_thread().getName())    global n    #加锁的代码串行运行    lock.acquire()    temp=n    time.sleep(0.5)    n=temp-1    lock.release()if __name__ == '__main__':    n=100    lock=Lock()    threads=[]    start_time=time.time()    for i in range(100):        t=Thread(target=task)        threads.append(t)        t.start()    for t in threads:        t.join()    stop_time=time.time()    print('主:%s n:%s' %(stop_time-start_time,n))'''Thread-1 is runningThread-2 is running......Thread-100 is running主:53.294203758239746 n:0'''#既然加锁会让运行变成串行,那么我在start之后立即使用join,就不用加锁了啊,也是串行的效果啊#没错:在start之后立刻使用jion,肯定会将100个任务的执行变成串行,毫无疑问,最终n的结果也肯定是0,是安全的,但问题是#start后立即join:任务内的所有代码都是串行执行的,而加锁,只是加锁的部分即修改共享数据的部分是串行的#单从保证数据安全方面,二者都可以实现,但很明显是加锁的效率更高.from threading import current_thread,Thread,Lockimport os,timedef task():    time.sleep(3)    print('%s start to run' %current_thread().getName())    global n    temp=n    time.sleep(0.5)    n=temp-1if __name__ == '__main__':    n=100    lock=Lock()    start_time=time.time()    for i in range(100):        t=Thread(target=task)        t.start()        t.join()    stop_time=time.time()    print('主:%s n:%s' %(stop_time-start_time,n))'''Thread-1 start to runThread-2 start to run......Thread-100 start to run主:350.6937336921692 n:0 #耗时是多么的恐怖'''

进程也有死锁与递归锁，在进程那里忘记说了，放到这里一切说了额

　　所谓死锁： 是指两个或两个以上的进程或线程在执行过程中，因争夺资源而造成的一种互相等待的现象，若无外力作用，它们都将无法推进下去。此时称系统处于死锁状态或系统产生了死锁，这些永远在互相等待的进程称为死锁进程，如下就是死锁

死锁

死锁from threading import Lock as Lockimport timemutexA=Lock()mutexA.acquire()mutexA.acquire()print(123)mutexA.release()mutexA.release()

　　解决方法，递归锁，在Python中为了支持在同一线程中多次请求同一资源，python提供了可重入锁RLock。

　　这个RLock内部维护着一个Lock和一个counter变量，counter记录了acquire的次数，从而使得资源可以被多次require。直到一个线程所有的acquire都被release，其他的线程才能获得资源。上面的例子如果使用RLock代替Lock，则不会发生死锁

递归锁RLock

from threading import RLock as Lockimport timemutexA=Lock()mutexA.acquire()mutexA.acquire()print(123)mutexA.release()mutexA.release()

递归锁解决死锁问题

import timefrom threading import Thread,RLockfork_lock = noodle_lock = RLock()def eat1(name):    noodle_lock.acquire()    print('%s 抢到了面条'%name)    fork_lock.acquire()    print('%s 抢到了叉子'%name)    print('%s 吃面'%name)    fork_lock.release()    noodle_lock.release()def eat2(name):    fork_lock.acquire()    print('%s 抢到了叉子' % name)    time.sleep(1)    noodle_lock.acquire()    print('%s 抢到了面条' % name)    print('%s 吃面' % name)    noodle_lock.release()    fork_lock.release()for name in ['哪吒','egon','yuan']:    t1 = Thread(target=eat1,args=(name,))    t2 = Thread(target=eat2,args=(name,))    t1.start()    t2.start()

同进程的一样

Semaphore管理一个内置的计数器，

每当调用acquire()时内置计数器-1；

调用release() 时内置计数器+1；

计数器不能小于0；当计数器为0时，acquire()将阻塞线程直到其他线程调用release()。

实例：(同时只有5个线程可以获得semaphore,即可以限制最大连接数为5)：

from threading import Thread,Semaphoreimport threadingimport time# def func():#     if sm.acquire():#         print (threading.currentThread().getName() + ' get semaphore')#         time.sleep(2)#         sm.release()def func():    sm.acquire()    print('%s get sm' %threading.current_thread().getName())    time.sleep(3)    sm.release()if __name__ == '__main__':    sm=Semaphore(5)    for i in range(23):        t=Thread(target=func)        t.start()

与进程池是完全不同的概念，进程池Pool(4)，最大只能产生4个进程，而且从头到尾都只是这四个进程，不会产生新的，而信号量是产生一堆线程/进程

事件

　　同进程的一样

　　线程的一个关键特性是每个线程都是独立运行且状态不可预测。如果程序中的其 他线程需要通过判断某个线程的状态来确定自己下一步的操作,这时线程同步问题就会变得非常棘手。为了解决这些问题,我们需要使用threading库中的Event对象。 对象包含一个可由线程设置的信号标志,它允许线程等待某些事件的发生。在 初始情况下,Event对象中的信号标志被设置为假。如果有线程等待一个Event对象, 而这个Event对象的标志为假,那么这个线程将会被一直阻塞直至该标志为真。一个线程如果将一个Event对象的信号标志设置为真,它将唤醒所有等待这个Event对象的线程。如果一个线程等待一个已经被设置为真的Event对象,那么它将忽略这个事件, 继续执行

event.isSet()：返回event的状态值；event.wait()：如果 event.isSet()==False将阻塞线程；event.set()： 设置event的状态值为True，所有阻塞池的线程激活进入就绪状态， 等待操作系统调度；event.clear()：恢复event的状态值为False。

例如，有多个工作线程尝试连接MySQL，我们想要在连接前确保MySQL服务正常才让那些工作线程去连接MySQL服务器，如果连接不成功，都会去尝试重新连接。那么我们就可以采用threading.Event机制来协调各个工作线程的连接操作

import threadingimport time,randomfrom threading import Thread,Eventdef conn_mysql():    count=1    while not event.is_set():        if count > 3:            raise TimeoutError('链接超时')        print('<%s>第%s次尝试链接' % (threading.current_thread().getName(), count))        event.wait(0.5)        count+=1    print('<%s>链接成功' %threading.current_thread().getName())def check_mysql():    print('\033[45m[%s]正在检查mysql\033[0m' % threading.current_thread().getName())    time.sleep(random.randint(2,4))    event.set()if __name__ == '__main__':    event=Event()    conn1=Thread(target=conn_mysql)    conn2=Thread(target=conn_mysql)    check=Thread(target=check_mysql)    conn1.start()    conn2.start()    check.start()

使得线程等待，只有满足某条件时，才释放n个线程

说明

Python提供的Condition对象提供了对复杂线程同步问题的支持。Condition被称为条件变量，除了提供与Lock类似的acquire和release方法外，还提供了wait和notify方法。线程首先acquire一个条件变量，然后判断一些条件。如果条件不满足则wait；如果条件满足，进行一些处理改变条件后，通过notify方法通知其他线程，其他处于wait状态的线程接到通知后会重新判断条件。不断的重复这一过程，从而解决复杂的同步问题。

实例

import threadingdef run(n):    con.acquire()    con.wait()    print("run the thread: %s" % n)    con.release()if __name__ == '__main__':    con = threading.Condition()    for i in range(10):        t = threading.Thread(target=run, args=(i,))        t.start()    while True:        inp = input('>>>')        if inp == 'q':            break        con.acquire()        con.notify(int(inp))        con.release()        print('****')

定时器，指定n秒后执行某个操作

from threading import Timer def hello():    print("hello, world") t = Timer(1, hello)t.start()  # after 1 seconds, "hello, world" will be printed

queue队列 ：使用import queue，用法与进程Queue一样

queue is especially useful in threaded programming when information must be exchanged safely between multiple threads.

class queue.Queue(maxsize=0) #先进先出

import queueq=queue.Queue()q.put('first')q.put('second')q.put('third')print(q.get())print(q.get())print(q.get())'''结果(先进先出):firstsecondthird'''

class queue.LifoQueue(maxsize=0) #后进先出

import queueq=queue.LifoQueue()q.put('first')q.put('second')q.put('third')print(q.get())print(q.get())print(q.get())'''结果(后进先出):thirdsecondfirst'''

class queue.PriorityQueue(maxsize=0) #存储数据时可设置优先级的队列

优先级队列

import queueq=queue.PriorityQueue()#put进入一个元组,元组的第一个元素是优先级(通常是数字,也可以是非数字之间的比较),数字越小优先级越高q.put((20,'a'))q.put((10,'b'))q.put((30,'c'))print(q.get())print(q.get())print(q.get())'''结果(数字越小优先级越高,优先级高的优先出队):(10, 'b')(20, 'a')(30, 'c')'''

更多方法说明

Constructor for a priority queue. maxsize is an integer that sets the upperbound limit on the number of items that can be placed in the queue. Insertion will block once this size has been reached, until queue items are consumed. If maxsize is less than or equal to zero, the queue size is infinite.The lowest valued entries are retrieved first (the lowest valued entry is the one returned by sorted(list(entries))[0]). A typical pattern for entries is a tuple in the form: (priority_number, data).exception queue.EmptyException raised when non-blocking get() (or get_nowait()) is called on a Queue object which is empty.exception queue.FullException raised when non-blocking put() (or put_nowait()) is called on a Queue object which is full.Queue.qsize()Queue.empty() #return True if empty  Queue.full() # return True if full Queue.put(item, block=True, timeout=None)Put item into the queue. If optional args block is true and timeout is None (the default), block if necessary until a free slot is available. If timeout is a positive number, it blocks at most timeout seconds and raises the Full exception if no free slot was available within that time. Otherwise (block is false), put an item on the queue if a free slot is immediately available, else raise the Full exception (timeout is ignored in that case).Queue.put_nowait(item)Equivalent to put(item, False).Queue.get(block=True, timeout=None)Remove and return an item from the queue. If optional args block is true and timeout is None (the default), block if necessary until an item is available. If timeout is a positive number, it blocks at most timeout seconds and raises the Empty exception if no item was available within that time. Otherwise (block is false), return an item if one is immediately available, else raise the Empty exception (timeout is ignored in that case).Queue.get_nowait()Equivalent to get(False).Two methods are offered to support tracking whether enqueued tasks have been fully processed by daemon consumer threads.Queue.task_done()Indicate that a formerly enqueued task is complete. Used by queue consumer threads. For each get() used to fetch a task, a subsequent call to task_done() tells the queue that the processing on the task is complete.If a join() is currently blocking, it will resume when all items have been processed (meaning that a task_done() call was received for every item that had been put() into the queue).Raises a ValueError if called more times than there were items placed in the queue.Queue.join() block直到queue被消费完毕

#1 介绍concurrent.futures模块提供了高度封装的异步调用接口ThreadPoolExecutor：线程池，提供异步调用ProcessPoolExecutor: 进程池，提供异步调用Both implement the same interface, which is defined by the abstract Executor class.#2 基本方法#submit(fn, *args, **kwargs)异步提交任务#map(func, *iterables, timeout=None, chunksize=1) 取代for循环submit的操作#shutdown(wait=True) 相当于进程池的pool.close()+pool.join()操作wait=True，等待池内所有任务执行完毕回收完资源后才继续wait=False，立即返回，并不会等待池内的任务执行完毕但不管wait参数为何值，整个程序都会等到所有任务执行完毕submit和map必须在shutdown之前#result(timeout=None)取得结果#add_done_callback(fn)回调函数

ProcessPoolExecutor

#介绍The ProcessPoolExecutor class is an Executor subclass that uses a pool of processes to execute calls asynchronously. ProcessPoolExecutor uses the multiprocessing module, which allows it to side-step the Global Interpreter Lock but also means that only picklable objects can be executed and returned.class concurrent.futures.ProcessPoolExecutor(max_workers=None, mp_context=None)An Executor subclass that executes calls asynchronously using a pool of at most max_workers processes. If max_workers is None or not given, it will default to the number of processors on the machine. If max_workers is lower or equal to 0, then a ValueError will be raised.#用法from concurrent.futures import ThreadPoolExecutor,ProcessPoolExecutorimport os,time,randomdef task(n):    print('%s is runing' %os.getpid())    time.sleep(random.randint(1,3))    return n**2if __name__ == '__main__':    executor=ProcessPoolExecutor(max_workers=3)    futures=[]    for i in range(11):        future=executor.submit(task,i)        futures.append(future)    executor.shutdown(True)    print('+++>')    for future in futures:        print(future.result())

ThreadPoolExecutor

#介绍ThreadPoolExecutor is an Executor subclass that uses a pool of threads to execute calls asynchronously.class concurrent.futures.ThreadPoolExecutor(max_workers=None, thread_name_prefix='')An Executor subclass that uses a pool of at most max_workers threads to execute calls asynchronously.Changed in version 3.5: If max_workers is None or not given, it will default to the number of processors on the machine, multiplied by 5, assuming that ThreadPoolExecutor is often used to overlap I/O instead of CPU work and the number of workers should be higher than the number of workers for ProcessPoolExecutor.New in version 3.6: The thread_name_prefix argument was added to allow users to control the threading.Thread names for worker threads created by the pool for easier debugging.#用法与ProcessPoolExecutor相同

map的用法

from concurrent.futures import ThreadPoolExecutor,ProcessPoolExecutorimport os,time,randomdef task(n):    print('%s is runing' %os.getpid())    time.sleep(random.randint(1,3))    return n**2if __name__ == '__main__':    executor=ThreadPoolExecutor(max_workers=3)    # for i in range(11):    #     future=executor.submit(task,i)    executor.map(task,range(1,12)) #map取代了for+submit

回调函数

from concurrent.futures import ThreadPoolExecutor,ProcessPoolExecutorfrom multiprocessing import Poolimport requestsimport jsonimport osdef get_page(url):    print('<进程%s> get %s' %(os.getpid(),url))    respone=requests.get(url)    if respone.status_code == 200:        return {'url':url,'text':respone.text}def parse_page(res):    res=res.result()    print('<进程%s> parse %s' %(os.getpid(),res['url']))    parse_res='url:<%s> size:[%s]\n' %(res['url'],len(res['text']))    with open('db.txt','a') as f:        f.write(parse_res)if __name__ == '__main__':    urls=[        ';,        ';,        ';,        ';,        ';    ]    # p=Pool(3)    # for url in urls:    #     p.apply_async(get_page,args=(url,),callback=pasrse_page)    # p.close()    # p.join()    p=ProcessPoolExecutor(3)    for url in urls:        p.submit(get_page,url).add_done_callback(parse_page) #parse_page拿到的是一个future对象obj，需要用obj.result()拿到结果

#计算机##python##Python基础##互联网##科技新星创作营#