python: thead and processing

®Geovin Du Dream Park™ / 2023-07-16 / 原文

线程：

# encoding: utf-8
# 版权所有 2023 涂聚文有限公司
# 许可信息查看：
# 描述：
# Author    : geovindu,Geovin Du 涂聚文.
# IDE       : PyCharm 2023.1 python 311
# Datetime  : 2023/7/16 18:55
# User      : geovindu
# Product   : PyCharm
# Project   : pythonTkinterDemo
# File      : crawlerDouyu.py
# explain   : 学习


import json
import requests
import os
import threading


def saveimageUrl(url, dirName):
    """
    线程
    :param url: 链接
    :param dirName: 文件夹名字
    :return:
    """
    # if not os.path.exists('website'):
    # os.mkdir('website')
    try:
        if os.path.exists('website'):
            os.chdir('website')
        if not os.path.exists(f"{dirName}"):
            os.mkdir(dirName)
        content = requests.get(url)
        # print(content.text)
        data = json.loads(content.text)  # 转字典
        # print(data)

        cn = 1
        for d in data['data']['rl'][0:30]:  # 切片为前30个数据
            print(d['nn'], d['rs16'], d['rn'])
            #if not d['av']:
                #av=d['Info'][av]
            av = 'https://apic.douyucdn.cn/upload' + '/' + d['av'] + '_big' + '.jpg'
            print(av)
            name = d['nn']
            # 创建多线程
            t = threading.Thread(target=loadAvator, args=(av, dirName, name))
            t.start()
            # img1=Image.open(av)
            # img1=img1.save('img'+cn+'.jpg')
            # imagedata = requests.get(av)
            # with open(fr'{dirName}\{d["nn"]}.jpg','wb') as fp:
            # fp.write(imagedata)

    except Exception as ex:
        print(ex)


def loadAvator(av, dirname, name):
    """

    :param av:图片
    :param dirname:文件夹名字
    :param name:博主名称
    :return:
    """
    print("jpg:", fr'{dirname}\{name}.jpg')
    imagedata = requests.get(av)
    try:
        with open(fr'{dirname}\{name}.jpg', 'wb') as fp:
            fp.write(imagedata.content)
            # fp.close()
    except Exception as ex:
        print(ex)

进程：

# encoding: utf-8
# 版权所有 2023 涂聚文有限公司
# 许可信息查看：
# 描述：
# Author    : geovindu,Geovin Du 涂聚文.
# IDE       : PyCharm 2023.1 python 311
# Datetime  : 2023/7/16 19:10
# User      : geovindu
# Product   : PyCharm
# Project   : pythonTkinterDemo
# File      : crawierProcessing.py
# explain   : 学习


import multiprocessing
import threading
import time
import json
import requests
import os
import sys


def saveimageUrl(url, dirName):
    """
    进程  并发
    :param url: 链接
    :param dirName: 文件夹名字
    :return:
    """
    # if not os.path.exists('website'):
    # os.mkdir('website')
    try:
        if os.path.exists('website'):
            os.chdir('website')
        if not os.path.exists(f"{dirName}"):
            os.mkdir(dirName)
        content = requests.get(url)
        # print(content.text)
        data = json.loads(content.text)  # 转字典
        # print(data)

        cn = 1
        for d in data['data']['rl'][0:30]:  # 切片为前30个数据
            print(d['nn'], d['rs16'], d['rn'])
            #if not d['av']:
                #av=d['Info'][av]
            av = 'https://apic.douyucdn.cn/upload' + '/' + d['av'] + '_big' + '.jpg'
            print(av)
            name = d['nn']
            # 创建进程
            t = multiprocessing.Process(target=loadAvator, args=(av, dirName, name))
            t.start()
            # img1=Image.open(av)
            # img1=img1.save('img'+cn+'.jpg')
            # imagedata = requests.get(av)
            # with open(fr'{dirName}\{d["nn"]}.jpg','wb') as fp:
            # fp.write(imagedata)

    except Exception as ex:
        print(ex)


def loadAvator(av, dirname, name):
    """

    :param av:图片
    :param dirname:文件夹名字
    :param name:博主名称
    :return:
    """
    print("jpg:", fr'{dirname}\{name}.jpg')
    imagedata = requests.get(av)
    try:
        with open(fr'{dirname}\{name}.jpg', 'wb') as fp:
            fp.write(imagedata.content)
            # fp.close()
    except Exception as ex:
        print(ex)
















def dance():
    """

    :return:
    """
    while True:
        print('dance')
        time.sleep(0.5)


def sing():
    """

    :return:
    """
    while True:
        print('sing')
        time.sleep(0.5)


def main():
    """
    线程  是并发  多CPU,多核
    :return:
    """
    '''
    print(multiprocessing.current_process().name)
    print(multiprocessing.current_process().ident)
    p1=multiprocessing.Process(target=dance)
    p2=multiprocessing.Process(target=sing)
    p1.start()
    p2.start()
    print("cpu个数：",multiprocessing.cpu_count)
    '''

    for _ in range(10):
        t = threadSing.sing()
        t2 = threadSing.dance()

    for _ in range(10):
        p1 = multiprocessing.Process(target=dance)
        p2 = multiprocessing.Process(target=sing)
        p1.start()
        p2.start()

调用：

    if not os.path.exists('website'):
        os.mkdir('website')
    os.chdir('website')
    #Common.crawlerDouyu.saveimageUrl("https://www.douyu.com/gapi/rkc/directory/mixList/2_1/2", 'geovindu')
    #Common.crawierProcessing.saveimageUrl("https://www.douyu.com/gapi/rkc/directory/mixList/2_1/2", 'process')
    #Common.crawierProcessing.saveimageUrl("https://www.douyu.com/wgapi/ordnc/live/web/room/yzList/1", 'face')
    #Common.crawierProcessing.saveimageUrl("https://www.douyu.com/gapi/rkc/directory/mixList/2_194/2", 'sugar')
    Common.crawierProcessing.saveimageUrl("https://www.douyu.com/wgapi/ordnc/live/web/room/mixList/2/1008/0/2", 'stars')