#在今日头条首页搜索“街拍”关键字后抓取图片
from urllib.parse import  urlencode
import requests
import os
import random

#https://www.toutiao.com/search_content/?offset=60&format=json&keyword=%E8%A1%97%E6%8B%8D&autoload=true&count=20&cur_tab=1&from=search_tab&pd=synthesis
def get_page(page):
    params = {
        'offset' : page,
        'format' : 'json',
        'keyword' : '街拍',
        'autoload' : 'true',
        'count' : '20',
        'cur_tab' : '1',
        'from' : 'search_tab',
        'pd' : 'synthesis',
    }

    url = 'https://www.toutiao.com/search_content/?' + urlencode(params)
    headers = {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36',
        'Connection': 'close'
    }
    response = requests.get(url = url,headers = headers)

    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.json()
    except requests.ConnectionError:
        return None

def parse_page(res_json):
    if res_json.get('data'):
        list_image = []
        for item in res_json.get('data'):
            title = item.get('title')
            images = item.get('image_list')
            if images == None:
                continue
            for image in images:
                dic_image = {}
                dic_image['title'] = title
                dic_image['image'] = 'https:' + image.get('url')
                #可以使用yield每次返回一个字典；也可以在每次循环后，将字典添加到一个列表中，循环结束之后，一次性将列表返回
                #数据量大的时候，最好使用yield
                yield dic_image
    #             list_image.append(dic_image)
    # return list_image

def save_image(list_image):
    for item in list_image:
        path = 'image' + os.path.sep + item.get('title')
        if not os.path.exists(path):
            os.makedirs(path)
        try:
            response = requests.get(item.get('image'))
            if response.status_code == 200:
                #防止图片名称相同，导致文件下载失败，在文件名称后面添加一个随机数，防止重复
                file_path = path +os.path.sep+ item.get('title') + str(random.random()) + '.jpg'
                if not os.path.exists(file_path):
                    with open(file_path,'wb') as f_obj:
                        f_obj.write(response.content)    #二进制文件写入
                        print('image save sucessful', file_path)
                else:
                    print('Already Downloaded ', file_path)
        except requests.ConnectionError:
            print('Failed to Save Image')

if __name__ == '__main__':
    for i in range(0,1):
        res_json = get_page(20*i)
        list_image = parse_page(res_json)
        # for image in list_image:
        #     print(type(image))
        #     print(image.get('title'))    #遍历获取字典中的值，使用get时，一定使用小括号(),中括号会报错
        save_image(list_image)