scrapy

from scrapy import Field, Item
class BookItem(Item):
    authors = Field()
    catalog = Field()
    comments = Field()
    cover = Field()
    id = Field()
    introduction = Field()
    isbn = Field()
    name = Field()
    page_number = Field()
    price = Field()
    published_at = Field()
    publisher = Field()
    score = Field()
    tags = Field()
    translator = Field()

from scrapy import Request, Spider
from ..items import BookItem
from loguru import logger


class BookSpider(Spider):
    name = 'book'
    allowed_domains = ['antispider7.scrape.center']
    base_url = 'https://antispider7.scrape.center'
    max_page = 512

    def start_requests(self):
        for page in range(1, self.max_page+1):
            url = f'{self.base_url}/api/book/?limit=18&offset={(page-1)*18}'
            yield Request(url, callback=self.parse_index)

    def parse_index(self, res):
        data = res.json()
        results = data.get('results')
        for result in results:
            id = result.get('id')
            url = f'{self.base_url}/api/book/{id}/'
            yield Request(url, callback=self.parse_detail, priority=2) # priority表示该Requst在调度器中具有优先权

    def parse_detail(self, res):
        data = res.json()
        item = BookItem()
        for field in item.fields:
            item[field] = data.get(field)
        yield item

from concurrent.futures import thread
from redis import StrictRedis
from requests import Session
import requests
from flask import Flask
import random
import time
import re
from multiprocessing import Process


url = 'https://antispider7.scrape.center/api/login'
url_center = 'https://antispider7.scrape.center/api/book/?limit=18&offset=0'
db = StrictRedis(host='localhost', port=6379, db=0,
                 decode_responses=True)
s = Session()
app = Flask(__name__)

def Getcookie():  # 没有cookie就获取cookie并填入
    while True:
        for username in db.hkeys('account'):
            if not db.hexists('cookie', username):
                r = s.post(url=url, data={'username': username,
                                          'password': db.hget('account', username)})
                # for cookie in s.cookies:
                #     result.append(f'{cookie.name}={cookie.value}')
                # result = ';'.join(result)
                print(r.status_code, username)
                result = r.json().get('token', 'fail')
                db.hset('cookie', username, result)

        time.sleep(600)


def Testcookie():  # 根据res状态码把库中cookie无效的username键值对删除
    while True:
        if db.hgetall('cookie'):
            print('exist')
        else:
            print('not exist')
            db.hset('cookie', '1', '1')
            print('已创建cookie')
        for username, cookie in db.hgetall('cookie').items():
            res = requests.get(
                url_center, headers={'Authorization': f'jwt {cookie}'}, allow_redirects=False)
            if res.status_code != 200:
                db.hdel('cookie', username)
                print('cookie', username, res.status_code, '已失效！')
            else:
                print('cookie', username, res.status_code, '可用！')
        time.sleep(600)


@app.route('/')  # 接口功能
def cookie_api():
    result = db.hvals('cookie')
    result = [i for i in result if i != '']
    result = random.choice(result)
    return result


def api():
    app.run(threaded=True)


def schedule():
    p1 = Process(target=Getcookie)
    p2 = Process(target=Testcookie)
    p3 = Process(target=api)
    p1.start()
    p2.start()
    p3.start()
    p1.join()
    p2.join()
    p3.join()


if __name__ == '__main__':
    schedule()

import requests
class AuthorizationMiddleware():
    accountpool_url = 'http://127.0.0.1:5000/'
    tunnel = "tps656.kdlapi.com:15818"
    username = "t15160185660737"
    password = "vjopqphj"
    proxies = "http://%(user)s:%(pwd)s@%(proxy)s/" % {
        "user": username, "pwd": password, "proxy": tunnel}

    def process_request(self, request, spider):
        with requests.get(self.accountpool_url) as res:
            authorization = res.text
            authorization = f'jwt {authorization}'
            request.headers['authorization'] = authorization # jwt认证参数加入
            request.meta['proxy'] = self.proxies # 代理参数加入

from itemadapter import ItemAdapter
import pymongo


class ScrapycompositedemoPipeline:
    def process_item(self, item, spider):
        return item


class MongoDBPipeline(object):
    @classmethod
    def from_crawler(cls, crawler):
        cls.connect_string = crawler.settings.get('MONGODB_CONNECTION_STRING')
        cls.database = crawler.settings.get('MONGODB_DATABASE')
        cls.collection = crawler.settings.get('MONGODB_COLLECTION')
        return cls()

    def open_spider(self, spider):
        self.client = pymongo.MongoClient(self.connect_string)
        self.db = self.client[self.database]

    def process_item(self, item, spider):
        self.db[self.collection].update_one(
            {'id': item['id']}, {'$set': dict(item)}, True)
        return item

    def close_spider(self, spider):
        self.client.close()

ROBOTSTXT_OBEY = False # 不尊从robot协议
CONCURRENT_REQUESTS = 5 # 最大并发数
DOWNLOADER_MIDDLEWARES = {
    'scrapycompositedemo.middlewares.AuthorizationMiddleware': 543 # 下载中间件的优先度设置，最小约靠近engine
}
RETRY_HTTP_CODES = [401, 403, 500, 502, 503, 504] # 重试情况，增加爬取成功率
DOWNLOAD_TIMEOUT = 10 # 请求超时设置
RETRY_TIMES = 10 # 重试次数设置
# mongodb常量设置
MONGODB_CONNECTION_STRING = 'localhost'
MONGODB_DATABASE = 'books'
MONGODB_COLLECTION = 'books'
ITEM_PIPELINES = {
    'scrapycompositedemo.pipelines.MongoDBPipeline': 300 # 数据管道优先度设置，数字越小越靠近engine
}