"""
-------------------------------------------------
   File Name：     get_salt
   Description :
   Author :       willis
   date：          2019/1/22
-------------------------------------------------
   Change Activity:
                   2019/1/22:

-------------------------------------------------

"""
__author__ = 'willis'

import logging.handlers
import logging
import sys
import os
from bs4 import BeautifulSoup
import requests
from tqdm import tqdm
from urllib.parse import urljoin
import time


def Log_Out_func():
    '''
    日志输出
    :return:
    '''

    log_file = os.path.realpath(sys.argv[0]) + '.log'

    # 定义对应的程序模块名name，默认是root
    logs = logging.getLogger()
    logs.setLevel(logging.DEBUG)

    # 定义日志回滚
    logrota = logging.handlers.RotatingFileHandler(log_file, maxBytes=10 * 1024 * 1024, backupCount=20)

    # 设置日志等级
    logrota.setLevel(logging.DEBUG)

    # 日志输出到屏幕控制台
    console = logging.StreamHandler()
    console.setLevel(logging.DEBUG)

    # 定义日志格式
    format = ('%(asctime)s|%(name)s|%(levelname)s|'
              '%(pathname)s|%(lineno)d|%(thread)s|%(process)s|%(message)s')

    # 实例化handler
    formatter = logging.Formatter(format)

    # 格式化handler
    logrota.setFormatter(formatter)
    console.setFormatter(formatter)

    # 添加handler
    logs.addHandler(logrota)
    logs.addHandler(console)

    # logs.debug('my is debug')
    # logs.info('my is info')
    # logs.warning('my is warning')
    # logs.error('my is error')
    # logs.critical('my is critical')
    logs.info('Loging start ...')

    return logs

def get_url_func(salt_url):
    ''' 打开url '''
    headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36'}
    resp = requests.get(salt_url, headers=headers, stream=True, timeout=30)
    # logs.debug(resp.content)
    return resp

def merge_url_func(url, potion):
    '''
     拼接 url
     为避免手动配置url后的斜线，手动进行处理
    '''
    if url[-1:] == '/' and potion[:1] == '/':
        return url + potion[:1]
    if url[-1:] == '/' and potion[:1] != '/':
        return url + potion
    if url[-1:] != '/' and potion[:1] != '/':
        return url + "/" + potion
    if url[-1:] != '/' and potion[:1] == '/':
        return url + potion

def save_url_func(salt_url, file):
    ''' 保存html文件 '''
    if not os.path.isfile(file):
        logs.info('开始下载文件{}'.format(file))
        get_text = get_url_func(salt_url)

        logs.info('开始保存文件{}'.format(file))
        with open(file, 'wb') as fd:
            for file_line in tqdm(get_text.iter_content()):
            # for file_line in get_text.iter_content():
                fd.write(file_line)

# def read_url_file_func():
#     ''' 读取html文件 '''
#     logs.info('开始读取Html文件。')
#     with open('salt.html', 'rb') as rs:
#         return rs
#
#

def get_html_file_url_func():
    ''' 获取文件连接 '''
    bs = BeautifulSoup(open('salt.html'))
    # logs.debug(bs)

    url_list = []
    for k in bs.find_all('a'):
        v = k['href']
        url_list.append(v)

    logs.debug(url_list)
    return url_list

def get_html_url_func(salt_url, furl='', url_list=''):
    ''' 获取文件连接 '''
    logs.info('开始获取html:{}'.format(salt_url))
    resp = get_url_func(salt_url).text
    bs = BeautifulSoup(resp)

    logs.info('开始解析html')
    # 将子目录中的文件也写入url_list队列
    # 第一次处理时为空值，默认值为空，将变量定义为列表
    # 如果不为空时，需要判断是否为列表类型
    if url_list == '':
        url_list = []
    for k in bs.find_all('a'):
        v = k['href']
        # 如果href值开头一个字母中有问号
        # 或是一个开头为'/'的路径时，不处理他们，页面中默认为文件下载路径，不需要处理
        if v[:1] == '?' or v[:1] == '/':
            continue
        # 如果结尾是一个'/'，说明是子目录，开始迭代处理
        if v[-1:] == '/':
            time.sleep(1)
            logs.debug(v)
            logs.debug(salt_url)

            # 迭代子目录时，使用子目录的url
            new_url = merge_url_func(salt_url, v)
            # 将子目录记录，并随文件名一起写入队列
            furl = new_url[salt_url_len:]
            logs.debug(new_url)
            get_html_url_func(new_url, furl=furl, url_list=url_list)
            # 初始化为空，避免子目录处理完后，将子目录带给父文件
            # 父目录默认为空
            furl = ''
        else:
            logs.debug('添加文件{}到队列中'.format(furl + v))
            url_list.append(furl + v)

    logs.debug(url_list)
    return url_list

def get_salt_url_len(salt_url):
    ''' 计算Url长度 '''
    if salt_url[-1:] == '/':
        salt_url_len = len(salt_url[1:])
    else:
        salt_url_len = len(salt_url)
    return salt_url_len

def check_dir_func(download_dir, files):
    ''' 检查目录 '''
    file_dir = os.path.dirname(files)
    dir = merge_url_func(download_dir, file_dir)

    if not os.path.isfile(merge_url_func(download_dir, files)):
        if not os.path.exists(download_dir):
            os.mkdir(download_dir)
        if file_dir and not os.path.exists(dir):
            os.makedirs(dir)

def download_file(salt_url, download_dir):
    ''' 下载文件 '''
    for files in tqdm(get_html_url_func(salt_url)):
        save_file = merge_url_func(download_dir, files)
        check_dir_func(download_dir, files)

        file_url = merge_url_func(salt_url, files)
        logs.info('开始下载URL：{}'.format(file_url))
        save_url_func(file_url, save_file)

if __name__ == '__main__':
    salt_url = 'https://repo.saltstack.com/apt/ubuntu/12.04/amd64/latest'
    salt_url_len = int(get_salt_url_len(salt_url))

    logs = Log_Out_func()
    # 保存html文件
    # file = 'salt.html'
    # save_url_func(salt_url, file)
    # 处理html文件, 获取文件连接
    # get_html_file_url_func()

    # 获取html并解析
    download_dir = 'downfile'
    download_file(salt_url, download_dir)