自动下载服务器各目录文件
import threading
from multiprocessing import Pool,Process
import requests
import re,os
# new_content = r.content.decode('utf-8')
#第一次调用生成子目录
def request_dirname(url,pattern):
path_new = []
r = requests.get(url)
new = re.findall(pattern, str(r.content))
i = 0
while i < len(new):
aa = os.path.join(url, new[i])
# print(aa)
path_new.append(aa)
i = i + 1
return path_new
#第二次调用生产文件目录
def request_filename(url, pattern):
# dict_new = {}
# num = 0
file_compare_list = []
path_url = request_dirname(url, pattern)
for item in path_url:
path_fileurl = request_dirname(item,pattern)
# item_new = re.findall(r'8081/(.*?)/', item)
# dict_new[num] = item_new #对应目录编号
# num = num + 1
file_compare_list.append(path_fileurl[1:])
return file_compare_list
#第三次循环写入text文件，文件url
def filename_urls(url, pattern):
f_urls = []
file_compare_list = request_filename(url, pattern)
with open('filename_url.txt', 'w') as f_txt:
f_txt.write('')
for list01 in file_compare_list:
for list02 in list01:
f_urls.append(list02)
with open('filename_url.txt','a') as f_txt:
f_txt.write(list02+'\n')
return f_urls
#创建文件夹并下载
def build_dir(url,ICBC,current_path):
path = url.split('/')[-2] #子目录
filename = url.split('/')[-1] #文件名称
dir_path = os.path.join(current_path,ICBC,path)
if os.path.exists(dir_path):
download_files(url,ICBC,current_path)
else:
# os.makedirs('ICBC/'+path)
os.makedirs(dir_path)
print('%s已创建'%path)
build_dir(url,ICBC,current_path)
#下载文件
def download_files(url,ICBC,current_path):
filename = url.split('/')[-1]
path = url.split('/')[-2]
work_path = os.path.join(current_path,ICBC,path)
print(work_path)
os.chdir(work_path)
#查看当前文件
r = requests.get(url)
with open(filename,'wb') as f:
f.write(r.content)
print(os.listdir('.'))
def run(ICBC,urls_results,current_path):
# global urls_results
for xxl in urls_results:
build_dir(xxl,ICBC,current_path)
if __name__ == '__main__':
current_path = os.getcwd()
pattern1 = r''
url1 = 'http://172.17.3.162:8081/'
ABC = filename_urls(url1, pattern1)
urls_results = ABC[3:]
#文件夹
# p1 = Process(target=run,args=('happy',urls_results,current_path))
# p1.start()
pool = Pool(processes=20)
for i in range(20):
msg = 'ICBC%d'%i
print(msg)
pool.apply_async(run,(msg,urls_results,current_path))
pool.close()
pool.join()
print('done')
结论：