54 lines
1.7 KiB
Python
54 lines
1.7 KiB
Python
import os
|
||
import requests
|
||
|
||
# Список URL для загрузки
|
||
file_urls = [
|
||
"https://example.com/file1.txt",
|
||
"https://example.com/file2.txt",
|
||
]
|
||
|
||
# Папка для сохранения файлов
|
||
download_folder = "downloads"
|
||
os.makedirs(download_folder, exist_ok=True)
|
||
|
||
# Функция для удаления строки 'payload:' и удаления дубликатов
|
||
def process_file(file_path):
|
||
with open(file_path, "r") as f:
|
||
lines = f.readlines()
|
||
|
||
# Удалить строки с "payload:" и убрать дубликаты
|
||
unique_lines = []
|
||
seen_lines = set()
|
||
for line in lines:
|
||
if "payload:" in line:
|
||
continue
|
||
if line not in seen_lines:
|
||
unique_lines.append(line)
|
||
seen_lines.add(line)
|
||
else:
|
||
unique_lines.append(f"# {line.strip()}\n") # Комментировать дубликат
|
||
|
||
# Записать обратно в файл
|
||
with open(file_path, "w") as f:
|
||
f.writelines(unique_lines)
|
||
|
||
# Скачивание файлов и обработка
|
||
for url in file_urls:
|
||
file_name = os.path.basename(url)
|
||
file_path = os.path.join(download_folder, file_name)
|
||
|
||
# Скачивание файла
|
||
print(f"Downloading {url}...")
|
||
response = requests.get(url)
|
||
if response.status_code == 200:
|
||
with open(file_path, "wb") as f:
|
||
f.write(response.content)
|
||
print(f"Saved to {file_path}")
|
||
|
||
# Обработка файла
|
||
print(f"Processing {file_path}...")
|
||
process_file(file_path)
|
||
else:
|
||
print(f"Failed to download {url} (Status Code: {response.status_code})")
|
||
|
||
print("All files processed.") |