批量清理日志文件脚本
#!/usr/bin/python
#coding:utf8
"""
1、新增删除白名单目录，防止误删除。
2、新增过滤掉.log日志结尾的日志文件。
3、新增加磁盘阈值告警,配置文件添加。
"""
import logging,time,datetime,os,sys,time,ConfigParser,os.path
import logging.config,subprocess
#通过列表获取删除路径，然后删除历史数据
def RemoveRoutine_logFile(DirPathList,ClearFileDate):
IsDirlist = [ DirName for DirName in DirPathList.split(",") if os.path.isdir(DirName)]
IsDirlist = list(set(IsDirlist))
logger.info("Need del dir list: " + str(IsDirlist))
for EmptyDirInfile in IsDirlist:
if Del_White_List_dir(EmptyDirInfile):
for del_file in get_all_files(EmptyDirInfile):
# if log_file_time(del_file,ClearFileDate) and is_log_file(del_file):
if log_file_time(del_file,ClearFileDate):
try:
if is_log_file(del_file) == False:
os.remove(del_file)
logger.info("already del file: " + del_file)
else:
logger.error("file wrong .log ending: " + del_file + " skip del this log file.")
except Exception as e:
logger.error("return info:" + str(e) )
else:
logger.error("del file name wrong time range: " + del_file)
else:
logger.error("This dir wrong White List dir: " + EmptyDirInfile)
#通过列表获取删除路径，然后删除历史数据
def RemovePersistence_datFile(DirPathList,ClearFileDate):
IsDirlist = [ DirName for DirName in DirPathList.split(",") if os.path.isdir(DirName)]
IsDirlist = list(set(IsDirlist))
logger.info("need del Persistent file dir list: " + str(IsDirlist))
for EmptyDirInfile in IsDirlist:
if Del_White_List_dir(EmptyDirInfile):
for del_file in get_all_files(EmptyDirInfile):
if log_file_time(del_file,ClearFileDate):
try:
os.remove(del_file)
logger.debug("already del file: " + del_file)
except Exception as e:
logger.error("del file error:" + str(e) )
else:
logger.error("del file name wrong time range: " + del_file)
else:
logger.error("This Persistent dir wrong White List dir: " + EmptyDirInfile)
#删除sdr话单和cdr文件
def RemoveSpeechList(SpeechListDir,RmoveSpeechListDate):
IsDirlist = [ DirName for DirName in SpeechListDir.split(",") if os.path.isdir(DirName)]
logger.info("Need to delete the bill of lading dir list: " + str(IsDirlist))
for EmptyDirInfile in IsDirlist:
if Del_White_List_dir(EmptyDirInfile):
Shell = "find " + EmptyDirInfile + " -name '*.dat' -mtime +" + RmoveSpeechListDate + " | xargs rm -f >/dev/null 2>&1"
logger.info("runing shell command: " + Shell )
os.popen(Shell)
#危险目录判断
def IfDangerDir(dir):
if dir in GetDangerDirList:
return True
else:
return False
def get_del_file_need_time(del_times):
New_time = time.time()
Not_del_file_time = 86400 * int(del_times)
Need_del_time = int(New_time) - Not_del_file_time
return Need_del_time
def log_file_time(filename,del_time):
need_file_status = False
Need_del_file_time = get_del_file_need_time(del_time)
if os.path.isfile(filename):
modification_time = os.path.getmtime(filename)
if int(modification_time) < Need_del_file_time:
need_file_status = True
return need_file_status
def get_all_files(directory):
files = []
for root, dirs, filenames in os.walk(directory):
for filename in filenames:
files.append(os.path.join(root, filename))
for num in range(0,int(Default_move_file_num)):
if len(files) != 0:
latest_file = max(files, key=os.path.getmtime)
files.remove(latest_file)
logger.info("Default move file not need del: " + str(latest_file))
else:
continue
return files
def is_log_file(file_path):
_, file_extension = os.path.splitext(file_path)
return file_extension == '.log'
def Del_White_List_dir(files):
file_status = False
for White_dir in GetDangerDirList:
if White_dir in files:
file_status = True
return file_status
def get_disk_usage_gb(path):
"""
获取指定路径的磁盘使用信息并转换为GB
:param path: 路径字符串, 如 '/' 或 '/home'
:return: 返回一个包含使用率和可用空间(以GB为单位)的字典
"""
# 使用 df 命令获取磁盘使用情况
result = subprocess.check_output(['df', path])
lines = result.split('\n')
parts = lines[1].split()
# 解析所需数据，忽略表头行
total_blocks = int(parts[1]) # 总块数
used_blocks = int(parts[2]) # 已使用的块数
available_blocks = int(parts[3]) # 可用块数
use_percent = parts[4] # 使用百分比
# 计算大小，通常每个块是1KB
total_gb = total_blocks / 1024.0 / 1024.0
used_gb = int(used_blocks / 1024.0 / 1024.0)
available_gb = available_blocks / 1024.0 / 1024.0
bisk_Use_percent = use_percent.split("%")[0]
if int(bisk_Use_percent) > int(disk_usage):
Msg_info = path + " 磁盘使用率大于设定阈值，当前为:" + str(used_gb) + "%" + ", 还剩余空间为: " + str(int(round(available_gb,2))) + "GB,请及时清理，避免磁盘满，导致服务不可用."
logger.info(Msg_info)
SendApex(Msg_info)
else:
Msg_info = path + " 磁盘使用率正常，当前为:" + str(used_gb) + "%" + ", 还剩余空间为: " + str(int(round(available_gb,2))) + "GB."
logger.info(Msg_info)
#发送网管告警
def SendApex(ApexInfo):
GetDomainMsg = ApexInfo
# if "True" == IsSendApex:
shell = "/bin/logger -p local0.crit \"ccod: result=ERROR "+ GetDomainMsg + " \""
os.system(shell)
if __name__ == '__main__':
curpath = os.path.dirname(os.path.realpath(__file__))
cfgpath = os.path.join(curpath, "cfg/config.ini")
conf = ConfigParser.ConfigParser()
conf.read(cfgpath)
GetLogDir = conf.get("Base","LocalLogDir")
LogRunSentence = " [ ! -d " + GetLogDir + " ] && mkdir -p " + GetLogDir
os.system(LogRunSentence)
logging.config.fileConfig("./cfg/logger.conf")
logger = logging.getLogger("rotatfile")
MoveFilePath = conf.get("Base", "MoveFilePath")
BatFilePath = conf.get("Base", "BatFilePath")
MoveFileDate = conf.get("Base", "MoveFileDate")
MoveBatFileDate = conf.get("Base", "MoveBatFileDate")
Default_move_file_num = conf.get("Base", "Default_move_file_num")
RmoveSpeechListDate = conf.get("Base","RmoveSpeechListDate")
SpeechListPath = conf.get("Base","SpeechListPath")
# SpecifyfilePath = conf.get("Base","SpecifyfilePath")
RemoveSpecifyfileTime = conf.get("Base","RemoveSpecifyfileTime")
DangerDirList = conf.get("Base","DangerDirList")
SleepTime = conf.get("Base","SleepTime")
GetDangerDirList = DangerDirList.split(",")
Check_disk_list = conf.get("Base","Check_disk_list")
disk_usage = conf.get("Base","disk_usage")
Check_disk_list = Check_disk_list.split(",")
logger.info("clear log app start...")
logger.info("当前磁盘监控告警阈值: " + str(disk_usage) + "%")
logger.info("The whitelist includes a directory deletion list: " + str(GetDangerDirList))
RemoveRoutine_logFile(MoveFilePath,MoveFileDate)
RemovePersistence_datFile(BatFilePath,MoveBatFileDate)
RemoveSpeechList(SpeechListPath,RmoveSpeechListDate)
for disk_dir in Check_disk_list:
if os.path.exists(disk_dir):
get_disk_usage_gb(disk_dir)
logger.info("clear log file end...")
[Base]
#常规日志文件路径,以逗号分割,切勿少写导致是根,而进行删除，确保路径的完整性
MoveFilePath=/home/portal/resin-2/logs,/usr/local/freeswitch/log,/home/ccod/resin-1/logs
#删除持久化文件路径,以逗号分割
BatFilePath=/home/test/ccaa/dat,/home/cc/xxx/dat,/home/auto/fabric/update/Job/ClearFile/test,/home/test/Platform/kafka_stat
#删除历史话单文件路径,以逗号分割
SpeechListPath=/home/sdr,/home/test/sdr
#删除常规日文件多少天之前的日志
MoveFileDate=30
#删除持久化日志文件多少天之前的日志
MoveBatFileDate=60
#删除历史话单文件多少天之前的日志
RmoveSpeechListDate=365
#默认移除列表中指定的最新文件数,不进行删除，解决不刷新日志时，把日志文件删除掉的问题。
Default_move_file_num=3
#日志目录,代码已写死.
LocalLogDir=./log
#临时删除指定目录下文件日期,下面案例是一天前。
RemoveSpecifyfileTime=1
#白名单目录列表中目录包含则进行删除文件操作
DangerDirList=/log,/logs,/RunLog,/Log,/dat,/mpdat,/cctrackmsg
#循环slee时间
SleepTime = 10
#过滤需要检查挂载目录,以逗号分割
Check_disk_list = /,/home,/test
#磁盘大于95%则进行发送Rsyslog进行告警
disk_usage = 95
[devops@db1 CleanUpFile]$ sudo ./CleanUpFile-v4.py
2025-03-06 14:29:40 139641581000512 CleanUpFile-v4.py:222 INFO clear log app start...
2025-03-06 14:29:40 139641581000512 CleanUpFile-v4.py:223 INFO 当前设置监控告警阈值: 95%
2025-03-06 14:29:40 139641581000512 CleanUpFile-v4.py:224 INFO The whitelist includes a directory deletion list: ['/log', '/logs', '/RunLog', '/Log', '/ccod45/micro', '/dat', '/mpdat', '/cctrackmsg']
2025-03-06 14:29:40 139641581000512 CleanUpFile-v4.py:37 INFO Need del dir list: []
2025-03-06 14:29:40 139641581000512 CleanUpFile-v4.py:60 INFO need del Persistent file dir list: []
2025-03-06 14:29:40 139641581000512 CleanUpFile-v4.py:92 INFO Need to delete the bill of lading dir list: []
2025-03-06 14:29:40 139641581000512 CleanUpFile-v4.py:239 INFO clear log file end...
评论区