清理历史日志和垃圾文件脚本
#!/usr/bin/python
#coding:utf8
"""
1、新增危险目录，判断不删除操作
"""
import logging,time,datetime,os,sys,time,ConfigParser,os.path
import logging.config
#获取磁盘大于阈值返回磁盘挂载点
def DsikUseRate(DiskPercent):
logger.info("当前设置清理文件阈值是: " + DiskPercent + "%")
#获取磁盘分区信息
ALLpartitions = psutil.disk_partitions()
#列表推导树,获取挂载点
MountPointList = [Rate[1] for Rate in ALLpartitions if "mountpoint" in str(Rate)]
logger.info("获取磁盘挂载点有: " + str(MountPointList))
#常规写法
# for Rate in ALLpartitions:
# if mountpoint in str(Rate):
# MountPoint = str(Rate[0]).split(",")[1].split("'")[1]
# MountPointList.append(MountPoint)
#使用挂载点获取磁盘的使用率,在判断是否大于设置阈值，并返回挂载点列表
GetMountPoint = [Rate[1] for Point in MountPointList if int(DiskPercent) < int(psutil.disk_usage(Point)[3])]
logger.info("大于设置阈值: "+ DiskPercent + "% 磁盘列表有: " + str(GetMountPoint))
return GetMountPoint
#通过列表获取删除路径，然后删除历史数据
def RemoveRoutineFile(DirPathList,ClearFileDate):
IsDirlist = [ DirName for DirName in DirPathList.split(",") if os.path.isdir(DirName)]
logger.info("需要执行删除操作的常规目录有: " + str(IsDirlist))
for EmptyDirInfile in IsDirlist:
#判断删除目录是否为危险目录
GetDangerValues = IfDangerDir(EmptyDirInfile)
if GetDangerValues == False:
Shell = "find " + EmptyDirInfile + " -name '*' -mtime +" + ClearFileDate + " | xargs rm -f >/dev/null 2>&1"
Shell2 = "find " + EmptyDirInfile + " -name '*' -mtime +" + ClearFileDate
logger.info("将要执行的Shell命令: " + Shell )
GetMsg = os.popen(Shell2)
[ logger.info("需要删除日常日志目录文件如下: " + Msg.strip('\n')) for Msg in GetMsg]
os.popen(Shell)
else:
logger.info("该目录: " + EmptyDirInfile +" 是危险目录不执行删除操作..." )
#通过列表获取删除路径，然后删除历史数据
def RemovePersistenceFile(DirPathList,ClearFileDate):
IsDirlist = [ DirName for DirName in DirPathList.split(",") if os.path.isdir(DirName)]
logger.info("需要执行删除操作的持久化目录下文件有: " + str(IsDirlist))
for EmptyDirInfile in IsDirlist:
Shell = "find " + EmptyDirInfile + " -name '*' -mtime +" + ClearFileDate + " | xargs rm -f >/dev/null 2>&1"
Shell2 = "find " + EmptyDirInfile + " -name '*' -mtime +" + ClearFileDate
logger.info("将要执行的Shell命令: " + Shell )
GetMsg = os.popen(Shell2)
[ logger.info("需要删除持久化文件如下: " + Msg.strip('\n')) for Msg in GetMsg]
os.popen(Shell)
#指定删除临时目录下文件
def RemoveTmpFile(TmpDir):
logger.info("进入删除临时目录中")
TmpDirSentence = " [ ! -d " + TmpDir + " ] && mkdir -p " + TmpDir
os.system(TmpDirSentence)
TmpfileList = os.listdir(TmpDir)
if len(TmpfileList):
logger.info("删除前判断是否为危险目录: " )
Shell = "find " + TmpDir + " -name '*' -mtime +1 | xargs rm -f >/dev/null 2>&1"
logger.info("将要执行删除临时的Shell命令: " + Shell)
GetMsg = os.popen(Shell)
else:
logger.info("当前 " +TmpDir + " 中,没有文件可删除..." )
#删除sdr话单和cdr文件
def RemoveSpeechList(SpeechListDir,RmoveSpeechListDate):
IsDirlist = [ DirName for DirName in SpeechListDir.split(",") if os.path.isdir(DirName)]
logger.info("需要执行删除操作的话单目录下文件有: " + str(IsDirlist))
for EmptyDirInfile in IsDirlist:
Shell = "find " + EmptyDirInfile + " -name '*.dat' -mtime +" + RmoveSpeechListDate + " | xargs rm -f >/dev/null 2>&1"
Shell2 = "find " + EmptyDirInfile + " -name '*.dat' -mtime +" + RmoveSpeechListDate
logger.info("将要执行的Shell命令: " + Shell )
GetMsg = os.popen(Shell2)
[ logger.info("需要删除文件如下: " + Msg.strip('\n')) for Msg in GetMsg]
os.popen(Shell)
#删除指定目录下文件
def RemoveFile(SpecifyfilePath,RemoveSpecifyfileTime):
IsDirlist = [ DirName for DirName in SpecifyfilePath.split(",") if os.path.isdir(DirName)]
logger.info("需要执行删除操作的常规目录有: " + str(IsDirlist))
for EmptyDirInfile in IsDirlist:
Shell = "find " + EmptyDirInfile + " -name '*' -mtime +" + RemoveSpecifyfileTime + " | xargs rm -f >/dev/null 2>&1"
Shell2 = "find " + EmptyDirInfile + " -name '*' -mtime +" + RemoveSpecifyfileTime
logger.info("将要执行的Shell命令: " + Shell )
GetMsg = os.popen(Shell2)
[ logger.info("需要删除日常日志目录文件如下: " + Msg.strip('\n')) for Msg in GetMsg]
os.popen(Shell)
#危险目录判断
def IfDangerDir(dir):
if dir in GetDangerDirList:
return True
else:
return False
if __name__ == '__main__':
curpath = os.path.dirname(os.path.realpath(__file__))
cfgpath = os.path.join(curpath, "cfg/config.ini")
conf = ConfigParser.ConfigParser()
conf.read(cfgpath)
GetLogDir = conf.get("Base","LocalLogDir")
LogRunSentence = " [ ! -d " + GetLogDir + " ] && mkdir -p " + GetLogDir
os.system(LogRunSentence)
logging.config.fileConfig("./cfg/logger.conf")
logger = logging.getLogger("rotatfile")
MoveFilePath = conf.get("Base", "MoveFilePath")
BatFilePath = conf.get("Base", "BatFilePath")
MoveFileDate = conf.get("Base", "MoveFileDate")
MoveBatFileDate = conf.get("Base", "MoveBatFileDate")
DiskPercent = conf.get("Base", "DiskPercent")
MoveTmpFileDir = conf.get("Base", "MoveTmpFileDir")
RmoveSpeechListDate = conf.get("Base","RmoveSpeechListDate")
SpeechListPath = conf.get("Base","SpeechListPath")
SpecifyfilePath = conf.get("Base","SpecifyfilePath")
RemoveSpecifyfileTime = conf.get("Base","RemoveSpecifyfileTime")
DangerDirList = conf.get("Base","DangerDirList")
GetDangerDirList = DangerDirList.split(",")
logger.info("清理文件程序执行开始...")
logger.info("勿删除危险目录列表: " + str(GetDangerDirList))
GetThresholdReachedDir = DsikUseRate(DiskPercent)
# 返回的阈值目录数量大于等于1，则执行删除操作
if len(GetThresholdReachedDir) >= 1:
RemoveRoutineFile(MoveFilePath,MoveFileDate)
RemovePersistenceFile(BatFilePath,MoveBatFileDate)
RemoveFile(SpecifyfilePath,RemoveSpecifyfileTime)
RemoveSpeechList(SpeechListPath,RmoveSpeechListDate)
RemoveTmpFile(MoveTmpFileDir)
logger.info("清理文件程序执行结束...")
config.ini
[Base]
#常规日志文件路径,以逗号分割,切勿少写导致是根,而进行删除，确保路径的完整性
MoveFilePath=/home/portal/apache-tomcat-6.0.35/logs,/var/ccod-runtime/ccod45/cloud10/dds,/var/ccod-runtime/ccod45/cloud10/de/log
#删除持久化文件路径,以逗号分割
BatFilePath=/home/ccodrunner/Platform/dat,/home/test/Platform/dat
#删除历史话单文件路径,以逗号分割
SpeechListPath=/home/sdr,/home/cdr
#删除常规日文件多少天之前的日志
MoveFileDate=21
#删除持久化日志文件多少天之前的日志
MoveBatFileDate=50
#删除历史话单文件多少天之前的日志
RmoveSpeechListDate=730
#定义磁盘大小阈值，此配置项不生效
DiskPercent=80
#日志目录
LocalLogDir=./log
#指定临时目录删除,后续把单个要删除的文件可以丢在此目录下
MoveTmpFileDir=/tmp/TmpFileDir/
#临时删除指定目录下文件
SpecifyfilePath=/home/ccodrunner/monitorproxy/log,/home/ccodrunner/monitorproxy/mpdat/monitorproxy
#临时删除指定目录下文件日期
RemoveSpecifyfileTime=1
#危险目录列表中目录则不删除
DangerDirList=/bin,/boot,/dev,/etc,/home,/lib,/lib64,/opt,/proc,/root,/,/run,/sbin,/selinux,/srv,/sys,/tmp,/usr,/var,/home/portal,/home/ccodrunner/,/home/111,/home/agent,/home/agent,/home/cdesk,/home/oracle/,/home/mongodb,/home/umg,/home/storm,/home/kafka,/home/redis,/home/home,/home/slee,/home/SD,/home/scan,/home/call,/home/ccodrunner/Platform,/home/222/Platform,/home/SD/SmartDialer,/home/call/Platform,/home/mongodb/data,/home/mongodb/config,/home/ccodrunner/resin-3.0.25/webapps,/home/portal/resin-2/webapps,/home/ccodrunner/apache-tomcat-5.5.27/webapps,/home/portal/resin-4.0.13/webapps,/var/lib/mysql
logger.conf
###############################################
[loggers]
keys=root,rotatfile
[logger_root]
level=DEBUG
handlers=hand01,hand02
[logger_rotatfile]
handlers=hand01,hand02
qualname=rotatfile
propagate=0
###############################################
[handlers]
keys=hand01,hand02
[handler_hand01]
class=handlers.RotatingFileHandler
level=DEBUG
formatter=form01
args=('./log/CleanUpFile.log', 'a', 50*1024*1024, 100)
[handler_hand02]
class=StreamHandler
level=DEBUG
formatter=form01
args=(sys.stderr,)
###############################################
[formatters]
keys=form01
[formatter_form01]
format=%(asctime)s %(thread)d %(filename)s:%(lineno)d %(levelname)s %(message)s
datefmt=%Y-%m-%d %H:%M:%S
[root@dev ClanUpFile]# crontab -l
0 1 * * * cd /home/shell/CleanUpFile && ./CleanUpFile-v2.py
[root@dev ClanUpFile]# ./CleanUpFile-v2.py
2024-08-15 17:57:45 140701211469632 CleanUpFile-v2.py:162 INFO 清理文件程序执行开始...
2024-08-15 17:57:45 140701211469632 CleanUpFile-v2.py:163 INFO 勿删除危险目录列表: ['/bin', '/boot', '/dev', '/etc', '/home', '/lib', '/lib64', '/opt', '/proc', '/root', '/', '/run', '/sbin', '/selinux', '/srv', '/sys', '/tmp', '/usr', '/var', '/home/portal', '/home/ccodrunner/', '/home/ccodqnsoft', '/home/agent', '/home/agent', '/home/cdesk', '/home/oracle/', '/home/mongodb', '/home/umg', '/home/storm', '/home/kafka', '/home/redis', '/home/home', '/home/slee', '/home/SD', '/home/scan', '/home/call', '/home/ccodrunner/Platform', '/home/ccodqnsoft/Platform', '/home/SD/SmartDialer', '/home/call/Platform', '/home/mongodb/data', '/home/mongodb/config', '/home/ccodrunner/resin-3.0.25/webapps', '/home/portal/resin-2/webapps', '/home/ccodrunner/apache-tomcat-5.5.27/webapps', '/home/portal/resin-4.0.13/webapps', '/var/lib/mysql']
2024-08-15 17:57:45 140701211469632 CleanUpFile-v2.py:35 INFO 需要执行删除操作的常规目录有: ['/home/agentproxy/resin/logs', '/home/agentproxy/resin/log']
2024-08-15 17:57:45 140701211469632 CleanUpFile-v2.py:42 INFO 将要执行的Shell命令: find /home/agentproxy/resin/logs -name '*' -mtime +21 | xargs rm -f >/dev/null 2>&1
2024-08-15 17:57:45 140701211469632 CleanUpFile-v2.py:44 INFO 需要删除日常日志目录文件如下: /home/agentproxy/resin/logs
2024-08-15 17:57:45 140701211469632 CleanUpFile-v2.py:42 INFO 将要执行的Shell命令: find /home/agentproxy/resin/log -name '*' -mtime +21 | xargs rm -f >/dev/null 2>&1
2024-08-15 17:57:45 140701211469632 CleanUpFile-v2.py:52 INFO 需要执行删除操作的持久化目录下文件有: []
2024-08-15 17:57:45 140701211469632 CleanUpFile-v2.py:90 INFO 需要执行删除操作的常规目录有: []
2024-08-15 17:57:45 140701211469632 CleanUpFile-v2.py:78 INFO 需要执行删除操作的话单目录下文件有: []
2024-08-15 17:57:45 140701211469632 CleanUpFile-v2.py:63 INFO 进入删除临时目录中
2024-08-15 17:57:45 140701211469632 CleanUpFile-v2.py:73 INFO 当前 /tmp/TmpFileDir/ 中,没有文件可删除...
2024-08-15 17:57:45 140701211469632 CleanUpFile-v2.py:122 INFO 需要指定异常目录有: []
2024-08-15 17:57:45 140701211469632 CleanUpFile-v2.py:176 INFO 清理文件程序执行结束...
评论区