- 新增重试和失败告警功能(支持自动重试N次,失败后钉钉/企微通知) - 新增密钥管理(安全存储API Key等敏感信息) - 新增脚本模板库(预置常用脚本模板) - 新增脚本版本管理(自动保存历史版本,支持回滚) - 新增执行统计(成功率、平均耗时、7日趋势) - SDK 新增多租户遍历能力(get_tenants/get_tenant_config/get_all_tenant_configs) - SDK 新增密钥读取方法(get_secret)
This commit is contained in:
@@ -31,10 +31,81 @@ def get_db_session() -> Session:
|
||||
return SessionLocal()
|
||||
|
||||
|
||||
async def send_alert(webhook: str, task_name: str, error_message: str):
|
||||
"""发送失败告警通知"""
|
||||
try:
|
||||
# 自动判断钉钉或企微
|
||||
if "dingtalk" in webhook or "oapi.dingtalk.com" in webhook:
|
||||
data = {
|
||||
"msgtype": "markdown",
|
||||
"markdown": {
|
||||
"title": "定时任务执行失败",
|
||||
"text": f"### ⚠️ 定时任务执行失败\n\n**任务名称**:{task_name}\n\n**错误信息**:{error_message[:500]}\n\n**时间**:{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
|
||||
}
|
||||
}
|
||||
else:
|
||||
# 企微格式
|
||||
data = {
|
||||
"msgtype": "markdown",
|
||||
"markdown": {
|
||||
"content": f"### ⚠️ 定时任务执行失败\n\n**任务名称**:{task_name}\n\n**错误信息**:{error_message[:500]}\n\n**时间**:{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
|
||||
}
|
||||
}
|
||||
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
await client.post(webhook, json=data)
|
||||
logger.info(f"Alert sent for task {task_name}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to send alert: {e}")
|
||||
|
||||
|
||||
async def execute_task_with_retry(task_id: int, retry_count: int = 0, max_retries: int = 0, retry_interval: int = 60):
|
||||
"""带重试的任务执行"""
|
||||
success = await execute_task_once(task_id)
|
||||
|
||||
if not success and retry_count < max_retries:
|
||||
logger.info(f"Task {task_id} failed, scheduling retry {retry_count + 1}/{max_retries} in {retry_interval}s")
|
||||
await asyncio.sleep(retry_interval)
|
||||
await execute_task_with_retry(task_id, retry_count + 1, max_retries, retry_interval)
|
||||
elif not success:
|
||||
# 所有重试都失败,发送告警
|
||||
db = get_db_session()
|
||||
try:
|
||||
result = db.execute(
|
||||
text("SELECT task_name, alert_on_failure, alert_webhook, last_run_message FROM platform_scheduled_tasks WHERE id = :id"),
|
||||
{"id": task_id}
|
||||
)
|
||||
task = result.mappings().first()
|
||||
if task and task["alert_on_failure"] and task["alert_webhook"]:
|
||||
await send_alert(task["alert_webhook"], task["task_name"], task["last_run_message"] or "未知错误")
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
async def execute_task(task_id: int):
|
||||
"""执行定时任务"""
|
||||
"""执行定时任务入口(处理重试配置)"""
|
||||
db = get_db_session()
|
||||
try:
|
||||
result = db.execute(
|
||||
text("SELECT retry_count, retry_interval FROM platform_scheduled_tasks WHERE id = :id"),
|
||||
{"id": task_id}
|
||||
)
|
||||
task = result.mappings().first()
|
||||
if task:
|
||||
max_retries = task.get("retry_count", 0) or 0
|
||||
retry_interval = task.get("retry_interval", 60) or 60
|
||||
await execute_task_with_retry(task_id, 0, max_retries, retry_interval)
|
||||
else:
|
||||
await execute_task_once(task_id)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
async def execute_task_once(task_id: int) -> bool:
|
||||
"""执行一次定时任务,返回是否成功"""
|
||||
db = get_db_session()
|
||||
log_id = None
|
||||
success = False
|
||||
|
||||
try:
|
||||
# 1. 查询任务配置
|
||||
@@ -46,7 +117,7 @@ async def execute_task(task_id: int):
|
||||
|
||||
if not task:
|
||||
logger.warning(f"Task {task_id} not found or disabled")
|
||||
return
|
||||
return True # 不需要重试
|
||||
|
||||
# 2. 更新任务状态为运行中
|
||||
db.execute(
|
||||
@@ -161,9 +232,11 @@ async def execute_task(task_id: int):
|
||||
db.commit()
|
||||
|
||||
logger.info(f"Task {task_id} executed with status: {status}")
|
||||
success = (status == "success")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Task {task_id} execution error: {str(e)}")
|
||||
success = False
|
||||
|
||||
# 更新失败状态
|
||||
try:
|
||||
@@ -190,6 +263,8 @@ async def execute_task(task_id: int):
|
||||
pass
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
return success
|
||||
|
||||
|
||||
def add_task_to_scheduler(task: Dict[str, Any]):
|
||||
|
||||
Reference in New Issue
Block a user