feat: 集成MinIO对象存储服务
All checks were successful
continuous-integration/drone/push Build is passing

- 新增storage_service.py封装MinIO操作
- 修改upload.py使用storage_service上传文件
- 修改course_service.py使用storage_service删除文件
- 适配preview.py支持从MinIO获取文件
- 适配knowledge_analysis_v2.py支持MinIO存储
- 在config.py添加MinIO配置项
- 添加minio依赖到requirements.txt

支持特性:
- 自动降级到本地存储(MinIO不可用时)
- 保持URL格式兼容(/static/uploads/)
- 文件自动缓存到本地(用于预览和分析)

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
yuliang_guo
2026-02-03 14:06:22 +08:00
parent fca82e2d44
commit 2f47193059
13 changed files with 1071 additions and 629 deletions

View File

@@ -41,7 +41,7 @@ UPLOAD_DIR=uploads
COZE_OAUTH_CLIENT_ID=1114009328887
COZE_OAUTH_PUBLIC_KEY_ID=GGs9pw0BDHx2k9vGGehUyRgKV-PyUWLBncDs-YNNN_I
COZE_OAUTH_PRIVATE_KEY_PATH=/app/secrets/coze_private_key.pem
COZE_PRACTICE_BOT_ID=7560643598174683145
COZE_PRACTICE_BOT_ID=7602204855037591602
# Dify 工作流 API Key 配置
# 01-知识点分析

View File

@@ -1,6 +1,8 @@
"""
文件预览API
提供课程资料的在线预览功能
支持MinIO和本地文件系统两种存储后端
"""
import logging
from pathlib import Path
@@ -15,6 +17,7 @@ from app.core.config import settings
from app.models.user import User
from app.models.course import CourseMaterial
from app.services.document_converter import document_converter
from app.services.storage_service import storage_service
logger = logging.getLogger(__name__)
router = APIRouter()
@@ -81,10 +84,12 @@ def get_preview_type(file_ext: str) -> str:
return PreviewType.DOWNLOAD
def get_file_path_from_url(file_url: str) -> Optional[Path]:
async def get_file_path_from_url(file_url: str) -> Optional[Path]:
"""
从文件URL获取本地文件路径
支持MinIO和本地文件系统。如果文件在MinIO中会先下载到本地缓存。
Args:
file_url: 文件URL如 /static/uploads/courses/1/xxx.pdf
@@ -94,11 +99,12 @@ def get_file_path_from_url(file_url: str) -> Optional[Path]:
try:
# 移除 /static/uploads/ 前缀
if file_url.startswith('/static/uploads/'):
relative_path = file_url.replace('/static/uploads/', '')
full_path = Path(settings.UPLOAD_PATH) / relative_path
return full_path
object_name = file_url.replace('/static/uploads/', '')
# 使用storage_service获取文件路径自动处理MinIO下载
return await storage_service.get_file_path(object_name)
return None
except Exception:
except Exception as e:
logger.error(f"获取文件路径失败: {e}")
return None
@@ -158,7 +164,7 @@ async def get_material_preview(
# 根据预览类型处理
if preview_type == PreviewType.TEXT:
# 文本类型,读取文件内容
file_path = get_file_path_from_url(material.file_url)
file_path = await get_file_path_from_url(material.file_url)
if file_path and file_path.exists():
try:
with open(file_path, 'r', encoding='utf-8') as f:
@@ -176,7 +182,7 @@ async def get_material_preview(
elif preview_type == PreviewType.EXCEL_HTML:
# Excel文件转换为HTML预览
file_path = get_file_path_from_url(material.file_url)
file_path = await get_file_path_from_url(material.file_url)
if file_path and file_path.exists():
converted_url = document_converter.convert_excel_to_html(
str(file_path),
@@ -200,7 +206,7 @@ async def get_material_preview(
elif preview_type == PreviewType.PDF and document_converter.is_convertible(file_ext):
# Office文档需要转换为PDF
file_path = get_file_path_from_url(material.file_url)
file_path = await get_file_path_from_url(material.file_url)
if file_path and file_path.exists():
# 执行转换
converted_url = document_converter.convert_to_pdf(

View File

@@ -1,5 +1,9 @@
"""
文件上传API接口
支持两种存储后端:
1. MinIO对象存储生产环境推荐
2. 本地文件系统(开发环境或降级方案)
"""
import os
import shutil
@@ -17,6 +21,7 @@ from app.models.user import User
from app.models.course import Course
from app.schemas.base import ResponseModel
from app.core.logger import get_logger
from app.services.storage_service import storage_service
logger = get_logger(__name__)
@@ -93,16 +98,13 @@ async def upload_file(
# 生成唯一文件名
unique_filename = generate_unique_filename(file.filename)
# 获取上传路径
upload_path = get_upload_path(file_type)
file_path = upload_path / unique_filename
# 保存文件
with open(file_path, "wb") as f:
f.write(contents)
# 生成文件访问URL
file_url = f"/static/uploads/{file_type}/{unique_filename}"
# 使用storage_service上传文件
object_name = f"{file_type}/{unique_filename}"
file_url = await storage_service.upload(
contents,
object_name,
content_type=file.content_type
)
logger.info(
"文件上传成功",
@@ -111,6 +113,7 @@ async def upload_file(
saved_filename=unique_filename,
file_size=file_size,
file_type=file_type,
storage="minio" if storage_service.is_minio_enabled else "local",
)
return ResponseModel(
@@ -184,17 +187,13 @@ async def upload_course_material(
# 生成唯一文件名
unique_filename = generate_unique_filename(file.filename)
# 创建课程专属目录
course_upload_path = Path(settings.UPLOAD_PATH) / "courses" / str(course_id)
course_upload_path.mkdir(parents=True, exist_ok=True)
# 保存文件
file_path = course_upload_path / unique_filename
with open(file_path, "wb") as f:
f.write(contents)
# 生成文件访问URL
file_url = f"/static/uploads/courses/{course_id}/{unique_filename}"
# 使用storage_service上传文件
object_name = f"courses/{course_id}/{unique_filename}"
file_url = await storage_service.upload(
contents,
object_name,
content_type=file.content_type
)
logger.info(
"课程资料上传成功",
@@ -203,6 +202,7 @@ async def upload_course_material(
original_filename=file.filename,
saved_filename=unique_filename,
file_size=file_size,
storage="minio" if storage_service.is_minio_enabled else "local",
)
return ResponseModel(
@@ -243,24 +243,24 @@ async def delete_file(
detail="无效的文件URL"
)
# 转换为实际文件路径
relative_path = file_url.replace("/static/uploads/", "")
file_path = Path(settings.UPLOAD_PATH) / relative_path
# 从URL中提取对象名称
object_name = file_url.replace("/static/uploads/", "")
# 检查文件是否存在
if not file_path.exists():
if not await storage_service.exists(object_name):
raise HTTPException(
status_code=status.HTTP_404_NOT_FOUND,
detail="文件不存在"
)
# 删除文件
os.remove(file_path)
# 使用storage_service删除文件
await storage_service.delete(object_name)
logger.info(
"文件删除成功",
user_id=current_user.id,
file_url=file_url,
storage="minio" if storage_service.is_minio_enabled else "local",
)
return ResponseModel(data=True, message="文件删除成功")

View File

@@ -107,6 +107,14 @@ class Settings(BaseSettings):
import os
return os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), self.UPLOAD_DIR)
# MinIO对象存储配置
MINIO_ENABLED: bool = Field(default=True, description="是否启用MinIO存储")
MINIO_ENDPOINT: str = Field(default="kaopeilian-minio:9000", description="MinIO服务地址")
MINIO_ACCESS_KEY: str = Field(default="kaopeilian_admin", description="MinIO访问密钥")
MINIO_SECRET_KEY: str = Field(default="KplMinio2026!@#", description="MinIO秘密密钥")
MINIO_SECURE: bool = Field(default=False, description="是否使用HTTPS")
MINIO_PUBLIC_URL: str = Field(default="", description="MinIO公开访问URL留空则使用Nginx代理")
# Coze 平台配置(陪练对话、播课等)
COZE_API_BASE: Optional[str] = Field(default="https://api.coze.cn")
COZE_WORKSPACE_ID: Optional[str] = Field(default=None)

View File

@@ -8,6 +8,7 @@
- 写入数据库
提供稳定可靠的知识点分析能力。
支持MinIO和本地文件系统两种存储后端。
"""
import logging
@@ -20,6 +21,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from app.core.config import settings
from app.core.exceptions import ExternalServiceError
from app.schemas.course import KnowledgePointCreate
from app.services.storage_service import storage_service
from .ai_service import AIService, AIResponse
from .llm_json_parser import parse_with_fallback, clean_llm_output
@@ -92,8 +94,8 @@ class KnowledgeAnalysisServiceV2:
f"file_url: {file_url}"
)
# 1. 解析文件路径
file_path = self._resolve_file_path(file_url)
# 1. 解析文件路径支持MinIO和本地文件系统
file_path = await self._resolve_file_path(file_url)
if not file_path.exists():
raise FileNotFoundError(f"文件不存在: {file_path}")
@@ -160,11 +162,20 @@ class KnowledgeAnalysisServiceV2:
)
raise ExternalServiceError(f"知识点分析失败: {e}")
def _resolve_file_path(self, file_url: str) -> Path:
"""解析文件 URL 为本地路径"""
async def _resolve_file_path(self, file_url: str) -> Path:
"""
解析文件 URL 为本地路径
支持MinIO和本地文件系统。如果文件在MinIO中会先下载到本地缓存。
"""
if file_url.startswith(STATIC_UPLOADS_PREFIX):
relative_path = file_url.replace(STATIC_UPLOADS_PREFIX, '')
return Path(self.upload_path) / relative_path
object_name = file_url.replace(STATIC_UPLOADS_PREFIX, '')
# 使用storage_service获取文件路径自动处理MinIO下载
file_path = await storage_service.get_file_path(object_name)
if file_path:
return file_path
# 如果storage_service返回None尝试本地路径兼容旧数据
return Path(self.upload_path) / object_name
elif file_url.startswith('/'):
# 绝对路径
return Path(file_url)

View File

@@ -465,9 +465,7 @@ class CourseService(BaseService[Course]):
Returns:
是否删除成功
"""
import os
from pathlib import Path
from app.core.config import settings
from app.services.storage_service import storage_service
# 先确认课程存在
course = await self.get_by_id(db, course_id)
@@ -498,20 +496,17 @@ class CourseService(BaseService[Course]):
db.add(material)
await db.commit()
# 删除物理文件
# 删除物理文件使用storage_service
if file_url and file_url.startswith("/static/uploads/"):
try:
# 从URL中提取相对路径
relative_path = file_url.replace("/static/uploads/", "")
file_path = Path(settings.UPLOAD_PATH) / relative_path
# 检查文件是否存在并删除
if file_path.exists() and file_path.is_file():
os.remove(file_path)
object_name = file_url.replace("/static/uploads/", "")
await storage_service.delete(object_name)
logger.info(
"删除物理文件成功",
file_path=str(file_path),
object_name=object_name,
material_id=material_id,
storage="minio" if storage_service.is_minio_enabled else "local",
)
except Exception as e:
# 物理文件删除失败不影响业务流程,仅记录日志

View File

@@ -0,0 +1,422 @@
"""
统一文件存储服务
支持MinIO对象存储兼容本地文件系统
使用方式:
from app.services.storage_service import storage_service
# 上传文件
file_url = await storage_service.upload(file_data, "courses/1/doc.pdf")
# 下载文件
file_data = await storage_service.download("courses/1/doc.pdf")
# 删除文件
await storage_service.delete("courses/1/doc.pdf")
"""
import os
import io
import logging
from pathlib import Path
from typing import Optional, Union, BinaryIO
from datetime import timedelta
from minio import Minio
from minio.error import S3Error
from app.core.config import settings
logger = logging.getLogger(__name__)
class StorageService:
"""
统一文件存储服务
支持两种存储后端:
1. MinIO对象存储推荐生产环境
2. 本地文件系统开发环境或MinIO不可用时的降级方案
"""
def __init__(self):
self._client: Optional[Minio] = None
self._initialized = False
self._use_minio = False
def _ensure_initialized(self):
"""确保服务已初始化"""
if self._initialized:
return
self._initialized = True
# 检查是否启用MinIO
if not settings.MINIO_ENABLED:
logger.info("MinIO未启用使用本地文件存储")
self._use_minio = False
return
try:
self._client = Minio(
settings.MINIO_ENDPOINT,
access_key=settings.MINIO_ACCESS_KEY,
secret_key=settings.MINIO_SECRET_KEY,
secure=settings.MINIO_SECURE,
)
# 验证连接并确保bucket存在
bucket_name = self._get_bucket_name()
if not self._client.bucket_exists(bucket_name):
self._client.make_bucket(bucket_name)
logger.info(f"创建MinIO bucket: {bucket_name}")
# 设置bucket策略为公开读取
self._set_bucket_public_read(bucket_name)
self._use_minio = True
logger.info(f"MinIO存储服务初始化成功 - endpoint: {settings.MINIO_ENDPOINT}, bucket: {bucket_name}")
except Exception as e:
logger.warning(f"MinIO初始化失败降级为本地存储: {e}")
self._use_minio = False
def _get_bucket_name(self) -> str:
"""获取当前租户的bucket名称"""
return f"kpl-{settings.TENANT_CODE}"
def _set_bucket_public_read(self, bucket_name: str):
"""设置bucket为公开读取"""
try:
# 设置匿名读取策略
policy = {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {"AWS": "*"},
"Action": ["s3:GetObject"],
"Resource": [f"arn:aws:s3:::{bucket_name}/*"]
}
]
}
import json
self._client.set_bucket_policy(bucket_name, json.dumps(policy))
except Exception as e:
logger.warning(f"设置bucket公开读取策略失败: {e}")
def _normalize_object_name(self, object_name: str) -> str:
"""标准化对象名称,移除前缀斜杠"""
if object_name.startswith('/'):
object_name = object_name[1:]
if object_name.startswith('static/uploads/'):
object_name = object_name.replace('static/uploads/', '')
return object_name
def _get_file_url(self, object_name: str) -> str:
"""获取文件访问URL"""
object_name = self._normalize_object_name(object_name)
# 统一返回 /static/uploads/ 格式的URL由Nginx代理到MinIO
return f"/static/uploads/{object_name}"
def _get_local_path(self, object_name: str) -> Path:
"""获取本地文件路径"""
object_name = self._normalize_object_name(object_name)
return Path(settings.UPLOAD_PATH) / object_name
async def upload(
self,
file_data: Union[bytes, BinaryIO],
object_name: str,
content_type: Optional[str] = None,
) -> str:
"""
上传文件
Args:
file_data: 文件数据bytes或文件对象
object_name: 对象名称(如 courses/1/doc.pdf
content_type: 文件MIME类型
Returns:
文件访问URL
"""
self._ensure_initialized()
object_name = self._normalize_object_name(object_name)
# 转换为bytes
if isinstance(file_data, bytes):
data = file_data
else:
data = file_data.read()
if self._use_minio:
return await self._upload_to_minio(data, object_name, content_type)
else:
return await self._upload_to_local(data, object_name)
async def _upload_to_minio(
self,
data: bytes,
object_name: str,
content_type: Optional[str] = None,
) -> str:
"""上传到MinIO"""
try:
bucket_name = self._get_bucket_name()
# 自动检测content_type
if not content_type:
content_type = self._guess_content_type(object_name)
self._client.put_object(
bucket_name,
object_name,
io.BytesIO(data),
length=len(data),
content_type=content_type,
)
file_url = self._get_file_url(object_name)
logger.info(f"文件上传到MinIO成功: {object_name} -> {file_url}")
return file_url
except S3Error as e:
logger.error(f"MinIO上传失败: {e}")
# 降级到本地存储
return await self._upload_to_local(data, object_name)
async def _upload_to_local(self, data: bytes, object_name: str) -> str:
"""上传到本地文件系统"""
try:
file_path = self._get_local_path(object_name)
file_path.parent.mkdir(parents=True, exist_ok=True)
with open(file_path, 'wb') as f:
f.write(data)
file_url = self._get_file_url(object_name)
logger.info(f"文件上传到本地成功: {object_name} -> {file_url}")
return file_url
except Exception as e:
logger.error(f"本地文件上传失败: {e}")
raise
async def download(self, object_name: str) -> Optional[bytes]:
"""
下载文件
Args:
object_name: 对象名称
Returns:
文件数据如果文件不存在返回None
"""
self._ensure_initialized()
object_name = self._normalize_object_name(object_name)
if self._use_minio:
return await self._download_from_minio(object_name)
else:
return await self._download_from_local(object_name)
async def _download_from_minio(self, object_name: str) -> Optional[bytes]:
"""从MinIO下载"""
try:
bucket_name = self._get_bucket_name()
response = self._client.get_object(bucket_name, object_name)
data = response.read()
response.close()
response.release_conn()
return data
except S3Error as e:
if e.code == 'NoSuchKey':
logger.warning(f"MinIO文件不存在: {object_name}")
# 尝试从本地读取(兼容迁移过渡期)
return await self._download_from_local(object_name)
logger.error(f"MinIO下载失败: {e}")
return None
async def _download_from_local(self, object_name: str) -> Optional[bytes]:
"""从本地文件系统下载"""
try:
file_path = self._get_local_path(object_name)
if not file_path.exists():
logger.warning(f"本地文件不存在: {file_path}")
return None
with open(file_path, 'rb') as f:
return f.read()
except Exception as e:
logger.error(f"本地文件下载失败: {e}")
return None
async def delete(self, object_name: str) -> bool:
"""
删除文件
Args:
object_name: 对象名称
Returns:
是否删除成功
"""
self._ensure_initialized()
object_name = self._normalize_object_name(object_name)
success = True
# MinIO删除
if self._use_minio:
try:
bucket_name = self._get_bucket_name()
self._client.remove_object(bucket_name, object_name)
logger.info(f"MinIO文件删除成功: {object_name}")
except S3Error as e:
if e.code != 'NoSuchKey':
logger.error(f"MinIO文件删除失败: {e}")
success = False
# 同时删除本地文件(确保彻底清理)
try:
file_path = self._get_local_path(object_name)
if file_path.exists():
os.remove(file_path)
logger.info(f"本地文件删除成功: {file_path}")
except Exception as e:
logger.warning(f"本地文件删除失败: {e}")
return success
async def exists(self, object_name: str) -> bool:
"""
检查文件是否存在
Args:
object_name: 对象名称
Returns:
文件是否存在
"""
self._ensure_initialized()
object_name = self._normalize_object_name(object_name)
if self._use_minio:
try:
bucket_name = self._get_bucket_name()
self._client.stat_object(bucket_name, object_name)
return True
except S3Error:
pass
# 检查本地文件
file_path = self._get_local_path(object_name)
return file_path.exists()
async def get_file_path(self, object_name: str) -> Optional[Path]:
"""
获取文件的本地路径(用于需要本地文件操作的场景)
如果文件在MinIO中会先下载到临时目录
Args:
object_name: 对象名称
Returns:
本地文件路径如果文件不存在返回None
"""
self._ensure_initialized()
object_name = self._normalize_object_name(object_name)
# 先检查本地是否存在
local_path = self._get_local_path(object_name)
if local_path.exists():
return local_path
# 如果MinIO启用尝试下载到本地缓存
if self._use_minio:
try:
data = await self._download_from_minio(object_name)
if data:
# 保存到本地缓存
local_path.parent.mkdir(parents=True, exist_ok=True)
with open(local_path, 'wb') as f:
f.write(data)
logger.info(f"从MinIO下载文件到本地缓存: {object_name}")
return local_path
except Exception as e:
logger.error(f"下载MinIO文件到本地失败: {e}")
return None
def get_presigned_url(self, object_name: str, expires: int = 3600) -> Optional[str]:
"""
获取预签名URL用于直接访问MinIO
Args:
object_name: 对象名称
expires: 过期时间(秒)
Returns:
预签名URL如果MinIO未启用返回None
"""
self._ensure_initialized()
if not self._use_minio:
return None
object_name = self._normalize_object_name(object_name)
try:
bucket_name = self._get_bucket_name()
url = self._client.presigned_get_object(
bucket_name,
object_name,
expires=timedelta(seconds=expires)
)
return url
except S3Error as e:
logger.error(f"获取预签名URL失败: {e}")
return None
def _guess_content_type(self, filename: str) -> str:
"""根据文件名猜测MIME类型"""
ext = filename.rsplit('.', 1)[-1].lower() if '.' in filename else ''
content_types = {
'pdf': 'application/pdf',
'doc': 'application/msword',
'docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
'xls': 'application/vnd.ms-excel',
'xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
'ppt': 'application/vnd.ms-powerpoint',
'pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
'txt': 'text/plain',
'md': 'text/markdown',
'html': 'text/html',
'htm': 'text/html',
'csv': 'text/csv',
'json': 'application/json',
'xml': 'application/xml',
'zip': 'application/zip',
'png': 'image/png',
'jpg': 'image/jpeg',
'jpeg': 'image/jpeg',
'gif': 'image/gif',
'webp': 'image/webp',
'mp3': 'audio/mpeg',
'wav': 'audio/wav',
'mp4': 'video/mp4',
'webm': 'video/webm',
}
return content_types.get(ext, 'application/octet-stream')
@property
def is_minio_enabled(self) -> bool:
"""检查MinIO是否启用"""
self._ensure_initialized()
return self._use_minio
# 全局单例
storage_service = StorageService()

View File

@@ -31,6 +31,9 @@ PyMySQL==1.1.0
httpx==0.27.2
aiofiles==23.2.1
# 对象存储MinIO
minio>=7.2.0
# 日志
structlog==23.2.0

View File

@@ -64,9 +64,7 @@ export interface TrendData {
export interface LevelDistribution {
levels: number[]
counts: number[]
}
// 实时动态
}// 实时动态
export interface ActivityItem {
id: number
user_id: number

View File

@@ -112,4 +112,3 @@ export function deleteTask(id: number): Promise<ResponseModel<void>> {
export function sendTaskReminder(id: number): Promise<ResponseModel<void>> {
return http.post(`/api/v1/manager/tasks/${id}/remind`)
}