#!/bin/bash # Specify bash explicitly to avoid shell compatibility issues # 添加更详细的日志 log_info() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] INFO: $1" } log_error() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: $1" } # 检查环境变量 if [ -z "$HF_TOKEN" ] || [ -z "$DATASET_ID" ]; then log_error "未启用备份功能 - 缺少HF_TOKEN或DATASET_ID环境变量" # 修复字符串切片语法 if [ -n "$HF_TOKEN" ]; then TOKEN_PREFIX=$(echo "$HF_TOKEN" | cut -c1-3) log_info "HF_TOKEN=${TOKEN_PREFIX}... DATASET_ID=${DATASET_ID}" else log_info "HF_TOKEN未设置 DATASET_ID=${DATASET_ID}" fi exit 0 fi # 创建临时目录 TEMP_DIR="/tmp/cloudreve_backup" DB_FILE="/app/cloudreve.db" # 确保目录存在并有正确权限 mkdir -p $TEMP_DIR chmod -R 777 $TEMP_DIR log_info "临时目录: $TEMP_DIR" log_info "数据库文件: $DB_FILE" # 修复字符串切片语法 if [ -n "$HF_TOKEN" ]; then TOKEN_PREFIX=$(echo "$HF_TOKEN" | cut -c1-5) log_info "HF_TOKEN: ${TOKEN_PREFIX}..." else log_info "HF_TOKEN未设置" fi log_info "DATASET_ID: $DATASET_ID" # 安装python和huggingface_hub if ! command -v python3 > /dev/null 2>&1; then log_info "正在安装Python..." apk add --no-cache python3 py3-pip else log_info "Python3已安装: $(python3 --version)" fi # 确保pip已安装 if ! command -v pip3 > /dev/null 2>&1; then log_info "正在安装pip..." apk add --no-cache py3-pip else log_info "Pip3已安装: $(pip3 --version)" fi # 安装或更新huggingface_hub log_info "正在安装/更新huggingface_hub..." pip3 install --no-cache-dir --upgrade huggingface_hub log_info "huggingface_hub安装完成" # 测试huggingface_hub是否正常工作 if ! python3 -c "import huggingface_hub; print('huggingface_hub版本:', huggingface_hub.__version__)" > /dev/null 2>&1; then log_error "huggingface_hub导入失败,正在重试安装..." pip3 install --no-cache-dir huggingface_hub fi # 测试权限是否正常 touch "${TEMP_DIR}/test_file" && rm "${TEMP_DIR}/test_file" if [ $? -ne 0 ]; then log_error "临时目录权限测试失败,正在修复权限..." chmod -R 777 $TEMP_DIR fi # 测试与HuggingFace API的连接 log_info "正在测试与HuggingFace API的连接..." python3 -c " from huggingface_hub import HfApi try: api = HfApi(token='$HF_TOKEN') user_info = api.whoami() print(f'成功连接到HuggingFace API,用户: {user_info}') except Exception as e: print(f'连接HuggingFace API失败: {str(e)}') exit(1) " if [ $? -ne 0 ]; then log_error "HuggingFace API连接测试失败,请检查令牌是否有效" else log_info "HuggingFace API连接测试成功" fi # 生成唯一的测试文件名 TEST_FILE_NAME="test_file_$(date +%s)" # 测试数据集权限 log_info "正在测试Dataset权限..." python3 -c " from huggingface_hub import HfApi try: api = HfApi(token='$HF_TOKEN') # 创建本地测试文件 with open('$TEMP_DIR/test_file', 'w') as f: f.write('test') # 上传测试文件 test_file_name = '$TEST_FILE_NAME' print(f'正在上传测试文件: {test_file_name}') api.upload_file( path_or_fileobj='$TEMP_DIR/test_file', path_in_repo=test_file_name, repo_id='$DATASET_ID', repo_type='dataset' ) print('成功上传测试文件到Dataset') # 删除已上传的测试文件 print('正在删除测试文件...') api.delete_file( path_in_repo=test_file_name, repo_id='$DATASET_ID', repo_type='dataset' ) print('已成功删除测试文件') except Exception as e: print(f'Dataset权限测试失败: {str(e)}') exit(1) " if [ $? -ne 0 ]; then log_error "Dataset权限测试失败,请检查DATASET_ID是否正确且有写入权限" else log_info "Dataset权限测试成功,测试文件已清理" fi # 确保本地测试文件被删除 rm -f "$TEMP_DIR/test_file" # 上传备份 upload_backup() { file_path="$1" file_name="$2" if [ ! -f "$file_path" ]; then log_error "备份文件不存在: $file_path" return 1 fi log_info "开始上传备份: $file_name ($(du -h $file_path | cut -f1))" python3 -c " from huggingface_hub import HfApi import sys import os import time def manage_backups(api, repo_id, max_files=10): try: files = api.list_repo_files(repo_id=repo_id, repo_type='dataset') backup_files = [f for f in files if f.startswith('cloudreve_backup_') and f.endswith('.db')] backup_files.sort() if len(backup_files) >= max_files: files_to_delete = backup_files[:(len(backup_files) - max_files + 1)] for file_to_delete in files_to_delete: try: api.delete_file(path_in_repo=file_to_delete, repo_id=repo_id, repo_type='dataset') print(f'已删除旧备份: {file_to_delete}') except Exception as e: print(f'删除 {file_to_delete} 时出错: {str(e)}') except Exception as e: print(f'管理备份文件时出错: {str(e)}') token='$HF_TOKEN' repo_id='$DATASET_ID' try: api = HfApi(token=token) # 检查文件大小 file_size = os.path.getsize('$file_path') print(f'备份文件大小: {file_size / (1024*1024):.2f} MB') # 确认Dataset存在 try: dataset_info = api.dataset_info(repo_id=repo_id) print(f'Dataset信息: {dataset_info.id}') except Exception as e: print(f'获取Dataset信息失败: {str(e)}') start_time = time.time() print(f'开始上传: {start_time}') # 上传文件 api.upload_file( path_or_fileobj='$file_path', path_in_repo='$file_name', repo_id=repo_id, repo_type='dataset' ) end_time = time.time() print(f'上传完成,耗时: {end_time - start_time:.2f} 秒') print(f'成功上传 $file_name') # 管理备份 manage_backups(api, repo_id) except Exception as e: print(f'上传文件时出错: {str(e)}') sys.exit(1) " if [ $? -ne 0 ]; then log_error "备份上传失败" return 1 else log_info "备份上传成功" return 0 fi } # 下载最新备份 download_latest_backup() { log_info "开始下载最新备份..." python3 -c " from huggingface_hub import HfApi import sys import os import shutil import time try: api = HfApi(token='$HF_TOKEN') print('已创建API实例') # 列出仓库文件 try: files = api.list_repo_files(repo_id='$DATASET_ID', repo_type='dataset') print(f'仓库文件数量: {len(files)}') except Exception as e: print(f'列出仓库文件失败: {str(e)}') sys.exit(1) backup_files = [f for f in files if f.startswith('cloudreve_backup_') and f.endswith('.db')] print(f'找到备份文件数量: {len(backup_files)}') if not backup_files: print('未找到备份文件') sys.exit(0) # 按名称排序(实际上是按时间戳排序) latest_backup = sorted(backup_files)[-1] print(f'最新备份文件: {latest_backup}') # 创建临时目录 temp_dir = '$TEMP_DIR/download' os.makedirs(temp_dir, exist_ok=True) start_time = time.time() print(f'开始下载: {start_time}') # 下载文件 try: filepath = api.hf_hub_download( repo_id='$DATASET_ID', filename=latest_backup, repo_type='dataset', local_dir=temp_dir ) print(f'文件下载到: {filepath}') except Exception as e: print(f'下载文件失败: {str(e)}') sys.exit(1) end_time = time.time() print(f'下载完成,耗时: {end_time - start_time:.2f} 秒') if filepath and os.path.exists(filepath): # 检查目标目录是否存在 os.makedirs(os.path.dirname('$DB_FILE'), exist_ok=True) # 备份现有数据库(如果存在) if os.path.exists('$DB_FILE'): backup_path = '$DB_FILE.bak' shutil.copy2('$DB_FILE', backup_path) print(f'已备份现有数据库到: {backup_path}') # 复制下载的文件到目标位置 shutil.copy2(filepath, '$DB_FILE') print(f'已恢复数据库到: $DB_FILE') # 确保权限正确 os.chmod('$DB_FILE', 0o666) print(f'已设置数据库文件权限') else: print('下载的文件路径无效') sys.exit(1) except Exception as e: print(f'下载备份过程中出错: {str(e)}') sys.exit(1) " if [ $? -ne 0 ]; then log_error "备份下载失败" return 1 else log_info "备份下载成功" return 0 fi } # 首次启动时下载最新备份 log_info "正在从HuggingFace下载最新备份..." download_latest_backup # 同步函数 sync_data() { log_info "数据同步服务已启动" while true; do log_info "开始同步进程,时间: $(date)" if [ -f "$DB_FILE" ]; then timestamp=$(date +%Y%m%d_%H%M%S) backup_file="cloudreve_backup_${timestamp}.db" backup_path="${TEMP_DIR}/${backup_file}" log_info "创建备份文件: $backup_path" # 复制数据库文件 cp "$DB_FILE" "$backup_path" if [ $? -ne 0 ]; then log_error "复制数据库文件失败" else log_info "数据库文件复制成功: $(du -h $backup_path | cut -f1)" # 上传备份 log_info "正在上传备份到HuggingFace..." upload_backup "$backup_path" "$backup_file" # 删除临时备份文件 rm -f "$backup_path" log_info "已删除临时备份文件" fi else log_error "数据库文件不存在: $DB_FILE" fi # 设置同步间隔 SYNC_INTERVAL=${SYNC_INTERVAL:-3600} log_info "下次同步将在 ${SYNC_INTERVAL} 秒后进行..." sleep $SYNC_INTERVAL done } # 启动同步进程 sync_data