shishantbiswas/v-hono
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add chunk file upload

Browse Source
This commit is contained in:
yangshangbo
2025-07-10 10:03:57 +08:00
parent 684866baa6
commit d59c268492
15 changed files with 2394 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README_CHUNK_UPLOAD.md
+394
View File
@@ -0,0 +1,394 @@
# V-Hono 大文件分片上传系统
## 概述
V-Hono 大文件分片上传系统是一个基于 V 语言和 Hono 框架构建的高性能文件上传解决方案。支持大文件分片上传、断点续传、秒传、自动合并等功能。
## 主要特性
- ✅ **分片上传**：支持大文件分片上传，默认分片大小 2MB
- ✅ **断点续传**：支持上传中断后继续上传
- ✅ **秒传功能**：基于文件哈希的秒传检测
- ✅ **自动合并**：所有分片上传完成后自动合并文件
- ✅ **文件去重**：基于文件哈希的去重机制
- ✅ **数据库存储**：使用 SQLite 存储文件元数据
- ✅ **RESTful API**：完整的 REST API 接口
- ✅ **Web 界面**：提供友好的 Web 上传界面
## 系统架构
```
前端 (Web/移动端)
↓
V-Hono 服务器
↓
分片存储 (./uploads/chunks/)
↓
文件合并 (./uploads/files/)
↓
数据库 (SQLite)
```
## 安装和运行
### 1. 环境要求
- V 语言 0.4.x 或更高版本
- Windows/Linux/macOS
### 2. 启动服务器
```bash
# 克隆项目
git clone <repository-url>
cd v-hono
# 启动分片上传服务器
v run chunk_upload_example.v
```
服务器将在 `http://localhost:8080` 启动。
### 3. 访问 Web 界面
打开浏览器访问 `http://localhost:8080` 即可使用 Web 上传界面。
## API 接口文档
### 1. 分片上传接口
**接口地址：** `POST /upload/chunk`
**请求参数：**
- `file_hash` (string): 文件哈希值
- `chunk_index` (int): 分片索引，从 0 开始
- `filename` (string): 原始文件名
- `file_size` (int): 文件总大小（字节）
- `chunk_size` (int): 分片大小（字节）
- `chunk_hash` (string): 分片哈希值
- `chunk` (file): 分片文件数据
**响应格式：**
上传完成（自动合并）：
```json
{
"success": true,
"all_chunk_uploaded": true,
"file_path": ".\\uploads\\files\\xxx.zip",
"file_uuid": "xxx-xxx-xxx",
"message": "File merged successfully"
}
```
上传中：
```json
{
"success": true,
"chunk_index": 1,
"all_chunk_uploaded": false,
"message": "Chunk uploaded successfully"
}
```
### 2. 分片存在检查接口
**接口地址：** `GET /upload/chunk_exists`
**请求参数：**
- `file_hash` (string): 文件哈希值
- `chunk_index` (int): 分片索引
- `chunk_hash` (string): 分片哈希值
- `file_size` (int): 文件总大小
- `trunk_size` (int): 分片大小
**响应格式：**
```json
{
"exists": true,
"all_chunk_uploaded": false
}
```
### 3. 上传状态查询接口
**接口地址：** `GET /upload/status`
**请求参数：**
- `file_hash` (string): 文件哈希值
**响应格式：**
```json
{
"file_hash": "xxx",
"filename": "example.zip",
"total_chunks": 0,
"file_size": 1048576,
"chunk_size": 2097152,
"uploaded_chunks": [0, 1, 2],
"status": "uploading",
"created_at": 1640995200,
"updated_at": 1640995300
}
```
### 4. 已上传分片查询接口
**接口地址：** `GET /upload/chunks`
**请求参数：**
- `file_hash` (string): 文件哈希值
**响应格式：**
```json
{
"uploaded_chunks": [0, 1, 2],
"total_chunks": 5,
"completed": false
}
```
### 5. 文件管理接口
#### 获取所有文件
**接口地址：** `GET /api/files`
#### 根据 UUID 获取文件
**接口地址：** `GET /api/files/{uuid}`
#### 根据哈希获取文件
**接口地址：** `GET /api/files/hash/{hash}`
#### 删除文件
**接口地址：** `DELETE /api/files/{uuid}`
## 配置说明
### ChunkUploadConfig 配置项
```v
pub struct ChunkUploadConfig {
chunk_size: int = 1024 * 1024 // 默认分片大小 1MB
max_file_size: int = 1024 * 1024 * 1024 // 最大文件大小 1GB
temp_dir: string = './uploads/chunks' // 临时分片目录
upload_dir: string = './uploads/files' // 最终文件目录
cleanup_delay: int = 3600 // 清理延迟时间（秒）
clear_chunks_on_complete: bool = false // 完成后是否清理分片
db_path: string = './uploads/files.db' // 数据库文件路径
}
```
## 文件存储结构
```
uploads/
├── chunks/ # 分片文件目录
│ └── {file_hash}/ # 按文件哈希分组
│ └── {chunk_size}/ # 按分片大小分组
│ ├── chunk_0.part # 分片文件
│ ├── chunk_1.part
│ └── ...
├── files/ # 最终文件目录
│ ├── {file_hash}.{ext} # 合并后的文件
│ └── ...
└── files.db # SQLite 数据库文件
```
## 使用示例
### 1. 前端 JavaScript 示例
```javascript
// 计算文件哈希
async function calculateFileHash(file) {
return new Promise(resolve => {
const spark = new SparkMD5.ArrayBuffer();
const reader = new FileReader();
reader.onload = e => {
spark.append(e.target.result);
resolve(spark.end());
};
reader.readAsArrayBuffer(file);
});
}
// 上传分片
async function uploadChunk(file, chunk, { fileHash, chunkIndex, chunkHash }) {
const form = new FormData();
form.append('file_hash', fileHash);
form.append('chunk_index', chunkIndex);
form.append('filename', file.name);
form.append('file_size', file.size);
form.append('chunk_size', CHUNK_SIZE);
form.append('chunk_hash', chunkHash);
form.append('chunk', chunk);
const response = await fetch('/upload/chunk', {
method: 'POST',
body: form
});
const result = await response.json();
if (result.all_chunk_uploaded) {
console.log('上传完成，文件已合并:', result.file_path);
return true; // 上传完成
}
return false; // 继续上传
}
// 分片上传主函数
async function uploadFile(file) {
const fileHash = await calculateFileHash(file);
const chunkSize = 2 * 1024 * 1024; // 2MB
const totalChunks = Math.ceil(file.size / chunkSize);
for (let i = 0; i < totalChunks; i++) {
const chunk = file.slice(i * chunkSize, (i + 1) * chunkSize);
const chunkHash = await calculateFileHash(chunk);
const completed = await uploadChunk(file, chunk, {
fileHash,
chunkIndex: i,
chunkHash
});
if (completed) {
break; // 上传完成
}
}
}
```
### 2. cURL 示例
```bash
# 上传分片
curl -X POST http://localhost:8080/upload/chunk \
-F "file_hash=abc123" \
-F "chunk_index=0" \
-F "filename=large_file.zip" \
-F "file_size=10485760" \
-F "chunk_size=2097152" \
-F "chunk_hash=def456" \
-F "chunk=@chunk_0.part"
# 检查分片是否存在
curl "http://localhost:8080/upload/chunk_exists?file_hash=abc123&chunk_index=0&chunk_hash=def456&file_size=10485760&trunk_size=2097152"
```
## 核心算法
### 1. 分片合并判断
系统使用以下逻辑判断是否所有分片都已上传：
```v
// 计算理论上需要的分片数量
expected_chunks := (expected_file_size + u64(chunk_size) - 1) / u64(chunk_size)
// 只有当分片数量达到预期且总大小 >= 文件大小时，才认为上传完成
if chunk_count >= int(expected_chunks) && total_chunk_size >= expected_file_size {
all_chunk_uploaded = true
}
```
### 2. 文件去重机制
- 基于文件哈希进行去重
- 相同哈希的文件只存储一份
- 支持同一文件的不同文件名
### 3. 秒传检测
- 前端计算文件哈希
- 后端检查文件是否已存在
- 如果存在则直接返回文件信息，无需上传
## 性能优化
### 1. 内存管理
- 分片文件存储在磁盘，不占用大量内存
- 上传状态使用内存缓存，提高查询速度
- 支持配置清理策略，自动清理临时文件
### 2. 并发处理
- 支持多用户同时上传
- 分片级别的并发控制
- 文件级别的锁机制
### 3. 错误处理
- 网络异常自动重试
- 分片损坏自动重新上传
- 完整的错误日志记录
## 监控和日志
### 1. 调试日志
系统提供详细的调试日志，包括：
- 分片上传状态
- 文件合并过程
- 错误信息追踪
### 2. 性能监控
- 上传速度统计
- 分片成功率
- 系统资源使用情况
## 故障排除
### 1. 常见问题
**Q: 上传大文件时出现内存不足**
A: 检查分片大小配置，建议设置为 1-5MB
**Q: 分片上传后文件合并失败**
A: 检查磁盘空间和文件权限
**Q: 秒传功能不工作**
A: 确认前端哈希算法与后端一致
### 2. 日志分析
查看服务器日志获取详细错误信息：
```bash
# 启动时查看详细日志
v run chunk_upload_example.v
```
## 扩展功能
### 1. 云存储集成
可以扩展支持：
- AWS S3
- 阿里云 OSS
- 腾讯云 COS
### 2. 文件处理
可以添加：
- 图片压缩
- 视频转码
- 文档预览
### 3. 权限控制
可以集成：
- 用户认证
- 文件权限
- 访问控制
## 贡献指南
欢迎提交 Issue 和 Pull Request 来改进这个项目。
## 许可证
本项目采用 MIT 许可证。
README_DATABASE.md
+166
View File
@@ -0,0 +1,166 @@
# V-Hono 文件上传系统 - 数据库集成
## 概述
V-Hono 文件上传系统现在集成了 SQLite 数据库来管理文件信息，支持文件去重、版本管理和元数据存储。
## 新功能
### 1. 文件路径格式更新
合并后的文件现在使用新的命名格式：
```
./uploads/files/{filehash}.{filetype}
```
例如：
- `./uploads/files/a1b2c3d4e5f6.txt`
- `./uploads/files/1234567890abcdef.jpg`
### 2. 数据库表结构
```sql
CREATE TABLE file_info (
id INTEGER PRIMARY KEY AUTOINCREMENT,
file_uuid TEXT UNIQUE NOT NULL, -- 文件唯一标识符
file_hash TEXT NOT NULL, -- 文件哈希值
file_name TEXT NOT NULL, -- 原始文件名
file_size INTEGER NOT NULL, -- 文件大小
file_type TEXT NOT NULL, -- 文件扩展名
created_at INTEGER NOT NULL, -- 创建时间
updated_at INTEGER NOT NULL -- 更新时间
);
```
### 3. 文件去重逻辑
- **相同 hash + 相同文件名**：更新现有记录
- **相同 hash + 不同文件名**：创建新记录（支持同一文件的不同名称）
- **不同 hash**：创建新记录
### 4. 新增 API 端点
#### 获取所有文件信息
```http
GET /api/files
```
#### 根据文件UUID获取文件信息
```http
GET /api/files/{uuid}
```
#### 根据文件hash获取文件信息
```http
GET /api/files/hash/{hash}
```
#### 删除文件信息
```http
DELETE /api/files/{uuid}
```
## 配置
在 `ChunkUploadConfig` 中添加了数据库配置：
```v
pub struct ChunkUploadConfig {
// ... 其他配置
db_path string = './uploads/files.db' // 数据库文件路径
}
```
## 使用示例
### 1. 启动服务器
```bash
v run chunk_upload_example.v
```
### 2. 测试数据库功能
```bash
v run test_database.v
```
### 3. 上传文件
文件上传完成后，系统会：
1. 将文件保存为 `{filehash}.{filetype}` 格式
2. 在数据库中记录文件信息
3. 返回文件UUID
### 4. 查询文件信息
```bash
# 获取所有文件
curl http://localhost:8080/api/files
# 根据UUID获取文件信息
curl http://localhost:8080/api/files/{uuid}
# 根据hash获取文件信息
curl http://localhost:8080/api/files/hash/{hash}
```
## 数据库操作
### 插入或更新文件信息
```v
file_info := db.insert_or_update_file(file_hash, file_name, file_size, file_type)
```
### 查询文件信息
```v
// 根据hash查询
file_info := db.get_file_by_hash(file_hash)
// 根据UUID查询
file_info := db.get_file_by_uuid(file_uuid)
// 获取所有文件
files := db.get_all_files()
```
### 删除文件信息
```v
db.delete_file(file_uuid)
```
## 依赖
项目现在依赖 `vlang/sqlite` 模块：
```v
// v.mod
dependencies: ['vlang/sqlite']
```
## 文件结构
```
v-hono/
├── hono/
│ ├── database.v # 数据库管理模块
│ ├── upload.v # 上传模块（已更新）
│ └── ...
├── chunk_upload_example.v # 示例应用（已更新）
├── test_database.v # 数据库测试
└── uploads/
├── files/ # 最终文件目录
├── chunks/ # 分片文件目录
└── files.db # SQLite数据库文件
```
## 注意事项
1. 数据库文件会自动创建在 `uploads/` 目录下
2. 文件去重基于文件hash，支持同一文件的不同名称
3. 每个文件都有唯一的UUID，用于API访问
4. 文件路径使用hash命名，避免文件名冲突
5. 数据库操作失败不会影响文件上传功能
chunk_upload_example.v
+264
View File
@@ -0,0 +1,264 @@
module main
import hono
import net.http
import os
import json
fn main() {
mut app := hono.Hono.new()
// 创建分片上传管理器
mut upload_manager := hono.new_chunk_upload_manager(hono.ChunkUploadConfig{})
// 分片上传接口
app.post('/upload/chunk', fn [mut upload_manager] (mut c hono.Context) http.Response {
return upload_manager.handle_chunk_upload(mut c)
})
// 查询上传状态接口
app.get('/upload/status', fn [mut upload_manager] (mut c hono.Context) http.Response {
return upload_manager.get_upload_status(mut c)
})
// 查询已上传分片接口
app.get('/upload/chunks', fn [mut upload_manager] (mut c hono.Context) http.Response {
file_hash := c.query['file_hash'] or {
c.status(400)
return c.json('{"error": "Missing file_hash parameter"}')
}
upload_status := upload_manager.uploads[file_hash] or {
// 如果内存中没有记录，检查是否有最终文件（已完成上传）
final_path := './uploads/${file_hash.trim_space()}'
if os.exists(final_path) {
// 文件已存在，返回所有分片已上传
mut uploaded_chunks := []int{}
uploaded_chunks << 0 // 假设只有一个分片
return c.json('{"uploaded_chunks": ' + json.encode(uploaded_chunks) + ', "total_chunks": 1, "completed": true}')
}
c.status(404)
return c.json('{"error": "Upload not found"}')
}
// 检查是否所有分片都已上传（基于分片大小总和）
chunk_dir := os.join_path(os.join_path('./uploads/chunks', file_hash.trim_space()), upload_status.chunk_size.str())
mut total_chunk_size := u64(0)
mut chunk_count := 0
if os.exists(chunk_dir) {
for i := 0; ; i++ {
chunk_path := os.join_path(chunk_dir, 'chunk_${i}.part')
if !os.exists(chunk_path) {
break
}
chunk_info := os.stat(chunk_path) or { continue }
total_chunk_size += chunk_info.size
chunk_count++
}
}
is_completed := total_chunk_size >= u64(upload_status.file_size)
return c.json('{"uploaded_chunks": ' + json.encode(upload_status.uploaded_chunks) + ', "total_chunks": $chunk_count, "completed": $is_completed}')
})
// 检查分片是否存在接口（秒传功能）
app.get('/upload/chunk_exists', fn [mut upload_manager] (mut c hono.Context) http.Response {
file_hash := c.query['file_hash'] or {
c.status(400)
return c.json('{"error": "Missing file_hash parameter"}')
}
chunk_index := c.query['chunk_index'] or {
c.status(400)
return c.json('{"error": "Missing chunk_index parameter"}')
}
chunk_hash := c.query['chunk_hash'] or {
c.status(400)
return c.json('{"error": "Missing chunk_hash parameter"}')
}
file_size_str := c.query['file_size'] or {
c.status(400)
return c.json('{"error": "Missing file_size parameter"}')
}
file_size := file_size_str.int()
// 新增 trunk_size 参数
trunk_size_str := c.query['trunk_size'] or { '' }
mut chunk_size := upload_manager.config.chunk_size
if trunk_size_str != '' {
chunk_size = trunk_size_str.int()
} else if upload_status := upload_manager.uploads[file_hash] {
chunk_size = upload_status.chunk_size
}
chunk_path := os.join_path(os.join_path(os.join_path('./uploads/chunks', file_hash.trim_space()), chunk_size.str()), 'chunk_${chunk_index}.part')
println('[DEBUG] Checking chunk: $chunk_path')
println('[DEBUG] File hash: $file_hash')
println('[DEBUG] Chunk index: $chunk_index')
println('[DEBUG] Expected hash: $chunk_hash')
// 清理无效的上传状态
upload_manager.cleanup_invalid_status()
mut exists := os.exists(chunk_path)
println('[DEBUG] File exists: $exists')
// 计算是否所有分片都已上传
mut all_chunk_uploaded := false
// 方案1：优先检查最终文件是否存在（新格式：filehash.filetype）
uploads_dir := './uploads/files'
if os.exists(uploads_dir) {
// 从上传状态中获取文件扩展名
mut file_ext := ''
if upload_status := upload_manager.uploads[file_hash] {
file_ext = get_file_extension(upload_status.filename)
}
// 如果内存中没有记录，尝试从数据库获取
if file_ext == '' {
if file_info := upload_manager.db.get_file_by_hash(file_hash) {
file_ext = file_info.file_type
}
}
// 构建最终文件名
final_filename := '${file_hash.trim_space()}${file_ext}'
final_path := os.join_path(uploads_dir, final_filename)
if os.exists(final_path) {
println('[DEBUG] Found final file: $final_path')
all_chunk_uploaded = true
}
}
// 方案2：如果最终文件不存在，检查分片文件
if !all_chunk_uploaded {
chunk_dir := os.join_path(os.join_path('./uploads/chunks', file_hash.trim_space()), chunk_size.str())
if os.exists(chunk_dir) {
// 计算所有分片文件的大小总和
mut total_chunk_size := u64(0)
mut chunk_count := 0
// 遍历所有分片文件
for i := 0; ; i++ {
chunk_file_path := '${chunk_dir}/chunk_${i}.part'
if !os.exists(chunk_file_path) {
break
}
// 获取分片文件大小
chunk_info := os.stat(chunk_file_path) or { break }
total_chunk_size += chunk_info.size
chunk_count++
}
// 使用传入的文件大小
expected_file_size := u64(file_size)
println('[DEBUG] Total chunk size: $total_chunk_size, Expected file size: $expected_file_size, Chunk count: $chunk_count')
// 计算理论上需要的分片数量
expected_chunks := (expected_file_size + u64(chunk_size) - 1) / u64(chunk_size)
println('[DEBUG] Expected chunks: $expected_chunks')
// 只有当分片数量达到预期且总大小 >= 文件大小时，才认为上传完成
if chunk_count >= int(expected_chunks) && total_chunk_size >= expected_file_size {
all_chunk_uploaded = true
println('[DEBUG] All chunks uploaded: chunk_count=$chunk_count >= expected_chunks=$expected_chunks && total_size=$total_chunk_size >= file_size=$expected_file_size')
} else {
println('[DEBUG] Not all chunks uploaded: chunk_count=$chunk_count < expected_chunks=$expected_chunks || total_size=$total_chunk_size < file_size=$expected_file_size')
}
}
}
println('[DEBUG] All chunk uploaded: $all_chunk_uploaded')
// chunk_exists 接口不应该自动触发合并，只返回状态信息
// 合并操作应该在 /upload/chunk 接口中处理
// 只检查文件是否存在，不验证 hash（因为前后端 hash 算法可能不一致）
// 如果需要严格验证，可以在这里添加 hash 检查逻辑
return c.json('{"exists": $exists, "all_chunk_uploaded": $all_chunk_uploaded}')
})
// 获取所有文件信息
app.get('/api/files', fn [mut upload_manager] (mut c hono.Context) http.Response {
files := upload_manager.db.get_all_files() or {
c.status(500)
return c.json('{"error": "Failed to get files", "message": "$err"}')
}
return c.json(json.encode(files))
})
// 根据文件UUID获取文件信息
app.get('/api/files/:uuid', fn [mut upload_manager] (mut c hono.Context) http.Response {
file_uuid := c.params['uuid'] or {
c.status(400)
return c.json('{"error": "Missing file UUID"}')
}
file_info := upload_manager.db.get_file_by_uuid(file_uuid) or {
c.status(404)
return c.json('{"error": "File not found"}')
}
return c.json(json.encode(file_info))
})
// 根据文件hash获取文件信息
app.get('/api/files/hash/:hash', fn [mut upload_manager] (mut c hono.Context) http.Response {
file_hash := c.params['hash'] or {
c.status(400)
return c.json('{"error": "Missing file hash"}')
}
file_info := upload_manager.db.get_file_by_hash(file_hash) or {
c.status(404)
return c.json('{"error": "File not found"}')
}
return c.json(json.encode(file_info))
})
// 删除文件信息
app.delete('/api/files/:uuid', fn [mut upload_manager] (mut c hono.Context) http.Response {
file_uuid := c.params['uuid'] or {
c.status(400)
return c.json('{"error": "Missing file UUID"}')
}
upload_manager.db.delete_file(file_uuid) or {
c.status(500)
return c.json('{"error": "Failed to delete file", "message": "$err"}')
}
return c.json('{"success": true, "message": "File deleted successfully"}')
})
// 静态文件服务
app.use(hono.serve_static_path('/files', './uploads'))
// HTML 示例页面
app.get('/', fn (mut c hono.Context) http.Response {
html_content := os.read_file('public/upload.html') or {
return c.text('Error: Cannot read upload.html file')
}
return c.html(html_content)
})
println('Chunk upload server running at http://localhost:8080')
app.listen(':8080')
}
// 获取文件扩展名
fn get_file_extension(filename string) string {
println('[DEBUG] Getting extension for filename: "$filename"')
parts := filename.split('.')
println('[DEBUG] Split parts: $parts')
if parts.len > 1 {
ext := '.${parts.last()}'
println('[DEBUG] Extracted extension: "$ext"')
return ext
}
println('[DEBUG] No extension found, returning empty string')
return ''
}
hono/database.v
+212
View File
@@ -0,0 +1,212 @@
module hono
import db.sqlite
import time
import crypto.rand
// 文件信息结构
pub struct FileInfo {
pub:
file_uuid string
file_hash string
file_name string
file_size int
file_type string
created_at int
updated_at int
}
// 数据库管理器
pub struct DatabaseManager {
mut:
db sqlite.DB
}
// 创建数据库管理器
pub fn new_database_manager(db_path string) !DatabaseManager {
mut db := sqlite.connect(db_path) or {
return error('Failed to connect to database: $err')
}
// 创建文件信息表
db.exec('CREATE TABLE IF NOT EXISTS file_info (
id INTEGER PRIMARY KEY AUTOINCREMENT,
file_uuid TEXT UNIQUE NOT NULL,
file_hash TEXT NOT NULL,
file_name TEXT NOT NULL,
file_size INTEGER NOT NULL,
file_type TEXT NOT NULL,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);') or {
return error('Failed to create table: $err')
}
db.exec('CREATE INDEX IF NOT EXISTS idx_file_hash ON file_info(file_hash);') or {
return error('Failed to create index: $err')
}
db.exec('CREATE INDEX IF NOT EXISTS idx_file_uuid ON file_info(file_uuid);') or {
return error('Failed to create index: $err')
}
return DatabaseManager{
db: db
}
}
// 生成文件UUID
pub fn generate_file_uuid() string {
// 生成16字节的随机数据
random_bytes := rand.bytes(16) or { return '' }
// 转换为UUID格式 (8-4-4-4-12)
mut uuid := ''
for i, byte in random_bytes {
if i == 4 || i == 6 || i == 8 || i == 10 {
uuid += '-'
}
uuid += '${byte:02x}'
}
return uuid
}
// 插入或更新文件信息
pub fn (mut db DatabaseManager) insert_or_update_file(file_hash string, file_name string, file_size int, file_type string) !FileInfo {
now := int(time.now().unix())
// 检查是否已存在相同的file_hash
existing_file := db.get_file_by_hash(file_hash) or { FileInfo{} }
if existing_file.file_uuid != '' {
// 如果存在相同的file_hash但文件名不同，创建新记录
if existing_file.file_name != file_name {
// 生成新的UUID
file_uuid := generate_file_uuid()
// 插入新记录
db.db.exec('INSERT INTO file_info (file_uuid, file_hash, file_name, file_size, file_type, created_at, updated_at) VALUES ("$file_uuid", "$file_hash", "$file_name", $file_size, "$file_type", $now, $now)') or {
return error('Failed to insert file info: $err')
}
return FileInfo{
file_uuid: file_uuid
file_hash: file_hash
file_name: file_name
file_size: file_size
file_type: file_type
created_at: now
updated_at: now
}
} else {
// 文件名相同，更新现有记录
db.db.exec('UPDATE file_info SET file_size = $file_size, file_type = "$file_type", updated_at = $now WHERE file_hash = "$file_hash"') or {
return error('Failed to update file info: $err')
}
return FileInfo{
file_uuid: existing_file.file_uuid
file_hash: file_hash
file_name: file_name
file_size: file_size
file_type: file_type
created_at: existing_file.created_at
updated_at: now
}
}
} else {
// 不存在相同的file_hash，创建新记录
file_uuid := generate_file_uuid()
db.db.exec('INSERT INTO file_info (file_uuid, file_hash, file_name, file_size, file_type, created_at, updated_at) VALUES ("$file_uuid", "$file_hash", "$file_name", $file_size, "$file_type", $now, $now)') or {
return error('Failed to insert file info: $err')
}
return FileInfo{
file_uuid: file_uuid
file_hash: file_hash
file_name: file_name
file_size: file_size
file_type: file_type
created_at: now
updated_at: now
}
}
}
// 根据文件hash获取文件信息
pub fn (db DatabaseManager) get_file_by_hash(file_hash string) !FileInfo {
rows := db.db.exec('SELECT file_uuid, file_hash, file_name, file_size, file_type, created_at, updated_at FROM file_info WHERE file_hash = "$file_hash"') or {
return error('Failed to query file info: $err')
}
if rows.len > 0 {
row := rows[0]
return FileInfo{
file_uuid: row.vals[0]
file_hash: row.vals[1]
file_name: row.vals[2]
file_size: row.vals[3].int()
file_type: row.vals[4]
created_at: row.vals[5].int()
updated_at: row.vals[6].int()
}
}
return error('File not found')
}
// 根据文件UUID获取文件信息
pub fn (db DatabaseManager) get_file_by_uuid(file_uuid string) !FileInfo {
rows := db.db.exec('SELECT file_uuid, file_hash, file_name, file_size, file_type, created_at, updated_at FROM file_info WHERE file_uuid = "$file_uuid"') or {
return error('Failed to query file info: $err')
}
if rows.len > 0 {
row := rows[0]
return FileInfo{
file_uuid: row.vals[0]
file_hash: row.vals[1]
file_name: row.vals[2]
file_size: row.vals[3].int()
file_type: row.vals[4]
created_at: row.vals[5].int()
updated_at: row.vals[6].int()
}
}
return error('File not found')
}
// 获取所有文件信息
pub fn (db DatabaseManager) get_all_files() ![]FileInfo {
rows := db.db.exec('SELECT file_uuid, file_hash, file_name, file_size, file_type, created_at, updated_at FROM file_info ORDER BY created_at DESC') or {
return error('Failed to query all files: $err')
}
mut files := []FileInfo{}
for row in rows {
files << FileInfo{
file_uuid: row.vals[0]
file_hash: row.vals[1]
file_name: row.vals[2]
file_size: row.vals[3].int()
file_type: row.vals[4]
created_at: row.vals[5].int()
updated_at: row.vals[6].int()
}
}
return files
}
// 删除文件信息
pub fn (mut db DatabaseManager) delete_file(file_uuid string) ! {
db.db.exec('DELETE FROM file_info WHERE file_uuid = "$file_uuid"') or {
return error('Failed to delete file info: $err')
}
}
// 关闭数据库连接
pub fn (mut db DatabaseManager) close() {
db.db.close() or { }
}
hono/json_parser.v
+154
View File
@@ -0,0 +1,154 @@
module hono
// 简单的 JSON 解析器
pub struct JsonParser {
pub:
data string
}
// 创建 JSON 解析器
pub fn new_json_parser(data string) JsonParser {
return JsonParser{data: data}
}
// 解析 JSON 对象
pub fn (parser JsonParser) parse_object() !map[string]string {
mut result := map[string]string{}
// 移除外层的大括号
data := parser.data.trim_space()
if !data.starts_with('{') || !data.ends_with('}') {
return error('Invalid JSON object format')
}
content := data[1..data.len - 1]
// 分割键值对
pairs := split_json_pairs(content)
for pair in pairs {
key_value := split_key_value(pair) or { continue }
result[key_value.key] = key_value.value
}
return result
}
// 键值对结构
struct KeyValue {
pub:
key string
value string
}
// 分割 JSON 键值对
fn split_json_pairs(content string) []string {
mut pairs := []string{}
mut current := ''
mut brace_count := 0
mut in_string := false
mut escape_next := false
for i := 0; i < content.len; i++ {
ch := content[i]
if escape_next {
current += ch.str()
escape_next = false
continue
}
if ch == `\\` {
escape_next = true
current += ch.str()
continue
}
if ch == `"` && !escape_next {
in_string = !in_string
current += ch.str()
continue
}
if !in_string {
if ch == `{` {
brace_count++
} else if ch == `}` {
brace_count--
} else if ch == `,` && brace_count == 0 {
pairs << current.trim_space()
current = ''
continue
}
}
current += ch.str()
}
if current.trim_space() != '' {
pairs << current.trim_space()
}
return pairs
}
// 分割键值对
fn split_key_value(pair string) !KeyValue {
colon_index := pair.index(':') or {
return error('No colon found')
}
mut key := pair[..colon_index].trim_space()
mut value := pair[colon_index + 1..].trim_space()
// 移除键的引号
if key.starts_with('"') && key.ends_with('"') {
key = key[1..key.len - 1]
}
// 移除值的引号
if value.starts_with('"') && value.ends_with('"') {
value = value[1..value.len - 1]
}
return KeyValue{key: key, value: value}
}
pub struct MergeRequest {
pub:
file_hash string
filename string
total_chunks int
}
// 解析合并请求
pub fn parse_merge_request(body string) !MergeRequest {
if body.len == 0 {
return error('Empty request body')
}
parser := new_json_parser(body)
obj := parser.parse_object() or {
return error('Failed to parse JSON')
}
file_hash := obj['file_hash'] or {
return error('Missing file_hash')
}
filename := obj['filename'] or {
return error('Missing filename')
}
total_chunks_str := obj['total_chunks'] or {
return error('Missing total_chunks')
}
if !total_chunks_str.is_int() {
return error('Invalid total_chunks')
}
total_chunks := total_chunks_str.int()
return MergeRequest{
file_hash: file_hash
filename: filename
total_chunks: total_chunks
}
}
hono/multipart.v
+171
View File
@@ -0,0 +1,171 @@
module hono
import net.http
// Multipart 表单数据项
pub struct MultipartItem {
pub:
name string
filename string
content string
content_type string
}
// Multipart 解析器
pub struct MultipartParser {
pub:
boundary string
data string
}
// 创建 Multipart 解析器
pub fn new_multipart_parser(content_type string, data string) !MultipartParser {
// 从 Content-Type 中提取 boundary
boundary := extract_boundary(content_type) or {
return error('Failed to extract boundary')
}
return MultipartParser{
boundary: boundary
data: data
}
}
// 解析 multipart 数据
pub fn (parser MultipartParser) parse() ![]MultipartItem {
mut items := []MultipartItem{}
// 更健壮的分割方式，兼容不同换行
parts := parser.data.split('--${parser.boundary}')
for _, part in parts {
if part.trim_space() == '' || part.trim_space() == '--' || part.trim_space().starts_with('--') {
continue
}
item := parser.parse_part(part) or {
continue
}
items << item
}
return items
}
// 解析单个部分
fn (parser MultipartParser) parse_part(part string) !MultipartItem {
// 分离头部和内容
header_content := part.split('\r\n\r\n')
if header_content.len < 2 {
return error('Invalid part format')
}
header := header_content[0]
content := header_content[1..].join('\r\n\r\n')
// 解析头部
name := extract_header_value(header, 'name') or {
return error('Missing name')
}
filename := extract_header_value(header, 'filename') or { '' }
content_type := extract_header_value(header, 'Content-Type') or { 'text/plain' }
return MultipartItem{
name: name
filename: filename
content: content
content_type: content_type
}
}
// 从 Content-Type 中提取 boundary
fn extract_boundary(content_type string) !string {
if !content_type.starts_with('multipart/form-data') {
return error('Not multipart/form-data')
}
boundary_start := content_type.index('boundary=') or {
return error('No boundary found')
}
mut boundary := content_type[boundary_start + 9..]
// 移除引号
if boundary.starts_with('"') && boundary.ends_with('"') {
boundary = boundary[1..boundary.len - 1]
}
return boundary
}
// 从头部中提取值
fn extract_header_value(header string, key string) !string {
// 先找 Content-Disposition 行
for line in header.split('\r\n') {
if line.starts_with('Content-Disposition:') {
// 查找 key="value"
key_eq := '${key}="'
idx := line.index(key_eq) or { continue }
start := idx + key_eq.len
end := line.index_after('"', start) or { line.len }
return line[start..end]
}
}
return error('Key not found: $key')
}
// 解析 multipart 表单数据（便捷函数）
pub fn parse_multipart_form(req http.Request) !map[string]MultipartItem {
content_type := req.header.get_custom('Content-Type') or {
return error('No Content-Type header')
}
parser := new_multipart_parser(content_type, req.data) or {
return error('Failed to create parser: $err')
}
items := parser.parse() or {
return error('Failed to parse multipart data: $err')
}
// 转换为 map
mut result := map[string]MultipartItem{}
for item in items {
result[item.name] = item
}
return result
}
// 获取文件数据
pub fn (items map[string]MultipartItem) get_file(key string) !string {
item := items[key] or {
return error('File not found: $key')
}
return item.content
}
// 获取表单字段值
pub fn (items map[string]MultipartItem) get(key string) !string {
item := items[key] or {
return error('Field not found: $key')
}
return item.content
}
// 检查是否为文件
pub fn (items map[string]MultipartItem) is_file(key string) bool {
item := items[key] or { return false }
return item.filename != ''
}
// 获取文件名
pub fn (items map[string]MultipartItem) get_filename(key string) !string {
item := items[key] or {
return error('Item not found: $key')
}
return item.filename
}
// 获取内容类型
pub fn (items map[string]MultipartItem) get_content_type(key string) !string {
item := items[key] or {
return error('Item not found: $key')
}
return item.content_type
}
hono/request.v
+4 -4
View File
@@ -187,12 +187,12 @@ pub fn (mut c Context) file_with_options(file_path string, options FileOptions)
// FileOptions struct for configuring file serving // FileOptions struct for configuring file serving
pub struct FileOptions { pub struct FileOptions {
pub: pub:
status_code int = 0 // 自定义状态码，0表示使用默认200 status_code int // 自定义状态码，0表示使用默认200
content_type string = '' // 自定义Content-Type content_type string // 自定义Content-Type
last_modified bool = true // 是否设置Last-Modified头 last_modified bool = true // 是否设置Last-Modified头
etag bool = true // 是否设置ETag头 etag bool = true // 是否设置ETag头
max_age int = 0 // 缓存时间（秒） max_age int // 缓存时间（秒）
no_cache bool = false // 是否禁用缓存 no_cache bool // 是否禁用缓存
headers map[string]string // 自定义响应头 headers map[string]string // 自定义响应头
} }
hono/router.v
+1 -2
View File
@@ -1,7 +1,6 @@
module hono module hono
import regex import regex
import time
// 路由节点类型 // 路由节点类型
enum RouteType { enum RouteType {
@@ -142,7 +141,7 @@ fn (mut router ContextHybridRouter) match_dynamic_route(method string, path stri
} }
for handler in router.dynamic_routes { for handler in router.dynamic_routes {
match_result, replaced_path_reg, param_names := router.match_path_with_regex(path, handler.path) match_result, replaced_path_reg, _ := router.match_path_with_regex(path, handler.path)
if match_result { if match_result {
mut param_map := map[string]string{} mut param_map := map[string]string{}
hono/static.v
-1
View File
@@ -2,7 +2,6 @@ module hono
import os import os
import net.http import net.http
import net.urllib
// 静态文件服务配置 // 静态文件服务配置
pub struct StaticOptions { pub struct StaticOptions {
hono/trie_router.v
+4 -4
View File
@@ -52,7 +52,7 @@ pub fn (mut tr ContextTrieRouter) add_route(method string, path string, handler
} }
segments := path.split('/').filter(it != '') segments := path.split('/').filter(it != '')
mut current := tr.method_trees[method] mut current := tr.method_trees[method] or { return }
for i, seg in segments { for i, seg in segments {
if seg.starts_with(':') { if seg.starts_with(':') {
@@ -72,7 +72,7 @@ pub fn (mut tr ContextTrieRouter) add_route(method string, path string, handler
if seg !in current.children { if seg !in current.children {
current.children[seg] = ContextTrieNode.new(seg, TrieNodeType.static, '') current.children[seg] = ContextTrieNode.new(seg, TrieNodeType.static, '')
} }
current = current.children[seg] current = current.children[seg] or { return }
} }
// 在最后一个节点设置处理器 // 在最后一个节点设置处理器
@@ -94,13 +94,13 @@ pub fn (mut tr ContextTrieRouter) match_route(method string, path string) ?Conte
} }
segments := path.split('/').filter(it != '') segments := path.split('/').filter(it != '')
mut current := tr.method_trees[method] mut current := tr.method_trees[method] or { return none }
mut params := map[string]string{} mut params := map[string]string{}
for seg in segments { for seg in segments {
// 先尝试静态匹配 // 先尝试静态匹配
if seg in current.children { if seg in current.children {
current = current.children[seg] current = current.children[seg] or { return none }
} else if current.param_child != unsafe { nil } { } else if current.param_child != unsafe { nil } {
// 参数匹配 // 参数匹配
params[current.param_child.param_name] = seg params[current.param_child.param_name] = seg
hono/upload.v
+498
View File
@@ -0,0 +1,498 @@
module hono
import net.http
import os
import crypto.md5
import json
import time
// 分片上传配置
pub struct ChunkUploadConfig {
pub:
chunk_size int = 1024 * 1024 // 1MB 默认分片大小
max_file_size int = 1024 * 1024 * 1024 // 1GB 最大文件大小
temp_dir string = './uploads/chunks' // 临时分片目录（分片保存在 temp_dir/filehash/chunksize/ 下）
upload_dir string = './uploads/files' // 最终文件目录
cleanup_delay int = 3600 // 1小时后清理临时文件
clear_chunks_on_complete bool // 上传完成后是否清空分片，默认不清空
db_path string = './uploads/files.db' // 数据库文件路径
}
// 分片信息
pub struct ChunkInfo {
pub:
file_hash string
chunk_index int
total_chunks int
filename string
file_size int
chunk_size int
upload_time int
}
// 文件上传状态
pub struct FileUploadStatus {
pub:
file_hash string
filename string
total_chunks int
file_size int
chunk_size int
created_at int
pub mut:
uploaded_chunks []int
status string
updated_at int
}
// 分片上传管理器
pub struct ChunkUploadManager {
pub mut:
config ChunkUploadConfig
uploads map[string]FileUploadStatus
db DatabaseManager
}
// 创建分片上传管理器
pub fn new_chunk_upload_manager(config ChunkUploadConfig) ChunkUploadManager {
// 确保目录存在
os.mkdir_all(config.temp_dir) or { panic('Failed to create temp directory') }
os.mkdir_all(config.upload_dir) or { panic('Failed to create upload directory') }
// 创建数据库管理器
db := new_database_manager(config.db_path) or { panic('Failed to create database manager: $err') }
return ChunkUploadManager{
config: config
uploads: map[string]FileUploadStatus{}
db: db
}
}
// 处理分片上传
pub fn (mut manager ChunkUploadManager) handle_chunk_upload(mut ctx Context) http.Response {
// 解析 multipart 表单数据
form_data := hono.parse_multipart_form(ctx.req) or {
ctx.status(400)
return ctx.json('{"error": "Invalid form data", "message": "$err"}')
}
// 获取必要参数
file_hash := form_data.get('file_hash') or {
ctx.status(400)
return ctx.json('{"error": "Missing file_hash"}')
}
chunk_index_str := form_data.get('chunk_index') or {
ctx.status(400)
return ctx.json('{"error": "Missing chunk_index"}')
}
filename := form_data.get('filename') or {
ctx.status(400)
return ctx.json('{"error": "Missing filename"}')
}
file_size_str := form_data.get('file_size') or {
ctx.status(400)
return ctx.json('{"error": "Missing file_size"}')
}
// 解析数值
chunk_index := chunk_index_str.int()
file_size := file_size_str.int()
// 验证文件大小
if file_size > manager.config.max_file_size {
ctx.status(413)
return ctx.json('{"error": "File too large", "max_size": "${manager.config.max_file_size}"}')
}
// 获取前端传递的分片大小参数
chunk_size_str := form_data.get('chunk_size') or {
ctx.status(400)
return ctx.json('{"error": "Missing chunk_size"}')
}
chunk_size := chunk_size_str.int()
// 验证分片大小（允许更大的分片大小，但不超过10MB）
max_allowed_chunk_size := 10 * 1024 * 1024 // 10MB
if chunk_size > max_allowed_chunk_size {
ctx.status(413)
return ctx.json('{"error": "Chunk size too large", "max_chunk_size": "$max_allowed_chunk_size"}')
}
// 获取文件数据
file_data := form_data.get_file('chunk') or {
ctx.status(400)
return ctx.json('{"error": "Missing chunk file"}')
}
// 创建按文件hash和分片大小分组的目录
chunk_dir := os.join_path(manager.config.temp_dir, file_hash.trim_space(), chunk_size.str())
os.mkdir_all(chunk_dir) or {
ctx.status(500)
return ctx.json('{"error": "Failed to create chunk directory"}')
}
// 保存分片文件到hash/chunksize子目录
chunk_path := os.join_path(chunk_dir, 'chunk_${chunk_index}.part')
println('[DEBUG] Saving chunk to: $chunk_path')
println('[DEBUG] Chunk dir exists: ${os.exists(chunk_dir)}')
println('[DEBUG] Chunk data size: ${file_data.len} bytes')
os.write_file(chunk_path, file_data) or {
println('[DEBUG] Failed to save chunk: $err')
ctx.status(500)
return ctx.json('{"error": "Failed to save chunk"}')
}
// 更新上传状态
manager.update_upload_status(file_hash, filename, chunk_index, file_size, chunk_size)
println('[DEBUG] Updated upload status for file: $file_hash, chunk: $chunk_index')
// 判断是否所有分片都已上传，自动合并
// 基于分片大小总和来判断是否合并
mut all_chunk_uploaded := false
merge_chunk_dir := os.join_path(manager.config.temp_dir, file_hash.trim_space(), chunk_size.str())
if os.exists(merge_chunk_dir) {
mut total_chunk_size := u64(0)
mut chunk_count := 0
mut max_chunk_index := -1
// 遍历所有分片文件，计算总大小和找到最大分片索引
for i := 0; ; i++ {
merge_chunk_path := os.join_path(merge_chunk_dir, 'chunk_${i}.part')
if !os.exists(merge_chunk_path) {
break
}
chunk_info := os.stat(merge_chunk_path) or { continue }
total_chunk_size += chunk_info.size
chunk_count++
max_chunk_index = i
}
println('[DEBUG] [MergeCheck] total_chunk_size=$total_chunk_size, file_size=$file_size, chunk_count=$chunk_count, max_chunk_index=$max_chunk_index')
// 如果分片文件大小总和 >= file_size，认为可以合并
if total_chunk_size >= u64(file_size) {
all_chunk_uploaded = true
println('[DEBUG] All chunks uploaded based on size comparison')
}
}
println('[DEBUG] All chunks uploaded: $all_chunk_uploaded')
if all_chunk_uploaded {
// 重新计算分片数量用于合并
mut actual_total_chunks := 0
for i := 0; ; i++ {
check_chunk_path := os.join_path(merge_chunk_dir, 'chunk_${i}.part')
if !os.exists(check_chunk_path) {
break
}
actual_total_chunks++
}
// 获取文件扩展名
file_ext := get_file_extension(filename)
final_filename := '${file_hash.trim_space()}${file_ext}'
final_path := os.join_path(manager.config.upload_dir, final_filename)
// 检查最终文件是否已经存在，避免重复合并
if os.exists(final_path) {
println('[DEBUG] Final file already exists: $final_path, skipping merge')
} else {
println('[DEBUG] Merging chunks to: $final_path')
println('[DEBUG] Upload dir exists: ${os.exists(manager.config.upload_dir)}')
println('[DEBUG] Actual total chunks: $actual_total_chunks')
manager.merge_chunks(file_hash, actual_total_chunks, final_path, chunk_size) or {
println('[DEBUG] Merge failed: $err')
ctx.status(500)
return ctx.json('{"error": "Failed to merge chunks", "message": "$err"}')
}
}
// 在数据库中记录文件信息
file_info := manager.db.insert_or_update_file(file_hash, filename, file_size, file_ext) or {
println('[DEBUG] Failed to save file info to database: $err')
// 即使数据库保存失败，也不影响文件合并
FileInfo{}
}
manager.uploads[file_hash].status = 'completed'
manager.uploads[file_hash].updated_at = int(time.now().unix())
if manager.config.clear_chunks_on_complete {
manager.cleanup_chunks(file_hash, chunk_size)
}
clean_file_path := final_path.replace('\n', '').replace('\r', '').replace('\\', '\\\\').trim_space()
return ctx.json('{"success": true, "all_chunk_uploaded": true, "file_path": "$clean_file_path", "file_uuid": "${file_info.file_uuid}", "message": "File merged successfully"}')
}
// 未全部上传，正常返回
return ctx.json('{"success": true, "chunk_index": $chunk_index, "all_chunk_uploaded": false, "message": "Chunk uploaded successfully"}')
}
// 处理分片合并
pub fn (mut manager ChunkUploadManager) handle_chunk_merge(mut ctx Context) http.Response {
// 解析请求体
merge_request := hono.parse_merge_request(ctx.body) or {
ctx.status(400)
return ctx.json('{"error": "Invalid request body", "message": "$err"}')
}
file_hash := merge_request.file_hash
filename := merge_request.filename
total_chunks := merge_request.total_chunks
// 检查上传状态
upload_status := manager.uploads[file_hash] or {
ctx.status(404)
return ctx.json('{"error": "Upload not found"}')
}
// 验证所有分片是否上传完成
if upload_status.uploaded_chunks.len != total_chunks {
ctx.status(400)
return ctx.json('{"error": "Not all chunks uploaded", "uploaded": ${upload_status.uploaded_chunks.len}, "total": $total_chunks}')
}
// 合并文件
file_ext := get_file_extension(filename)
final_filename := '${file_hash.trim_space()}${file_ext}'
final_path := os.join_path(manager.config.upload_dir, final_filename)
// 从上传状态中获取分片大小
chunk_size := upload_status.chunk_size
manager.merge_chunks(file_hash, total_chunks, final_path, chunk_size) or {
ctx.status(500)
return ctx.json('{"error": "Failed to merge chunks", "message": "$err"}')
}
// 在数据库中记录文件信息
file_info := manager.db.insert_or_update_file(file_hash, filename, upload_status.file_size, file_ext) or {
println('[DEBUG] Failed to save file info to database: $err')
// 即使数据库保存失败，也不影响文件合并
FileInfo{}
}
// 更新状态为完成
manager.uploads[file_hash].status = 'completed'
manager.uploads[file_hash].updated_at = int(time.now().unix())
// 清理临时分片
manager.cleanup_chunks(file_hash, chunk_size)
// 返回成功响应
clean_file_path2 := final_path.replace('\n', '').replace('\r', '').replace('\\', '\\\\').trim_space()
return ctx.json('{"success": true, "file_path": "$clean_file_path2", "file_uuid": "${file_info.file_uuid}", "message": "File merged successfully"}')
}
// 获取上传状态
pub fn (manager ChunkUploadManager) get_upload_status(mut ctx Context) http.Response {
file_hash := ctx.query['file_hash'] or {
ctx.status(400)
return ctx.json('{"error": "Missing file_hash parameter"}')
}
upload_status := manager.uploads[file_hash] or {
ctx.status(404)
return ctx.json('{"error": "Upload not found"}')
}
return ctx.json(json.encode(upload_status))
}
// 更新上传状态
pub fn (mut manager ChunkUploadManager) update_upload_status(file_hash string, filename string, chunk_index int, file_size int, chunk_size int) {
now := int(time.now().unix())
if file_hash !in manager.uploads {
manager.uploads[file_hash] = FileUploadStatus{
file_hash: file_hash
filename: filename
total_chunks: 0 // 不再使用固定的total_chunks，改为动态计算
uploaded_chunks: []
file_size: file_size
chunk_size: chunk_size
status: 'uploading'
created_at: now
updated_at: now
}
}
// 添加已上传的分片索引
if chunk_index !in manager.uploads[file_hash].uploaded_chunks {
manager.uploads[file_hash].uploaded_chunks << chunk_index
}
manager.uploads[file_hash].updated_at = now
}
// 合并分片
fn (mut manager ChunkUploadManager) merge_chunks(file_hash string, total_chunks int, final_path string, chunk_size int) ! {
println('[DEBUG] Merge chunks called with:')
println('[DEBUG] file_hash: $file_hash')
println('[DEBUG] total_chunks: $total_chunks')
println('[DEBUG] final_path: $final_path')
println('[DEBUG] chunk_size: $chunk_size')
println('[DEBUG] upload_dir: ${manager.config.upload_dir}')
// 确保上传目录存在
os.mkdir_all(manager.config.upload_dir) or {
return error('Failed to create upload directory: $err')
}
// 检查目录权限
if !os.is_writable(manager.config.upload_dir) {
return error('Upload directory is not writable: ${manager.config.upload_dir}')
}
println('[DEBUG] Creating final file: $final_path')
println('[DEBUG] Final path length: ${final_path.len}')
println('[DEBUG] Final path bytes: ${final_path.bytes()}')
println('[DEBUG] Final path exists: ${os.exists(final_path)}')
println('[DEBUG] Final path is file: ${os.is_file(final_path)}')
println('[DEBUG] Final path is dir: ${os.is_dir(final_path)}')
// 如果文件已存在，直接返回成功
if os.exists(final_path) {
println('[DEBUG] Final file already exists, skipping creation')
return
}
// Try to create the file with explicit path cleaning
clean_path := final_path.trim_space()
println('[DEBUG] Clean path: "$clean_path"')
println('[DEBUG] Clean path length: ${clean_path.len}')
// Try to get absolute path
abs_path := os.abs_path(clean_path)
println('[DEBUG] Absolute path: "$abs_path"')
mut final_file := os.create(abs_path) or {
println('[DEBUG] File creation failed with error: $err')
return error('Failed to create final file: $err')
}
defer { final_file.close() }
for i in 0 .. total_chunks {
chunk_path := os.join_path(manager.config.temp_dir, file_hash.trim_space(), chunk_size.str(), 'chunk_${i}.part')
println('[DEBUG] Reading chunk: $chunk_path')
if !os.exists(chunk_path) {
return error('Chunk file not found: $chunk_path')
}
chunk_data := os.read_file(chunk_path) or {
return error('Failed to read chunk $i: $err')
}
final_file.write(chunk_data.bytes()) or {
return error('Failed to write chunk $i: $err')
}
println('[DEBUG] Chunk $i merged successfully, size: ${chunk_data.len} bytes')
}
println('[DEBUG] All chunks merged successfully to: $final_path')
}
// 清理临时分片
fn (mut manager ChunkUploadManager) cleanup_chunks(file_hash string, chunk_size int) {
chunk_dir := os.join_path(manager.config.temp_dir, file_hash.trim_space(), chunk_size.str())
if os.exists(chunk_dir) {
// 删除整个分片目录
os.rmdir_all(chunk_dir) or {
println('[DEBUG] Failed to remove chunk directory: $err')
}
}
}
// 公共清理方法
pub fn (mut manager ChunkUploadManager) cleanup_chunks_public(file_hash string, chunk_size int) {
manager.cleanup_chunks(file_hash, chunk_size)
}
// 内部合并处理方法
pub fn (mut manager ChunkUploadManager) handle_chunk_merge_internal(file_hash string, filename string, total_chunks int, final_path string, chunk_size int, file_size int, file_ext string) ! {
// 执行合并
manager.merge_chunks(file_hash, total_chunks, final_path, chunk_size) or {
return error('Failed to merge chunks: $err')
}
// 在数据库中记录文件信息
manager.db.insert_or_update_file(file_hash, filename, file_size, file_ext) or {
println('[DEBUG] Failed to save file info to database: $err')
// 即使数据库保存失败，也不影响文件合并
}
}
// 生成文件哈希
pub fn generate_file_hash(data string) string {
return md5.sum(data.bytes()).hex()
}
// 验证文件完整性
pub fn verify_file_integrity(file_path string, expected_hash string) bool {
file_data := os.read_file(file_path) or { return false }
actual_hash := generate_file_hash(file_data)
return actual_hash == expected_hash
}
// 获取文件扩展名
fn get_file_extension(filename string) string {
parts := filename.split('.')
if parts.len > 1 {
return '.${parts.last()}'
}
return ''
}
// 清理文件上传状态（当文件被删除时调用）
pub fn (mut manager ChunkUploadManager) cleanup_upload_status(file_hash string) {
if file_hash in manager.uploads {
manager.uploads.delete(file_hash)
}
}
// 检查并清理无效的上传状态
pub fn (mut manager ChunkUploadManager) cleanup_invalid_status() {
mut to_delete := []string{}
for file_hash, upload_status in manager.uploads {
// 检查最终文件是否存在
final_path := os.join_path(manager.config.upload_dir, upload_status.filename.trim_space())
if !os.exists(final_path) {
// 如果最终文件不存在，检查分片文件是否都存在
mut all_chunks_exist := true
chunk_size := upload_status.chunk_size
chunk_dir := os.join_path(manager.config.temp_dir, file_hash.trim_space(), chunk_size.str())
// 动态检查分片文件
for i := 0; ; i++ {
chunk_path := os.join_path(chunk_dir, 'chunk_${i}.part')
if !os.exists(chunk_path) {
break
}
// 如果找到至少一个分片，说明上传正在进行中
all_chunks_exist = false
break
}
// 如果没有找到任何分片文件，清理状态
if all_chunks_exist {
to_delete << file_hash
}
}
}
// 删除无效状态
for file_hash in to_delete {
manager.uploads.delete(file_hash)
}
}
public/upload.html
+366
View File
@@ -0,0 +1,366 @@
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>V-Hono 分片上传示例</title>
<style>
body {
font-family: Arial, sans-serif;
max-width: 800px;
margin: 0 auto;
padding: 20px;
background-color: #f5f5f5;
}
.container {
background: white;
padding: 30px;
border-radius: 10px;
box-shadow: 0 2px 10px rgba(0,0,0,0.1);
}
.upload-area {
border: 2px dashed #ccc;
border-radius: 8px;
padding: 40px;
text-align: center;
margin: 20px 0;
transition: border-color 0.3s;
}
.upload-area:hover {
border-color: #007bff;
}
.upload-area.dragover {
border-color: #007bff;
background-color: #f8f9fa;
}
.file-input {
display: none;
}
.btn {
background: #007bff;
color: white;
border: none;
padding: 12px 24px;
border-radius: 5px;
cursor: pointer;
font-size: 16px;
margin: 10px;
}
.btn:hover {
background: #0056b3;
}
.btn:disabled {
background: #6c757d;
cursor: not-allowed;
}
.progress-container {
margin: 20px 0;
display: none;
}
.progress-bar {
width: 100%;
height: 20px;
background-color: #e9ecef;
border-radius: 10px;
overflow: hidden;
}
.progress-fill {
height: 100%;
background: linear-gradient(90deg, #007bff, #28a745);
width: 0%;
transition: width 0.3s;
}
.status {
margin: 10px 0;
padding: 10px;
border-radius: 5px;
display: none;
}
.status.success {
background-color: #d4edda;
color: #155724;
border: 1px solid #c3e6cb;
}
.status.error {
background-color: #f8d7da;
color: #721c24;
border: 1px solid #f5c6cb;
}
.file-info {
margin: 10px 0;
padding: 15px;
background-color: #f8f9fa;
border-radius: 5px;
display: none;
}
.chunk-status {
margin: 10px 0;
font-size: 14px;
color: #666;
}
</style>
</head>
<body>
<div class="container">
<h1>🚀 V-Hono 分片上传示例</h1>
<p>支持大文件分片上传、断点续传、秒传功能</p>
<div class="upload-area" id="uploadArea">
<p>📁 拖拽文件到此处或点击选择文件</p>
<input type="file" id="fileInput" class="file-input" multiple>
<button class="btn" onclick="document.getElementById('fileInput').click()">选择文件</button>
</div>
<div class="file-info" id="fileInfo">
<h3>文件信息</h3>
<p><strong>文件名:</strong> <span id="fileName"></span></p>
<p><strong>文件大小:</strong> <span id="fileSize"></span></p>
<p><strong>分片数量:</strong> <span id="chunkCount"></span></p>
<p><strong>分片大小:</strong> <span id="chunkSize"></span></p>
</div>
<div class="progress-container" id="progressContainer">
<h3>上传进度</h3>
<div class="progress-bar">
<div class="progress-fill" id="progressFill"></div>
</div>
<div class="chunk-status" id="chunkStatus"></div>
</div>
<div class="status" id="status"></div>
<button class="btn" id="uploadBtn" onclick="startUpload()" style="display: none;">开始上传</button>
</div>
<script src="https://cdn.jsdelivr.net/npm/spark-md5@3.0.2/spark-md5.min.js"></script>
<script>
const CHUNK_SIZE = 2 * 1024 * 1024; // 2MB
let selectedFile = null;
let fileHash = null;
let totalChunks = 0;
let uploadedChunks = 0;
// 文件选择处理
document.getElementById('fileInput').addEventListener('change', handleFileSelect);
// 拖拽处理
const uploadArea = document.getElementById('uploadArea');
uploadArea.addEventListener('dragover', (e) => {
e.preventDefault();
uploadArea.classList.add('dragover');
});
uploadArea.addEventListener('dragleave', () => {
uploadArea.classList.remove('dragover');
});
uploadArea.addEventListener('drop', (e) => {
e.preventDefault();
uploadArea.classList.remove('dragover');
const files = e.dataTransfer.files;
if (files.length > 0) {
document.getElementById('fileInput').files = files;
handleFileSelect();
}
});
function handleFileSelect() {
const file = document.getElementById('fileInput').files[0];
if (!file) return;
// 重置上传状态
resetUploadState();
selectedFile = file;
totalChunks = Math.ceil(file.size / CHUNK_SIZE);
// 显示文件信息
document.getElementById('fileName').textContent = file.name;
document.getElementById('fileSize').textContent = formatFileSize(file.size);
document.getElementById('chunkCount').textContent = totalChunks;
document.getElementById('chunkSize').textContent = formatFileSize(CHUNK_SIZE);
document.getElementById('fileInfo').style.display = 'block';
document.getElementById('uploadBtn').style.display = 'inline-block';
// 计算文件hash
calculateFileHash(file).then(hash => {
fileHash = hash;
console.log('File hash:', hash);
});
}
// 重置上传状态
function resetUploadState() {
// 重置进度条
document.getElementById('progressFill').style.width = '0%';
document.getElementById('chunkStatus').textContent = '';
// 隐藏进度容器
document.getElementById('progressContainer').style.display = 'none';
// 重置状态信息
document.getElementById('status').style.display = 'none';
// 重置上传按钮
const uploadBtn = document.getElementById('uploadBtn');
uploadBtn.disabled = false;
uploadBtn.textContent = '开始上传';
// 重置变量
uploadedChunks = 0;
}
async function startUpload() {
if (!selectedFile || !fileHash) {
showStatus('请先选择文件', 'error');
return;
}
const uploadBtn = document.getElementById('uploadBtn');
uploadBtn.disabled = true;
uploadBtn.textContent = '上传中...';
document.getElementById('progressContainer').style.display = 'block';
showStatus('开始上传...', 'success');
try {
// 直接开始上传分片，每个分片都会检查是否存在
for (let i = 0; i < totalChunks; i++) {
const chunk = selectedFile.slice(i * CHUNK_SIZE, (i + 1) * CHUNK_SIZE);
const chunkHash = await calculateChunkHash(chunk);
// 检查分片是否已存在（包括秒传检测）
const checkResult = await checkChunkExists(fileHash, i, chunkHash);
// 如果检测到所有分片都已上传完成，立即结束上传
if (checkResult.all_uploaded) {
console.log('检测到秒传，所有分片都已上传完成！');
updateProgress(totalChunks, totalChunks); // 更新进度到100%
showStatus('秒传成功！文件已完整上传。', 'success');
uploadBtn.textContent = '上传完成';
return;
}
// 如果分片已存在，跳过上传
if (checkResult.exists) {
console.log(`分片 ${i} 已存在，跳过上传`);
updateProgress(i + 1, totalChunks);
continue;
}
// 上传分片
await uploadChunk(selectedFile, chunk, {
fileHash, chunkIndex: i, totalChunks, chunkHash
});
updateProgress(i + 1, totalChunks);
console.log(`分片 ${i} 上传完成`);
}
showStatus('上传完成！文件已自动合并。', 'success');
uploadBtn.textContent = '上传完成';
} catch (error) {
console.error('上传失败:', error);
showStatus('上传失败: ' + error.message, 'error');
uploadBtn.disabled = false;
uploadBtn.textContent = '重新上传';
}
}
// 计算文件hash
async function calculateFileHash(file) {
return new Promise(resolve => {
const spark = new SparkMD5.ArrayBuffer();
const reader = new FileReader();
reader.onload = e => {
spark.append(e.target.result);
resolve(spark.end());
};
reader.readAsArrayBuffer(file);
});
}
// 计算分片hash
async function calculateChunkHash(chunk) {
return new Promise(resolve => {
const spark = new SparkMD5.ArrayBuffer();
const reader = new FileReader();
reader.onload = e => {
spark.append(e.target.result);
resolve(spark.end());
};
reader.readAsArrayBuffer(chunk);
});
}
// 检查分片hash是否已存在
async function checkChunkExists(fileHash, chunkIndex, chunkHash) {
const res = await fetch('/upload/chunk_exists?file_hash=' + fileHash + '&chunk_index=' + chunkIndex + '&chunk_hash=' + chunkHash + '&file_size=' + selectedFile.size + '&trunk_size=' + CHUNK_SIZE);
if (!res.ok) return false;
const data = await res.json();
// 如果所有分片都已上传完成，直接返回 true 表示秒传成功
if (data.all_chunk_uploaded) {
console.log('所有分片都已上传完成，秒传成功！');
return { exists: true, all_uploaded: true };
}
return { exists: data.exists, all_uploaded: false };
}
// 上传分片
async function uploadChunk(file, chunk, { fileHash, chunkIndex, totalChunks, chunkHash }) {
const form = new FormData();
form.append('file_hash', fileHash);
form.append('chunk_index', chunkIndex);
form.append('total_chunks', totalChunks);
form.append('filename', file.name);
form.append('file_size', file.size);
form.append('chunk_size', CHUNK_SIZE);
form.append('chunk_hash', chunkHash);
form.append('chunk', chunk, file.name + '.part' + chunkIndex);
const response = await fetch('/upload/chunk', {
method: 'POST',
body: form
});
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || '上传失败');
}
const result = await response.json();
if (result.all_chunk_uploaded) {
console.log('所有分片已上传完成，文件已自动合并:', result.file_path);
}
}
// 更新进度
function updateProgress(current, total) {
const percentage = (current / total) * 100;
document.getElementById('progressFill').style.width = percentage + '%';
document.getElementById('chunkStatus').textContent = current + '/' + total + ' 分片已上传 (' + percentage.toFixed(1) + '%)';
}
// 显示状态
function showStatus(message, type) {
const status = document.getElementById('status');
status.textContent = message;
status.className = 'status ' + type;
status.style.display = 'block';
}
// 格式化文件大小
function formatFileSize(bytes) {
if (bytes === 0) return '0 Bytes';
const k = 1024;
const sizes = ['Bytes', 'KB', 'MB', 'GB'];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
}
</script>
</body>
</html>
test_no_total_chunks.v
+93
View File
@@ -0,0 +1,93 @@
module main
import hono
import os
fn main() {
println('Testing upload without total_chunks parameter...')
// Create upload manager
mut upload_manager := hono.new_chunk_upload_manager(hono.ChunkUploadConfig{})
// Test file info
file_hash := 'test123456789'
filename := 'test_file.zip'
chunk_index := 0
chunk_size := 2097152
file_size := 93569
// Create test chunk directory and file
chunk_dir := os.join_path(os.join_path('./uploads/chunks', file_hash), chunk_size.str())
os.mkdir_all(chunk_dir) or { panic('Failed to create chunk directory') }
// Create a test chunk file
chunk_path := os.join_path(chunk_dir, 'chunk_${chunk_index}.part')
test_data := 'This is test chunk data for testing the merge functionality.'
os.write_file(chunk_path, test_data) or { panic('Failed to create test chunk') }
println('Created test chunk at: $chunk_path')
// Test the update_upload_status function
upload_manager.update_upload_status(file_hash, filename, chunk_index, file_size, chunk_size)
println('Updated upload status')
// Check if the upload status was created correctly
if file_hash in upload_manager.uploads {
status := upload_manager.uploads[file_hash]
println('Upload status:')
println(' File hash: ${status.file_hash}')
println(' Filename: ${status.filename}')
println(' Total chunks: ${status.total_chunks}')
println(' Uploaded chunks: ${status.uploaded_chunks}')
println(' File size: ${status.file_size}')
println(' Chunk size: ${status.chunk_size}')
println(' Status: ${status.status}')
} else {
println('Upload status not found!')
return
}
// Test the merge logic
file_ext := get_file_extension(filename)
final_filename := '${file_hash.trim_space()}${file_ext}'
final_path := os.join_path('./uploads/files', final_filename)
println('Testing merge logic...')
println('Final path: $final_path')
// Calculate actual total chunks
mut actual_total_chunks := 0
for i := 0; ; i++ {
check_chunk_path := os.join_path(chunk_dir, 'chunk_${i}.part')
if !os.exists(check_chunk_path) {
break
}
actual_total_chunks++
}
println('Actual total chunks: $actual_total_chunks')
// Test merge
upload_manager.handle_chunk_merge_internal(file_hash, filename, actual_total_chunks, final_path, chunk_size, file_size, file_ext) or {
println('Merge failed: $err')
return
}
println('Merge successful!')
println('Final file exists: ${os.exists(final_path)}')
// Clean up
os.rmdir_all(chunk_dir) or { println('Failed to clean up chunk directory') }
os.rm(final_path) or { println('Failed to clean up final file') }
println('Test completed successfully!')
}
// 获取文件扩展名
fn get_file_extension(filename string) string {
parts := filename.split('.')
if parts.len > 1 {
return '.${parts.last()}'
}
return ''
}
test_specific_fix.v
+66
View File
@@ -0,0 +1,66 @@
module main
import hono
import os
fn main() {
println('Testing specific file fix...')
// Create upload manager
mut upload_manager := hono.new_chunk_upload_manager(hono.ChunkUploadConfig{})
// Use the exact same parameters from the failing case
file_hash := 'a45abf58302850765be03cf78bcf5129'
filename := 'test.zip' // This should give us .zip extension
total_chunks := 1
chunk_size := 2097152
file_size := 93569
// Create test chunk directory and file
chunk_dir := os.join_path(os.join_path('./uploads/chunks', file_hash), chunk_size.str())
os.mkdir_all(chunk_dir) or { panic('Failed to create chunk directory') }
// Create a test chunk file
chunk_path := os.join_path(chunk_dir, 'chunk_0.part')
test_data := 'This is test chunk data for testing the merge functionality.'
os.write_file(chunk_path, test_data) or { panic('Failed to create test chunk') }
println('Created test chunk at: $chunk_path')
// Test the file extension function
file_ext := get_file_extension(filename)
println('File extension: "$file_ext"')
// Test the merge functionality with the exact same path construction
final_filename := '${file_hash.trim_space()}${file_ext}'
final_path := os.join_path('./uploads/files', final_filename)
println('Attempting to merge to: $final_path')
upload_manager.handle_chunk_merge_internal(file_hash, filename, total_chunks, final_path, chunk_size, file_size, file_ext) or {
println('Merge failed: $err')
return
}
println('Merge successful!')
println('Final file exists: ${os.exists(final_path)}')
// Clean up
os.rmdir_all(chunk_dir) or { println('Failed to clean up chunk directory') }
os.rm(final_path) or { println('Failed to clean up final file') }
println('Test completed successfully!')
}
// 获取文件扩展名
fn get_file_extension(filename string) string {
println('[DEBUG] Getting extension for filename: "$filename"')
parts := filename.split('.')
println('[DEBUG] Split parts: $parts')
if parts.len > 1 {
ext := '.${parts.last()}'
println('[DEBUG] Extracted extension: "$ext"')
return ext
}
println('[DEBUG] No extension found, returning empty string')
return ''
}
v.mod
+1 -2
View File
@@ -1,7 +1,6 @@
Module { Module {
name: 'v-hono' name: 'v-hono'
description: 'A high-performance V language web framework with hybrid routing and LRU cache' description: 'A high-performance V language web framework with hybrid routing and LRU cache'
version: '0.1.0' version: '0.2.0'
license: 'MIT' license: 'MIT'
dependencies: []
} }