feat: first /upload/chunk_exists request alone
This commit is contained in:
@@ -24,6 +24,7 @@
|
||||
- 🚀 **双请求池系统**:活跃请求池 + 待请求池,支持并发控制和智能调度
|
||||
- ⚡ **并发优化**:可配置最大并发数,自动管理请求队列
|
||||
- 📊 **实时状态监控**:实时显示活跃请求数和等待请求数
|
||||
- ⚡ **性能优化**:分片大小记录文件,避免遍历文件计算总大小
|
||||
|
||||
## 安装
|
||||
|
||||
@@ -911,6 +912,7 @@ v run chunk_upload_example.v
|
||||
- **LRU 缓存**:自动缓存路由匹配结果
|
||||
- **Trie 路由树**:支持复杂路径匹配
|
||||
- **内存优化**:使用指针和堆分配优化性能
|
||||
- **分片上传优化**:分片大小记录文件,避免遍历文件计算总大小(O(1) vs O(n))
|
||||
|
||||
## 贡献
|
||||
|
||||
|
||||
+42
-8
@@ -214,7 +214,8 @@ uploads/
|
||||
│ └── {chunk_size}/ # 按分片大小分组
|
||||
│ ├── chunk_0.part # 分片文件
|
||||
│ ├── chunk_1.part
|
||||
│ └── ...
|
||||
│ ├── ...
|
||||
│ └── total_size.record # 分片大小记录文件(优化性能)
|
||||
├── files/ # 最终文件目录
|
||||
│ ├── {file_hash}.{ext} # 合并后的文件
|
||||
│ └── ...
|
||||
@@ -689,20 +690,52 @@ class ManagedRequestPool extends RequestPool {
|
||||
|
||||
## 核心算法
|
||||
|
||||
### 1. 分片合并判断
|
||||
### 1. 分片合并判断(优化版)
|
||||
|
||||
系统使用以下逻辑判断是否所有分片都已上传:
|
||||
系统使用分片大小记录文件来避免遍历分片文件计算总大小,提高性能:
|
||||
|
||||
```v
|
||||
// 计算理论上需要的分片数量
|
||||
expected_chunks := (expected_file_size + u64(chunk_size) - 1) / u64(chunk_size)
|
||||
// 更新分片大小记录
|
||||
fn (mut manager ChunkUploadManager) update_chunk_size_record(file_hash string, chunk_size int, current_chunk_size int) {
|
||||
chunk_dir := os.join_path(manager.config.temp_dir, file_hash.trim_space(), chunk_size.str())
|
||||
size_record_path := os.join_path(chunk_dir, 'total_size.record')
|
||||
|
||||
// 只有当分片数量达到预期且总大小 >= 文件大小时,才认为上传完成
|
||||
if chunk_count >= int(expected_chunks) && total_chunk_size >= expected_file_size {
|
||||
all_chunk_uploaded = true
|
||||
// 读取现有的总大小记录
|
||||
mut total_size := u64(0)
|
||||
if os.exists(size_record_path) {
|
||||
size_data := os.read_file(size_record_path) or { '0' }
|
||||
total_size = size_data.u64() or { u64(0) }
|
||||
}
|
||||
|
||||
// 更新总大小
|
||||
total_size += u64(current_chunk_size)
|
||||
|
||||
// 写入更新后的总大小
|
||||
os.write_file(size_record_path, total_size.str()) or {
|
||||
println('[DEBUG] Failed to write size record: $err')
|
||||
}
|
||||
}
|
||||
|
||||
// 获取分片大小记录
|
||||
fn (manager ChunkUploadManager) get_chunk_size_record(file_hash string, chunk_size int) u64 {
|
||||
chunk_dir := os.join_path(manager.config.temp_dir, file_hash.trim_space(), chunk_size.str())
|
||||
size_record_path := os.join_path(chunk_dir, 'total_size.record')
|
||||
|
||||
if os.exists(size_record_path) {
|
||||
size_data := os.read_file(size_record_path) or { '0' }
|
||||
return size_data.u64() or { u64(0) }
|
||||
}
|
||||
|
||||
return u64(0)
|
||||
}
|
||||
```
|
||||
|
||||
**性能优化说明:**
|
||||
- **传统方法**:每次需要遍历所有分片文件,计算总大小(O(n)复杂度)
|
||||
- **优化方法**:维护一个记录文件,实时更新总大小(O(1)复杂度)
|
||||
- **性能提升**:对于大文件(数百个分片),性能提升显著
|
||||
- **存储结构**:`./uploads/chunks/{file_hash}/{chunk_size}/total_size.record`
|
||||
|
||||
### 2. 文件去重机制
|
||||
|
||||
- 基于文件哈希进行去重
|
||||
@@ -722,6 +755,7 @@ if chunk_count >= int(expected_chunks) && total_chunk_size >= expected_file_size
|
||||
- 分片文件存储在磁盘,不占用大量内存
|
||||
- 上传状态使用内存缓存,提高查询速度
|
||||
- 支持配置清理策略,自动清理临时文件
|
||||
- **分片大小记录文件**:避免遍历分片文件计算总大小,提高合并判断性能
|
||||
|
||||
### 2. 并发处理
|
||||
|
||||
|
||||
@@ -1,276 +0,0 @@
|
||||
import hono
|
||||
import net.http
|
||||
import os
|
||||
import json
|
||||
|
||||
fn main() {
|
||||
mut app := hono.Hono.new()
|
||||
|
||||
// 创建分片上传管理器
|
||||
mut upload_manager := hono.new_chunk_upload_manager(hono.ChunkUploadConfig{})
|
||||
|
||||
// 分片上传接口
|
||||
app.post('/upload/chunk', fn [mut upload_manager] (mut c hono.Context) http.Response {
|
||||
return upload_manager.handle_chunk_upload(mut c)
|
||||
})
|
||||
|
||||
|
||||
// 查询上传状态接口
|
||||
app.get('/upload/status', fn [mut upload_manager] (mut c hono.Context) http.Response {
|
||||
return upload_manager.get_upload_status(mut c)
|
||||
})
|
||||
|
||||
// 查询已上传分片接口
|
||||
app.get('/upload/chunks', fn [mut upload_manager] (mut c hono.Context) http.Response {
|
||||
file_hash := c.query['file_hash'] or {
|
||||
c.status(400)
|
||||
return c.json('{"error": "Missing file_hash parameter"}')
|
||||
}
|
||||
upload_status := upload_manager.uploads[file_hash] or {
|
||||
// 如果内存中没有记录,检查是否有最终文件(已完成上传)
|
||||
final_path := './uploads/${file_hash.trim_space()}'
|
||||
if os.exists(final_path) {
|
||||
// 文件已存在,返回所有分片已上传
|
||||
mut uploaded_chunks := []int{}
|
||||
uploaded_chunks << 0 // 假设只有一个分片
|
||||
return c.json('{"uploaded_chunks": ' + json.encode(uploaded_chunks) + ', "total_chunks": 1, "completed": true}')
|
||||
}
|
||||
c.status(404)
|
||||
return c.json('{"error": "Upload not found"}')
|
||||
}
|
||||
|
||||
// 检查是否所有分片都已上传(基于分片大小总和)
|
||||
chunk_dir := os.join_path(os.join_path('./uploads/chunks', file_hash.trim_space()), upload_status.chunk_size.str())
|
||||
mut total_chunk_size := u64(0)
|
||||
mut chunk_count := 0
|
||||
|
||||
if os.exists(chunk_dir) {
|
||||
for i := 0; ; i++ {
|
||||
chunk_path := os.join_path(chunk_dir, 'chunk_${i}.part')
|
||||
if !os.exists(chunk_path) {
|
||||
break
|
||||
}
|
||||
chunk_info := os.stat(chunk_path) or { continue }
|
||||
total_chunk_size += chunk_info.size
|
||||
chunk_count++
|
||||
}
|
||||
}
|
||||
|
||||
is_completed := total_chunk_size >= u64(upload_status.file_size)
|
||||
|
||||
return c.json('{"uploaded_chunks": ' + json.encode(upload_status.uploaded_chunks) + ', "total_chunks": $chunk_count, "completed": $is_completed}')
|
||||
})
|
||||
|
||||
// 检查分片是否存在接口(秒传功能)
|
||||
app.get('/upload/chunk_exists', fn [mut upload_manager] (mut c hono.Context) http.Response {
|
||||
file_hash := c.query['file_hash'] or {
|
||||
c.status(400)
|
||||
return c.json('{"error": "Missing file_hash parameter"}')
|
||||
}
|
||||
chunk_index := c.query['chunk_index'] or {
|
||||
c.status(400)
|
||||
return c.json('{"error": "Missing chunk_index parameter"}')
|
||||
}
|
||||
chunk_hash := c.query['chunk_hash'] or {
|
||||
c.status(400)
|
||||
return c.json('{"error": "Missing chunk_hash parameter"}')
|
||||
}
|
||||
file_size_str := c.query['file_size'] or {
|
||||
c.status(400)
|
||||
return c.json('{"error": "Missing file_size parameter"}')
|
||||
}
|
||||
file_size := file_size_str.int()
|
||||
|
||||
// 新增 trunk_size 参数
|
||||
trunk_size_str := c.query['trunk_size'] or { '' }
|
||||
mut chunk_size := upload_manager.config.chunk_size
|
||||
if trunk_size_str != '' {
|
||||
chunk_size = trunk_size_str.int()
|
||||
} else if upload_status := upload_manager.uploads[file_hash] {
|
||||
chunk_size = upload_status.chunk_size
|
||||
}
|
||||
|
||||
chunk_path := os.join_path(os.join_path(os.join_path('./uploads/chunks', file_hash.trim_space()), chunk_size.str()), 'chunk_${chunk_index}.part')
|
||||
println('[DEBUG] Checking chunk: $chunk_path')
|
||||
println('[DEBUG] File hash: $file_hash')
|
||||
println('[DEBUG] Chunk index: $chunk_index')
|
||||
println('[DEBUG] Expected hash: $chunk_hash')
|
||||
|
||||
// 清理无效的上传状态
|
||||
upload_manager.cleanup_invalid_status()
|
||||
|
||||
mut exists := os.exists(chunk_path)
|
||||
println('[DEBUG] File exists: $exists')
|
||||
|
||||
// 计算是否所有分片都已上传
|
||||
mut all_chunk_uploaded := false
|
||||
|
||||
// 方案1:优先检查最终文件是否存在(新格式:filehash.filetype)
|
||||
uploads_dir := './uploads/files'
|
||||
if os.exists(uploads_dir) {
|
||||
// 从上传状态中获取文件扩展名
|
||||
mut file_ext := ''
|
||||
if upload_status := upload_manager.uploads[file_hash] {
|
||||
file_ext = get_file_extension(upload_status.filename)
|
||||
}
|
||||
|
||||
// 如果内存中没有记录,尝试从数据库获取
|
||||
if file_ext == '' {
|
||||
if file_info := upload_manager.db.get_file_by_hash(file_hash) {
|
||||
file_ext = file_info.file_type
|
||||
}
|
||||
}
|
||||
|
||||
// 构建最终文件名
|
||||
final_filename := '${file_hash.trim_space()}${file_ext}'
|
||||
final_path := os.join_path(uploads_dir, final_filename)
|
||||
|
||||
if os.exists(final_path) {
|
||||
println('[DEBUG] Found final file: $final_path')
|
||||
all_chunk_uploaded = true
|
||||
}
|
||||
}
|
||||
|
||||
// 方案2:如果最终文件不存在,检查分片文件
|
||||
if !all_chunk_uploaded {
|
||||
chunk_dir := os.join_path(os.join_path('./uploads/chunks', file_hash.trim_space()), chunk_size.str())
|
||||
if os.exists(chunk_dir) {
|
||||
// 计算所有分片文件的大小总和
|
||||
mut total_chunk_size := u64(0)
|
||||
mut chunk_count := 0
|
||||
// 遍历所有分片文件
|
||||
for i := 0; ; i++ {
|
||||
chunk_file_path := '${chunk_dir}/chunk_${i}.part'
|
||||
if !os.exists(chunk_file_path) {
|
||||
break
|
||||
}
|
||||
// 获取分片文件大小
|
||||
chunk_info := os.stat(chunk_file_path) or { break }
|
||||
total_chunk_size += chunk_info.size
|
||||
chunk_count++
|
||||
}
|
||||
// 使用传入的文件大小
|
||||
expected_file_size := u64(file_size)
|
||||
println('[DEBUG] Total chunk size: $total_chunk_size, Expected file size: $expected_file_size, Chunk count: $chunk_count')
|
||||
|
||||
// 计算理论上需要的分片数量
|
||||
expected_chunks := (expected_file_size + u64(chunk_size) - 1) / u64(chunk_size)
|
||||
println('[DEBUG] Expected chunks: $expected_chunks')
|
||||
|
||||
// 只有当分片数量达到预期且总大小 >= 文件大小时,才认为上传完成
|
||||
if chunk_count >= int(expected_chunks) && total_chunk_size >= expected_file_size {
|
||||
all_chunk_uploaded = true
|
||||
println('[DEBUG] All chunks uploaded: chunk_count=$chunk_count >= expected_chunks=$expected_chunks && total_size=$total_chunk_size >= file_size=$expected_file_size')
|
||||
} else {
|
||||
println('[DEBUG] Not all chunks uploaded: chunk_count=$chunk_count < expected_chunks=$expected_chunks || total_size=$total_chunk_size < file_size=$expected_file_size')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println('[DEBUG] All chunk uploaded: $all_chunk_uploaded')
|
||||
|
||||
// chunk_exists 接口不应该自动触发合并,只返回状态信息
|
||||
// 合并操作应该在 /upload/chunk 接口中处理
|
||||
|
||||
// 只检查文件是否存在,不验证 hash(因为前后端 hash 算法可能不一致)
|
||||
// 如果需要严格验证,可以在这里添加 hash 检查逻辑
|
||||
|
||||
return c.json('{"exists": $exists, "all_chunk_uploaded": $all_chunk_uploaded}')
|
||||
})
|
||||
|
||||
// 获取所有文件信息
|
||||
app.get('/api/files', fn [mut upload_manager] (mut c hono.Context) http.Response {
|
||||
files := upload_manager.db.get_all_files() or {
|
||||
c.status(500)
|
||||
return c.json('{"error": "Failed to get files", "message": "$err"}')
|
||||
}
|
||||
return c.json(json.encode(files))
|
||||
})
|
||||
|
||||
// 根据文件UUID获取文件信息
|
||||
app.get('/api/files/:uuid', fn [mut upload_manager] (mut c hono.Context) http.Response {
|
||||
file_uuid := c.params['uuid'] or {
|
||||
c.status(400)
|
||||
return c.json('{"error": "Missing file UUID"}')
|
||||
}
|
||||
|
||||
file_info := upload_manager.db.get_file_by_uuid(file_uuid) or {
|
||||
c.status(404)
|
||||
return c.json('{"error": "File not found"}')
|
||||
}
|
||||
|
||||
return c.json(json.encode(file_info))
|
||||
})
|
||||
|
||||
// 根据文件hash获取文件信息
|
||||
app.get('/api/files/hash/:hash', fn [mut upload_manager] (mut c hono.Context) http.Response {
|
||||
file_hash := c.params['hash'] or {
|
||||
c.status(400)
|
||||
return c.json('{"error": "Missing file hash"}')
|
||||
}
|
||||
|
||||
file_info := upload_manager.db.get_file_by_hash(file_hash) or {
|
||||
c.status(404)
|
||||
return c.json('{"error": "File not found"}')
|
||||
}
|
||||
|
||||
return c.json(json.encode(file_info))
|
||||
})
|
||||
|
||||
// 删除文件信息
|
||||
app.delete('/api/files/:uuid', fn [mut upload_manager] (mut c hono.Context) http.Response {
|
||||
file_uuid := c.params['uuid'] or {
|
||||
c.status(400)
|
||||
return c.json('{"error": "Missing file UUID"}')
|
||||
}
|
||||
|
||||
upload_manager.db.delete_file(file_uuid) or {
|
||||
c.status(500)
|
||||
return c.json('{"error": "Failed to delete file", "message": "$err"}')
|
||||
}
|
||||
|
||||
return c.json('{"success": true, "message": "File deleted successfully"}')
|
||||
})
|
||||
|
||||
// 获取上传配置信息
|
||||
app.get('/api/upload-config', fn [mut upload_manager] (mut c hono.Context) http.Response {
|
||||
config := {
|
||||
'chunk_size': upload_manager.config.chunk_size.str()
|
||||
'max_file_size': upload_manager.config.max_file_size.str()
|
||||
'max_chunk_size': upload_manager.config.max_chunk_size.str()
|
||||
'temp_dir': upload_manager.config.temp_dir
|
||||
'upload_dir': upload_manager.config.upload_dir
|
||||
'cleanup_delay': upload_manager.config.cleanup_delay.str()
|
||||
'clear_chunks_on_complete': upload_manager.config.clear_chunks_on_complete.str()
|
||||
}
|
||||
return c.json(json.encode(config))
|
||||
})
|
||||
|
||||
// 静态文件服务
|
||||
app.use(hono.serve_static_path('/files', './uploads'))
|
||||
|
||||
// HTML 示例页面
|
||||
app.get('/', fn (mut c hono.Context) http.Response {
|
||||
html_content := os.read_file('public/upload.html') or {
|
||||
return c.text('Error: Cannot read upload.html file')
|
||||
}
|
||||
return c.html(html_content)
|
||||
})
|
||||
|
||||
println('Chunk upload server running at http://localhost:8080')
|
||||
app.listen(':8080')
|
||||
}
|
||||
|
||||
// 获取文件扩展名
|
||||
fn get_file_extension(filename string) string {
|
||||
println('[DEBUG] Getting extension for filename: "$filename"')
|
||||
parts := filename.split('.')
|
||||
println('[DEBUG] Split parts: $parts')
|
||||
if parts.len > 1 {
|
||||
ext := '.${parts.last()}'
|
||||
println('[DEBUG] Extracted extension: "$ext"')
|
||||
return ext
|
||||
}
|
||||
println('[DEBUG] No extension found, returning empty string')
|
||||
return ''
|
||||
}
|
||||
BIN
Binary file not shown.
@@ -84,20 +84,17 @@ fn main() {
|
||||
return c.json('{"error": "Upload not found"}')
|
||||
}
|
||||
|
||||
// 检查是否所有分片都已上传(基于分片大小总和)
|
||||
// 检查是否所有分片都已上传(使用分片大小记录文件)
|
||||
chunk_dir := os.join_path(os.join_path('./uploads/chunks', file_hash.trim_space()), upload_status.chunk_size.str())
|
||||
mut total_chunk_size := u64(0)
|
||||
mut chunk_count := 0
|
||||
mut chunk_count := upload_status.uploaded_chunks.len
|
||||
|
||||
if os.exists(chunk_dir) {
|
||||
for i := 0; ; i++ {
|
||||
chunk_path := os.join_path(chunk_dir, 'chunk_${i}.part')
|
||||
if !os.exists(chunk_path) {
|
||||
break
|
||||
}
|
||||
chunk_info := os.stat(chunk_path) or { continue }
|
||||
total_chunk_size += chunk_info.size
|
||||
chunk_count++
|
||||
// 读取分片大小记录文件
|
||||
size_record_path := os.join_path(chunk_dir, 'total_size.record')
|
||||
if os.exists(size_record_path) {
|
||||
size_data := os.read_file(size_record_path) or { '0' }
|
||||
total_chunk_size = size_data.u64()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -108,6 +105,7 @@ fn main() {
|
||||
|
||||
// 检查分片是否存在接口(秒传功能)
|
||||
app.get('/upload/chunk_exists', fn [mut upload_manager] (mut c hono.Context) http.Response {
|
||||
mut chunk_path := ''
|
||||
file_hash := c.query['file_hash'] or {
|
||||
c.status(400)
|
||||
return c.json('{"error": "Missing file_hash parameter"}')
|
||||
@@ -135,7 +133,7 @@ fn main() {
|
||||
chunk_size = upload_status.chunk_size
|
||||
}
|
||||
|
||||
chunk_path := os.join_path(os.join_path(os.join_path('./uploads/chunks', file_hash.trim_space()), chunk_size.str()), 'chunk_${chunk_index}.part')
|
||||
chunk_path = os.join_path(os.join_path(os.join_path('./uploads/chunks', file_hash.trim_space()), chunk_size.str()), 'chunk_${chunk_index}.part')
|
||||
println('[DEBUG] Checking chunk: $chunk_path')
|
||||
println('[DEBUG] File hash: $file_hash')
|
||||
println('[DEBUG] Chunk index: $chunk_index')
|
||||
@@ -180,20 +178,34 @@ fn main() {
|
||||
if !all_chunk_uploaded {
|
||||
chunk_dir := os.join_path(os.join_path('./uploads/chunks', file_hash.trim_space()), chunk_size.str())
|
||||
if os.exists(chunk_dir) {
|
||||
// 计算所有分片文件的大小总和
|
||||
// 使用分片大小记录文件,避免遍历
|
||||
mut total_chunk_size := u64(0)
|
||||
mut chunk_count := 0
|
||||
// 遍历所有分片文件
|
||||
for i := 0; ; i++ {
|
||||
chunk_file_path := '${chunk_dir}/chunk_${i}.part'
|
||||
if !os.exists(chunk_file_path) {
|
||||
break
|
||||
}
|
||||
// 获取分片文件大小
|
||||
chunk_info := os.stat(chunk_file_path) or { break }
|
||||
total_chunk_size += chunk_info.size
|
||||
chunk_count++
|
||||
|
||||
// 读取分片大小记录文件
|
||||
size_record_path := os.join_path(chunk_dir, 'total_size.record')
|
||||
if os.exists(size_record_path) {
|
||||
size_data := os.read_file(size_record_path) or { '0' }
|
||||
total_chunk_size = size_data.u64()
|
||||
}
|
||||
|
||||
// 从内存中的上传状态获取分片数量
|
||||
if upload_status := upload_manager.uploads[file_hash] {
|
||||
chunk_count = upload_status.uploaded_chunks.len
|
||||
} else {
|
||||
// 如果内存中没有状态,遍历分片文件来计算实际分片数量
|
||||
mut actual_chunk_count := 0
|
||||
for i := 0; ; i++ {
|
||||
chunk_path = os.join_path(chunk_dir, 'chunk_${i}.part')
|
||||
if os.exists(chunk_path) {
|
||||
actual_chunk_count++
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
chunk_count = actual_chunk_count
|
||||
}
|
||||
|
||||
// 使用传入的文件大小
|
||||
expected_file_size := u64(file_size)
|
||||
println('[DEBUG] Total chunk size: $total_chunk_size, Expected file size: $expected_file_size, Chunk count: $chunk_count')
|
||||
@@ -220,6 +232,7 @@ fn main() {
|
||||
// 只检查文件是否存在,不验证 hash(因为前后端 hash 算法可能不一致)
|
||||
// 如果需要严格验证,可以在这里添加 hash 检查逻辑
|
||||
|
||||
// 返回时字段改回 all_chunk_uploaded
|
||||
return c.json('{"exists": $exists, "all_chunk_uploaded": $all_chunk_uploaded}')
|
||||
})
|
||||
|
||||
|
||||
+65
-25
@@ -153,28 +153,18 @@ pub fn (mut manager ChunkUploadManager) handle_chunk_upload(mut ctx Context) htt
|
||||
println('[DEBUG] Updated upload status for file: $file_hash, chunk: $chunk_index')
|
||||
|
||||
// 判断是否所有分片都已上传,自动合并
|
||||
// 基于分片大小总和来判断是否合并
|
||||
// 使用记录文件来避免遍历分片文件计算总大小
|
||||
mut all_chunk_uploaded := false
|
||||
merge_chunk_dir := os.join_path(manager.config.temp_dir, file_hash.trim_space(), chunk_size.str())
|
||||
|
||||
if os.exists(merge_chunk_dir) {
|
||||
mut total_chunk_size := u64(0)
|
||||
mut chunk_count := 0
|
||||
mut max_chunk_index := -1
|
||||
// 更新已上传分片的总大小记录
|
||||
manager.update_chunk_size_record(file_hash, chunk_size, file_data.len)
|
||||
|
||||
// 遍历所有分片文件,计算总大小和找到最大分片索引
|
||||
for i := 0; ; i++ {
|
||||
merge_chunk_path := os.join_path(merge_chunk_dir, 'chunk_${i}.part')
|
||||
if !os.exists(merge_chunk_path) {
|
||||
break
|
||||
}
|
||||
chunk_info := os.stat(merge_chunk_path) or { continue }
|
||||
total_chunk_size += chunk_info.size
|
||||
chunk_count++
|
||||
max_chunk_index = i
|
||||
}
|
||||
// 读取已上传分片的总大小
|
||||
total_chunk_size := manager.get_chunk_size_record(file_hash, chunk_size)
|
||||
|
||||
println('[DEBUG] [MergeCheck] total_chunk_size=$total_chunk_size, file_size=$file_size, chunk_count=$chunk_count, max_chunk_index=$max_chunk_index')
|
||||
println('[DEBUG] [MergeCheck] total_chunk_size=$total_chunk_size, file_size=$file_size, chunk_index=$chunk_index')
|
||||
|
||||
// 如果分片文件大小总和 >= file_size,认为可以合并
|
||||
if total_chunk_size >= u64(file_size) {
|
||||
@@ -186,15 +176,8 @@ pub fn (mut manager ChunkUploadManager) handle_chunk_upload(mut ctx Context) htt
|
||||
println('[DEBUG] All chunks uploaded: $all_chunk_uploaded')
|
||||
|
||||
if all_chunk_uploaded {
|
||||
// 重新计算分片数量用于合并
|
||||
mut actual_total_chunks := 0
|
||||
for i := 0; ; i++ {
|
||||
check_chunk_path := os.join_path(merge_chunk_dir, 'chunk_${i}.part')
|
||||
if !os.exists(check_chunk_path) {
|
||||
break
|
||||
}
|
||||
actual_total_chunks++
|
||||
}
|
||||
// 使用内存中的上传状态来获取分片数量,避免遍历文件
|
||||
actual_total_chunks := manager.uploads[file_hash].uploaded_chunks.len
|
||||
|
||||
// 获取文件扩展名
|
||||
file_ext := get_file_extension(filename)
|
||||
@@ -406,6 +389,9 @@ fn (mut manager ChunkUploadManager) merge_chunks(file_hash string, total_chunks
|
||||
fn (mut manager ChunkUploadManager) cleanup_chunks(file_hash string, chunk_size int) {
|
||||
chunk_dir := os.join_path(manager.config.temp_dir, file_hash.trim_space(), chunk_size.str())
|
||||
if os.exists(chunk_dir) {
|
||||
// 清理分片大小记录
|
||||
manager.cleanup_chunk_size_record(file_hash, chunk_size)
|
||||
|
||||
// 删除整个分片目录
|
||||
os.rmdir_all(chunk_dir) or {
|
||||
println('[DEBUG] Failed to remove chunk directory: $err')
|
||||
@@ -453,6 +439,60 @@ fn get_file_extension(filename string) string {
|
||||
return ''
|
||||
}
|
||||
|
||||
// 更新分片大小记录
|
||||
pub fn (mut manager ChunkUploadManager) update_chunk_size_record(file_hash string, chunk_size int, current_chunk_size int) {
|
||||
chunk_dir := os.join_path(manager.config.temp_dir, file_hash.trim_space(), chunk_size.str())
|
||||
size_record_path := os.join_path(chunk_dir, 'total_size.record')
|
||||
|
||||
// 确保目录存在
|
||||
os.mkdir_all(chunk_dir) or {
|
||||
println('[DEBUG] Failed to create chunk directory: $err')
|
||||
return
|
||||
}
|
||||
|
||||
// 读取现有的总大小记录
|
||||
mut total_size := u64(0)
|
||||
if os.exists(size_record_path) {
|
||||
size_data := os.read_file(size_record_path) or { '0' }
|
||||
total_size = size_data.u64()
|
||||
}
|
||||
|
||||
// 更新总大小
|
||||
total_size += u64(current_chunk_size)
|
||||
|
||||
// 写入更新后的总大小
|
||||
os.write_file(size_record_path, total_size.str()) or {
|
||||
println('[DEBUG] Failed to write size record: $err')
|
||||
}
|
||||
|
||||
println('[DEBUG] Updated size record: $total_size bytes')
|
||||
}
|
||||
|
||||
// 获取分片大小记录
|
||||
pub fn (manager ChunkUploadManager) get_chunk_size_record(file_hash string, chunk_size int) u64 {
|
||||
chunk_dir := os.join_path(manager.config.temp_dir, file_hash.trim_space(), chunk_size.str())
|
||||
size_record_path := os.join_path(chunk_dir, 'total_size.record')
|
||||
|
||||
if os.exists(size_record_path) {
|
||||
size_data := os.read_file(size_record_path) or { '0' }
|
||||
return size_data.u64()
|
||||
}
|
||||
|
||||
return u64(0)
|
||||
}
|
||||
|
||||
// 清理分片大小记录
|
||||
pub fn (mut manager ChunkUploadManager) cleanup_chunk_size_record(file_hash string, chunk_size int) {
|
||||
chunk_dir := os.join_path(manager.config.temp_dir, file_hash.trim_space(), chunk_size.str())
|
||||
size_record_path := os.join_path(chunk_dir, 'total_size.record')
|
||||
|
||||
if os.exists(size_record_path) {
|
||||
os.rm(size_record_path) or {
|
||||
println('[DEBUG] Failed to remove size record: $err')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 清理文件上传状态(当文件被删除时调用)
|
||||
pub fn (mut manager ChunkUploadManager) cleanup_upload_status(file_hash string) {
|
||||
if file_hash in manager.uploads {
|
||||
|
||||
+37
-16
@@ -1106,29 +1106,50 @@ app.listen(':8080')
|
||||
let completedChunks = 0;
|
||||
let failedChunks = [];
|
||||
let instantSuccess = false;
|
||||
let cancelledTasks = false; // 新增:标记是否已取消任务
|
||||
|
||||
// 创建所有分片的上传任务并添加到请求池
|
||||
// 先串行检查第一个分片,检测秒传
|
||||
console.log('先检查第一个分片,检测秒传...');
|
||||
const firstChunk = selectedFile.slice(0, CHUNK_SIZE);
|
||||
const firstChunkHash = await calculateChunkHash(firstChunk);
|
||||
const firstCheckResult = await checkChunkExists(fileHash, 0, firstChunkHash);
|
||||
|
||||
// 如果第一个分片检测到秒传,直接返回
|
||||
if (firstCheckResult.all_chunk_uploaded) {
|
||||
console.log('检测到秒传,所有分片都已上传完成!');
|
||||
updateProgress(totalChunks, totalChunks); // 更新进度到100%
|
||||
showUploadStatus('秒传成功!文件已完整上传。', 'success');
|
||||
uploadBtn.textContent = '上传完成';
|
||||
return;
|
||||
}
|
||||
|
||||
// 如果第一个分片已存在,记录进度
|
||||
if (firstCheckResult.exists) {
|
||||
console.log('分片 0 已存在,跳过上传');
|
||||
completedChunks++;
|
||||
updateProgress(completedChunks, totalChunks);
|
||||
} else {
|
||||
// 第一个分片不存在,需要上传
|
||||
await uploadChunk(selectedFile, firstChunk, {
|
||||
fileHash, chunkIndex: 0, totalChunks, chunkHash: firstChunkHash
|
||||
});
|
||||
completedChunks++;
|
||||
updateProgress(completedChunks, totalChunks);
|
||||
console.log('分片 0 上传完成');
|
||||
}
|
||||
|
||||
// 创建剩余分片的上传任务并添加到请求池(并发处理)
|
||||
const uploadPromises = [];
|
||||
for (let i = 0; i < totalChunks; i++) {
|
||||
for (let i = 1; i < totalChunks; i++) {
|
||||
const chunkIndex = i;
|
||||
const promise = requestPool.addRequest(async () => {
|
||||
try {
|
||||
const chunk = selectedFile.slice(chunkIndex * CHUNK_SIZE, (chunkIndex + 1) * CHUNK_SIZE);
|
||||
const chunkHash = await calculateChunkHash(chunk);
|
||||
|
||||
// 检查分片是否已存在(包括秒传检测)
|
||||
// 检查分片是否已存在
|
||||
const checkResult = await checkChunkExists(fileHash, chunkIndex, chunkHash);
|
||||
|
||||
// 如果检测到所有分片都已上传完成,立即结束上传
|
||||
if (checkResult.all_uploaded) {
|
||||
console.log('检测到秒传,所有分片都已上传完成!');
|
||||
updateProgress(totalChunks, totalChunks); // 更新进度到100%
|
||||
showUploadStatus('秒传成功!文件已完整上传。', 'success');
|
||||
uploadBtn.textContent = '上传完成';
|
||||
instantSuccess = true;
|
||||
return { success: true, chunkIndex, isInstant: true };
|
||||
}
|
||||
|
||||
// 如果分片已存在,跳过上传
|
||||
if (checkResult.exists) {
|
||||
console.log(`分片 ${chunkIndex} 已存在,跳过上传`);
|
||||
@@ -1233,16 +1254,16 @@ app.listen(':8080')
|
||||
}
|
||||
|
||||
const res = await fetch(url.toString());
|
||||
if (!res.ok) return { exists: false, all_uploaded: false };
|
||||
if (!res.ok) return { exists: false, all_chunk_uploaded: false };
|
||||
const data = await res.json();
|
||||
|
||||
// 如果所有分片都已上传完成,直接返回 true 表示秒传成功
|
||||
if (data.all_chunk_uploaded) {
|
||||
console.log('所有分片都已上传完成,秒传成功!');
|
||||
return { exists: true, all_uploaded: true };
|
||||
return { exists: true, all_chunk_uploaded: true };
|
||||
}
|
||||
|
||||
return { exists: data.exists, all_uploaded: false };
|
||||
return { exists: data.exists, all_chunk_uploaded: false };
|
||||
}
|
||||
|
||||
// 上传分片
|
||||
|
||||
Reference in New Issue
Block a user