文件上传、分块与断点续传

package main

import (
    "fmt"
    "log"
    "path/filepath"

    "github.com/gin-gonic/gin"
)

func main() {
    r := gin.Default()

    // 步骤1：限制multipart内存为8MB
    r.MaxMultipartMemory = 8 << 20

    // 步骤2：FormData文件上传接口
    r.POST("/upload/formdata", func(c *gin.Context) {
        // 步骤3：获取上传的文件
        file, err := c.FormFile("file")
        if err != nil {
            c.JSON(400, gin.H{"error": err.Error()})
            return
        }

        // 步骤4：获取附加的文本字段
        userId := c.PostForm("userId")
        description := c.PostForm("description")
        log.Printf("用户%s上传文件: %s, 描述: %s", userId, file.Filename, description)

        // 步骤5：保存文件到指定目录
        savePath := filepath.Join("./uploads", file.Filename)
        if err := c.SaveUploadedFile(file, savePath); err != nil {
            c.JSON(500, gin.H{"error": "保存失败"})
            return
        }

        c.JSON(200, gin.H{
            "code":     0,
            "message":  "上传成功",
            "filename": file.Filename,
            "size":     file.Size,
        })
    })

    r.Run(":8080")
}

package main

import (
    "crypto/md5"
    "fmt"
    "io"
    "os"

    "github.com/gin-gonic/gin"
)

func main() {
    r := gin.Default()

    // 步骤1：Binary文件上传接口
    r.POST("/upload/binary", func(c *gin.Context) {
        // 步骤2：从Header获取文件名
        filename := c.GetHeader("X-File-Name")
        if filename == "" {
            filename = "unnamed.bin"
        }

        // 步骤3：读取原始二进制数据
        data, err := c.GetRawData()
        if err != nil {
            c.JSON(400, gin.H{"error": err.Error()})
            return
        }

        // 步骤4：计算MD5用于后续校验
        hash := fmt.Sprintf("%x", md5.Sum(data))

        // 步骤5：写入文件
        savePath := fmt.Sprintf("./uploads/%s", filename)
        if err := os.WriteFile(savePath, data, 0644); err != nil {
            c.JSON(500, gin.H{"error": "保存失败"})
            return
        }

        c.JSON(200, gin.H{
            "code":     0,
            "message":  "上传成功",
            "filename": filename,
            "size":     len(data),
            "md5":      hash,
        })
    })

    r.Run(":8080")
}

package main

import (
    "fmt"
    "log"
    "os"
    "path/filepath"
    "strconv"

    "github.com/gin-gonic/gin"
)

// 步骤1：定义分块上传存储目录
const chunkDir = "./chunks"

func main() {
    r := gin.Default()
    os.MkdirAll(chunkDir, 0755)
    os.MkdirAll("./uploads", 0755)

    // 步骤2：接收单个分块
    r.POST("/upload/chunk", func(c *gin.Context) {
        fileHash := c.PostForm("fileHash")
        chunkIndex := c.PostForm("chunkIndex")
        chunkTotal := c.PostForm("chunkTotal")

        // 步骤3：为每个文件创建独立的分块目录
        fileChunkDir := filepath.Join(chunkDir, fileHash)
        os.MkdirAll(fileChunkDir, 0755)

        // 步骤4：接收分块文件
        file, err := c.FormFile("chunk")
        if err != nil {
            c.JSON(400, gin.H{"error": err.Error()})
            return
        }

        // 步骤5：保存分块，文件名用序号命名
        chunkPath := filepath.Join(fileChunkDir, chunkIndex)
        if err := c.SaveUploadedFile(file, chunkPath); err != nil {
            c.JSON(500, gin.H{"error": "保存分块失败"})
            return
        }

        log.Printf("文件%s的分块%s/%s上传成功", fileHash, chunkIndex, chunkTotal)

        c.JSON(200, gin.H{
            "code":        0,
            "message":     "分块上传成功",
            "chunkIndex":  chunkIndex,
            "chunkTotal":  chunkTotal,
        })
    })

    // 步骤6：合并分块
    r.POST("/upload/merge", func(c *gin.Context) {
        fileHash := c.PostForm("fileHash")
        fileName := c.PostForm("fileName")
        chunkTotal, _ := strconv.Atoi(c.PostForm("chunkTotal"))

        fileChunkDir := filepath.Join(chunkDir, fileHash)
        outputPath := filepath.Join("./uploads", fileName)

        // 步骤7：创建合并后的输出文件
        outFile, err := os.Create(outputPath)
        if err != nil {
            c.JSON(500, gin.H{"error": "创建输出文件失败"})
            return
        }
        defer outFile.Close()

        // 步骤8：按序号顺序合并所有分块
        for i := 0; i < chunkTotal; i++ {
            chunkPath := filepath.Join(fileChunkDir, strconv.Itoa(i))
            chunkData, err := os.ReadFile(chunkPath)
            if err != nil {
                c.JSON(500, gin.H{"error": fmt.Sprintf("读取分块%d失败", i)})
                return
            }
            outFile.Write(chunkData)
        }

        log.Printf("文件%s合并完成，共%d块", fileName, chunkTotal)

        c.JSON(200, gin.H{
            "code":     0,
            "message":  "合并成功",
            "fileName": fileName,
            "fileHash": fileHash,
        })
    })

    r.Run(":8080")
}

package main

import (
    "crypto/md5"
    "fmt"
    "io"
    "log"
    "os"
    "path/filepath"
    "sort"
    "strconv"
    "time"

    "github.com/gin-gonic/gin"
    "github.com/redis/go-redis/v9"
    "golang.org/x/net/context"
)

// 步骤1：定义全局变量
var (
    rdb       *redis.Client
    chunkDir  = "./chunks"
    uploadDir = "./uploads"
)

func main() {
    // 步骤2：初始化Redis
    rdb = redis.NewClient(&redis.Options{
        Addr: "localhost:6379",
    })

    os.MkdirAll(chunkDir, 0755)
    os.MkdirAll(uploadDir, 0755)

    r := gin.Default()

    // 步骤3：注册三个核心接口
    r.GET("/upload/status", handleChunkStatus)
    r.POST("/upload/chunk", handleChunkUpload)
    r.POST("/upload/merge", handleMergeChunks)

    r.Run(":8080")
}

// 步骤4：查询已上传的分块
func handleChunkStatus(c *gin.Context) {
    fileHash := c.Query("fileHash")

    ctx := context.Background()

    // 步骤5：检查文件是否已存在（秒传检查）
    fileKey := fmt.Sprintf("file:%s", fileHash)
    if exists, _ := rdb.Exists(ctx, fileKey).Result(); exists > 0 {
        filePath, _ := rdb.Get(ctx, fileKey).Result()
        c.JSON(200, gin.H{
            "code":     0,
            "uploaded": true,
            "url":      filePath,
            "message":  "文件已存在，秒传成功",
        })
        return
    }

    // 步骤6：查询已上传的分块列表
    chunkKey := fmt.Sprintf("chunks:%s", fileHash)
    uploadedChunks, err := rdb.SMembers(ctx, chunkKey).Result()
    if err != nil {
        c.JSON(500, gin.H{"error": "查询失败"})
        return
    }

    // 步骤7：同时检查磁盘上实际存在的分块
    fileChunkDir := filepath.Join(chunkDir, fileHash)
    diskChunks := []string{}
    if entries, err := os.ReadDir(fileChunkDir); err == nil {
        for _, entry := range entries {
            diskChunks = append(diskChunks, entry.Name())
        }
    }

    // 步骤8：合并Redis和磁盘的记录，返回已传块列表
    chunkSet := make(map[string]bool)
    for _, ch := range uploadedChunks {
        chunkSet[ch] = true
    }
    for _, ch := range diskChunks {
        chunkSet[ch] = true
    }

    var result []string
    for ch := range chunkSet {
        result = append(result, ch)
    }
    sort.Strings(result)

    c.JSON(200, gin.H{
        "code":           0,
        "uploaded":       false,
        "uploadedChunks": result,
    })
}

// 步骤9：接收单个分块
func handleChunkUpload(c *gin.Context) {
    fileHash := c.PostForm("fileHash")
    chunkIndex := c.PostForm("chunkIndex")
    chunkTotal := c.PostForm("chunkTotal")

    // 步骤10：为每个文件创建独立的分块目录
    fileChunkDir := filepath.Join(chunkDir, fileHash)
    os.MkdirAll(fileChunkDir, 0755)

    // 步骤11：接收分块文件
    file, err := c.FormFile("chunk")
    if err != nil {
        c.JSON(400, gin.H{"error": err.Error()})
        return
    }

    // 步骤12：保存分块到磁盘
    chunkPath := filepath.Join(fileChunkDir, chunkIndex)
    if err := c.SaveUploadedFile(file, chunkPath); err != nil {
        c.JSON(500, gin.H{"error": "保存分块失败"})
        return
    }

    // 步骤13：将已传块记录到Redis
    ctx := context.Background()
    chunkKey := fmt.Sprintf("chunks:%s", fileHash)
    rdb.SAdd(ctx, chunkKey, chunkIndex)

    // 步骤14：设置过期时间，7天后自动清理
    rdb.Expire(ctx, chunkKey, 7*24*3600*time.Second)

    c.JSON(200, gin.H{
        "code":       0,
        "message":    "分块上传成功",
        "chunkIndex": chunkIndex,
        "chunkTotal": chunkTotal,
    })
}

// 步骤15：合并所有分块
func handleMergeChunks(c *gin.Context) {
    fileHash := c.PostForm("fileHash")
    fileName := c.PostForm("fileName")
    chunkTotal, _ := strconv.Atoi(c.PostForm("chunkTotal"))

    fileChunkDir := filepath.Join(chunkDir, fileHash)
    outputPath := filepath.Join(uploadDir, fileName)

    // 步骤16：创建合并后的输出文件
    outFile, err := os.Create(outputPath)
    if err != nil {
        c.JSON(500, gin.H{"error": "创建输出文件失败"})
        return
    }
    defer outFile.Close()

    // 步骤17：按序号顺序合并所有分块
    for i := 0; i < chunkTotal; i++ {
        chunkPath := filepath.Join(fileChunkDir, strconv.Itoa(i))
        chunkData, err := os.ReadFile(chunkPath)
        if err != nil {
            c.JSON(500, gin.H{"error": fmt.Sprintf("读取分块%d失败", i)})
            return
        }
        outFile.Write(chunkData)
    }

    // 步骤18：计算合并后文件的MD5，校验完整性
    mergedHash, err := calcFileMD5(outputPath)
    if err != nil {
        c.JSON(500, gin.H{"error": "计算MD5失败"})
        return
    }

    if mergedHash != fileHash {
        // 步骤19：MD5不匹配，删除合并文件和分块
        os.Remove(outputPath)
        os.RemoveAll(fileChunkDir)
        c.JSON(400, gin.H{"error": "文件MD5校验失败，请重新上传"})
        return
    }

    // 步骤20：MD5校验通过，记录到Redis（供后续秒传）
    ctx := context.Background()
    fileKey := fmt.Sprintf("file:%s", fileHash)
    rdb.Set(ctx, fileKey, outputPath, 0)

    // 步骤21：清理分块文件
    os.RemoveAll(fileChunkDir)
    rdb.Del(ctx, fmt.Sprintf("chunks:%s", fileHash))

    log.Printf("文件%s合并并校验成功", fileName)

    c.JSON(200, gin.H{
        "code":     0,
        "message":  "合并成功",
        "fileName": fileName,
        "fileHash": fileHash,
        "url":      outputPath,
    })
}

// 步骤22：计算文件的MD5
func calcFileMD5(filePath string) (string, error) {
    file, err := os.Open(filePath)
    if err != nil {
        return "", err
    }
    defer file.Close()

    hash := md5.New()
    if _, err := io.Copy(hash, file); err != nil {
        return "", err
    }

    return fmt.Sprintf("%x", hash.Sum(nil)), nil
}

package main

import (
    "github.com/gin-gonic/gin"
    "golang.org/x/net/context"
    "fmt"
)

// 步骤1：秒传预检接口
func handleInstantUpload(c *gin.Context) {
    fileHash := c.PostForm("fileHash")
    fileName := c.PostForm("fileName")
    fileSize := c.PostForm("fileSize")

    ctx := context.Background()

    // 步骤2：查Redis，文件MD5是否已存在
    fileKey := fmt.Sprintf("file:%s", fileHash)
    if filePath, err := rdb.Get(ctx, fileKey).Result(); err == nil {
        // 步骤3：文件已存在，直接返回成功（秒传）
        c.JSON(200, gin.H{
            "code":     0,
            "message":  "秒传成功，文件已存在",
            "fileName": fileName,
            "fileHash": fileHash,
            "fileSize": fileSize,
            "url":      filePath,
        })
        return
    }

    // 步骤4：文件不存在，需要正常上传
    c.JSON(200, gin.H{
        "code":    1,
        "message": "文件不存在，请上传",
    })
}

package main

import (
    "crypto/md5"
    "fmt"
    "io"
    "log"
    "os"
    "sync"
    "time"
)

// 步骤1：定义分块任务
type ChunkTask struct {
    Index      int
    Data       []byte
    FileHash   string
    ChunkHash  string
    Retries    int
}

// 步骤2：定义上传结果
type UploadResult struct {
    Index  int
    Success bool
    Error  error
}

// 步骤3：并发上传函数
func concurrentUpload(filePath string, chunkSize int64, concurrency int) error {
    // 步骤4：打开文件并计算MD5
    file, err := os.Open(filePath)
    if err != nil {
        return err
    }
    defer file.Close()

    // 步骤5：读取文件并分块
    stat, _ := file.Stat()
    totalChunks := int(stat.Size()/chunkSize) + 1
    if stat.Size()%chunkSize == 0 {
        totalChunks = int(stat.Size() / chunkSize)
    }

    // 步骤6：创建任务通道和结果通道
    taskCh := make(chan ChunkTask, totalChunks)
    resultCh := make(chan UploadResult, totalChunks)

    // 步骤7：启动worker pool
    var wg sync.WaitGroup
    for w := 0; w < concurrency; w++ {
        wg.Add(1)
        go func(workerId int) {
            defer wg.Done()
            for task := range taskCh {
                // 步骤8：上传分块（带重试）
                err := uploadChunkWithRetry(task, 3)
                if err != nil {
                    resultCh <- UploadResult{Index: task.Index, Success: false, Error: err}
                } else {
                    resultCh <- UploadResult{Index: task.Index, Success: true}
                }
            }
        }(w)
    }

    // 步骤9：分配任务——读取文件并分块
    go func() {
        for i := 0; i < totalChunks; i++ {
            offset := int64(i) * chunkSize
            chunkData := make([]byte, chunkSize)
            file.ReadAt(chunkData, offset)

            // 步骤10：计算分块MD5
            chunkHash := fmt.Sprintf("%x", md5.Sum(chunkData))

            taskCh <- ChunkTask{
                Index:     i,
                Data:      chunkData,
                ChunkHash: chunkHash,
            }
        }
        close(taskCh)
    }()

    // 步骤11：等待所有worker完成
    go func() {
        wg.Wait()
        close(resultCh)
    }()

    // 步骤12：收集结果并更新进度
    uploaded := 0
    failed := 0
    for result := range resultCh {
        if result.Success {
            uploaded++
        } else {
            failed++
            log.Printf("分块%d上传失败: %v", result.Index, result.Error)
        }
        // 步骤13：进度回调
        progress := float64(uploaded+failed) / float64(totalChunks) * 100
        log.Printf("进度: %.1f%% (%d/%d)", progress, uploaded+failed, totalChunks)
    }

    if failed > 0 {
        return fmt.Errorf("%d个分块上传失败", failed)
    }

    log.Printf("全部%d个分块上传完成", uploaded)
    return nil
}

// 步骤14：带重试的上传函数
func uploadChunkWithRetry(task ChunkTask, maxRetries int) error {
    var lastErr error
    for retry := 0; retry <= maxRetries; retry++ {
        if retry > 0 {
            // 步骤15：指数退避等待
            time.Sleep(time.Duration(retry*retry) * time.Second)
        }
        // 步骤16：模拟上传分块
        err := doUploadChunk(task)
        if err == nil {
            return nil
        }
        lastErr = err
        log.Printf("分块%d第%d次上传失败: %v", task.Index, retry+1, err)
    }
    return lastErr
}

// 步骤17：实际上传分块的HTTP请求（简化示例）
func doUploadChunk(task ChunkTask) error {
    // 实际项目中这里发送HTTP POST请求到服务端
    // 包含fileHash, chunkIndex, chunkData等信息
    return nil
}