From 49b709df9f85d2e2db7d6e773c5e91c78b2c7eb2 Mon Sep 17 00:00:00 2001 From: nepiedg <806669289@qq.com> Date: Sat, 3 Jan 2026 23:50:30 +0000 Subject: [PATCH] Add video download failure reporting feature - Introduced a new API endpoint `POST /api/v1/video/remove_watermark/report_failure` for reporting download failures. - Added a new database table `video_download_failures` to store details about failed downloads, including domain, URL, error message, and reporting metadata. - Updated the video handler to process failure reports and save them to the database. - Enhanced documentation to include details about the new reporting feature and its usage. --- cmd/api/main.go | 1 + docs/remove_watermark/API.md | 22 +++++++ docs/remove_watermark/README.md | 3 +- docs/sql/remove_watermark.sql | 16 +++++ .../remove_watermark/handler/video_handler.go | 60 +++++++++++++++++++ .../remove_watermark/model/video_parse.go | 22 +++++++ .../remove_watermark/service/video_service.go | 41 ++++++++++++- internal/routes/remove_watermark_routes.go | 6 +- internal/routes/routes.go | 2 +- 9 files changed, 168 insertions(+), 5 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index b65b569..16ce700 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -45,6 +45,7 @@ func main() { &membershipmodel.MembershipRedemption{}, &rmmodel.VideoParseLog{}, &rmmodel.VideoParseUnlock{}, + &rmmodel.VideoDownloadFailure{}, &smokemodel.SmokeLog{}, &smokemodel.SmokeAIAdvice{}, &smokemodel.SmokeAIAdviceUnlock{}, diff --git a/docs/remove_watermark/API.md b/docs/remove_watermark/API.md index e264095..513281c 100644 --- a/docs/remove_watermark/API.md +++ b/docs/remove_watermark/API.md @@ -99,3 +99,25 @@ curl -X POST 'http://127.0.0.1:8080/api/v1/video/remove_watermark/unlock' \ - 每次解析调用都会写入 `video_parse_logs`，其中包含 `request_content`、`parsed_url`、第三方响应、调用耗时等字段，便于审计和配额统计。 - 第三方返回的 JSON 直接保存在 `video_parse_logs.third_party_payload` 字段，可通过 SQL 查询和脱敏。 - SQL DDL 位于 `docs/sql/remove_watermark.sql`，部署数据库时执行即可。 + +## 5. 下载失败上报 + +`POST /api/v1/video/remove_watermark/report_failure` + +| 项目 | 说明 | +| --- | --- | +| Header | 可为空（供内部服务调用），也可附加 `Content-Type: application/json` | +| 请求体 | `{ "domain": "example.com", "failedUrl": "https://example.com/video.mp4", "errorMessage": "403 from CDN", "timestamp": 1728034710000, "userAgent": "miniprogram" }` | +| 响应 | `{"code":200,"message":"success","data":{"reported":true}}` | + +字段说明： + +| 字段 | 类型 | 说明 | +| --- | --- | --- | +| `domain` | string，可选 | 失败 URL 所属域名，若缺失会自动从 `failedUrl` 解析 | +| `failedUrl` | string，必填 | 下载失败的完整地址 | +| `errorMessage` | string，可选 | 失败原因描述，建议包含第三方响应 | +| `timestamp` | number，可选 | 失败发生时间，毫秒级 Unix 时间戳。默认使用服务端接收时间。 | +| `userAgent` | string，可选 | 报告端的 UA，默认取 HTTP 头 | + +服务端会额外记录请求来源 IP（`client_ip`），并存入 `video_download_failures` 表，便于后续排查白名单或 CDN 问题。 diff --git a/docs/remove_watermark/README.md b/docs/remove_watermark/README.md index a2b0910..7e40755 100644 --- a/docs/remove_watermark/README.md +++ b/docs/remove_watermark/README.md @@ -97,6 +97,7 @@ Authorization: Bearer - 项目已将 `SHORT_VIDEO_API_KEY`、`SHORT_VIDEO_FREE_QUOTA`、`SHORT_VIDEO_TIMEOUT_SECONDS` 等变量加入配置，可通过 `.env` 控制。 - 新增 `video_parse_logs` / `video_parse_unlocks` 表（DDL 见 `docs/sql/remove_watermark.sql`）：分别记录每次解析详情和“观看广告解锁”状态。 +- 新增 `video_download_failures` 表，用于记录第三方下载失败的域名、URL、错误原因、上报端 UA/IP、失败时间，方便排查白名单问题。 - 用户在登录后可使用登录接口返回的 `session_key` 作为 `Authorization: Bearer ` 调用受保护的接口。 - 成功解析后会将第三方原始响应以 JSON 形式直接写入 `video_parse_logs.third_party_payload` 字段，方便统一检索。 -- API 列表和请求/响应示例详见 `docs/remove_watermark/API.md`。 +- API 列表和请求/响应示例详见 `docs/remove_watermark/API.md`，其中包含新的 `POST /api/v1/video/remove_watermark/report_failure` 上报接口。 diff --git a/docs/sql/remove_watermark.sql b/docs/sql/remove_watermark.sql index 7abfe1d..4b8dee0 100644 --- a/docs/sql/remove_watermark.sql +++ b/docs/sql/remove_watermark.sql @@ -31,3 +31,19 @@ CREATE TABLE IF NOT EXISTS `video_parse_unlocks` ( PRIMARY KEY (`id`), UNIQUE KEY `uniq_video_unlock_date` (`mini_program_id`,`user_id`,`unlock_date`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +-- 下载失败上报 +CREATE TABLE IF NOT EXISTS `video_download_failures` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT, + `created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `updated_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `deleted_at` datetime DEFAULT NULL, + `domain` varchar(255) DEFAULT NULL, + `failed_url` varchar(1000) NOT NULL, + `error_message` text, + `reported_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `user_agent` varchar(255) DEFAULT NULL, + `client_ip` varchar(64) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `idx_video_download_domain` (`domain`,`reported_at`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/internal/remove_watermark/handler/video_handler.go b/internal/remove_watermark/handler/video_handler.go index 7b680e3..7e33fdf 100644 --- a/internal/remove_watermark/handler/video_handler.go +++ b/internal/remove_watermark/handler/video_handler.go @@ -3,6 +3,9 @@ package handler import ( "errors" "net/http" + "net/url" + "strings" + "time" "github.com/gin-gonic/gin" @@ -25,6 +28,14 @@ type removeWatermarkRequest struct { Content string `json:"content" binding:"required"` } +type reportDownloadFailureRequest struct { + Domain string `json:"domain"` + FailedURL string `json:"failedUrl" binding:"required"` + ErrorMessage string `json:"errorMessage"` + Timestamp int64 `json:"timestamp"` + UserAgent string `json:"userAgent"` +} + func (h *VideoHandler) RemoveWatermark(c *gin.Context) { var req removeWatermarkRequest if err := c.ShouldBindJSON(&req); err != nil { @@ -84,3 +95,52 @@ func (h *VideoHandler) UnlockQuota(c *gin.Context) { "unlocked": true, })) } + +func (h *VideoHandler) ReportDownloadFailure(c *gin.Context) { + var req reportDownloadFailureRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, model.Error(http.StatusBadRequest, "请求参数错误")) + return + } + + failedURL := strings.TrimSpace(req.FailedURL) + if failedURL == "" { + c.JSON(http.StatusBadRequest, model.Error(http.StatusBadRequest, "failedUrl 不能为空")) + return + } + + reportedAt := time.Now() + if req.Timestamp > 0 { + reportedAt = time.UnixMilli(req.Timestamp) + } + + domain := strings.TrimSpace(req.Domain) + if domain == "" { + if parsed, err := url.Parse(failedURL); err == nil { + domain = parsed.Host + } + } + + userAgent := strings.TrimSpace(req.UserAgent) + if userAgent == "" { + userAgent = c.Request.UserAgent() + } + + report := service.DownloadFailureReport{ + Domain: domain, + FailedURL: failedURL, + ErrorMessage: req.ErrorMessage, + ReportedAt: reportedAt, + UserAgent: userAgent, + ClientIP: c.ClientIP(), + } + + if err := h.videoService.ReportDownloadFailure(c.Request.Context(), report); err != nil { + c.JSON(http.StatusInternalServerError, model.Error(http.StatusInternalServerError, "上报失败，请稍后重试")) + return + } + + c.JSON(http.StatusOK, model.Success(gin.H{ + "reported": true, + })) +} diff --git a/internal/remove_watermark/model/video_parse.go b/internal/remove_watermark/model/video_parse.go index d1ab222..d730bd5 100644 --- a/internal/remove_watermark/model/video_parse.go +++ b/internal/remove_watermark/model/video_parse.go @@ -50,3 +50,25 @@ func (VideoParseUnlock) TableName() string { func (VideoParseUnlock) TableComment() string { return "短视频去水印-每日广告解锁" } + +type VideoDownloadFailure struct { + ID uint `gorm:"primarykey" json:"id"` + CreatedAt time.Time `gorm:"comment:创建时间" json:"created_at"` + UpdatedAt time.Time `gorm:"comment:更新时间" json:"updated_at"` + DeletedAt gorm.DeletedAt `gorm:"index;comment:删除时间" json:"-"` + + Domain string `gorm:"size:255;index;comment:来源域名" json:"domain"` + FailedURL string `gorm:"size:1000;comment:下载失败的URL" json:"failed_url"` + ErrorMessage string `gorm:"type:text;comment:失败原因" json:"error_message"` + ReportedAt time.Time `gorm:"index;comment:失败发生时间" json:"reported_at"` + UserAgent string `gorm:"size:255;comment:上报端UA" json:"user_agent"` + ClientIP string `gorm:"size:64;comment:上报IP" json:"client_ip"` +} + +func (VideoDownloadFailure) TableName() string { + return "video_download_failures" +} + +func (VideoDownloadFailure) TableComment() string { + return "短视频去水印-下载失败上报" +} diff --git a/internal/remove_watermark/service/video_service.go b/internal/remove_watermark/service/video_service.go index 32406d3..57c7378 100644 --- a/internal/remove_watermark/service/video_service.go +++ b/internal/remove_watermark/service/video_service.go @@ -19,7 +19,7 @@ import ( rmmodel "wx_service/internal/remove_watermark/model" ) -const removeWatermarkEndpoint = "https://api.23bt.cn/api/d1w/index" +const removeWatermarkEndpoint = "https://api.23bt.cn/api/dsp/index" var ( // 从用户输入的文本里“抓取 URL”的简单正则（找到第一个 http/https 链接） @@ -56,6 +56,15 @@ func (e *ThirdPartyError) Error() string { return fmt.Sprintf("third-party api error: status=%d message=%s", e.StatusCode, e.Message) } +type DownloadFailureReport struct { + Domain string + FailedURL string + ErrorMessage string + ReportedAt time.Time + UserAgent string + ClientIP string +} + func NewVideoService(db *gorm.DB, cfg config.ShortVideoConfig) (*VideoService, error) { timeout := cfg.RequestTimeout if timeout <= 0 { @@ -253,3 +262,33 @@ func truncateString(input string, max int) string { } return input[:max] } + +func (s *VideoService) ReportDownloadFailure(ctx context.Context, report DownloadFailureReport) error { + if report.ReportedAt.IsZero() { + report.ReportedAt = time.Now() + } + + if report.Domain == "" && report.FailedURL != "" { + if parsed, err := url.Parse(report.FailedURL); err == nil { + report.Domain = parsed.Host + } + } + + entry := rmmodel.VideoDownloadFailure{ + Domain: report.Domain, + FailedURL: report.FailedURL, + ErrorMessage: truncateString(report.ErrorMessage, 2000), + ReportedAt: report.ReportedAt, + UserAgent: truncateString(report.UserAgent, 255), + ClientIP: truncateString(report.ClientIP, 64), + } + + if entry.FailedURL == "" { + return errors.New("failed_url is required") + } + + if err := s.db.WithContext(ctx).Create(&entry).Error; err != nil { + return fmt.Errorf("save download failure report: %w", err) + } + return nil +} diff --git a/internal/routes/remove_watermark_routes.go b/internal/routes/remove_watermark_routes.go index 20f5ff6..d263e0f 100644 --- a/internal/routes/remove_watermark_routes.go +++ b/internal/routes/remove_watermark_routes.go @@ -6,8 +6,10 @@ import ( rmhandler "wx_service/internal/remove_watermark/handler" ) -func registerRemoveWatermarkRoutes(protected *gin.RouterGroup, videoHandler *rmhandler.VideoHandler) { - // 去水印相关接口（保持原有路径不变） +func registerRemoveWatermarkRoutes(api *gin.RouterGroup, protected *gin.RouterGroup, videoHandler *rmhandler.VideoHandler) { + // 去水印解析与广告解锁：需要登录鉴权 protected.POST("/video/remove_watermark", videoHandler.RemoveWatermark) protected.POST("/video/remove_watermark/unlock", videoHandler.UnlockQuota) + // 下载失败上报：供其他服务调用，无需鉴权 + api.POST("/video/remove_watermark/report_failure", videoHandler.ReportDownloadFailure) } diff --git a/internal/routes/routes.go b/internal/routes/routes.go index 32c8f20..c4e1c71 100644 --- a/internal/routes/routes.go +++ b/internal/routes/routes.go @@ -43,7 +43,7 @@ func Register( protected.Use(middleware.AuthMiddleware(db, sessionCache)) { registerCommonRoutes(protected, uploadHandler) - registerRemoveWatermarkRoutes(protected, videoHandler) + registerRemoveWatermarkRoutes(api, protected, videoHandler) registerMembershipRoutes(protected, redeemCodeHandler) registerSmokeRoutes(protected, smokeHandler) }