解决紫金获取验证码失败问题

tenant/insurance/quotation-zijin/src/main/java/com/jzg/quotation/zijin/crawler/component/ZiJinCrawlerRequestComponent.java

@@ -9,6 +9,7 @@ import com.jzg.quotation.zijin.crawler.entity.response.*;
 import com.jzg.quotation.zijin.crawler.enums.RequestUrl;
 import com.jzg.quotation.zijin.crawler.properties.ZiJinCrawlerProperties;
 import com.jzg.quotation.zijin.crawler.util.DESEncryption;
+import lombok.Data;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.jetbrains.annotations.NotNull;
@@ -100,7 +101,6 @@ public class ZiJinCrawlerRequestComponent {
         log.info("ziJinCrawlerCommonLoginRequest请求参数的值：{}", JSON.toJSONString(ziJinCrawlerCommonLoginRequest));
 
         HttpHeaders httpHeaders = setHttpHeaders();
-        httpHeaders.setContentLength(144);
         String jsonString = JSON.toJSONString(ziJinCrawlerCommonLoginRequest);
         ResponseEntity<ZiJinCrawlerCommonLoginResponse> responseEntity = requestComponent.post(RequestUrl.COMMONLOGIN.getRequestUrl(ziJinCrawlerProperties.getBaseUrl()), jsonString, ZiJinCrawlerCommonLoginResponse.class, httpHeaders);
         return responseEntity.getBody();
@@ -116,7 +116,7 @@ public class ZiJinCrawlerRequestComponent {
     public ZiJinCrawlerCheckCaptchaResponse checkCaptchaAndGetToken(ZiJinCrawlerCheckCaptchaRequest ziJinCrawlerCheckCaptchaRequest, String token) throws IOException {
         log.info("ziJinCrawlerCheckCaptchaRequest请求参数的值：{}", JSON.toJSONString(ziJinCrawlerCheckCaptchaRequest));
         HttpHeaders httpHeaders = setHttpHeaders();
-        httpHeaders.setContentLength(289);
+        // 注意：不要手动设置Content-Length，由底层WebClient根据body自动计算，否则会被WAF判定为异常请求(412)
         // 需要单独设置这个
         httpHeaders.set("Authorization", token);
         httpHeaders.set("loginToken", token);
@@ -137,7 +137,7 @@ public class ZiJinCrawlerRequestComponent {
     public ZiJinCrawlerCheckTokenResponse checkToken(ZiJinCrawlerCheckTokenRequest ziJinCrawlerCheckTokenRequest, String token) throws IOException {
         log.info("ziJinCrawlerCheckTokenRequest请求参数的值：{}", JSON.toJSONString(ziJinCrawlerCheckTokenRequest));
         HttpHeaders httpHeaders = setHttpHeaders();
-        httpHeaders.setContentLength(66);
+        // 注意：不要手动设置Content-Length，由底层WebClient根据body自动计算，否则会被WAF判定为异常请求(412)
         // 需要单独设置这个
         httpHeaders.set("Authorization", token);
         httpHeaders.set("loginToken", token);
@@ -514,13 +514,40 @@ public class ZiJinCrawlerRequestComponent {
 
 
     /**
-     * 获取Cookie的值
-     * 通过调用Python外放接口获取Cookie
+     * Python外放接口返回的Cookie信息
+     * 用于动态设置请求头中的Cookie、x-csrf-token、User-Agent等，避免硬编码导致WAF校验失败
+     */
+    @Data
+    public static class CookieInfo {
+        /** 完整的Cookie字符串，包含WAF挑战Cookie和csrfToken */
+        private String cookie;
+        /** CSRF Token值，用于x-csrf-token请求头 */
+        private String csrfToken;
+        /** User-Agent字符串，需与获取Cookie时Python使用的UA保持一致以通过WAF指纹校验 */
+        private String ua;
+    }
+
+    /**
+     * 调用Python外放接口获取Cookie及关联信息
+     * 返回包含cookie、csrfToken、ua的完整信息对象，用于动态构建请求头
+     * <p>
+     * Python接口返回示例：
+     * <pre>{@code
+     * {
+     *     "ok": true,
+     *     "cookie": "c7IBxtKJpAH2O=...; c7IBxtKJpAH2P=...; csrfToken=uZvxgMXq-TPNbLU8jJhSOpxmaynt9YmmuxH0",
+     *     "pLen": 343,
+     *     "csrfToken": "uZvxgMXq-TPNbLU8jJhSOpxmaynt9YmmuxH0",
+     *     "ua": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+     *     "impersonate": "chrome131",
+     *     "elapsed_ms": 3801
+     * }
+     * }</pre>
      *
-     * @return String Cookie字符串
+     * @return CookieInfo Cookie信息对象
      * @throws IOException 如果获取Cookie时发生错误
      */
-    public String getCookie() throws IOException {
+    public CookieInfo getCookieInfo() throws IOException {
         String cookieUrl = ziJinCrawlerProperties.getCookiePath();
         if (cookieUrl == null || cookieUrl.trim().isEmpty()) {
             throw new IOException("紫金crawler.cookie-path未配置");
@@ -531,17 +558,24 @@ public class ZiJinCrawlerRequestComponent {
             if (body == null || body.trim().isEmpty()) {
                 throw new IOException("获取Cookie失败: Python接口返回空内容");
             }
-            // 解析JSON响应，提取cookie字段
+            log.info("获取Cookie响应：{}", body);
+            // 解析JSON响应，提取cookie、csrfToken、ua字段
             com.alibaba.fastjson2.JSONObject jsonObject = JSON.parseObject(body);
             if (!jsonObject.getBooleanValue("ok")) {
                 throw new IOException("获取Cookie失败: Python接口返回ok=false");
             }
-            String cookie = jsonObject.getString("cookie");
-            if (cookie == null || cookie.trim().isEmpty()) {
+            CookieInfo cookieInfo = new CookieInfo();
+            cookieInfo.setCookie(jsonObject.getString("cookie"));
+            cookieInfo.setCsrfToken(jsonObject.getString("csrfToken"));
+            cookieInfo.setUa(jsonObject.getString("ua"));
+            if (cookieInfo.getCookie() == null || cookieInfo.getCookie().trim().isEmpty()) {
                 throw new IOException("获取Cookie失败: 响应中cookie字段为空");
             }
-            log.info("紫金Cookie获取成功");
-            return cookie.trim();
+            if (cookieInfo.getCsrfToken() == null || cookieInfo.getCsrfToken().trim().isEmpty()) {
+                throw new IOException("获取Cookie失败: 响应中csrfToken字段为空");
+            }
+            log.info("紫金Cookie获取成功，耗时: {}ms", jsonObject.getString("elapsed_ms"));
+            return cookieInfo;
         } catch (IOException e) {
             throw e;
         } catch (Exception e) {
@@ -550,6 +584,45 @@ public class ZiJinCrawlerRequestComponent {
         }
     }
 
+    /**
+     * 获取Cookie字符串
+     * 通过调用Python外放接口获取Cookie，返回完整的Cookie字符串（包含WAF挑战Cookie和csrfToken）
+     *
+     * @return String Cookie字符串
+     * @throws IOException 如果获取Cookie时发生错误
+     */
+    public String getCookie() throws IOException {
+        return getCookieInfo().getCookie();
+    }
+
+    /**
+     * 从User-Agent字符串中解析Chrome主版本号
+     * 例如: "Mozilla/5.0 ... Chrome/131.0.0.0 Safari/537.36" → "131"
+     * 用于构造与UA一致的sec-ch-ua请求头，避免版本号不匹配导致WAF校验失败
+     *
+     * @param ua User-Agent字符串
+     * @return Chrome主版本号字符串，解析失败时返回"131"
+     */
+    private String extractChromeVersion(String ua) {
+        if (ua == null) {
+            return "131";
+        }
+        String marker = "Chrome/";
+        int idx = ua.indexOf(marker);
+        if (idx < 0) {
+            return "131";
+        }
+        int start = idx + marker.length();
+        int end = ua.indexOf(".", start);
+        if (end > start) {
+            String version = ua.substring(start, end);
+            if (!version.isEmpty()) {
+                return version;
+            }
+        }
+        return "131";
+    }
+
     /**
      * 设置HTTP请求头
      *
@@ -564,14 +637,16 @@ public class ZiJinCrawlerRequestComponent {
     @NotNull
     private HttpHeaders setHttpHeaders(String token) throws IOException {
         HttpHeaders headers = new HttpHeaders();
-        String cookie = getCookie();
-        headers.set("Cookie", "csrfToken=CrwLEmSfUevAb8PcN7kXff-I; " + cookie);
+        // 动态获取Cookie信息（包含WAF挑战Cookie、csrfToken、ua），避免硬编码导致WAF校验失败
+        CookieInfo cookieInfo = getCookieInfo();
+        // Cookie字符串已包含csrfToken，直接使用Python返回的完整Cookie
+        headers.set("Cookie", cookieInfo.getCookie());
         headers.set("Accept", "application/json, text/plain, */*, application/problem+json");
         // headers.set("Accept-Encoding", "gzip, deflate, br, zstd");
         headers.set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8");
         headers.set("Cache-Control", "no-cache");
         headers.set("Connection", "keep-alive");
-        headers.set("Content-Length", "23");
+        // 注意：不要手动设置Content-Length，由底层WebClient根据body自动计算，否则会被WAF判定为异常请求(412)
         headers.set("Host", "kscd.zking.com");
         headers.set("Origin", "https://kscd.zking.com");
         headers.set("Pragma", "no-cache");
@@ -580,11 +655,19 @@ public class ZiJinCrawlerRequestComponent {
         headers.set("Sec-Fetch-Mode", "cors");
         headers.set("Sec-Fetch-Site", "same-origin");
         headers.set("Tokentype", "DEFULT");
-        headers.set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36");
-        headers.set("sec-ch-ua", "\"Google Chrome\";v=\"141\", \"Not?A_Brand\";v=\"8\", \"Chromium\";v=\"141\"");
+        // User-Agent需与Python获取Cookie时使用的UA保持一致，否则WAF指纹校验会失败
+        String ua = cookieInfo.getUa();
+        if (ua == null || ua.trim().isEmpty()) {
+            ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
+        }
+        headers.set("User-Agent", ua);
+        // 从UA中解析Chrome版本号，构造与UA一致的sec-ch-ua头，避免版本号不匹配
+        String chromeVersion = extractChromeVersion(ua);
+        headers.set("sec-ch-ua", "\"Google Chrome\";v=\"" + chromeVersion + "\", \"Not?A_Brand\";v=\"8\", \"Chromium\";v=\"" + chromeVersion + "\"");
         headers.set("sec-ch-ua-mobile", "?0");
         headers.set("sec-ch-ua-platform", "\"Windows\"");
-        headers.set("x-csrf-token", "CrwLEmSfUevAb8PcN7kXff-I");
+        // x-csrf-token使用Python返回的动态值，不再硬编码
+        headers.set("x-csrf-token", cookieInfo.getCsrfToken());
         headers.set("content-type", "application/json");
         if (StringUtils.hasText(token)) {
             headers.set("Authorization", token);