|
|
@@ -18,11 +18,12 @@ import org.springframework.http.HttpHeaders;
|
|
|
import org.springframework.http.ResponseEntity;
|
|
|
import org.springframework.stereotype.Component;
|
|
|
|
|
|
-import java.time.Duration;
|
|
|
+import java.io.IOException;
|
|
|
+import java.net.HttpURLConnection;
|
|
|
+import java.net.URL;
|
|
|
import java.util.HashMap;
|
|
|
import java.util.List;
|
|
|
import java.util.Map;
|
|
|
-import java.util.stream.Collectors;
|
|
|
|
|
|
/**
|
|
|
* @description:
|
|
|
@@ -39,6 +40,8 @@ public class HtCrawlerRequestComponent {
|
|
|
|
|
|
private final RedissonClient redissonClient;
|
|
|
|
|
|
+ private final HuaTaiCrawlerCacheComponent huaTaiCrawlerCacheComponent;
|
|
|
+
|
|
|
/**
|
|
|
* 登录
|
|
|
* @param username
|
|
|
@@ -47,22 +50,27 @@ public class HtCrawlerRequestComponent {
|
|
|
* @param loginsNumber
|
|
|
* @return
|
|
|
*/
|
|
|
- public List<String> authenticate(String username, String password, String comcode, int loginsNumber) {
|
|
|
+ public String authenticate(String username, String password, String comcode, int loginsNumber) {
|
|
|
+ String haTaiCookie = getHaTaiCookie(RequestUrl.NE_COOKIE.getRequestUrl(huaTaiCrawlerProperties.getBaseUrl()));
|
|
|
+ System.out.println(haTaiCookie);
|
|
|
HtCrawlerLoginRequest htCrawlerLoginRequest = new HtCrawlerLoginRequest(username, password, comcode);
|
|
|
- ResponseEntity<HtCrawlerLoginResponse> loginResponseEntity = requestComponent.objectPost(RequestUrl.AUTHENTICATE.getRequestUrl(huaTaiCrawlerProperties.getBaseUrl()), htCrawlerLoginRequest, HtCrawlerLoginResponse.class);
|
|
|
- List<String> cookies = loginResponseEntity.getHeaders().get("Set-Cookie");
|
|
|
- if (cookies != null && !cookies.isEmpty()) {
|
|
|
- Map<String, Object> authorizationMap = buildCache(loginResponseEntity);
|
|
|
- RMap<Object, Object> map = redissonClient.getMap(CacheConstant.HENGBANG_AUTHORIZATION + username);
|
|
|
- map.putAll(authorizationMap);
|
|
|
- map.expire(Duration.ofDays(CacheConstant.GR_SPECIAL_AGREEMENT_TIME));
|
|
|
- return (List<String>) authorizationMap.get(HtRequestConstant.COOKIE);
|
|
|
- }
|
|
|
- loginsNumber = loginsNumber - 1;
|
|
|
- if (loginsNumber <= 0) {
|
|
|
- throw new SystemException("华泰财险登录异常");
|
|
|
- }
|
|
|
- return authenticate(username, password, comcode, loginsNumber);
|
|
|
+ HttpHeaders httpHeaders = getHeaders();
|
|
|
+ httpHeaders.add("referer",RequestUrl.COOKIE.getUrl());
|
|
|
+ ResponseEntity<String> loginResponseEntity = requestComponent.objectPost(RequestUrl.NE_AUTHENTICATE.getRequestUrl(huaTaiCrawlerProperties.getBaseUrl()), htCrawlerLoginRequest, String.class,httpHeaders);
|
|
|
+ System.out.println(loginResponseEntity);
|
|
|
+ return loginResponseEntity.getBody();
|
|
|
+ }
|
|
|
+
|
|
|
+ /**
|
|
|
+ * session_keep
|
|
|
+ * @return
|
|
|
+ */
|
|
|
+ public String sessionKeep(String referer){
|
|
|
+ HttpHeaders httpHeaders = getHeaders();
|
|
|
+ httpHeaders.add("referer",referer);
|
|
|
+ String response = requestComponent.get(RequestUrl.NE_SESSION_KEEP.getRequestUrl(huaTaiCrawlerProperties.getBaseUrl()),null,null, String.class,httpHeaders);
|
|
|
+ System.out.println(response);
|
|
|
+ return response;
|
|
|
}
|
|
|
|
|
|
public Map<String, Object> buildCache(ResponseEntity<HtCrawlerLoginResponse> loginResponseEntity) {
|
|
|
@@ -74,24 +82,135 @@ public class HtCrawlerRequestComponent {
|
|
|
|
|
|
/**
|
|
|
* 获取cookie
|
|
|
- * @param username
|
|
|
* @return
|
|
|
*/
|
|
|
- public List<String> getCookie(String username) {
|
|
|
- return (List<String>) redissonClient.getMap(CacheConstant.HUATAI_AUTHORIZATION + username).get(HtRequestConstant.COOKIE);
|
|
|
+ public String getCookie() {
|
|
|
+ return huaTaiCrawlerCacheComponent.getCookie();
|
|
|
+ }
|
|
|
+
|
|
|
+ public String getHaTaiCookie(String link) {
|
|
|
+ try {
|
|
|
+ // 请求头设置
|
|
|
+ String[] headers = {
|
|
|
+ "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
|
|
|
+ "Accept-Language: zh-CN,zh;q=0.9,en;q=0.8",
|
|
|
+ "Cache-Control: no-cache",
|
|
|
+ "Connection: keep-alive",
|
|
|
+ "Pragma: no-cache",
|
|
|
+ "Upgrade-Insecure-Requests: 1",
|
|
|
+ "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
|
|
|
+ };
|
|
|
+ // 创建URL对象
|
|
|
+ URL url = new URL(link);
|
|
|
+
|
|
|
+ // 打开连接
|
|
|
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
|
|
|
+
|
|
|
+ // 设置请求方法为GET
|
|
|
+ connection.setRequestMethod("GET");
|
|
|
+ // 添加请求头
|
|
|
+ for (String header : headers) {
|
|
|
+ String[] parts = header.split(":", 2);
|
|
|
+ if (parts.length == 2) {
|
|
|
+ connection.setRequestProperty(parts[0].trim(), parts[1].trim());
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ // 获取响应码
|
|
|
+ int responseCode = connection.getResponseCode();
|
|
|
+// Map<String, List<String>> requestProperties = connection.getRequestProperties();
|
|
|
+// String cookie1 = connection.getRequestProperty("Cookie");
|
|
|
+ System.out.println("响应码: " + responseCode);
|
|
|
+
|
|
|
+ // 获取所有响应头
|
|
|
+ Map<String, List<String>> headersList = connection.getHeaderFields();
|
|
|
+ StringBuilder builder = new StringBuilder();
|
|
|
+ // 检查并获取Cookie
|
|
|
+ if (headersList.containsKey("Set-Cookie")) {
|
|
|
+ List<String> cookies = headersList.get("Set-Cookie");
|
|
|
+ System.out.println("获取到的Cookie:");
|
|
|
+ String s = cookies.get(0).split(";")[0];
|
|
|
+ String s1 = cookies.get(1).split(";")[0];
|
|
|
+ builder.append(s1).append(";").append(s);
|
|
|
+ huaTaiCrawlerCacheComponent.cacheCookie(builder.toString());
|
|
|
+ return builder.toString();
|
|
|
+ } else {
|
|
|
+ System.out.println("响应头中没有找到Cookie");
|
|
|
+ }
|
|
|
+
|
|
|
+ // 关闭连接
|
|
|
+ connection.disconnect();
|
|
|
+
|
|
|
+ } catch (IOException e) {
|
|
|
+ System.err.println("请求过程中发生错误: " + e.getMessage());
|
|
|
+ e.printStackTrace();
|
|
|
+ }
|
|
|
+
|
|
|
+ // 创建WebClient实例
|
|
|
+// WebClient webClient = WebClient.create();
|
|
|
+//
|
|
|
+// // 发送GET请求,获取完整响应(包括响应头)
|
|
|
+// Mono<ClientResponse> responseMono = webClient.get()
|
|
|
+// .uri("http://127.0.0.1/ht")
|
|
|
+// .header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
|
|
|
+// .header("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
|
|
|
+// .header("Cache-Control", "no-cache")
|
|
|
+// .header("Connection", "keep-alive")
|
|
|
+// .header("Pragma", "no-cache")
|
|
|
+// .header("Upgrade-Insecure-Requests", "1")
|
|
|
+// .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36")
|
|
|
+// .exchangeToMono(response -> Mono.just(response));
|
|
|
+//
|
|
|
+// // 阻塞获取完整响应
|
|
|
+// ClientResponse response = responseMono.block();
|
|
|
+//
|
|
|
+// if (response != null) {
|
|
|
+// // 获取响应状态码
|
|
|
+// System.out.println("响应状态码: " + response.statusCode());
|
|
|
+//
|
|
|
+// // 获取响应头中的Cookie
|
|
|
+// HttpHeaders headers = response.headers().asHttpHeaders();
|
|
|
+// List<String> cookies = headers.get("Set-Cookie");
|
|
|
+//
|
|
|
+// if (cookies != null && !cookies.isEmpty()) {
|
|
|
+// System.out.println("获取到的Cookie:");
|
|
|
+// for (String cookie : cookies) {
|
|
|
+// System.out.println(cookie);
|
|
|
+// }
|
|
|
+// return cookies;
|
|
|
+// } else {
|
|
|
+// System.out.println("响应头中没有Cookie");
|
|
|
+// }
|
|
|
+//
|
|
|
+// // 读取响应体
|
|
|
+// String responseBody = response.bodyToMono(String.class).block();
|
|
|
+// System.out.println("\n响应体内容: " + responseBody);
|
|
|
+// }
|
|
|
+
|
|
|
+ return null;
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* 获取请求头
|
|
|
* @return
|
|
|
*/
|
|
|
- public HttpHeaders getHeaders(String username, String password, String comcode) {
|
|
|
- List<String> cookie = getCookie(username);
|
|
|
- if (cookie == null || cookie.isEmpty()) {
|
|
|
- cookie = authenticate(username, password, comcode, HtRequestConstant.REQUEST_NUMBER);
|
|
|
+ public HttpHeaders getHeaders() {
|
|
|
+ String cookie = getCookie();
|
|
|
+ if (cookie.isEmpty()) {
|
|
|
+ cookie = getHaTaiCookie(RequestUrl.NE_COOKIE.getRequestUrl(huaTaiCrawlerProperties.getBaseUrl()));
|
|
|
}
|
|
|
HttpHeaders headers = new HttpHeaders();
|
|
|
- headers.add(HtRequestConstant.COOKIE, cookie.stream().collect(Collectors.joining(";")));
|
|
|
+ headers.add(HtRequestConstant.COOKIE, cookie);
|
|
|
+ headers.add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7");
|
|
|
+ headers.add("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8");
|
|
|
+ headers.add("Cache-Control", "no-cache");
|
|
|
+ headers.add("Connection","keep-alive");
|
|
|
+ headers.add("Content-Type", "application/x-www-form-urlencoded");
|
|
|
+ headers.add("Origin", "http://172.16.100.27:9000");
|
|
|
+ headers.add("Pragma", "no-cache");
|
|
|
+ headers.add("Upgrade-Insecure-Requests", "1");
|
|
|
+ headers.add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36");
|
|
|
+
|
|
|
return headers;
|
|
|
}
|
|
|
|
|
|
@@ -102,7 +221,7 @@ public class HtCrawlerRequestComponent {
|
|
|
* @return
|
|
|
*/
|
|
|
public HtCrawlerCarQueryResponse.DataDTO carQuery(HtCrawlerCarQueryRequest htCrawlerCarQueryRequest, HttpHeaders headers) throws Exception {
|
|
|
- ResponseEntity<HtCrawlerCarQueryResponse> carQueryResponseEntity = requestComponent.objectPost(RequestUrl.CAR_QUERY.getRequestUrl(huaTaiCrawlerProperties.getBaseUrl()), htCrawlerCarQueryRequest, HtCrawlerCarQueryResponse.class, headers);
|
|
|
+ ResponseEntity<HtCrawlerCarQueryResponse> carQueryResponseEntity = requestComponent.objectPost(RequestUrl.NE_CAR_QUERY.getRequestUrl(huaTaiCrawlerProperties.getBaseUrl()), htCrawlerCarQueryRequest, HtCrawlerCarQueryResponse.class, headers);
|
|
|
HtCrawlerCarQueryResponse carQueryResponse = carQueryResponseEntity.getBody();
|
|
|
String decryptKey = HuaTaiCryptUtil.RSADecrypt(carQueryResponse.getAccessToken());
|
|
|
String decryptData = HuaTaiCryptUtil.AESDecrypt(carQueryResponse.getData(), decryptKey);
|