|
|
@@ -118,8 +118,6 @@ public class HtCrawlerRequestComponent {
|
|
|
|
|
|
// 获取响应码
|
|
|
int responseCode = connection.getResponseCode();
|
|
|
-// Map<String, List<String>> requestProperties = connection.getRequestProperties();
|
|
|
-// String cookie1 = connection.getRequestProperty("Cookie");
|
|
|
System.out.println("响应码: " + responseCode);
|
|
|
|
|
|
// 获取所有响应头
|
|
|
@@ -145,48 +143,6 @@ public class HtCrawlerRequestComponent {
|
|
|
System.err.println("请求过程中发生错误: " + e.getMessage());
|
|
|
e.printStackTrace();
|
|
|
}
|
|
|
-
|
|
|
- // 创建WebClient实例
|
|
|
-// WebClient webClient = WebClient.create();
|
|
|
-//
|
|
|
-// // 发送GET请求,获取完整响应(包括响应头)
|
|
|
-// Mono<ClientResponse> responseMono = webClient.get()
|
|
|
-// .uri("http://127.0.0.1/ht")
|
|
|
-// .header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
|
|
|
-// .header("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8")
|
|
|
-// .header("Cache-Control", "no-cache")
|
|
|
-// .header("Connection", "keep-alive")
|
|
|
-// .header("Pragma", "no-cache")
|
|
|
-// .header("Upgrade-Insecure-Requests", "1")
|
|
|
-// .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36")
|
|
|
-// .exchangeToMono(response -> Mono.just(response));
|
|
|
-//
|
|
|
-// // 阻塞获取完整响应
|
|
|
-// ClientResponse response = responseMono.block();
|
|
|
-//
|
|
|
-// if (response != null) {
|
|
|
-// // 获取响应状态码
|
|
|
-// System.out.println("响应状态码: " + response.statusCode());
|
|
|
-//
|
|
|
-// // 获取响应头中的Cookie
|
|
|
-// HttpHeaders headers = response.headers().asHttpHeaders();
|
|
|
-// List<String> cookies = headers.get("Set-Cookie");
|
|
|
-//
|
|
|
-// if (cookies != null && !cookies.isEmpty()) {
|
|
|
-// System.out.println("获取到的Cookie:");
|
|
|
-// for (String cookie : cookies) {
|
|
|
-// System.out.println(cookie);
|
|
|
-// }
|
|
|
-// return cookies;
|
|
|
-// } else {
|
|
|
-// System.out.println("响应头中没有Cookie");
|
|
|
-// }
|
|
|
-//
|
|
|
-// // 读取响应体
|
|
|
-// String responseBody = response.bodyToMono(String.class).block();
|
|
|
-// System.out.println("\n响应体内容: " + responseBody);
|
|
|
-// }
|
|
|
-
|
|
|
return null;
|
|
|
}
|
|
|
|
|
|
@@ -206,7 +162,27 @@ public class HtCrawlerRequestComponent {
|
|
|
headers.add("Cache-Control", "no-cache");
|
|
|
headers.add("Connection","keep-alive");
|
|
|
headers.add("Content-Type", "application/x-www-form-urlencoded");
|
|
|
- headers.add("Origin", "http://172.16.100.27:9000");
|
|
|
+ headers.add("Origin", "http://172.16.100.37:9999");
|
|
|
+ headers.add("Pragma", "no-cache");
|
|
|
+ headers.add("Upgrade-Insecure-Requests", "1");
|
|
|
+ headers.add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36");
|
|
|
+
|
|
|
+ return headers;
|
|
|
+ }
|
|
|
+
|
|
|
+ public HttpHeaders getHeaders1() {
|
|
|
+ String cookie = getCookie();
|
|
|
+ if (cookie.isEmpty()) {
|
|
|
+ cookie = getHaTaiCookie(RequestUrl.NE_COOKIE.getRequestUrl(huaTaiCrawlerProperties.getBaseUrl()));
|
|
|
+ }
|
|
|
+ HttpHeaders headers = new HttpHeaders();
|
|
|
+ headers.add(HtRequestConstant.COOKIE, cookie);
|
|
|
+ headers.add("Accept", "application/json, text/javascript, */*; q=0.01");
|
|
|
+ headers.add("Accept-Language","zh-CN,zh;q=0.9,en;q=0.8");
|
|
|
+ headers.add("Cache-Control", "no-cache");
|
|
|
+ headers.add("Connection","keep-alive");
|
|
|
+ headers.add("Content-Type", "application/x-www-form-urlencoded");
|
|
|
+ headers.add("Origin", "http://172.16.100.37:9999");
|
|
|
headers.add("Pragma", "no-cache");
|
|
|
headers.add("Upgrade-Insecure-Requests", "1");
|
|
|
headers.add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36");
|