HtmlUnit爬虫

Signed-off-by: tangsong <86121657@qq.com>

unis-web-app/pom.xml

@@ -74,6 +74,12 @@
             <version>${dependency-check-maven.version}</version>
         </dependency>
 
+        <!--htmlunit爬虫-->
+        <dependency>
+            <groupId>org.htmlunit</groupId>
+            <artifactId>htmlunit</artifactId>
+            <version>4.3.0</version>
+        </dependency>
 
         <!-- https://mvnrepository.com/artifact/io.netty/netty-common -->
         <dependency>

unis-web-app/src/test/java/htmlUnit/HtmlUnitTest.java

@@ -0,0 +1,42 @@
+package htmlUnit;
+
+import org.htmlunit.BrowserVersion;
+import org.htmlunit.WebClient;
+import org.htmlunit.html.HtmlBody;
+import org.htmlunit.html.HtmlPage;
+import org.junit.Test;
+
+import java.util.List;
+
+/**
+ * htmlUnit测试
+ * @Date: 2024-07-19 09:51
+ * @Author: TangSong
+ * @Description:
+ */
+public class HtmlUnitTest {
+
+
+
+    @Test
+    public  void testHtml() {
+        WebClient webClient=null;
+        try {
+            webClient= new WebClient(BrowserVersion.CHROME);    //定义一个WebClient
+            final HtmlPage page=webClient.getPage("http://fgw.qinghai.gov.cn//");    //从指定URL获取HtmlPage
+
+            List<HtmlBody> divList=page.getByXPath("//body");
+            for (HtmlBody htmlBody : divList) {
+                System.out.println("***********************************************8");
+                System.out.println("输出："+htmlBody.asXml());
+            }
+
+        } catch (Exception e) {
+            // TODO: handle exception
+            e.printStackTrace();
+        }finally {
+            webClient.close();    //关闭客户端
+        }
+    }
+
+}