抓取国务院要文

Signed-off-by: tangsong <86121657@qq.com>

src/main/java/com/unis/news/service/BizSinStateCouncilNoticeService.java

@@ -30,4 +30,9 @@ public interface BizSinStateCouncilNoticeService extends IService<BizSinStateCou
      * @date  2024/07/23 10:18
      */
     void add(BizSinStateCouncilNotice addParam);
+
+    /**
+     * 使用定时任务抓取数据
+     */
+    void taskAction();
 }

src/main/java/com/unis/news/service/impl/BizSinQinghaiNewsServiceImpl.java

@@ -42,8 +42,6 @@ import java.util.List;
 public class BizSinQinghaiNewsServiceImpl extends ServiceImpl<BizSinQinghaiNewsMapper, BizSinQinghaiNews> implements BizSinQinghaiNewsService {
     @Resource
     private NewsUrl newsUrl;
-    @Resource
-    private WebClient webClient;
     @Transactional(rollbackFor = Exception.class)
     @Override
     public void add(BizSinQinghaiNews addParam) {
@@ -66,12 +64,14 @@ public class BizSinQinghaiNewsServiceImpl extends ServiceImpl<BizSinQinghaiNewsM
             List<HtmlParagraph> divList=page.getByXPath("//p[@class='item']");
             int i = 0;
             for (HtmlParagraph node : divList) {
+                if (i==8){ //只抓取最近8条数
+                    break;
+                }
                 HtmlSpan span = node.getFirstByXPath("span[@class='gray']"); //得到发布日期
                 HtmlAnchor anchor = node.getFirstByXPath("a[@target='_blank']"); //得到发布标题与地址
                 if (span == null || anchor == null){
                     continue;
                 }
-                i++;
                 //先查询是否有相同标题内容，存在跳过
                 QueryWrapper<BizSinQinghaiNews> queryWrapper = new QueryWrapper<>();
                 queryWrapper.lambda().eq(BizSinQinghaiNews::getTitle,anchor.getAttribute("title"));
@@ -79,10 +79,10 @@ public class BizSinQinghaiNewsServiceImpl extends ServiceImpl<BizSinQinghaiNewsM
                 if (!list.isEmpty()){ //表示此内容已经存在
                     continue;
                 }
+                i++;
                 BizSinQinghaiNews bizSinQinghaiNews  = new BizSinQinghaiNews();
                 //发布日期
                 String[] string = span.getTextContent().split("]");
-                System.out.println("[[["+string[0].replace("[","")+"1111");
                 Date publishDate = DateUtils.parseDate(string[0].replace("[",""),"yyyy/MM/dd");
                 bizSinQinghaiNews.setCreateTime(new Date());
                 bizSinQinghaiNews.setUpdateTime(new Date());
@@ -92,9 +92,7 @@ public class BizSinQinghaiNewsServiceImpl extends ServiceImpl<BizSinQinghaiNewsM
                 bizSinQinghaiNews.setIsRedirect(1);
                 bizSinQinghaiNews.setStatus(2); //默认使用提交状态
                 super.save(bizSinQinghaiNews);
-                if (i==8){ //只抓取最近8条数
-                    break;
-                }
+
             }
 
         } catch (Exception e) {

src/main/java/com/unis/news/service/impl/BizSinStateCouncilNoticeServiceImpl.java

@@ -12,13 +12,24 @@
  */
 package com.unis.news.service.impl;
 
+import com.alibaba.fastjson2.util.DateUtils;
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
 import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.unis.config.NewsUrl;
+import com.unis.news.entity.BizSinQinghaiNews;
 import com.unis.news.entity.BizSinStateCouncilNotice;
 import com.unis.news.mapper.BizSinStateCouncilNoticeMapper;
 import com.unis.news.service.BizSinStateCouncilNoticeService;
+import org.htmlunit.BrowserVersion;
+import org.htmlunit.WebClient;
+import org.htmlunit.html.*;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
 
+import javax.annotation.Resource;
+import java.util.Date;
+import java.util.List;
+
 /**
  * 要闻概况-国务院公告管理表Service接口实现类
  *
@@ -27,11 +38,69 @@ import org.springframework.transaction.annotation.Transactional;
  **/
 @Service
 public class BizSinStateCouncilNoticeServiceImpl extends ServiceImpl<BizSinStateCouncilNoticeMapper, BizSinStateCouncilNotice> implements BizSinStateCouncilNoticeService {
+    @Resource
+    private NewsUrl newsUrl;
 
     @Transactional(rollbackFor = Exception.class)
     @Override
     public void add(BizSinStateCouncilNotice addParam) {
-
+        // 在添加
+        this.save(addParam);
     }
 
+    /**
+     * 使用定时任务抓取数据
+     */
+    @Override
+    public void taskAction() {
+        WebClient webClient = null;
+        try {
+            webClient = new WebClient(BrowserVersion.CHROME);
+            // 配置 WebClient 的参数，例如 JavaScript 支持
+            webClient.getOptions().setJavaScriptEnabled(false);
+            webClient.getOptions().setCssEnabled(false);
+            final HtmlPage page=webClient.getPage(newsUrl.getStateCouncil());    //从指定URL获取HtmlPage
+
+            List<HtmlHeading4> h4List=page.getByXPath("//h4");
+            int i = 0;
+            for (HtmlHeading4 node : h4List) {
+                if (i==8){ //只抓取最近8条数
+                    break;
+                }
+                    HtmlSpan span = node.getFirstByXPath("span[@class='date']"); //得到发布日期
+                    HtmlAnchor anchor = node.getFirstByXPath("a[@target='_blank']"); //得到发布标题与地址
+                    if (span == null || anchor == null){
+                        continue;
+                    }
+                    i++;
+                    //先查询是否有相同标题内容，存在跳过
+                    String title = anchor.getTextContent().trim();
+                    QueryWrapper<BizSinStateCouncilNotice> queryWrapper = new QueryWrapper<>();
+                    queryWrapper.lambda().eq(BizSinStateCouncilNotice::getTitle,title);
+                    List<BizSinStateCouncilNotice> list =  this.list(queryWrapper);
+                    if (!list.isEmpty()){ //表示此内容已经存在
+                        continue;
+                    }
+                    BizSinStateCouncilNotice notice  = new BizSinStateCouncilNotice();
+                    //发布日期
+                    Date publishDate = DateUtils.parseDate(span.getTextContent().trim(),"yyyy-MM-dd");
+                    notice.setCreateTime(new Date());
+                    notice.setUpdateTime(new Date());
+                    notice.setTitle(title);
+                    notice.setPublishDate(publishDate);
+                    notice.setRedirectUrl(anchor.getAttribute("href").trim());
+                    notice.setIsRedirect(1);
+                    notice.setStatus(2); //默认使用提交状态
+                    super.save(notice);
+
+            }
+
+
+        } catch (Exception e) {
+            e.printStackTrace();
+        }finally {
+            webClient.close();    //关闭客户端
+     }
+
+    }
 }

src/main/java/com/unis/news/tasksNews/TasksNewsRunning.java

@@ -3,6 +3,7 @@ package com.unis.news.tasksNews;
 import com.unis.config.NewsUrl;
 import com.unis.news.entity.BizSinQinghaiNews;
 import com.unis.news.service.BizSinQinghaiNewsService;
+import com.unis.news.service.BizSinStateCouncilNoticeService;
 import lombok.extern.slf4j.Slf4j;
 import org.htmlunit.BrowserVersion;
 import org.htmlunit.Page;
@@ -26,67 +27,30 @@ public class TasksNewsRunning {
     @Resource
     private BizSinQinghaiNewsService newsService;
     @Resource
+    private BizSinStateCouncilNoticeService bizSinStateCouncilNoticeService;
+    @Resource
     private NewsUrl newsUrl;
     @Resource
     private WebClient webClient;
 
-    /**
-     * test
-     */
-    //  n秒 直接在方法上使用注解即可完成扫描
-    @Scheduled(cron = "*/20 * * * * ?")
-    public void run() throws Exception {
-        // 测试查询
-        List<BizSinQinghaiNews> list = newsService.list();
-        log.info("test get all list for news: {}", list);
-
-        log.info("newsUrl stateCouncil: {}", newsUrl.getStateCouncil());
-        log.info("newsUrl qinghai: {}", newsUrl.getQinghai());
 
-        final HtmlPage page = webClient.getPage(newsUrl.getQinghai());
-        log.info("page: {}", page);
-
-        List<HtmlAnchor> divList = page.getByXPath("//a");
-        /*for (HtmlAnchor node : divList) {
-            log.info("webClient stateCouncil asXml: {}", node.asXml());
-            log.info("webClient stateCouncil href: {}", node.getAttribute("href"));
-            log.info("webClient stateCouncil href: {}", node.getAttribute("href"));
-            log.info("webClient stateCouncil src: {}", node.getAttribute("src"));
-            log.info("webClient stateCouncil getLocalName: {}", node.getLocalName());
-        }*/
-
-        log.info("----每天n秒执行---");
-    }
     /**
      * 抓取青海省政务网站数据，晚上3点执行一次
      */
     @Scheduled(cron = "0 0 3 * * ?")
-    public void qinghaiNewsrun() throws Exception {
+    public void qinghaiNewsun()  {
         newsService.taskAction();
-        log.info("----每天n秒执行---");
+        log.info("----抓取青海省政务网站数据，晚上3点执行一次---");
+    }
+    /**
+     * 抓取国务院网站数据，晚上3点执行一次
+     */
+    @Scheduled(cron = "0 0 3 * * ?")
+    public void stateCouncilNoticeun()   {
+        bizSinStateCouncilNoticeService.taskAction();
+        log.info("----抓取国务院网站数据，晚上3点执行一次---");
     }
 
-    public static void main(String[] args) {
-        WebClient webClient=null;
-        try {
-            webClient= new WebClient(BrowserVersion.CHROME);    //定义一个WebClient
-            final HtmlPage page=webClient.getPage("http://fgw.qinghai.gov.cn//");    //从指定URL获取HtmlPage
-
-            List<HtmlAnchor> divList=page.getByXPath("//a");
-            for (HtmlAnchor node : divList) {
-                System.out.println("***********************************************8");
-                System.out.println("输出："+node.asXml());
-                System.out.println("输出1："+node.getAttribute("href"));
-                System.out.println("输出2："+node.getAttribute("src"));
-                System.out.println("输出3："+node.getLocalName());
-            }
 
-        } catch (Exception e) {
-            // TODO: handle exception
-            e.printStackTrace();
-        }finally {
-            webClient.close();    //关闭客户端
-        }
-    }
 
 }