Ver código fonte

first commit

hanqingsong 1 ano atrás
commit
68edbb188e
24 arquivos alterados com 1139 adições e 0 exclusões
  1. 33 0
      .gitignore
  2. 19 0
      .mvn/wrapper/maven-wrapper.properties
  3. 259 0
      mvnw
  4. 149 0
      mvnw.cmd
  5. 91 0
      pom.xml
  6. 15 0
      src/main/java/com/unis/PortalSiteBackCrawlerApplication.java
  7. 24 0
      src/main/java/com/unis/config/HtmlUnitConfig.java
  8. 22 0
      src/main/java/com/unis/config/NewsUrl.java
  9. 19 0
      src/main/java/com/unis/config/SpringBeanConfig.java
  10. 84 0
      src/main/java/com/unis/news/entity/BizSinGrainBureauNotice.java
  11. 69 0
      src/main/java/com/unis/news/entity/BizSinQinghaiNews.java
  12. 28 0
      src/main/java/com/unis/news/mapper/BizSinGrainBureauNoticeMapper.java
  13. 28 0
      src/main/java/com/unis/news/mapper/BizSinQinghaiNewsMapper.java
  14. 5 0
      src/main/java/com/unis/news/mapper/mapping/BizSinGrainBureauNoticeMapper.xml
  15. 5 0
      src/main/java/com/unis/news/mapper/mapping/BizSinQinghaiNewsMapper.xml
  16. 34 0
      src/main/java/com/unis/news/service/BizSinGrainBureauNoticeService.java
  17. 34 0
      src/main/java/com/unis/news/service/BizSinQinghaiNewsService.java
  18. 41 0
      src/main/java/com/unis/news/service/impl/BizSinGrainBureauNoticeServiceImpl.java
  19. 41 0
      src/main/java/com/unis/news/service/impl/BizSinQinghaiNewsServiceImpl.java
  20. 84 0
      src/main/java/com/unis/news/tasksNews/TasksNewsRunning.java
  21. 6 0
      src/main/resources/application-dev.yml
  22. 6 0
      src/main/resources/application-prod.yml
  23. 30 0
      src/main/resources/application.yml
  24. 13 0
      src/test/java/com/unis/PortalSiteBackCrawlerApplicationTests.java

+ 33 - 0
.gitignore

@@ -0,0 +1,33 @@
1
+HELP.md
2
+target/
3
+!.mvn/wrapper/maven-wrapper.jar
4
+!**/src/main/**/target/
5
+!**/src/test/**/target/
6
+
7
+### STS ###
8
+.apt_generated
9
+.classpath
10
+.factorypath
11
+.project
12
+.settings
13
+.springBeans
14
+.sts4-cache
15
+
16
+### IntelliJ IDEA ###
17
+.idea
18
+*.iws
19
+*.iml
20
+*.ipr
21
+
22
+### NetBeans ###
23
+/nbproject/private/
24
+/nbbuild/
25
+/dist/
26
+/nbdist/
27
+/.nb-gradle/
28
+build/
29
+!**/src/main/**/build/
30
+!**/src/test/**/build/
31
+
32
+### VS Code ###
33
+.vscode/

+ 19 - 0
.mvn/wrapper/maven-wrapper.properties

@@ -0,0 +1,19 @@
1
+# Licensed to the Apache Software Foundation (ASF) under one
2
+# or more contributor license agreements.  See the NOTICE file
3
+# distributed with this work for additional information
4
+# regarding copyright ownership.  The ASF licenses this file
5
+# to you under the Apache License, Version 2.0 (the
6
+# "License"); you may not use this file except in compliance
7
+# with the License.  You may obtain a copy of the License at
8
+#
9
+#   https://www.apache.org/licenses/LICENSE-2.0
10
+#
11
+# Unless required by applicable law or agreed to in writing,
12
+# software distributed under the License is distributed on an
13
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+# KIND, either express or implied.  See the License for the
15
+# specific language governing permissions and limitations
16
+# under the License.
17
+wrapperVersion=3.3.2
18
+distributionType=only-script
19
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.7/apache-maven-3.9.7-bin.zip

+ 259 - 0
mvnw

@@ -0,0 +1,259 @@
1
+#!/bin/sh
2
+# ----------------------------------------------------------------------------
3
+# Licensed to the Apache Software Foundation (ASF) under one
4
+# or more contributor license agreements.  See the NOTICE file
5
+# distributed with this work for additional information
6
+# regarding copyright ownership.  The ASF licenses this file
7
+# to you under the Apache License, Version 2.0 (the
8
+# "License"); you may not use this file except in compliance
9
+# with the License.  You may obtain a copy of the License at
10
+#
11
+#    https://www.apache.org/licenses/LICENSE-2.0
12
+#
13
+# Unless required by applicable law or agreed to in writing,
14
+# software distributed under the License is distributed on an
15
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16
+# KIND, either express or implied.  See the License for the
17
+# specific language governing permissions and limitations
18
+# under the License.
19
+# ----------------------------------------------------------------------------
20
+
21
+# ----------------------------------------------------------------------------
22
+# Apache Maven Wrapper startup batch script, version 3.3.2
23
+#
24
+# Optional ENV vars
25
+# -----------------
26
+#   JAVA_HOME - location of a JDK home dir, required when download maven via java source
27
+#   MVNW_REPOURL - repo url base for downloading maven distribution
28
+#   MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven
29
+#   MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output
30
+# ----------------------------------------------------------------------------
31
+
32
+set -euf
33
+[ "${MVNW_VERBOSE-}" != debug ] || set -x
34
+
35
+# OS specific support.
36
+native_path() { printf %s\\n "$1"; }
37
+case "$(uname)" in
38
+CYGWIN* | MINGW*)
39
+  [ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")"
40
+  native_path() { cygpath --path --windows "$1"; }
41
+  ;;
42
+esac
43
+
44
+# set JAVACMD and JAVACCMD
45
+set_java_home() {
46
+  # For Cygwin and MinGW, ensure paths are in Unix format before anything is touched
47
+  if [ -n "${JAVA_HOME-}" ]; then
48
+    if [ -x "$JAVA_HOME/jre/sh/java" ]; then
49
+      # IBM's JDK on AIX uses strange locations for the executables
50
+      JAVACMD="$JAVA_HOME/jre/sh/java"
51
+      JAVACCMD="$JAVA_HOME/jre/sh/javac"
52
+    else
53
+      JAVACMD="$JAVA_HOME/bin/java"
54
+      JAVACCMD="$JAVA_HOME/bin/javac"
55
+
56
+      if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then
57
+        echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2
58
+        echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2
59
+        return 1
60
+      fi
61
+    fi
62
+  else
63
+    JAVACMD="$(
64
+      'set' +e
65
+      'unset' -f command 2>/dev/null
66
+      'command' -v java
67
+    )" || :
68
+    JAVACCMD="$(
69
+      'set' +e
70
+      'unset' -f command 2>/dev/null
71
+      'command' -v javac
72
+    )" || :
73
+
74
+    if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then
75
+      echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2
76
+      return 1
77
+    fi
78
+  fi
79
+}
80
+
81
+# hash string like Java String::hashCode
82
+hash_string() {
83
+  str="${1:-}" h=0
84
+  while [ -n "$str" ]; do
85
+    char="${str%"${str#?}"}"
86
+    h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296))
87
+    str="${str#?}"
88
+  done
89
+  printf %x\\n $h
90
+}
91
+
92
+verbose() { :; }
93
+[ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; }
94
+
95
+die() {
96
+  printf %s\\n "$1" >&2
97
+  exit 1
98
+}
99
+
100
+trim() {
101
+  # MWRAPPER-139:
102
+  #   Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds.
103
+  #   Needed for removing poorly interpreted newline sequences when running in more
104
+  #   exotic environments such as mingw bash on Windows.
105
+  printf "%s" "${1}" | tr -d '[:space:]'
106
+}
107
+
108
+# parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties
109
+while IFS="=" read -r key value; do
110
+  case "${key-}" in
111
+  distributionUrl) distributionUrl=$(trim "${value-}") ;;
112
+  distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;;
113
+  esac
114
+done <"${0%/*}/.mvn/wrapper/maven-wrapper.properties"
115
+[ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in ${0%/*}/.mvn/wrapper/maven-wrapper.properties"
116
+
117
+case "${distributionUrl##*/}" in
118
+maven-mvnd-*bin.*)
119
+  MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/
120
+  case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in
121
+  *AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;;
122
+  :Darwin*x86_64) distributionPlatform=darwin-amd64 ;;
123
+  :Darwin*arm64) distributionPlatform=darwin-aarch64 ;;
124
+  :Linux*x86_64*) distributionPlatform=linux-amd64 ;;
125
+  *)
126
+    echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2
127
+    distributionPlatform=linux-amd64
128
+    ;;
129
+  esac
130
+  distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip"
131
+  ;;
132
+maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;;
133
+*) MVN_CMD="mvn${0##*/mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;;
134
+esac
135
+
136
+# apply MVNW_REPOURL and calculate MAVEN_HOME
137
+# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-<version>,maven-mvnd-<version>-<platform>}/<hash>
138
+[ -z "${MVNW_REPOURL-}" ] || distributionUrl="$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*"$_MVNW_REPO_PATTERN"}"
139
+distributionUrlName="${distributionUrl##*/}"
140
+distributionUrlNameMain="${distributionUrlName%.*}"
141
+distributionUrlNameMain="${distributionUrlNameMain%-bin}"
142
+MAVEN_USER_HOME="${MAVEN_USER_HOME:-${HOME}/.m2}"
143
+MAVEN_HOME="${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string "$distributionUrl")"
144
+
145
+exec_maven() {
146
+  unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || :
147
+  exec "$MAVEN_HOME/bin/$MVN_CMD" "$@" || die "cannot exec $MAVEN_HOME/bin/$MVN_CMD"
148
+}
149
+
150
+if [ -d "$MAVEN_HOME" ]; then
151
+  verbose "found existing MAVEN_HOME at $MAVEN_HOME"
152
+  exec_maven "$@"
153
+fi
154
+
155
+case "${distributionUrl-}" in
156
+*?-bin.zip | *?maven-mvnd-?*-?*.zip) ;;
157
+*) die "distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'" ;;
158
+esac
159
+
160
+# prepare tmp dir
161
+if TMP_DOWNLOAD_DIR="$(mktemp -d)" && [ -d "$TMP_DOWNLOAD_DIR" ]; then
162
+  clean() { rm -rf -- "$TMP_DOWNLOAD_DIR"; }
163
+  trap clean HUP INT TERM EXIT
164
+else
165
+  die "cannot create temp dir"
166
+fi
167
+
168
+mkdir -p -- "${MAVEN_HOME%/*}"
169
+
170
+# Download and Install Apache Maven
171
+verbose "Couldn't find MAVEN_HOME, downloading and installing it ..."
172
+verbose "Downloading from: $distributionUrl"
173
+verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName"
174
+
175
+# select .zip or .tar.gz
176
+if ! command -v unzip >/dev/null; then
177
+  distributionUrl="${distributionUrl%.zip}.tar.gz"
178
+  distributionUrlName="${distributionUrl##*/}"
179
+fi
180
+
181
+# verbose opt
182
+__MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR=''
183
+[ "${MVNW_VERBOSE-}" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v
184
+
185
+# normalize http auth
186
+case "${MVNW_PASSWORD:+has-password}" in
187
+'') MVNW_USERNAME='' MVNW_PASSWORD='' ;;
188
+has-password) [ -n "${MVNW_USERNAME-}" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;;
189
+esac
190
+
191
+if [ -z "${MVNW_USERNAME-}" ] && command -v wget >/dev/null; then
192
+  verbose "Found wget ... using wget"
193
+  wget ${__MVNW_QUIET_WGET:+"$__MVNW_QUIET_WGET"} "$distributionUrl" -O "$TMP_DOWNLOAD_DIR/$distributionUrlName" || die "wget: Failed to fetch $distributionUrl"
194
+elif [ -z "${MVNW_USERNAME-}" ] && command -v curl >/dev/null; then
195
+  verbose "Found curl ... using curl"
196
+  curl ${__MVNW_QUIET_CURL:+"$__MVNW_QUIET_CURL"} -f -L -o "$TMP_DOWNLOAD_DIR/$distributionUrlName" "$distributionUrl" || die "curl: Failed to fetch $distributionUrl"
197
+elif set_java_home; then
198
+  verbose "Falling back to use Java to download"
199
+  javaSource="$TMP_DOWNLOAD_DIR/Downloader.java"
200
+  targetZip="$TMP_DOWNLOAD_DIR/$distributionUrlName"
201
+  cat >"$javaSource" <<-END
202
+	public class Downloader extends java.net.Authenticator
203
+	{
204
+	  protected java.net.PasswordAuthentication getPasswordAuthentication()
205
+	  {
206
+	    return new java.net.PasswordAuthentication( System.getenv( "MVNW_USERNAME" ), System.getenv( "MVNW_PASSWORD" ).toCharArray() );
207
+	  }
208
+	  public static void main( String[] args ) throws Exception
209
+	  {
210
+	    setDefault( new Downloader() );
211
+	    java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() );
212
+	  }
213
+	}
214
+	END
215
+  # For Cygwin/MinGW, switch paths to Windows format before running javac and java
216
+  verbose " - Compiling Downloader.java ..."
217
+  "$(native_path "$JAVACCMD")" "$(native_path "$javaSource")" || die "Failed to compile Downloader.java"
218
+  verbose " - Running Downloader.java ..."
219
+  "$(native_path "$JAVACMD")" -cp "$(native_path "$TMP_DOWNLOAD_DIR")" Downloader "$distributionUrl" "$(native_path "$targetZip")"
220
+fi
221
+
222
+# If specified, validate the SHA-256 sum of the Maven distribution zip file
223
+if [ -n "${distributionSha256Sum-}" ]; then
224
+  distributionSha256Result=false
225
+  if [ "$MVN_CMD" = mvnd.sh ]; then
226
+    echo "Checksum validation is not supported for maven-mvnd." >&2
227
+    echo "Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2
228
+    exit 1
229
+  elif command -v sha256sum >/dev/null; then
230
+    if echo "$distributionSha256Sum  $TMP_DOWNLOAD_DIR/$distributionUrlName" | sha256sum -c >/dev/null 2>&1; then
231
+      distributionSha256Result=true
232
+    fi
233
+  elif command -v shasum >/dev/null; then
234
+    if echo "$distributionSha256Sum  $TMP_DOWNLOAD_DIR/$distributionUrlName" | shasum -a 256 -c >/dev/null 2>&1; then
235
+      distributionSha256Result=true
236
+    fi
237
+  else
238
+    echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." >&2
239
+    echo "Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2
240
+    exit 1
241
+  fi
242
+  if [ $distributionSha256Result = false ]; then
243
+    echo "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised." >&2
244
+    echo "If you updated your Maven version, you need to update the specified distributionSha256Sum property." >&2
245
+    exit 1
246
+  fi
247
+fi
248
+
249
+# unzip and move
250
+if command -v unzip >/dev/null; then
251
+  unzip ${__MVNW_QUIET_UNZIP:+"$__MVNW_QUIET_UNZIP"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -d "$TMP_DOWNLOAD_DIR" || die "failed to unzip"
252
+else
253
+  tar xzf${__MVNW_QUIET_TAR:+"$__MVNW_QUIET_TAR"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -C "$TMP_DOWNLOAD_DIR" || die "failed to untar"
254
+fi
255
+printf %s\\n "$distributionUrl" >"$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/mvnw.url"
256
+mv -- "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" "$MAVEN_HOME" || [ -d "$MAVEN_HOME" ] || die "fail to move MAVEN_HOME"
257
+
258
+clean || :
259
+exec_maven "$@"

+ 149 - 0
mvnw.cmd

@@ -0,0 +1,149 @@
1
+<# : batch portion
2
+@REM ----------------------------------------------------------------------------
3
+@REM Licensed to the Apache Software Foundation (ASF) under one
4
+@REM or more contributor license agreements.  See the NOTICE file
5
+@REM distributed with this work for additional information
6
+@REM regarding copyright ownership.  The ASF licenses this file
7
+@REM to you under the Apache License, Version 2.0 (the
8
+@REM "License"); you may not use this file except in compliance
9
+@REM with the License.  You may obtain a copy of the License at
10
+@REM
11
+@REM    https://www.apache.org/licenses/LICENSE-2.0
12
+@REM
13
+@REM Unless required by applicable law or agreed to in writing,
14
+@REM software distributed under the License is distributed on an
15
+@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16
+@REM KIND, either express or implied.  See the License for the
17
+@REM specific language governing permissions and limitations
18
+@REM under the License.
19
+@REM ----------------------------------------------------------------------------
20
+
21
+@REM ----------------------------------------------------------------------------
22
+@REM Apache Maven Wrapper startup batch script, version 3.3.2
23
+@REM
24
+@REM Optional ENV vars
25
+@REM   MVNW_REPOURL - repo url base for downloading maven distribution
26
+@REM   MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven
27
+@REM   MVNW_VERBOSE - true: enable verbose log; others: silence the output
28
+@REM ----------------------------------------------------------------------------
29
+
30
+@IF "%__MVNW_ARG0_NAME__%"=="" (SET __MVNW_ARG0_NAME__=%~nx0)
31
+@SET __MVNW_CMD__=
32
+@SET __MVNW_ERROR__=
33
+@SET __MVNW_PSMODULEP_SAVE=%PSModulePath%
34
+@SET PSModulePath=
35
+@FOR /F "usebackq tokens=1* delims==" %%A IN (`powershell -noprofile "& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}"`) DO @(
36
+  IF "%%A"=="MVN_CMD" (set __MVNW_CMD__=%%B) ELSE IF "%%B"=="" (echo %%A) ELSE (echo %%A=%%B)
37
+)
38
+@SET PSModulePath=%__MVNW_PSMODULEP_SAVE%
39
+@SET __MVNW_PSMODULEP_SAVE=
40
+@SET __MVNW_ARG0_NAME__=
41
+@SET MVNW_USERNAME=
42
+@SET MVNW_PASSWORD=
43
+@IF NOT "%__MVNW_CMD__%"=="" (%__MVNW_CMD__% %*)
44
+@echo Cannot start maven from wrapper >&2 && exit /b 1
45
+@GOTO :EOF
46
+: end batch / begin powershell #>
47
+
48
+$ErrorActionPreference = "Stop"
49
+if ($env:MVNW_VERBOSE -eq "true") {
50
+  $VerbosePreference = "Continue"
51
+}
52
+
53
+# calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties
54
+$distributionUrl = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionUrl
55
+if (!$distributionUrl) {
56
+  Write-Error "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties"
57
+}
58
+
59
+switch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) {
60
+  "maven-mvnd-*" {
61
+    $USE_MVND = $true
62
+    $distributionUrl = $distributionUrl -replace '-bin\.[^.]*$',"-windows-amd64.zip"
63
+    $MVN_CMD = "mvnd.cmd"
64
+    break
65
+  }
66
+  default {
67
+    $USE_MVND = $false
68
+    $MVN_CMD = $script -replace '^mvnw','mvn'
69
+    break
70
+  }
71
+}
72
+
73
+# apply MVNW_REPOURL and calculate MAVEN_HOME
74
+# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-<version>,maven-mvnd-<version>-<platform>}/<hash>
75
+if ($env:MVNW_REPOURL) {
76
+  $MVNW_REPO_PATTERN = if ($USE_MVND) { "/org/apache/maven/" } else { "/maven/mvnd/" }
77
+  $distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace '^.*'+$MVNW_REPO_PATTERN,'')"
78
+}
79
+$distributionUrlName = $distributionUrl -replace '^.*/',''
80
+$distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$',''
81
+$MAVEN_HOME_PARENT = "$HOME/.m2/wrapper/dists/$distributionUrlNameMain"
82
+if ($env:MAVEN_USER_HOME) {
83
+  $MAVEN_HOME_PARENT = "$env:MAVEN_USER_HOME/wrapper/dists/$distributionUrlNameMain"
84
+}
85
+$MAVEN_HOME_NAME = ([System.Security.Cryptography.MD5]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join ''
86
+$MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME"
87
+
88
+if (Test-Path -Path "$MAVEN_HOME" -PathType Container) {
89
+  Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME"
90
+  Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD"
91
+  exit $?
92
+}
93
+
94
+if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) {
95
+  Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl"
96
+}
97
+
98
+# prepare tmp dir
99
+$TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile
100
+$TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir"
101
+$TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null
102
+trap {
103
+  if ($TMP_DOWNLOAD_DIR.Exists) {
104
+    try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null }
105
+    catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" }
106
+  }
107
+}
108
+
109
+New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null
110
+
111
+# Download and Install Apache Maven
112
+Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..."
113
+Write-Verbose "Downloading from: $distributionUrl"
114
+Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName"
115
+
116
+$webclient = New-Object System.Net.WebClient
117
+if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) {
118
+  $webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD)
119
+}
120
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
121
+$webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null
122
+
123
+# If specified, validate the SHA-256 sum of the Maven distribution zip file
124
+$distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum
125
+if ($distributionSha256Sum) {
126
+  if ($USE_MVND) {
127
+    Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties."
128
+  }
129
+  Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash
130
+  if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) {
131
+    Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property."
132
+  }
133
+}
134
+
135
+# unzip and move
136
+Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null
137
+Rename-Item -Path "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" -NewName $MAVEN_HOME_NAME | Out-Null
138
+try {
139
+  Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null
140
+} catch {
141
+  if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) {
142
+    Write-Error "fail to move MAVEN_HOME"
143
+  }
144
+} finally {
145
+  try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null }
146
+  catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" }
147
+}
148
+
149
+Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD"

+ 91 - 0
pom.xml

@@ -0,0 +1,91 @@
1
+<?xml version="1.0" encoding="UTF-8"?>
2
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
3
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
4
+    <modelVersion>4.0.0</modelVersion>
5
+    <parent>
6
+        <groupId>org.springframework.boot</groupId>
7
+        <artifactId>spring-boot-starter-parent</artifactId>
8
+        <version>2.5.15</version>
9
+        <relativePath/> <!-- lookup parent from repository -->
10
+    </parent>
11
+    <groupId>com.unis</groupId>
12
+    <artifactId>portal-site-back-crawler</artifactId>
13
+    <version>0.0.1-SNAPSHOT</version>
14
+    <name>portal-site-back-crawler</name>
15
+    <description>portal-site-back-crawler project for Spring Boot</description>
16
+    <properties>
17
+        <java.version>1.8</java.version>
18
+        <skipTests>true</skipTests>
19
+        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
20
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
21
+        <fastjson.version>2.0.24</fastjson.version>
22
+        <mybatis.plus.version>3.5.3.1</mybatis.plus.version>
23
+        <mybatis.version>3.5.10</mybatis.version>
24
+        <mysql.connector.java.version>8.0.28</mysql.connector.java.version>
25
+        <lombok.versin>1.18.22</lombok.versin>
26
+        <htmlunit.version>4.3.0</htmlunit.version>
27
+    </properties>
28
+    <dependencies>
29
+        <!-- https://mvnrepository.com/artifact/org.htmlunit/htmlunit -->
30
+        <!--htmlunit爬虫-->
31
+        <dependency>
32
+            <groupId>org.htmlunit</groupId>
33
+            <artifactId>htmlunit</artifactId>
34
+            <version>${htmlunit.version}</version>
35
+        </dependency>
36
+        <!-- mybatis -->
37
+        <dependency>
38
+            <groupId>org.mybatis</groupId>
39
+            <artifactId>mybatis</artifactId>
40
+            <version>${mybatis.version}</version>
41
+        </dependency>
42
+        <!-- mybatis-plus -->
43
+        <dependency>
44
+            <groupId>com.baomidou</groupId>
45
+            <artifactId>mybatis-plus-boot-starter</artifactId>
46
+            <version>${mybatis.plus.version}</version>
47
+        </dependency>
48
+        <!-- lombok -->
49
+        <dependency>
50
+            <groupId>org.projectlombok</groupId>
51
+            <artifactId>lombok</artifactId>
52
+            <version>${lombok.versin}</version>
53
+        </dependency>
54
+        <!-- mysql -->
55
+        <dependency>
56
+            <groupId>mysql</groupId>
57
+            <artifactId>mysql-connector-java</artifactId>
58
+            <version>${mysql.connector.java.version}</version>
59
+        </dependency>
60
+        <!-- fastjson -->
61
+        <dependency>
62
+            <groupId>com.alibaba</groupId>
63
+            <artifactId>fastjson</artifactId>
64
+            <version>${fastjson.version}</version>
65
+        </dependency>
66
+        <dependency>
67
+            <groupId>org.springframework.boot</groupId>
68
+            <artifactId>spring-boot-configuration-processor</artifactId>
69
+            <optional>true</optional>
70
+        </dependency>
71
+        <dependency>
72
+            <groupId>org.springframework.boot</groupId>
73
+            <artifactId>spring-boot-starter-web</artifactId>
74
+        </dependency>
75
+        <dependency>
76
+            <groupId>org.springframework.boot</groupId>
77
+            <artifactId>spring-boot-starter-test</artifactId>
78
+            <scope>test</scope>
79
+        </dependency>
80
+    </dependencies>
81
+
82
+    <build>
83
+        <plugins>
84
+            <plugin>
85
+                <groupId>org.springframework.boot</groupId>
86
+                <artifactId>spring-boot-maven-plugin</artifactId>
87
+            </plugin>
88
+        </plugins>
89
+    </build>
90
+
91
+</project>

+ 15 - 0
src/main/java/com/unis/PortalSiteBackCrawlerApplication.java

@@ -0,0 +1,15 @@
1
+package com.unis;
2
+
3
+import org.springframework.boot.SpringApplication;
4
+import org.springframework.boot.autoconfigure.SpringBootApplication;
5
+import org.springframework.scheduling.annotation.EnableScheduling;
6
+
7
+@EnableScheduling // 开启任务调度
8
+@SpringBootApplication
9
+public class PortalSiteBackCrawlerApplication {
10
+
11
+    public static void main(String[] args) {
12
+        SpringApplication.run(PortalSiteBackCrawlerApplication.class, args);
13
+    }
14
+
15
+}

+ 24 - 0
src/main/java/com/unis/config/HtmlUnitConfig.java

@@ -0,0 +1,24 @@
1
+package com.unis.config;
2
+
3
+import org.htmlunit.BrowserVersion;
4
+import org.htmlunit.WebClient;
5
+import org.springframework.context.annotation.Bean;
6
+import org.springframework.context.annotation.Configuration;
7
+
8
+/**
9
+ * @author qingsong.han
10
+ * @description: Bean在Spring Boot应用中使用 HtmlUnit
11
+ * @create 2024-07-22 14:35
12
+ */
13
+@Configuration
14
+public class HtmlUnitConfig {
15
+
16
+    @Bean
17
+    public WebClient webClient() {
18
+        WebClient webClient = new WebClient(BrowserVersion.CHROME);
19
+        // 配置 WebClient 的参数,例如 JavaScript 支持
20
+         webClient.getOptions().setJavaScriptEnabled(true);
21
+        return webClient;
22
+    }
23
+
24
+}

+ 22 - 0
src/main/java/com/unis/config/NewsUrl.java

@@ -0,0 +1,22 @@
1
+package com.unis.config;
2
+
3
+import lombok.Data;
4
+import org.springframework.boot.context.properties.ConfigurationProperties;
5
+import org.springframework.stereotype.Component;
6
+
7
+/**
8
+ * @author qingsong.han
9
+ * @description: 获取新闻一级url
10
+ * @create 2024-07-22 14:53
11
+ */
12
+
13
+@Data
14
+@Component
15
+@ConfigurationProperties(prefix = "news")
16
+public class NewsUrl {
17
+    // 国务院
18
+    private String stateCouncil;
19
+
20
+    // 青海省人民政府
21
+    private String qinghai;
22
+}

+ 19 - 0
src/main/java/com/unis/config/SpringBeanConfig.java

@@ -0,0 +1,19 @@
1
+package com.unis.config;
2
+
3
+import org.springframework.context.annotation.Bean;
4
+import org.springframework.context.annotation.Configuration;
5
+import org.springframework.web.client.RestTemplate;
6
+
7
+/**
8
+ * @author qingsong.han
9
+ * @description: spring ben config
10
+ * @create 2024-07-22 10:29
11
+ */
12
+@Configuration
13
+public class SpringBeanConfig {
14
+
15
+    @Bean
16
+    public RestTemplate restTemplate() {
17
+        return new RestTemplate();
18
+    }
19
+}

+ 84 - 0
src/main/java/com/unis/news/entity/BizSinGrainBureauNotice.java

@@ -0,0 +1,84 @@
1
+/*
2
+ * Copyright [2022] [https://www.xiaonuo.vip]
3
+ *
4
+ * Snowy采用APACHE LICENSE 2.0开源协议,您在使用过程中,需要注意以下几点:
5
+ *
6
+ * 1.请不要删除和修改根目录下的LICENSE文件。
7
+ * 2.请不要删除和修改Snowy源码头部的版权声明。
8
+ * 3.本项目代码可免费商业使用,商业使用请保留源码和相关描述文件的项目出处,作者声明等。
9
+ * 4.分发源码时候,请注明软件出处 https://www.xiaonuo.vip
10
+ * 5.不可二次分发开源参与同类竞品,如有想法可联系团队xiaonuobase@qq.com商议合作。
11
+ * 6.若您的项目无法满足以上几点,需要更多功能代码,获取Snowy商业授权许可,请在官网购买授权,地址为 https://www.xiaonuo.vip
12
+ */
13
+package com.unis.news.entity;
14
+
15
+import com.baomidou.mybatisplus.annotation.FieldFill;
16
+import com.baomidou.mybatisplus.annotation.TableField;
17
+import com.baomidou.mybatisplus.annotation.TableId;
18
+import com.baomidou.mybatisplus.annotation.TableName;
19
+import lombok.Getter;
20
+import lombok.Setter;
21
+
22
+import java.util.Date;
23
+
24
+/**
25
+ * 要闻概况-粮食局公告管理表实体
26
+ *
27
+ * @author qingsong.han
28
+ * @date  2024/07/22 12:02
29
+ **/
30
+@Getter
31
+@Setter
32
+@TableName("biz_sin_grain_bureau_notice")
33
+public class BizSinGrainBureauNotice {
34
+
35
+    /** 公告唯一标识符 */
36
+    @TableId
37
+    private String id;
38
+
39
+    /** 创建时间 */
40
+    @TableField(fill = FieldFill.INSERT)
41
+    private Date createTime;
42
+
43
+    /** 创建用户 */
44
+    @TableField(fill = FieldFill.INSERT)
45
+    private String createUser;
46
+
47
+    /** 修改时间 */
48
+    @TableField(fill = FieldFill.UPDATE)
49
+    private Date updateTime;
50
+
51
+    /** 修改用户 */
52
+    @TableField(fill = FieldFill.UPDATE)
53
+    private String updateUser;
54
+
55
+    /** 标题 */
56
+    private String title;
57
+
58
+    /** 索引号 */
59
+    private String indexNumber;
60
+
61
+    /** 主题分类;dev_dict的id; */
62
+    private String subjectClassificationId;
63
+
64
+    /** 发布日期 */
65
+    private Date publishDate;
66
+
67
+    /** 发布机构 */
68
+    private String publishInstitution;
69
+
70
+    /** 发文字号 */
71
+    private String documentNumber;
72
+
73
+    /** 是否有效,0:否;1:是; */
74
+    private Integer isValid;
75
+
76
+    /** 公告内容 */
77
+    private String noticeContent;
78
+
79
+    /** 附件,多个附件用","分割 */
80
+    private String attachments;
81
+
82
+    /** 状态,1:保存;2:提交(启用);3:禁用 */
83
+    private Integer status;
84
+}

+ 69 - 0
src/main/java/com/unis/news/entity/BizSinQinghaiNews.java

@@ -0,0 +1,69 @@
1
+/*
2
+ * Copyright [2022] [https://www.xiaonuo.vip]
3
+ *
4
+ * Snowy采用APACHE LICENSE 2.0开源协议,您在使用过程中,需要注意以下几点:
5
+ *
6
+ * 1.请不要删除和修改根目录下的LICENSE文件。
7
+ * 2.请不要删除和修改Snowy源码头部的版权声明。
8
+ * 3.本项目代码可免费商业使用,商业使用请保留源码和相关描述文件的项目出处,作者声明等。
9
+ * 4.分发源码时候,请注明软件出处 https://www.xiaonuo.vip
10
+ * 5.不可二次分发开源参与同类竞品,如有想法可联系团队xiaonuobase@qq.com商议合作。
11
+ * 6.若您的项目无法满足以上几点,需要更多功能代码,获取Snowy商业授权许可,请在官网购买授权,地址为 https://www.xiaonuo.vip
12
+ */
13
+package com.unis.news.entity;
14
+
15
+import com.baomidou.mybatisplus.annotation.FieldFill;
16
+import com.baomidou.mybatisplus.annotation.TableField;
17
+import com.baomidou.mybatisplus.annotation.TableId;
18
+import com.baomidou.mybatisplus.annotation.TableName;
19
+import lombok.Getter;
20
+import lombok.Setter;
21
+
22
+import java.util.Date;
23
+
24
+/**
25
+ * 要闻概况-青海要闻管理表实体
26
+ *
27
+ * @author qingsong.han
28
+ * @date  2024/07/22 13:56
29
+ **/
30
+@Getter
31
+@Setter
32
+@TableName("biz_sin_qinghai_news")
33
+public class BizSinQinghaiNews {
34
+
35
+    /** 要闻管理唯一标识符 */
36
+    @TableId
37
+    private String id;
38
+
39
+    /** 创建时间 */
40
+    @TableField(fill = FieldFill.INSERT)
41
+    private Date createTime;
42
+
43
+    /** 创建用户 */
44
+    @TableField(fill = FieldFill.INSERT)
45
+    private String createUser;
46
+
47
+    /** 修改时间 */
48
+    @TableField(fill = FieldFill.UPDATE)
49
+    private Date updateTime;
50
+
51
+    /** 修改用户 */
52
+    @TableField(fill = FieldFill.UPDATE)
53
+    private String updateUser;
54
+
55
+    /** 标题 */
56
+    private String title;
57
+
58
+    /** 发布日期 */
59
+    private Date publishDate;
60
+
61
+    /** 是否跳转,0:否;1:是; */
62
+    private Integer isRedirect;
63
+
64
+    /** 跳转地址 */
65
+    private String redirectUrl;
66
+
67
+    /** 状态,1:保存;2:提交(启用);3:禁用 */
68
+    private Integer status;
69
+}

+ 28 - 0
src/main/java/com/unis/news/mapper/BizSinGrainBureauNoticeMapper.java

@@ -0,0 +1,28 @@
1
+/*
2
+ * Copyright [2022] [https://www.xiaonuo.vip]
3
+ *
4
+ * Snowy采用APACHE LICENSE 2.0开源协议,您在使用过程中,需要注意以下几点:
5
+ *
6
+ * 1.请不要删除和修改根目录下的LICENSE文件。
7
+ * 2.请不要删除和修改Snowy源码头部的版权声明。
8
+ * 3.本项目代码可免费商业使用,商业使用请保留源码和相关描述文件的项目出处,作者声明等。
9
+ * 4.分发源码时候,请注明软件出处 https://www.xiaonuo.vip
10
+ * 5.不可二次分发开源参与同类竞品,如有想法可联系团队xiaonuobase@qq.com商议合作。
11
+ * 6.若您的项目无法满足以上几点,需要更多功能代码,获取Snowy商业授权许可,请在官网购买授权,地址为 https://www.xiaonuo.vip
12
+ */
13
+package com.unis.news.mapper;
14
+
15
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
16
+import com.unis.news.entity.BizSinGrainBureauNotice;
17
+import org.apache.ibatis.annotations.Mapper;
18
+
19
+/**
20
+ * 要闻概况-粮食局公告管理表Mapper接口
21
+ *
22
+ * @author qingsong.han
23
+ * @date  2024/07/22 12:02
24
+ **/
25
+@Mapper
26
+public interface BizSinGrainBureauNoticeMapper extends BaseMapper<BizSinGrainBureauNotice> {
27
+
28
+}

+ 28 - 0
src/main/java/com/unis/news/mapper/BizSinQinghaiNewsMapper.java

@@ -0,0 +1,28 @@
1
+/*
2
+ * Copyright [2022] [https://www.xiaonuo.vip]
3
+ *
4
+ * Snowy采用APACHE LICENSE 2.0开源协议,您在使用过程中,需要注意以下几点:
5
+ *
6
+ * 1.请不要删除和修改根目录下的LICENSE文件。
7
+ * 2.请不要删除和修改Snowy源码头部的版权声明。
8
+ * 3.本项目代码可免费商业使用,商业使用请保留源码和相关描述文件的项目出处,作者声明等。
9
+ * 4.分发源码时候,请注明软件出处 https://www.xiaonuo.vip
10
+ * 5.不可二次分发开源参与同类竞品,如有想法可联系团队xiaonuobase@qq.com商议合作。
11
+ * 6.若您的项目无法满足以上几点,需要更多功能代码,获取Snowy商业授权许可,请在官网购买授权,地址为 https://www.xiaonuo.vip
12
+ */
13
+package com.unis.news.mapper;
14
+
15
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
16
+import com.unis.news.entity.BizSinQinghaiNews;
17
+import org.apache.ibatis.annotations.Mapper;
18
+
19
+/**
20
+ * 要闻概况-青海要闻管理表Mapper接口
21
+ *
22
+ * @author qingsong.han
23
+ * @date  2024/05/21 15:34
24
+ **/
25
+@Mapper
26
+public interface BizSinQinghaiNewsMapper extends BaseMapper<BizSinQinghaiNews> {
27
+
28
+}

+ 5 - 0
src/main/java/com/unis/news/mapper/mapping/BizSinGrainBureauNoticeMapper.xml

@@ -0,0 +1,5 @@
1
+<?xml version="1.0" encoding="UTF-8"?>
2
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
3
+<mapper namespace="com.unis.news.mapper.BizSinGrainBureauNoticeMapper">
4
+
5
+</mapper>

+ 5 - 0
src/main/java/com/unis/news/mapper/mapping/BizSinQinghaiNewsMapper.xml

@@ -0,0 +1,5 @@
1
+<?xml version="1.0" encoding="UTF-8"?>
2
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
3
+<mapper namespace="com.unis.news.mapper.BizSinQinghaiNewsMapper">
4
+
5
+</mapper>

+ 34 - 0
src/main/java/com/unis/news/service/BizSinGrainBureauNoticeService.java

@@ -0,0 +1,34 @@
1
+/*
2
+ * Copyright [2022] [https://www.xiaonuo.vip]
3
+ *
4
+ * Snowy采用APACHE LICENSE 2.0开源协议,您在使用过程中,需要注意以下几点:
5
+ *
6
+ * 1.请不要删除和修改根目录下的LICENSE文件。
7
+ * 2.请不要删除和修改Snowy源码头部的版权声明。
8
+ * 3.本项目代码可免费商业使用,商业使用请保留源码和相关描述文件的项目出处,作者声明等。
9
+ * 4.分发源码时候,请注明软件出处 https://www.xiaonuo.vip
10
+ * 5.不可二次分发开源参与同类竞品,如有想法可联系团队xiaonuobase@qq.com商议合作。
11
+ * 6.若您的项目无法满足以上几点,需要更多功能代码,获取Snowy商业授权许可,请在官网购买授权,地址为 https://www.xiaonuo.vip
12
+ */
13
+package com.unis.news.service;
14
+
15
+import com.baomidou.mybatisplus.extension.service.IService;
16
+import com.unis.news.entity.BizSinGrainBureauNotice;
17
+
18
+/**
19
+ * 要闻概况-粮食局公告管理表Service接口
20
+ *
21
+ * @author qingsong.han
22
+ * @date  2024/07/22 12:02
23
+ **/
24
+public interface BizSinGrainBureauNoticeService extends IService<BizSinGrainBureauNotice> {
25
+
26
+    /**
27
+     * 添加要闻概况-粮食局公告管理表
28
+     *
29
+     * @author shine
30
+     * @date  2024/05/21 15:32
31
+     */
32
+    void add(BizSinGrainBureauNotice addParam);
33
+
34
+}

+ 34 - 0
src/main/java/com/unis/news/service/BizSinQinghaiNewsService.java

@@ -0,0 +1,34 @@
1
+/*
2
+ * Copyright [2022] [https://www.xiaonuo.vip]
3
+ *
4
+ * Snowy采用APACHE LICENSE 2.0开源协议,您在使用过程中,需要注意以下几点:
5
+ *
6
+ * 1.请不要删除和修改根目录下的LICENSE文件。
7
+ * 2.请不要删除和修改Snowy源码头部的版权声明。
8
+ * 3.本项目代码可免费商业使用,商业使用请保留源码和相关描述文件的项目出处,作者声明等。
9
+ * 4.分发源码时候,请注明软件出处 https://www.xiaonuo.vip
10
+ * 5.不可二次分发开源参与同类竞品,如有想法可联系团队xiaonuobase@qq.com商议合作。
11
+ * 6.若您的项目无法满足以上几点,需要更多功能代码,获取Snowy商业授权许可,请在官网购买授权,地址为 https://www.xiaonuo.vip
12
+ */
13
+package com.unis.news.service;
14
+
15
+import com.baomidou.mybatisplus.extension.service.IService;
16
+import com.unis.news.entity.BizSinQinghaiNews;
17
+
18
+/**
19
+ * 要闻概况-青海要闻管理表Service接口
20
+ *
21
+ * @author qingsong.han
22
+ * @date  2024/07/22 13:56
23
+ **/
24
+public interface BizSinQinghaiNewsService extends IService<BizSinQinghaiNews> {
25
+
26
+    /**
27
+     * 添加要闻概况-青海要闻管理表
28
+     *
29
+     * @author qingsong.han
30
+     * @date  2024/07/22 13:56
31
+     */
32
+    void add(BizSinQinghaiNews addParam);
33
+
34
+}

+ 41 - 0
src/main/java/com/unis/news/service/impl/BizSinGrainBureauNoticeServiceImpl.java

@@ -0,0 +1,41 @@
1
+/*
2
+ * Copyright [2022] [https://www.xiaonuo.vip]
3
+ *
4
+ * Snowy采用APACHE LICENSE 2.0开源协议,您在使用过程中,需要注意以下几点:
5
+ *
6
+ * 1.请不要删除和修改根目录下的LICENSE文件。
7
+ * 2.请不要删除和修改Snowy源码头部的版权声明。
8
+ * 3.本项目代码可免费商业使用,商业使用请保留源码和相关描述文件的项目出处,作者声明等。
9
+ * 4.分发源码时候,请注明软件出处 https://www.xiaonuo.vip
10
+ * 5.不可二次分发开源参与同类竞品,如有想法可联系团队xiaonuobase@qq.com商议合作。
11
+ * 6.若您的项目无法满足以上几点,需要更多功能代码,获取Snowy商业授权许可,请在官网购买授权,地址为 https://www.xiaonuo.vip
12
+ */
13
+package com.unis.news.service.impl;
14
+
15
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
16
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
17
+import com.unis.news.entity.BizSinGrainBureauNotice;
18
+import com.unis.news.mapper.BizSinGrainBureauNoticeMapper;
19
+import com.unis.news.service.BizSinGrainBureauNoticeService;
20
+import org.springframework.stereotype.Service;
21
+import org.springframework.transaction.annotation.Transactional;
22
+
23
+
24
+/**
25
+ * 要闻概况-粮食局公告管理表Service接口实现类
26
+ *
27
+ * @author qingsong.han
28
+ * @date 2024/07/22 12:02
29
+ **/
30
+@Service
31
+public class BizSinGrainBureauNoticeServiceImpl extends ServiceImpl<BizSinGrainBureauNoticeMapper, BizSinGrainBureauNotice> implements BizSinGrainBureauNoticeService {
32
+
33
+    @Transactional(rollbackFor = Exception.class)
34
+    @Override
35
+    public void add(BizSinGrainBureauNotice addParam) {
36
+        // 先清除数据 TODO
37
+        this.remove(new QueryWrapper<>());
38
+        // 在保存
39
+        this.save(addParam);
40
+    }
41
+}

+ 41 - 0
src/main/java/com/unis/news/service/impl/BizSinQinghaiNewsServiceImpl.java

@@ -0,0 +1,41 @@
1
+/*
2
+ * Copyright [2022] [https://www.xiaonuo.vip]
3
+ *
4
+ * Snowy采用APACHE LICENSE 2.0开源协议,您在使用过程中,需要注意以下几点:
5
+ *
6
+ * 1.请不要删除和修改根目录下的LICENSE文件。
7
+ * 2.请不要删除和修改Snowy源码头部的版权声明。
8
+ * 3.本项目代码可免费商业使用,商业使用请保留源码和相关描述文件的项目出处,作者声明等。
9
+ * 4.分发源码时候,请注明软件出处 https://www.xiaonuo.vip
10
+ * 5.不可二次分发开源参与同类竞品,如有想法可联系团队xiaonuobase@qq.com商议合作。
11
+ * 6.若您的项目无法满足以上几点,需要更多功能代码,获取Snowy商业授权许可,请在官网购买授权,地址为 https://www.xiaonuo.vip
12
+ */
13
+package com.unis.news.service.impl;
14
+
15
+import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
16
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
17
+import com.unis.news.entity.BizSinQinghaiNews;
18
+import com.unis.news.mapper.BizSinQinghaiNewsMapper;
19
+import com.unis.news.service.BizSinQinghaiNewsService;
20
+import org.springframework.stereotype.Service;
21
+import org.springframework.transaction.annotation.Transactional;
22
+
23
+/**
24
+ * 要闻概况-青海要闻管理表Service接口实现类
25
+ *
26
+ * @author qingsong.han
27
+ * @date  2024/05/21 15:34
28
+ **/
29
+@Service
30
+public class BizSinQinghaiNewsServiceImpl extends ServiceImpl<BizSinQinghaiNewsMapper, BizSinQinghaiNews> implements BizSinQinghaiNewsService {
31
+
32
+    @Transactional(rollbackFor = Exception.class)
33
+    @Override
34
+    public void add(BizSinQinghaiNews addParam) {
35
+        // 先删除数据
36
+        this.remove(new QueryWrapper<>());
37
+        // 在添加
38
+        this.save(addParam);
39
+    }
40
+
41
+}

+ 84 - 0
src/main/java/com/unis/news/tasksNews/TasksNewsRunning.java

@@ -0,0 +1,84 @@
1
+package com.unis.news.tasksNews;
2
+
3
+import com.unis.config.NewsUrl;
4
+import com.unis.news.entity.BizSinQinghaiNews;
5
+import com.unis.news.service.BizSinQinghaiNewsService;
6
+import lombok.extern.slf4j.Slf4j;
7
+import org.htmlunit.BrowserVersion;
8
+import org.htmlunit.Page;
9
+import org.htmlunit.WebClient;
10
+import org.htmlunit.html.HtmlAnchor;
11
+import org.htmlunit.html.HtmlPage;
12
+import org.springframework.scheduling.annotation.Scheduled;
13
+import org.springframework.stereotype.Component;
14
+
15
+import javax.annotation.Resource;
16
+import java.util.List;
17
+
18
+/**
19
+ * @author qingsong.han
20
+ * @description: 静态定时任务执行
21
+ * @create 2024-07-22 11:17
22
+ */
23
+@Slf4j
24
+@Component
25
+public class TasksNewsRunning {
26
+    @Resource
27
+    private BizSinQinghaiNewsService newsService;
28
+    @Resource
29
+    private NewsUrl newsUrl;
30
+    @Resource
31
+    private WebClient webClient;
32
+
33
+    /**
34
+     * test
35
+     */
36
+    //  n秒 直接在方法上使用注解即可完成扫描
37
+    @Scheduled(cron = "*/20 * * * * ?")
38
+    public void run() throws Exception {
39
+        // 测试查询
40
+        List<BizSinQinghaiNews> list = newsService.list();
41
+        log.info("test get all list for news: {}", list);
42
+
43
+        log.info("newsUrl stateCouncil: {}", newsUrl.getStateCouncil());
44
+        log.info("newsUrl qinghai: {}", newsUrl.getQinghai());
45
+
46
+        final HtmlPage page = webClient.getPage(newsUrl.getQinghai());
47
+        log.info("page: {}", page);
48
+
49
+        List<HtmlAnchor> divList = page.getByXPath("//a");
50
+        /*for (HtmlAnchor node : divList) {
51
+            log.info("webClient stateCouncil asXml: {}", node.asXml());
52
+            log.info("webClient stateCouncil href: {}", node.getAttribute("href"));
53
+            log.info("webClient stateCouncil href: {}", node.getAttribute("href"));
54
+            log.info("webClient stateCouncil src: {}", node.getAttribute("src"));
55
+            log.info("webClient stateCouncil getLocalName: {}", node.getLocalName());
56
+        }*/
57
+
58
+        log.info("----每天n秒执行---");
59
+    }
60
+
61
+    public static void main(String[] args) {
62
+        WebClient webClient=null;
63
+        try {
64
+            webClient= new WebClient(BrowserVersion.CHROME);    //定义一个WebClient
65
+            final HtmlPage page=webClient.getPage("http://fgw.qinghai.gov.cn//");    //从指定URL获取HtmlPage
66
+
67
+            List<HtmlAnchor> divList=page.getByXPath("//a");
68
+            for (HtmlAnchor node : divList) {
69
+                System.out.println("***********************************************8");
70
+                System.out.println("输出:"+node.asXml());
71
+                System.out.println("输出1:"+node.getAttribute("href"));
72
+                System.out.println("输出2:"+node.getAttribute("src"));
73
+                System.out.println("输出3:"+node.getLocalName());
74
+            }
75
+
76
+        } catch (Exception e) {
77
+            // TODO: handle exception
78
+            e.printStackTrace();
79
+        }finally {
80
+            webClient.close();    //关闭客户端
81
+        }
82
+    }
83
+
84
+}

+ 6 - 0
src/main/resources/application-dev.yml

@@ -0,0 +1,6 @@
1
+spring:
2
+  datasource:
3
+    driver-class-name: com.mysql.cj.jdbc.Driver
4
+    url: jdbc:mysql://172.16.0.36:3306/portal_site?useUnicode=true&characterEncoding=utf-8&useSSL=false
5
+    username: root
6
+    password: admin@1234

+ 6 - 0
src/main/resources/application-prod.yml

@@ -0,0 +1,6 @@
1
+spring:
2
+  datasource:
3
+    driver-class-name: com.mysql.cj.jdbc.Driver
4
+    url: jdbc:mysql://172.18.93.244:3306/portal_site?useUnicode=true&characterEncoding=utf-8&useSSL=false
5
+    username: root
6
+    password: qh_admin@3579

+ 30 - 0
src/main/resources/application.yml

@@ -0,0 +1,30 @@
1
+server:
2
+  port: 85
3
+spring:
4
+  application:
5
+    name: portal-site-back-crawler
6
+  jackson:
7
+    date-format: yyyy-MM-dd HH:mm:ss
8
+    locale: zh_CN
9
+    time-zone: GMT+8
10
+  servlet:
11
+    multipart:
12
+      max-file-size: 100MB
13
+      max-request-size: 100MB
14
+  profiles:
15
+    active: dev
16
+mybatis-plus:
17
+  configuration:
18
+    jdbc-type-for-null: 'null'
19
+    log-impl: org.apache.ibatis.logging.stdout.StdOutImpl
20
+  global-config:
21
+    banner: false
22
+    db-config:
23
+      id-type: ASSIGN_ID
24
+    enable-sql-runner: true
25
+  mapper-locations: classpath*:com/unis/**/mapping/*.xml
26
+news:
27
+  # 国务院
28
+  stateCouncil: https://www.gov.cn/pushinfo/v150203/index.htm
29
+  # 青海省
30
+  qinghai: http://www.qinghai.gov.cn/zwgk/xwdt/qhyw/

+ 13 - 0
src/test/java/com/unis/PortalSiteBackCrawlerApplicationTests.java

@@ -0,0 +1,13 @@
1
+package com.unis;
2
+
3
+import org.junit.jupiter.api.Test;
4
+import org.springframework.boot.test.context.SpringBootTest;
5
+
6
+@SpringBootTest
7
+class PortalSiteBackCrawlerApplicationTests {
8
+
9
+    @Test
10
+    void contextLoads() {
11
+    }
12
+
13
+}