Merge pull request #30 from zenuo/fix_google_dom_changed

Fix google dom changed

Author: zenuo

README.md

@@ -147,3 +147,26 @@ $ python3 gogo.py stop
 ## 实例集合
 
 欢迎通过Issue分享实例供学习使用🏇
+
+## 实现思路
+
+```javascript
+// User-Agent: Mozilla/5.0 (Mobile; Nokia 8110 4G; rv:48.0) Gecko/48.0 Firefox/48.0 KAIOS/2.5
+
+let searchResultElements = Array.from(document.getElementsByTagName("a"))
+  .filter(a => a.hasAttribute("href") 
+    && a.getAttribute("href").startsWith("/url?") 
+    && a.childElementCount == 2
+    && a.childNodes[0].tagName == "H3");
+
+let entries = searchResultElements.map(e => {
+  let url = new URL(e).searchParams.get("q");
+  let name = e.children[0].textContent;
+  let desc = e.parentNode.parentElement.children[2].textContent;
+  return {
+    "url": url,
+    "name": name,
+    "desc": desc
+  };
+})
+```

gogo-server/src/main/java/zenuo/gogo/core/processor/impl/GoogleSearchResultProviderImpl.java

@@ -1,18 +1,17 @@
 package zenuo.gogo.core.processor.impl;
 
 import io.netty.handler.codec.http.HttpResponseStatus;
+import io.netty.handler.codec.http.QueryStringDecoder;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.http.client.methods.HttpGet;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
-import org.jsoup.select.Elements;
 import zenuo.gogo.core.processor.IHttpClientProvider;
 import zenuo.gogo.core.processor.ISearchResultProvider;
 import zenuo.gogo.model.Entry;
 import zenuo.gogo.model.SearchResponse;
-import zenuo.gogo.util.StringUtils;
 import zenuo.gogo.util.UserAgentUtils;
 
 import javax.inject.Inject;
@@ -21,6 +20,7 @@
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.stream.Collectors;
 
 /**
  * 谷歌搜索
@@ -54,45 +54,26 @@ public SearchResponse search(String key, int page) {
         } catch (IOException e) {
             throw new RuntimeException(e);
         }
-        final Elements webResults = document.getElementsByClass("g");
-        if (webResults.isEmpty()) {
-            log.error("pattern changed");
-            return patternChanged(builder);
-        }
-        final List<Entry> entries = new ArrayList<>();
-        //traverse search result entries
-        for (Element result : webResults) {
-            //entry builder
-            final Entry.EntryBuilder entryBuilder = Entry.builder();
-            //name
-            final Element name = result.getElementsByClass("LC20lb DKV0Md").first();
-            if (name == null) {
+        final List<Element> searchResultElements = document.getElementsByTag("a").stream()
+                .filter(a -> a.hasAttr("href")
+                        && a.attr("href").startsWith("/url?")
+                        && a.childrenSize() == 2
+                        && "h3".equals(a.child(0).tagName()))
+                .collect(Collectors.toList());
+        final List<Entry> entries = new ArrayList<>(searchResultElements.size());
+        builder.entries(entries);
+        for (Element element : searchResultElements) {
+            final QueryStringDecoder decoder = new QueryStringDecoder(element.attr("href"));
+            final List<String> q = decoder.parameters().get("q");
+            if (q == null) {
                 continue;
             }
-            entryBuilder.name(StringUtils.escapeHtmlEntities(name.text()));
-            //url
-            final Element url = name.parent();
-            entryBuilder.url(url.attr("href"));
-            //description
-            final Element desc = result.getElementsByClass("aCOpRe").first();
-            entryBuilder.desc(StringUtils.escapeHtmlEntities(desc.text()));
-            //build
-            final Entry entry = entryBuilder.build();
-            //name and url are not null
-            if (entry.getName() != null && entry.getUrl() != null) {
-                entries.add(entry);
-            }
+            final Entry entry = new Entry();
+            entries.add(entry);
+            entry.setUrl(q.get(0));
+            entry.setName(element.child(0).text());
+            entry.setDesc(element.parent().parent().child(2).text());
         }
-        final Elements videoResults = document.getElementsByClass("y8AWGd llvJ5e");
-        for (Element videoResult : videoResults) {
-            final Element a = videoResult.child(0);
-            entries.add(Entry.builder()
-                    .url(a.attr("href"))
-                    .name(a.child(1).text())
-                    .desc(videoResult.child(2).text())
-                    .build());
-        }
-        builder.entries(entries);
         return builder.status(HttpResponseStatus.OK).build();
     }
 

gogo-server/src/main/java/zenuo/gogo/core/processor/impl/LintProcessorImpl.java

@@ -63,7 +63,7 @@ public void process(ChannelHandlerContext ctx, FullHttpRequest request, QueryStr
                     request,
                     ResponseType.API,
                     body,
-                    response.getStatus());
+                    HttpResponseStatus.OK);
         }
     }
 
@@ -125,11 +125,10 @@ LintResponse response(final String key) {
         builder.key(key);
         try {
             final List<String> lints = lint(key);
-            builder.lints(lints).status(HttpResponseStatus.OK);
+            builder.lints(lints);
         } catch (Exception e) {
             log.error("lint {}", key, e);
-            builder.error(e.getMessage())
-                    .status(HttpResponseStatus.GATEWAY_TIMEOUT);
+            builder.error(e.getMessage());
         }
         return builder.build();
     }

gogo-server/src/main/java/zenuo/gogo/util/UserAgentUtils.java

@@ -11,11 +11,11 @@ public final class UserAgentUtils {
     /**
      * 用户代理字符串数组
      */
-    private static String[] USER_AGENTS = {
-            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36",
-            "Mozilla/5.0 (X11; FreeBSD amd64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36",
-            "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36",
-            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36",
+    private static final String[] USER_AGENTS = {
+            "Mozilla/5.0 (Mobile; Nokia 8110 4G; rv:46.0) Gecko/46.0 Firefox/46.0 KAIOS/2.5",
+            "Mozilla/5.0 (Mobile; Nokia 8110 4G; rv:47.0) Gecko/47.0 Firefox/47.0 KAIOS/2.5",
+            "Mozilla/5.0 (Mobile; Nokia 8110 4G; rv:48.0) Gecko/48.0 Firefox/48.0 KAIOS/2.5",
+            "Mozilla/5.0 (Mobile; Nokia 8110 4G; rv:49.0) Gecko/49.0 Firefox/49.0 KAIOS/2.5",
     };
     /**
      * 最后一次使用的索引