Merge branch 'development' of https://github.com/loklak/loklak_server…

….git into development
loklak · Aug 24, 2020 · f79c657 · f79c657
2 parents d6ea5ac + d866d7d
commit f79c657
Show file tree

Hide file tree

Showing 15 changed files with 66 additions and 699 deletions.
diff --git a/bin/upgrade.sh b/bin/upgrade.sh
@@ -1,9 +1,7 @@
 #!/usr/bin/env sh
 cd `dirname $0`/..
 echo "loading latest code changes"
-git pull origin master
-echo "clean up"
-./gradlew clean
-echo "building loklak"
-./gradlew build
+git pull -r
+echo "assembling loklak"
+./gradlew assemble
 bin/restart.sh
diff --git a/src/org/json/JSONObject.java b/src/org/json/JSONObject.java
@@ -297,7 +297,7 @@ public JSONObject(JSONTokener x) throws JSONException {
      */
     public JSONObject(Map<?, ?> m) {
         if (m == null) {
-            this.map = new HashMap<String, Object>();
+            this.map = new LinkedHashMap<String, Object>();
         } else {
             this.map = new HashMap<String, Object>(m.size());
         	for (final Entry<?, ?> e : m.entrySet()) {

diff --git a/src/org/loklak/Caretaker.java b/src/org/loklak/Caretaker.java
@@ -107,7 +107,7 @@ public void run() {
             if (SuggestServlet.cache.size() > 100) SuggestServlet.cache.clear();
 
             // sleep a bit to prevent that the DoS limit fires at backend server
-            try {Thread.sleep(busy ? 500 : 5000);} catch (InterruptedException e) {}
+            try {Thread.sleep(busy ? 500 : 10000);} catch (InterruptedException e) {}
             if (!this.shallRun) break beat;
             busy = false;
 
@@ -165,32 +165,36 @@ public void run() {
                     try {
                         pendingQueries = LoklakServer.harvester.get_harvest_queries();
                     } catch (IOException e) {
+                        DAO.severe("FAIL SUGGESTIONS cannot get queries from backend: " + e.getMessage(), e);
+                        break hloop;
                     }
 
-                    // in case we have queries
-                    if (pendingQueries > 0) {
-                        Thread[] rts = new Thread[Math.min(pendingQueries, retrieval_forbackend_concurrency)];
-                        final AtomicInteger acccount = new AtomicInteger(0);
-                        for (int j = 0; j < rts.length; j++) {
-                            rts[j] = new Thread() {
-                                public void run() {
-                                    TwitterTimeline tl = LoklakServer.harvester.harvest_timeline();
-                                    if (tl != null && tl.getQuery() != null) {
-                                        /* Thread t = */ LoklakServer.harvester.push_timeline_to_backend(tl);
-                                    }
-                                    int count = tl == null ? 0 : tl.size();
-                                    acccount.addAndGet(count);
+                    // without queries we cannot do harvesting
+                    if (pendingQueries == 0) break hloop;
+
+                    // start harvesting
+                    Thread[] rts = new Thread[Math.min(pendingQueries, retrieval_forbackend_concurrency)];
+                    final AtomicInteger acccount = new AtomicInteger(0);
+                    for (int j = 0; j < rts.length; j++) {
+                        rts[j] = new Thread() {
+                            public void run() {
+                                TwitterTimeline tl = LoklakServer.harvester.harvest_timeline();
+                                if (tl != null && tl.getQuery() != null) {
+                                    /* Thread t = */ LoklakServer.harvester.push_timeline_to_backend(tl);
                                 }
-                            };
-                            rts[j].start();
-                            try {Thread.sleep(retrieval_forbackend_sleep_base + random.nextInt(retrieval_forbackend_sleep_randomoffset));} catch (InterruptedException e) {}
-                        }
-                        for (Thread t: rts) t.join();
-                        if (acccount.get() < 0) break hloop;
+                                int count = tl == null ? 0 : tl.size();
+                                acccount.addAndGet(count);
+                            }
+                        };
+                        rts[j].start();
                         try {Thread.sleep(retrieval_forbackend_sleep_base + random.nextInt(retrieval_forbackend_sleep_randomoffset));} catch (InterruptedException e) {}
                     }
+                    for (Thread t: rts) t.join();
+                    if (acccount.get() < 0) break hloop;
+                    try {Thread.sleep(retrieval_forbackend_sleep_base + random.nextInt(retrieval_forbackend_sleep_randomoffset));} catch (InterruptedException e) {}
+
+                    busy = true;
                 }
-                busy = true;
             }
 
             // run some crawl steps

diff --git a/src/org/loklak/LoklakServer.java b/src/org/loklak/LoklakServer.java
@@ -104,7 +104,6 @@
 import org.loklak.api.p2p.HelloService;
 import org.loklak.api.p2p.PeersServlet;
 import org.loklak.api.p2p.PushServlet;
-import org.loklak.api.search.ConsoleService;
 import org.loklak.api.search.EventBriteCrawlerService;
 import org.loklak.api.search.GenericScraper;
 import org.loklak.api.search.GithubProfileScraper;
@@ -607,7 +606,6 @@ private static void setServerHandler(File dataFile){
                 HelloService.class,
 
                 // search
-                ConsoleService.class,
                 EventBriteCrawlerService.class,
                 MeetupsCrawlerService.class,
                 RSSReaderService.class,