mvolikas commented on code in PR #1544:
URL: https://github.com/apache/stormcrawler/pull/1544#discussion_r2117888390
##########
external/solr/src/main/java/org/apache/stormcrawler/solr/SolrConnection.java:
##########
@@ -32,55 +35,213 @@
import org.apache.solr.client.solrj.impl.LBHttp2SolrClient;
import org.apache.solr.client.solrj.impl.LBSolrClient;
import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.storm.shade.org.apache.commons.lang.StringUtils;
import org.apache.stormcrawler.util.ConfUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class SolrConnection {
+ private static final Logger LOG =
LoggerFactory.getLogger(SolrConnection.class);
- private SolrClient client;
- private SolrClient updateClient;
+ private final SolrClient client;
+ private final SolrClient updateClient;
- private boolean cloud;
- private String collection;
+ private final boolean cloud;
+ private final String collection;
+
+ private long lastUpdate;
+ private final Map<String, List<Update>> updateQueues;
+ private final Object lock = new Object();
+
+ private final int updateQueueSize = 10;
+
+ // Maximum time (ms) without updates before we flush all the queues
+ private final long noUpdateThreshold = 20_000;
Review Comment:
I guess these should be external properties. Any name recommendations?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: dev-unsubscr...@stormcrawler.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
