On 20 Feb 2018, at 09:47, [email protected] <mailto:[email protected]> wrote:
Repository: lucene-solr
Updated Branches:
refs/heads/master dfc0fe86e -> 4bfcbc5c6
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4bfcbc5c/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/config/SolrQueryConfigTest.java
----------------------------------------------------------------------
diff --git
a/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/config/SolrQueryConfigTest.java
b/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/config/SolrQueryConfigTest.java
new file mode 100644
index 0000000..c62d354
--- /dev/null
+++
b/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/config/SolrQueryConfigTest.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.prometheus.scraper.config;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.List;
+
+/**
+ * Unit test for SolrQueryConfig.
+ */
+public class SolrQueryConfigTest extends SolrTestCaseJ4 {
+ @Test
+ public void testQueryConfig() throws Exception {
+ SolrQueryConfig queryConfig = new SolrQueryConfig();
+
+ assertNotNull(queryConfig);
+ }
+
+ @Test
+ public void testGetCollection() throws Exception {
+ SolrQueryConfig queryConfig = new SolrQueryConfig();
+
+ String expected = "";
+ String actual = queryConfig.getCollection();
+ assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testSetCollection() throws Exception {
+ SolrQueryConfig queryConfig = new SolrQueryConfig();
+
+ queryConfig.setCollection("collection1");
+
+ String expected = "collection1";
+ String actual = queryConfig.getCollection();
+ assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testGetPath() throws Exception {
+ SolrQueryConfig queryConfig = new SolrQueryConfig();
+
+ String expected = "";
+ String actual = queryConfig.getPath();
+ assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testSetPath() throws Exception {
+ SolrQueryConfig queryConfig = new SolrQueryConfig();
+
+ queryConfig.setPath("/select");
+
+ String expected = "/select";
+ String actual = queryConfig.getPath();
+ assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testGetParams() throws Exception {
+ SolrQueryConfig queryConfig = new SolrQueryConfig();
+
+ List<LinkedHashMap<String, String>> expected = new ArrayList<>();
+ List<LinkedHashMap<String, String>> actual = queryConfig.getParams();
+ assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testSetParams() throws Exception {
+ SolrQueryConfig queryConfig = new SolrQueryConfig();
+
+ LinkedHashMap<String,String> param1 = new LinkedHashMap<>();
+ param1.put("q", "*:*");
+
+ LinkedHashMap<String,String> param2 = new LinkedHashMap<>();
+ param2.put("facet", "on");
+
+ queryConfig.setParams(Arrays.asList(param1, param2));
+
+ List<LinkedHashMap<String, String>> expected = Arrays.asList(param1,
param2);
+ List<LinkedHashMap<String, String>> actual = queryConfig.getParams();
+ assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testGetParamsString() throws Exception {
+ SolrQueryConfig queryConfig = new SolrQueryConfig();
+
+ LinkedHashMap<String,String> param1 = new LinkedHashMap<>();
+ param1.put("q", "*:*");
+ param1.put("fq", "manu:apple");
+
+ LinkedHashMap<String,String> param2 = new LinkedHashMap<>();
+ param2.put("facet", "on");
+
+ queryConfig.setParams(Arrays.asList(param1, param2));
+
+ String expected = "q=*:*&fq=manu:apple&facet=on";
+ String actual = queryConfig.getParamsString();
+ assertEquals(expected, actual);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4bfcbc5c/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/config/SolrScraperConfigTest.java
----------------------------------------------------------------------
diff --git
a/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/config/SolrScraperConfigTest.java
b/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/config/SolrScraperConfigTest.java
new file mode 100644
index 0000000..79d1204
--- /dev/null
+++
b/solr/contrib/prometheus-exporter/src/test/org/apache/solr/prometheus/scraper/config/SolrScraperConfigTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.prometheus.scraper.config;
+
+import org.apache.solr.prometheus.collector.config.SolrCollectorConfig;
+import org.apache.solr.SolrTestCaseJ4;
+import org.junit.Test;
+import org.yaml.snakeyaml.Yaml;
+
+import java.io.FileReader;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Unit test for SolrScraperConfig.
+ */
+public class SolrScraperConfigTest extends SolrTestCaseJ4 {
+ @Test
+ public void testScraperConfig() throws Exception {
+ String configFile = getFile("conf/config.yml").getAbsolutePath();
+
+ SolrCollectorConfig config = new Yaml().loadAs(new FileReader(configFile),
SolrCollectorConfig.class);
+
+ SolrScraperConfig scraperConfig = config.getMetrics();
+
+ assertNotNull(scraperConfig);
+ }
+
+ @Test
+ public void testGetJsonQueries() throws Exception {
+ String configFile = getFile("conf/config.yml").getAbsolutePath();
+
+ SolrCollectorConfig collectorConfig = new Yaml().loadAs(new FileReader(configFile),
SolrCollectorConfig.class);
+
+ SolrScraperConfig scraperConfig = collectorConfig.getMetrics();
+
+ assertNotNull(scraperConfig.getJsonQueries());
+ }
+
+ @Test
+ public void testSetJsonQueries() throws Exception {
+ List<String> jsonQueries = new ArrayList<>();
+
+ SolrScraperConfig scraperConfig = new SolrScraperConfig();
+
+ scraperConfig.setJsonQueries(jsonQueries);
+
+ assertNotNull(scraperConfig.getJsonQueries());
+ }
+
+ @Test
+ public void testGetQueryConfig() throws Exception {
+ String configFile = getFile("conf/config.yml").getAbsolutePath();
+
+ SolrCollectorConfig collectorConfig = new Yaml().loadAs(new FileReader(configFile),
SolrCollectorConfig.class);
+
+ SolrScraperConfig scraperConfig = collectorConfig.getMetrics();
+
+ assertNotNull(scraperConfig.getQuery());
+ }
+
+ @Test
+ public void testSetQueryConfig() throws Exception {
+ SolrQueryConfig queryConfig = new SolrQueryConfig();
+
+ SolrScraperConfig scraperConfig = new SolrScraperConfig();
+
+ scraperConfig.setQuery(queryConfig);
+
+ assertNotNull(scraperConfig.getQuery());
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4bfcbc5c/solr/solr-ref-guide/src/images/monitoring-solr-with-prometheus-and-grafana/grafana-solr-dashboard.png
----------------------------------------------------------------------
diff --git
a/solr/solr-ref-guide/src/images/monitoring-solr-with-prometheus-and-grafana/grafana-solr-dashboard.png
b/solr/solr-ref-guide/src/images/monitoring-solr-with-prometheus-and-grafana/grafana-solr-dashboard.png
new file mode 100644
index 0000000..69f21a4
Binary files /dev/null and
b/solr/solr-ref-guide/src/images/monitoring-solr-with-prometheus-and-grafana/grafana-solr-dashboard.png
differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4bfcbc5c/solr/solr-ref-guide/src/images/monitoring-solr-with-prometheus-and-grafana/solr-exporter-diagram.png
----------------------------------------------------------------------
diff --git
a/solr/solr-ref-guide/src/images/monitoring-solr-with-prometheus-and-grafana/solr-exporter-diagram.png
b/solr/solr-ref-guide/src/images/monitoring-solr-with-prometheus-and-grafana/solr-exporter-diagram.png
new file mode 100644
index 0000000..96efece
Binary files /dev/null and
b/solr/solr-ref-guide/src/images/monitoring-solr-with-prometheus-and-grafana/solr-exporter-diagram.png
differ
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4bfcbc5c/solr/solr-ref-guide/src/monitoring-solr-with-prometheus-and-grafana.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/monitoring-solr-with-prometheus-and-grafana.adoc
b/solr/solr-ref-guide/src/monitoring-solr-with-prometheus-and-grafana.adoc
new file mode 100644
index 0000000..1ac55a2
--- /dev/null
+++ b/solr/solr-ref-guide/src/monitoring-solr-with-prometheus-and-grafana.adoc
@@ -0,0 +1,250 @@
+= Monitoring Solr with Prometheus and Grafana
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+You can monitor Solr using solr-exporter that exposes Solr's metrics to
https://prometheus.io[Prometheus], and visualize metrics using https://grafana.com[Grafana].
+
+It allows users to monitor not only Solr metrics which come from
<<metrics-reporting.adoc#metrics-api,Metrics API>> but also facet counts which come from
<<searching.adoc#searching,Searching>>.
+
+.solr-exporter Diagram
+image::images/monitoring-solr-with-prometheus-and-grafana/solr-exporter-diagram.png[image,width=600]
+
+This feature is experimental status.
+
+== Running solr-exporter
+
+You can start solr-exporter by running `./bin/solr-exporter` from the
solr-exporter directory.
+
+[source,plain]
+----
+$ cd ./contrib/prometheus-exporter
+$ ./bin/solr-exporter -p 9983 -b http://localhost:8983/solr -f
./conf/config.yml -n 8
+----
+
+If you are on Windows platform, you can start solr-exporter by running `.\bin\solr-exporter.cmd`
instead.
+
+[source,plain]
+----
+> cd .\contrib\prometheus
+> .\bin\solr-exporter.cmd -p 9983 -b http://localhost:8983/solr -f
.\conf\config.yml -n 8
+----
+
+You can also connect to Solr in SolrCloud mode like this.
+
+[source,plain]
+----
+$ cd ./contrib/prometheus
+$ ./bin/solr-exporter -p 9983 -z localhost:2181/solr -f ./conf/config.yml -n 16
+----
+
+See command help:
+
+[source,plain]
+----
+$ ./bin/solr-exporter -h
+usage: SolrCollector [-h] [-v] [-p PORT] [-b BASE_URL] [-z ZK_HOST] [-f CONFIG]
+ [-n NUM_THREADS]
+
+Prometheus exporter for Apache Solr.
+
+optional arguments:
+ -h, --help show this help message and exit
+ -p PORT, --port PORT solr-exporter listen port
+ -b BASE_URL, --baseurl BASE_URL
+ specify Solr base URL when connecting to Solr in
standalone mode (for
+ example 'http://localhost:8983/solr')
+ -z ZK_HOST, --zkhost ZK_HOST
+ specify ZooKeeper connection string when
connecting to Solr in
+ SolrCloud mode (for example 'localhost:2181/solr')
+ -f CONFIG, --config-file CONFIG
+ specify configuration file
+ -n NUM_THREADS, --num-thread NUM_THREADS
+ specify number of threads
+----
+
+The Solr's metrics exposed by solr-exporter can see at the following URL.
+
+http://localhost:9983/metrics[http://localhost:9983/metrics]
+
+
+== Configuration
+
+The configuration is in `./config/config.yml`. An example with all possible
options:
+
+[source,plain]
+----
+ping:
+ query:
+ path: /admin/ping
+ jsonQueries:
+ - |-
+ . as $object | $object |
+ (if $object.status == "OK" then 1.0 else 0.0 end) as $value |
+ {
+ name : "solr_ping",
+ type : "GAUGE",
+ help : "See following URL:
https://lucene.apache.org/solr/guide/ping.html";,
+ label_names : [],
+ label_values : [],
+ value : $value
+ }
+
+metrics:
+ query:
+ path: /admin/metrics
+ params:
+ - group: 'all'
+ - type: 'all'
+ - prefix: ''
+ - property: ''
+ jsonQueries:
+ # solr_metrics_jetty_response_count
+ - |-
+ .metrics["solr.jetty"] | to_entries | .[] | select(.key |
startswith("org.eclipse.jetty.server.handler.DefaultHandler")) | select(.key |
endswith("xx-responses")) as $object |
+ $object.key | split(".") | last | split("-") | first as $status |
+ $object.value.count as $value |
+ {
+ name : "solr_metrics_jetty_response_count",
+ type : "gauge",
+ help : "See following URL:
https://lucene.apache.org/solr/guide/metrics-reporting.html";,
+ label_names : ["status"],
+ label_values : [$status],
+ value : $value
+ }
+
+...
+
+collections:
+ query:
+ path: /admin/collections
+ params:
+ - action: 'CLUSTERSTATUS'
+ jsonQueries:
+ # solr_collections_cluster_status_live_nodes
+ - |-
+ .cluster.live_nodes | length as $value|
+ {
+ name : "solr_collections_cluster_status_live_nodes",
+ type : "gauge",
+ help : "See following URL:
https://lucene.apache.org/solr/guide/collections-api.html#clusterstatus";,
+ label_names : [],
+ label_values : [],
+ value : $value
+ }
+
+...
+
+queries:
+ - query:
+ collection: collection1
+ path: /select
+ params:
+ - q: "*:*"
+ - start: 0
+ - rows: 0
+ - json.facet: |-
+ {
+ category: {
+ type: terms,
+ field: cat
+ }
+ }
+ jsonQueries:
+ # solr_facets_category
+ - |-
+ .facets.category.buckets[] as $object |
+ $object.val as $term |
+ $object.count as $value |
+ {
+ name : "solr_facets_category",
+ type : "gauge",
+ help : "Category facets",
+ label_names : ["term"],
+ label_values : [$term],
+ value : $value
+ }
+----
+
+|===
+|Name|Description
+
+|ping|Scrape <<ping.adoc#ping,Ping>> response.
+|metrics|Scrape <<metrics-reporting.adoc#metrics-api,Metrics API>> response.
+|collections|Scrape <<collections-api.adoc#collections-api,Collections API>>
response.
+|queries|Scrape <<searching.adoc#searching,Search API>> response.
+|*.query|Query parameter for each features. You can specify `collection`, `core`, `path`, and
`params`.
+|*.jsonQueries|JSON Query that is jq syntax. For more details, see
https://stedolan.github.io/jq/manual/[https://stedolan.github.io/jq/manual/].
+|===
+
+jq query has to output JSON in the following format.
+
+[source,json]
+----
+{
+ name : "solr_ping",
+ type : "GAUGE",
+ help : "See following URL:
https://lucene.apache.org/solr/guide/ping.html";,
+ label_names : ["base_url","core"],
+ label_values : ["http://localhost:8983/solr","collection1";],
+ value : 1.0
+}
+----
+
+It will be converted to the following exposition format.
+
+[source,plain]
+----
+# TYPE solr_ping gauge
+# HELP solr_ping See following URL:
https://lucene.apache.org/solr/guide/ping.html
+solr_ping{base_url="http://localhost:8983/solr",core="collection1"} 1.0
+----
+
+|===
+|Name|Description
+
+|name|The metric name to set. For more details, see
https://prometheus.io/docs/practices/naming/[https://prometheus.io/docs/practices/naming/].
+|type|The type of the metric, can be `COUNTER`, `GAUGE`, `SUMMARY`, `HISTOGRAM` or `UNTYPED`.
For more detauils, see
https://prometheus.io/docs/concepts/metric_types/[https://prometheus.io/docs/concepts/metric_types/].
+|help|Help text for the metric.
+|label_names|Label names for the metric. For more details, see
https://prometheus.io/docs/practices/naming/[https://prometheus.io/docs/practices/naming/].
+|label_values|Label values for the metric. For more details, see
https://prometheus.io/docs/practices/naming/[https://prometheus.io/docs/practices/naming/].
+|value|Value for the metric. Value must be set to Double type.
+|===
+
+
+== Prometheus Settings
+
+You need to specify the solr-exporter listen address into `scrape_configs` in `prometheus.yml`.
See following example:
+
+[source,plain]
+----
+scrape_configs:
+ - job_name: 'solr'
+ static_configs:
+ - targets: ['localhost:9983']
+----
+
+When you apply the above settings to prometheus, it will start to pull Solr's metrics from
solr-exporter.
+
+
+== Grafana Dashboard
+
+A Grafana sample dashboard is provided at the following JSON file.
+
+`./conf/grafana-solr-dashboard.json`
+
+.Grafana Dashboard
+image::images/monitoring-solr-with-prometheus-and-grafana/grafana-solr-dashboard.png[image,width=800]
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4bfcbc5c/solr/solr-ref-guide/src/monitoring-solr.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/monitoring-solr.adoc
b/solr/solr-ref-guide/src/monitoring-solr.adoc
index 0576e53..2fb7077 100644
--- a/solr/solr-ref-guide/src/monitoring-solr.adoc
+++ b/solr/solr-ref-guide/src/monitoring-solr.adoc
@@ -1,5 +1,5 @@
= Monitoring Solr
-:page-children: metrics-reporting, mbean-request-handler, configuring-logging,
using-jmx-with-solr, performance-statistics-reference
+:page-children: metrics-reporting, mbean-request-handler, configuring-logging,
using-jmx-with-solr, monitoring-solr-with-prometheus-and-grafana, performance-statistics-reference
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
@@ -30,6 +30,8 @@ Common administrative tasks include:
<<using-jmx-with-solr.adoc#using-jmx-with-solr,Using JMX with Solr>>: Describes how to use Java
Management Extensions with Solr.
+<<monitoring-solr-with-prometheus-and-grafana.adoc#monitoring-solr-with-prometheus-and-grafana,Monitoring
Solr with Prometheus and Grafana>>: Describes how to monitor Solr with Prometheus and Grafana.
+
<<performance-statistics-reference.adoc#performance-statistics-reference,Performance Statistics
Reference>>: Additional information on statistics returned from JMX.
