Apply limit and offset to IndexProvider.totals method

Fixes #3571

Signed-off-by: Oleksandr Porunov <alexandr.porunov@gmail.com>

Author: porunov

docs/changelog.md

@@ -110,6 +110,34 @@ For more information on features and bug fixes in 1.0.0, see the GitHub mileston
 The index management has received an overhaul which enables proper index removal.
 The schema action `REMOVE_INDEX` is no longer available and has been replaced by `DISCARD_INDEX`.
 
+##### `totals` for direct index queries now applies provided offset and limit
+
+Direct index queries which search count for totals (`vertexTotals`, `edgeTotals`, `propertyTotals` and direct execution of 
+`IndexProvider.totals`) now apply provided `limit` and `offset`. Previously provided `limit` and `offset` were ignored.    
+For example, previously the following query would return `500` if there were `500` indexed elements: 
+
+```groovy
+gremlin> graph.indexQuery("textIndex", "v.\"text\":fooBar").limit(10).vertexTotals()
+```
+
+Now the above query will return `10` elements because we limited the result to `10` elements only.
+Same applies to `offset`. Previously the following query would return `500` if there were `500` indexed elements:
+
+```groovy
+gremlin> graph.indexQuery("textIndex", "v.\"text\":fooBar").offset(10).vertexTotals()
+```
+
+Now the above query will return `490` as a result because we skip count of the first `10` elements.    
+`offset` provided with `limit` will apply `offset` first and `limit` last.  
+For example, the above query will return `10` elements now if there were `500` indexed elements:
+
+```groovy
+gremlin> graph.indexQuery("textIndex", "v.\"text\":fooBar").limit(10).offset(10).vertexTotals()
+```
+
+The new logic is applied similarly to Direct Index Queries `vertexTotals()`, `edgeTotals()`, `propertyTotals()` as well 
+as internal JanusGraph method `IndexProvider.totals`.
+
 ##### Add support for Java 11
 
 JanusGraph now officially supports Java 11 in addition to Java 8. We encourage everyone to update to Java 11.

docs/index-backend/direct-index-query.md

@@ -87,11 +87,18 @@ retrieving and ranking all documents. This shortcut is exposed through
 the ".vertexTotals()", ".edgeTotals()", and ".propertyTotals()" methods.
 
 The totals can be retrieved using the same query syntax as the
-indexQuery builder, but size is overwritten to be 0.
+indexQuery builder.
 ```groovy
 graph.indexQuery("vertexByText", "v.text:(farm uncle berry)").vertexTotals()
 ```
 
+!!! note
+    In JanusGraph versions earlier than 1.0.0 provided `limit` and `offset` was ignored
+    for any `totals` queries. Starting from JanusGraph 1.0.0 provided `limit` and `offset` are
+    applied to the final `totals` result (`vertexTotals()`, `edgeTotals()`, `propertyTotals()` as well
+    as internal JanusGraph method `IndexProvider.totals`). This was made to have consistent behaviour between
+    `totals` and `stream` direct index queries.
+
 ## Gotchas
 
 ### Property Key Names

janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphIndexTest.java

@@ -1938,12 +1938,14 @@ public void testRawQueries() {
         assertCount(numV / strings.length * 2, graph.indexQuery(VINDEX, "v.text:(beautiful are ducks)").vertexStream());
         assertCount(numV / strings.length * 2 - 10, graph.indexQuery(VINDEX, "v.text:(beautiful are ducks)").offset(10).vertexStream());
         long total = graph.indexQuery(VINDEX, "v.\"text\":(beautiful are ducks)").limit(Integer.MAX_VALUE).vertexStream().count();
-        assertCount(10, graph.indexQuery(VINDEX, "v.\"text\":(beautiful are ducks)").limit(10).vertexStream());
-        assertEquals(total, (long) graph.indexQuery(VINDEX, "v.\"text\":(beautiful are ducks)").limit(10).vertexTotals());
-        assertCount(10, graph.indexQuery(VINDEX, "v.\"text\":(beautiful are ducks)").limit(10).offset(10).vertexStream());
-        assertEquals(total, (long) graph.indexQuery(VINDEX, "v.\"text\":(beautiful are ducks)").limit(10).offset(10).vertexTotals());
-        assertCount(0, graph.indexQuery(VINDEX, "v.\"text\":(beautiful are ducks)").limit(10).offset(numV).vertexStream());
-        assertEquals(total, (long) graph.indexQuery(VINDEX, "v.\"text\":(beautiful are ducks)").limit(10).offset(numV).vertexTotals());
+        int limit = 10;
+        long expectedLimitedCount = Math.min(total, limit);
+        assertCount(expectedLimitedCount, graph.indexQuery(VINDEX, "v.\"text\":(beautiful are ducks)").limit(limit).vertexStream());
+        assertEquals(expectedLimitedCount, (long) graph.indexQuery(VINDEX, "v.\"text\":(beautiful are ducks)").limit(limit).vertexTotals());
+        assertCount(expectedLimitedCount, graph.indexQuery(VINDEX, "v.\"text\":(beautiful are ducks)").limit(limit).offset(10).vertexStream());
+        assertEquals(expectedLimitedCount, (long) graph.indexQuery(VINDEX, "v.\"text\":(beautiful are ducks)").limit(limit).offset(10).vertexTotals());
+        assertCount(0, graph.indexQuery(VINDEX, "v.\"text\":(beautiful are ducks)").limit(limit).offset(numV).vertexStream());
+        assertEquals(0, (long) graph.indexQuery(VINDEX, "v.\"text\":(beautiful are ducks)").limit(limit).offset(numV).vertexTotals());
         //Test name mapping
         assertCount(numV / strings.length * 2, graph.indexQuery(VINDEX, "xtext:ducks").vertexStream());
         assertCount(0, graph.indexQuery(VINDEX, "text:ducks").vertexStream());
@@ -1954,36 +1956,38 @@ public void testRawQueries() {
         //Same queries for edges
         assertCount(numV / strings.length * 2, graph.indexQuery(EINDEX, "e.text:ducks").edgeStream());
         total = graph.indexQuery(EINDEX, "e.text:ducks").limit(Integer.MAX_VALUE).edgeStream().count();
+        expectedLimitedCount = Math.min(total, limit);
         assertEquals(total, (long) numV / strings.length * 2, graph.indexQuery(EINDEX, "e.text:ducks").edgeTotals());
         assertCount(numV / strings.length * 2, graph.indexQuery(EINDEX, "e.text:(farm uncle berry)").edgeStream());
         assertCount(numV / strings.length, graph.indexQuery(EINDEX, "e.text:(farm uncle berry) AND e.name:\"Uncle Berry has a farm\"").edgeStream());
         assertCount(numV / strings.length * 2, graph.indexQuery(EINDEX, "e.text:(beautiful are ducks)").edgeStream());
         assertCount(numV / strings.length * 2 - 10, graph.indexQuery(EINDEX, "e.text:(beautiful are ducks)").offset(10).edgeStream());
         total = graph.indexQuery(EINDEX, "e.\"text\":(beautiful are ducks)").limit(Integer.MAX_VALUE).edgeStream().count();
-        assertCount(10, graph.indexQuery(EINDEX, "e.\"text\":(beautiful are ducks)").limit(10).edgeStream());
-        assertEquals(total, (long) graph.indexQuery(EINDEX, "e.\"text\":(beautiful are ducks)").limit(10).edgeTotals());
-        assertCount(10, graph.indexQuery(EINDEX, "e.\"text\":(beautiful are ducks)").limit(10).offset(10).edgeStream());
-        assertEquals(total, (long) graph.indexQuery(EINDEX, "e.\"text\":(beautiful are ducks)").limit(10).offset(10).edgeTotals());
-        assertCount(0, graph.indexQuery(EINDEX, "e.\"text\":(beautiful are ducks)").limit(10).offset(numV).edgeStream());
-        assertEquals(total, (long) graph.indexQuery(EINDEX, "e.\"text\":(beautiful are ducks)").limit(10).offset(numV).edgeTotals());
+        assertCount(expectedLimitedCount, graph.indexQuery(EINDEX, "e.\"text\":(beautiful are ducks)").limit(limit).edgeStream());
+        assertEquals(expectedLimitedCount, (long) graph.indexQuery(EINDEX, "e.\"text\":(beautiful are ducks)").limit(limit).edgeTotals());
+        assertCount(expectedLimitedCount, graph.indexQuery(EINDEX, "e.\"text\":(beautiful are ducks)").limit(limit).offset(10).edgeStream());
+        assertEquals(expectedLimitedCount, (long) graph.indexQuery(EINDEX, "e.\"text\":(beautiful are ducks)").limit(limit).offset(10).edgeTotals());
+        assertCount(0, graph.indexQuery(EINDEX, "e.\"text\":(beautiful are ducks)").limit(limit).offset(numV).edgeStream());
+        assertEquals(0, (long) graph.indexQuery(EINDEX, "e.\"text\":(beautiful are ducks)").limit(limit).offset(numV).edgeTotals());
         //Test name mapping
         assertCount(numV / strings.length * 2, graph.indexQuery(EINDEX, "text:ducks").edgeStream());
 
         //Same queries for properties
         assertCount(numV / strings.length * 2, graph.indexQuery(PINDEX, "p.text:ducks").propertyStream());
         total = graph.indexQuery(PINDEX, "p.text:ducks").limit(Integer.MAX_VALUE).propertyStream().count();
+        expectedLimitedCount = Math.min(total, limit);
         assertEquals(total, (long) numV / strings.length * 2, graph.indexQuery(PINDEX, "p.text:ducks").propertyTotals());
         assertCount(numV / strings.length * 2, graph.indexQuery(PINDEX, "p.text:(farm uncle berry)").propertyStream());
         assertCount(numV / strings.length, graph.indexQuery(PINDEX, "p.text:(farm uncle berry) AND p.name:\"Uncle Berry has a farm\"").propertyStream());
         assertCount(numV / strings.length * 2, graph.indexQuery(PINDEX, "p.text:(beautiful are ducks)").propertyStream());
         assertCount(numV / strings.length * 2 - 10, graph.indexQuery(PINDEX, "p.text:(beautiful are ducks)").offset(10).propertyStream());
         total = graph.indexQuery(PINDEX, "p.\"text\":(beautiful are ducks)").limit(Integer.MAX_VALUE).propertyStream().count();
-        assertCount(10, graph.indexQuery(PINDEX, "p.\"text\":(beautiful are ducks)").limit(10).propertyStream());
-        assertEquals(total, (long) graph.indexQuery(PINDEX, "p.\"text\":(beautiful are ducks)").limit(10).propertyTotals());
-        assertCount(10, graph.indexQuery(PINDEX, "p.\"text\":(beautiful are ducks)").limit(10).offset(10).propertyStream());
-        assertEquals(total, (long) graph.indexQuery(PINDEX, "p.\"text\":(beautiful are ducks)").limit(10).offset(10).propertyTotals());
-        assertCount(0, graph.indexQuery(PINDEX, "p.\"text\":(beautiful are ducks)").limit(10).offset(numV).propertyStream());
-        assertEquals(total, (long) graph.indexQuery(PINDEX, "p.\"text\":(beautiful are ducks)").limit(10).offset(numV).propertyTotals());
+        assertCount(expectedLimitedCount, graph.indexQuery(PINDEX, "p.\"text\":(beautiful are ducks)").limit(limit).propertyStream());
+        assertEquals(expectedLimitedCount, (long) graph.indexQuery(PINDEX, "p.\"text\":(beautiful are ducks)").limit(limit).propertyTotals());
+        assertCount(expectedLimitedCount, graph.indexQuery(PINDEX, "p.\"text\":(beautiful are ducks)").limit(limit).offset(10).propertyStream());
+        assertEquals(expectedLimitedCount, (long) graph.indexQuery(PINDEX, "p.\"text\":(beautiful are ducks)").limit(limit).offset(10).propertyTotals());
+        assertCount(0, graph.indexQuery(PINDEX, "p.\"text\":(beautiful are ducks)").limit(limit).offset(numV).propertyStream());
+        assertEquals(0, (long) graph.indexQuery(PINDEX, "p.\"text\":(beautiful are ducks)").limit(limit).offset(numV).propertyTotals());
         //Test name mapping
         assertCount(numV / strings.length * 2, graph.indexQuery(PINDEX, "text:ducks").propertyStream());
     }

janusgraph-core/src/main/java/org/janusgraph/diskstorage/indexing/IndexProvider.java

@@ -112,7 +112,7 @@ static void checkKeyValidity(String key) {
     Stream<RawQuery.Result<String>> query(RawQuery query, KeyInformation.IndexRetriever information, BaseTransaction tx) throws BackendException;
 
     /**
-     * Executes the given raw query against the index and returns the total hits. e.g. limit=0
+     * Executes the given raw query against the index and returns the total hits after specified offset and in limit scope if specified.
      *
      * @param query Query to execute
      * @param information Information on the keys used in the query accessible through {@link KeyInformation.IndexRetriever}.

janusgraph-core/src/main/java/org/janusgraph/graphdb/query/QueryUtil.java

@@ -407,6 +407,10 @@ public static long applyQueryLimitAfterCount(long count, Query query){
         return Math.max(0L, (query.hasLimit()) ? Math.min(count, query.getLimit()) : count);
     }
 
+    public static long applyOffsetWithQueryLimitAfterCount(long count, int offset, Query query){
+        return applyQueryLimitAfterCount((offset > 0) ? count - offset : count, query);
+    }
+
     public interface IndexCall<R> {
 
         Collection<R> call(int limit);

janusgraph-core/src/main/java/org/janusgraph/graphdb/query/graph/IndexQueryBuilder.java

@@ -216,7 +216,6 @@ private <E extends JanusGraphElement> Stream<Result<E>> execute(ElementCategory
     private Long executeTotals(ElementCategory resultType) {
         Preconditions.checkNotNull(indexName);
         Preconditions.checkNotNull(query);
-        this.setLimit(0);
         if (tx.hasModifications())
             log.warn("Modifications in this transaction might not be accurately reflected in this index query: {}",query);
         return serializer.executeTotals(this,resultType,tx.getTxHandle(),tx);

janusgraph-es/src/main/java/org/janusgraph/diskstorage/es/ElasticSearchIndex.java

@@ -1285,9 +1285,10 @@ private ElasticSearchResponse runCommonQuery(RawQuery query, KeyInformation.Inde
 
     private long runCountQuery(RawQuery query) throws BackendException{
         try {
-            return client.countTotal(
+            long countTotal = client.countTotal(
                 getIndexStoreName(query.getStore()),
                 compat.createRequestBody(compat.queryString(query.getQuery()), query.getParameters()));
+            return QueryUtil.applyOffsetWithQueryLimitAfterCount(countTotal, query.getOffset(), query);
         } catch (final IOException | UncheckedIOException e) {
             throw new PermanentBackendException(e);
         }

janusgraph-lucene/src/main/java/org/janusgraph/diskstorage/lucene/LuceneIndex.java

@@ -1029,11 +1029,9 @@ public Long totals(RawQuery query, KeyInformation.IndexRetriever information, Ba
 
             final long time = System.currentTimeMillis();
             // Lucene doesn't like limits of 0.  Also, it doesn't efficiently build a total list.
-            query.setLimit(1);
-            // We ignore offset and limit for totals
             final TopDocs docs = searcher.search(q, 1);
             log.debug("Executed query [{}] in {} ms", q, System.currentTimeMillis() - time);
-            return docs.totalHits.value;
+            return QueryUtil.applyOffsetWithQueryLimitAfterCount(docs.totalHits.value, query.getOffset(), query);
         } catch (final IOException e) {
             throw new TemporaryBackendException("Could not execute Lucene query", e);
         }

janusgraph-solr/src/main/java/org/janusgraph/diskstorage/solr/SolrIndex.java

@@ -802,7 +802,7 @@ public Long totals(RawQuery query, KeyInformation.IndexRetriever information,
             final QueryResponse response = solrClient.query(collection, runCommonQuery(query, information, tx,
                     collection, keyIdField));
             logger.debug("Executed query [{}] in {} ms", query.getQuery(), response.getElapsedTime());
-            return response.getResults().getNumFound();
+            return QueryUtil.applyOffsetWithQueryLimitAfterCount(response.getResults().getNumFound(), query.getOffset(), query);
         } catch (final IOException e) {
             logger.error("Query did not complete : ", e);
             throw new PermanentBackendException(e);