Mocking a "scroll" search

[elastic]

Mocking a "scroll" search

[abrin]

Ok! got it... for posterity, and in case it's useful... here's the final constructed test before I cleanup and add local functionality. Thanks!

// setup the mock and the client
const Mock = require("@elastic/elasticsearch-mock");
import { Client } from "@elastic/elasticsearch";
const esmock = new Mock();

const esclient = new Client({
  node: "http://localhost:9200",
  Connection: esmock.getConnection(),
});

// the base scrollSearch method from the documentation
async function* scrollSearch(params) {
  let response = await esclient.search(params);
  while (true) {
    const sourceHits = response.body.hits.hits;

    if (sourceHits.length === 0) {
      break;
    }

    for (const hit of sourceHits) {
      yield hit._source;
    }

    if (!response.body._scroll_id) {
      break;
    }

    response = await esclient.scroll({
      scroll: params.scroll,
      body: {
        scroll_id: response.body._scroll_id,
      },
    });
  }
}

describe("it tests elasticsearch", () => {
  it("tests scroll", async () => {

   // mock the search call (which should be the 1st call
    esmock.add(
      {
        method: ["POST"],
        path: "/_search",
      },
      () => {
        return {
          _scroll_id: "<id>",
          hits: {
            total: { value: 1, relation: "eq" },
            max_score: 1.0,
            hits: [
              {
                _index: "test",
                _type: "doc",
                _id: "component/f745b673-25fe-5b68-9e8a-361b4e13185a",
                _score: 1.0,
                _source: {
                  id: "component/f745b673-25fe-5b68-9e8a-361b4e13185a",
                },
              },
            ],
          },
        };
      }
    );


   // mock the subsequent "scroll" calls... we'll return a result the 1st time, and then the second return nothing.
    var calls = 0;
    esmock.add(
      {
        method: ["POST"],
        path: "/_search/scroll",
      },
      () => {
        if (calls > 0) {
          return {
            hits: { hits: [] },
            _scroll_id: "<id>",
            total: { value: 0, relation: "eq" },
          };
        }
        calls++;
        return {
          _scroll_id: "<id>",
          hits: {
            total: { value: 1, relation: "eq" },
            hits: [
              {
                _index: "test",
                _type: "doc",
                _id: "component/aaaaaaa-25fe-5b68-9e8a-361b4e13185a",
                _score: 1.0,
                _source: {
                  id: "component/aaaaaaa-25fe-5b68-9e8a-361b4e13185a",
                },
              },
            ],
          },
        };
      }
    );

//  this is the initial search
    let params = {
      scroll: "30s",
      size: 1,
      body: {
        query: {
          term: {
            rel_ids: "123",
          },
        },
      },
    };
    let count = 0;
    for await (const hit of scrollSearch(params)) {
      count++;
    }
    expect(count).toEqual(2);
  });
});

[delvedor]

Hello! It depends on how you are sending the scroll request. You can either send the _scroll_id as URL parameter, or in the body. I highly recommend using the body, as scroll ids can be quite long.

// send the scroll_id in the URL
client.scroll({
  scroll_id: '<id>',
  scroll: '30s'
}, console.log)

// send the scroll_id in the body
client.scroll({
  scroll: '30s',
  body: {
    scroll_id: '<id>'
  }
}, console.log)

If you want to use the URL params way, the mock should be updated as follows:

mock.add({
  method: 'GET',
  path: '/_search/scroll/:id'
}, () => {
  return {
    _scroll_id: 'id',
    hits: { hits: [] }
  }
})

[abrin]

@delvedor I gave that a try, but it doesn't seem to quite work... For clarification (using v7.9.1 of Elasticsearch and 0.3.0 of the mocking library), and following the pattern from here. Below is the full test-case.

// import * as elasticsearch from "./elasticsearch";
const Mock = require("@elastic/elasticsearch-mock");
import { Client } from "@elastic/elasticsearch";
const esmock = new Mock();

const esclient = new Client({
  node: "http://localhost:9200",
  Connection: esmock.getConnection()
});
// elasticsearch.setClient(esclient);

async function* scrollSearch(params) {
  let response = await esclient.search(params);
  while (true) {
    const sourceHits = response.body.hits.hits;

    if (sourceHits.length === 0) {
      break;
    }

    for (const hit of sourceHits) {
      yield hit._source;
    }

    if (!response.body._scroll_id) {
      break;
    }

    response = await esclient.scroll({
      scroll: params.scroll,
      body: {
        scroll_id: response.body._scroll_id
      }
    });
  }
}

describe("it tests elasticsearch", () => {
  it("tests scroll", async () => {
    esmock.add(
      {
        method: ["GET", "POST"],
        path: "/_search"
      },
      () => {
        return {
          _scroll_id: "<id>",
          hits: {
            total: { value: 1, relation: "eq" },
            max_score: 1.0,
            hits: [
              {
                _index: "test",
                _type: "doc",
                _id: "component/f745b673-25fe-5b68-9e8a-361b4e13185a",
                _score: 1.0,
                _source: {
                  id: "component/f745b673-25fe-5b68-9e8a-361b4e13185a"
                }
              }
            ]
          }
        };
      }
    );
    esmock.add(
      {
        method: "GET",
        path: "/_search/scroll/:id"
      },
      () => {
        return {
          hits: [
            {
              _index: "test",
              _type: "doc",
              _id: "component/aaaaaaa-25fe-5b68-9e8a-361b4e13185a",
              _score: 1.0,
              _source: {
                id: "component/aaaaaaa-25fe-5b68-9e8a-361b4e13185a"
              }
            }
          ]
        };
      }
    );
    let params = {
      scroll: "30s",
      size: 1,
      body: {
        query: {
          term: {
            rel_ids: "123"
          }
        }
      }
    };
    let count = 0;
    for await (const hit of scrollSearch(params)) {
      console.log(hit);
      count++;
    }
    expect(count).toEqual(2);
  });
});

The result in jest:

 FAIL  src/indexer/utils/elasticsearch.spec.js
  it tests elasticsearch
    ✕ tests scroll (41ms)

  ● it tests elasticsearch › tests scroll

    ResponseError: Response Error

      at Class.response.on (node_modules/@elastic/elasticsearch/lib/Transport.js:257:25)
      at endReadableNT (node_modules/readable-stream/lib/_stream_readable.js:1010:12)

  console.log src/indexer/utils/elasticsearch.spec.js:100
    { id: 'component/f745b673-25fe-5b68-9e8a-361b4e13185a' }

[delvedor]

@abrin you are now passing the scroll_id in the body, so you need to mock the POST method as well.

[abrin]

changing the mock to allow ['GET','POST'] still produces a transport the same error. Simply adding the _scroll_id in the initial search response seems to be what causes it:

    esmock.add(
      {
        method: ["GET", "POST"],
        path: "/_search",
      },
      () => {
        return {
          _scroll_id: "<id>",
          hits: {

[delvedor]

You must add the POST method to the scroll endpoint mock :)