We Keep Coding

Elastic Search MoreLikeThis Query Never Returns Results

I must be doing something fundamentally wrong here. I'm trying to get a "More Like This" query working in a search engine project we have that uses Elastic Search. The idea is that the CMS can write tags (like categories) to the page in a Meta tag or something, and we would read those into Elastic and use them to drive a "more like this" search based upon an input document id.
So if the input document has tags of catfish, chicken, goat I would expect Elastic Search to find other documents that share those tags and not return ones for racecar and airplane.
I've built a proof of concept console app by:
Getting a local Elastic Search 6.6.1 instance running in Docker by following the instructions on https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html
Creating a new .NET Framework 4.6.1 Console App
Adding the NuGet packages for NEST 6.5.0 and ElasticSearch.Net 6.5.0
Then I created a new elastic index that contains objects (Type "MyThing") that have a "Tags" property. This tag is a random comma-delimited set of words from a set of possible values. I've inserted anywhere from 100 to 5000 items in the index in testing. I've tried more and fewer possible words in the set.
No matter what I try the MoreLikeThis query never returns anything, and I don't understand why.
Query that isn't returning results:
var result = EsClient.Search<MyThing>(s => s
.Index(DEFAULT_INDEX)
.Query(esQuery =>
{
var mainQuery = esQuery
.MoreLikeThis(mlt => mlt
.Include(true)
.Fields(f => f.Field(ff => ff.Tags, 5))
.Like(l => l.Document(d => d.Id(id)))
);
return mainQuery;
}
Full "program.cs" source:
using Nest;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Test_MoreLikeThis_ES6
{
class Program
{
public class MyThing
{
public string Tags { get; set; }
}
const string ELASTIC_SERVER = "http://localhost:9200";
const string DEFAULT_INDEX = "my_index";
const int NUM_RECORDS = 1000;
private static Uri es_node = new Uri(ELASTIC_SERVER);
private static ConnectionSettings settings = new ConnectionSettings(es_node).DefaultIndex(DEFAULT_INDEX);
private static ElasticClient EsClient = new ElasticClient(settings);
private static Random rnd = new Random();
static void Main(string[] args)
{
Console.WriteLine("Rebuild index? (y):");
var answer = Console.ReadLine().ToLower();
if (answer == "y")
{
RebuildIndex();
for (int i = 0; i < NUM_RECORDS; i++)
{
AddToIndex();
}
}
Console.WriteLine("");
Console.WriteLine("Getting a Thing...");
var aThingId = GetARandomThingId();
Console.WriteLine("");
Console.WriteLine("Looking for something similar to document with id " + aThingId);
Console.WriteLine("");
Console.WriteLine("");
GetMoreLikeAThing(aThingId);
}
private static string GetARandomThingId()
{
var firstdocQuery = EsClient
.Search<MyThing>(s =>
s.Size(1)
.Query(q => {
return q.FunctionScore(fs => fs.Functions(fn => fn.RandomScore(rs => rs.Seed(DateTime.Now.Ticks).Field("_seq_no"))));
})
);
if (!firstdocQuery.IsValid || firstdocQuery.Hits.Count == 0) return null;
var hit = firstdocQuery.Hits.First();
Console.WriteLine("Found a thing with id '" + hit.Id + "' and tags: " + hit.Source.Tags);
return hit.Id;
}
private static void GetMoreLikeAThing(string id)
{
var result = EsClient.Search<MyThing>(s => s
.Index(DEFAULT_INDEX)
.Query(esQuery =>
{
var mainQuery = esQuery
.MoreLikeThis(mlt => mlt
.Include(true)
.Fields(f => f.Field(ff => ff.Tags, 5))
.Like(l => l.Document(d => d.Id(id)))
);
return mainQuery;
}
));
if (result.IsValid)
{
if (result.Hits.Count > 0)
{
Console.WriteLine("These things are similar:");
foreach (var hit in result.Hits)
{
Console.WriteLine(" " + hit.Id + " : " + hit.Source.Tags);
}
}
else
{
Console.WriteLine("No similar things found.");
}
}
else
{
Console.WriteLine("There was an error running the ES query.");
}
Console.WriteLine("");
Console.WriteLine("Enter (y) to get another thing, or anything else to exit");
var y = Console.ReadLine().ToLower();
if (y == "y")
{
var aThingId = GetARandomThingId();
GetMoreLikeAThing(aThingId);
}
Console.WriteLine("");
Console.WriteLine("Any key to exit...");
Console.ReadKey();
}
private static void RebuildIndex()
{
var existsResponse = EsClient.IndexExists(DEFAULT_INDEX);
if (existsResponse.Exists) //delete existing mapping (and data)
{
EsClient.DeleteIndex(DEFAULT_INDEX);
}
var rebuildResponse = EsClient.CreateIndex(DEFAULT_INDEX, c => c.Settings(s => s.NumberOfReplicas(1).NumberOfShards(5)));
var response2 = EsClient.Map<MyThing>(m => m.AutoMap());
}
private static void AddToIndex()
{
var myThing = new MyThing();
var tags = new List<string> {
"catfish",
"tractor",
"racecar",
"airplane",
"chicken",
"goat",
"pig",
"horse",
"goose",
"duck"
};
var randNum = rnd.Next(0, tags.Count);
//get randNum random tags
var rand = tags.OrderBy(o => Guid.NewGuid().ToString()).Take(randNum);
myThing.Tags = string.Join(", ", rand);
var ir = new IndexRequest<MyThing>(myThing);
var indexResponse = EsClient.Index(ir);
Console.WriteLine("Index response: " + indexResponse.Id + " : " + string.Join(" " , myThing.Tags));
}
}
}

The issue here is that the default min_term_freq value of 2 will never be satisfied for any of the terms of the prototype document because all documents contain only each tag (term) once. If you drop min_term_freq to 1, you'll get results. Might also want to set min_doc_freq to 1 too, and combine with a query that excludes the prototype document.
Here's an example to play with
const string ELASTIC_SERVER = "http://localhost:9200";
const string DEFAULT_INDEX = "my_index";
const int NUM_RECORDS = 1000;
private static readonly Random _random = new Random();
private static readonly IReadOnlyList<string> Tags =
new List<string>
{
"catfish",
"tractor",
"racecar",
"airplane",
"chicken",
"goat",
"pig",
"horse",
"goose",
"duck"
};
private static ElasticClient _client;
private static void Main()
{
var pool = new SingleNodeConnectionPool(new Uri(ELASTIC_SERVER));
var settings = new ConnectionSettings(pool)
.DefaultIndex(DEFAULT_INDEX);
_client = new ElasticClient(settings);
Console.WriteLine("Rebuild index? (y):");
var answer = Console.ReadLine().ToLower();
if (answer == "y")
{
RebuildIndex();
AddToIndex();
}
Console.WriteLine();
Console.WriteLine("Getting a Thing...");
var aThingId = GetARandomThingId();
Console.WriteLine();
Console.WriteLine("Looking for something similar to document with id " + aThingId);
Console.WriteLine();
Console.WriteLine();
GetMoreLikeAThing(aThingId);
}
public class MyThing
{
public List<string> Tags { get; set; }
}
private static string GetARandomThingId()
{
var firstdocQuery = _client
.Search<MyThing>(s =>
s.Size(1)
.Query(q => q
.FunctionScore(fs => fs
.Functions(fn => fn
.RandomScore(rs => rs
.Seed(DateTime.Now.Ticks)
.Field("_seq_no")
)
)
)
)
);
if (!firstdocQuery.IsValid || firstdocQuery.Hits.Count == 0) return null;
var hit = firstdocQuery.Hits.First();
Console.WriteLine($"Found a thing with id '{hit.Id}' and tags: {string.Join(", ", hit.Source.Tags)}");
return hit.Id;
}
private static void GetMoreLikeAThing(string id)
{
var result = _client.Search<MyThing>(s => s
.Index(DEFAULT_INDEX)
.Query(esQuery => esQuery
.MoreLikeThis(mlt => mlt
.Include(true)
.Fields(f => f.Field(ff => ff.Tags))
.Like(l => l.Document(d => d.Id(id)))
.MinTermFrequency(1)
.MinDocumentFrequency(1)
) && !esQuery
.Ids(ids => ids
.Values(id)
)
)
);
if (result.IsValid)
{
if (result.Hits.Count > 0)
{
Console.WriteLine("These things are similar:");
foreach (var hit in result.Hits)
{
Console.WriteLine($" {hit.Id}: {string.Join(", ", hit.Source.Tags)}");
}
}
else
{
Console.WriteLine("No similar things found.");
}
}
else
{
Console.WriteLine("There was an error running the ES query.");
}
Console.WriteLine();
Console.WriteLine("Enter (y) to get another thing, or anything else to exit");
var y = Console.ReadLine().ToLower();
if (y == "y")
{
var aThingId = GetARandomThingId();
GetMoreLikeAThing(aThingId);
}
Console.WriteLine();
Console.WriteLine("Any key to exit...");
}
private static void RebuildIndex()
{
var existsResponse = _client.IndexExists(DEFAULT_INDEX);
if (existsResponse.Exists) //delete existing mapping (and data)
{
_client.DeleteIndex(DEFAULT_INDEX);
}
var rebuildResponse = _client.CreateIndex(DEFAULT_INDEX, c => c
.Settings(s => s
.NumberOfShards(1)
)
.Mappings(m => m
.Map<MyThing>(mm => mm.AutoMap())
)
);
}
private static void AddToIndex()
{
var bulkAllObservable = _client.BulkAll(GetMyThings(), b => b
.RefreshOnCompleted()
.Size(1000));
var waitHandle = new ManualResetEvent(false);
Exception exception = null;
var bulkAllObserver = new BulkAllObserver(
onNext: r =>
{
Console.WriteLine($"Indexed page {r.Page}");
},
onError: e =>
{
exception = e;
waitHandle.Set();
},
onCompleted: () => waitHandle.Set());
bulkAllObservable.Subscribe(bulkAllObserver);
waitHandle.WaitOne();
if (exception != null)
{
throw exception;
}
}
private static IEnumerable<MyThing> GetMyThings()
{
for (int i = 0; i < NUM_RECORDS; i++)
{
var randomTags = Tags.OrderBy(o => Guid.NewGuid().ToString())
.Take(_random.Next(0, Tags.Count))
.OrderBy(t => t)
.ToList();
yield return new MyThing { Tags = randomTags };
}
}
And here's an example output
Found a thing with id 'Ugg9LGkBPK3n91HQD1d5' and tags: airplane, goat
These things are similar:
4wg9LGkBPK3n91HQD1l5: airplane, goat
9Ag9LGkBPK3n91HQD1l5: airplane, goat
Vgg9LGkBPK3n91HQD1d5: airplane, goat, goose
sQg9LGkBPK3n91HQD1d5: airplane, duck, goat
lQg9LGkBPK3n91HQD1h5: airplane, catfish, goat
9gg9LGkBPK3n91HQD1l5: airplane, catfish, goat
FQg9LGkBPK3n91HQD1p5: airplane, goat, goose
Jwg9LGkBPK3n91HQD1p5: airplane, goat, goose
Fwg9LGkBPK3n91HQD1d5: airplane, duck, goat, tractor
Kwg9LGkBPK3n91HQD1d5: airplane, goat, goose, horse

Parallel execution issue

I need a little education here with regards to the execution of parallel tasks.
I have created a small fiddle:
https://dotnetfiddle.net/JO2a4m
What I am trying to do send a few accounts to process in batches to another method and creating a unit of work (task) for each batch but when I execute the tasks, it only executes the last task which was added. This is something I am trying to break my head around.
Code:
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
public class Program
{
public static void Main()
{
var accounts = GenerateAccount();
var accountsProcess = new List<Account>();
var taskList = new List<Task>();
var batch = 4;
var count = 0;
foreach (var account in accounts)
{
if (count == batch)
{
taskList.Add(new Task(() => ProcessAccount(accountsProcess)));
count = 0;
accountsProcess.Clear();
}
count++;
accountsProcess.Add(account);
}
Parallel.ForEach(taskList, t =>
{
t.Start();
}
);
Task.WaitAll(taskList.ToArray());
if (accountsProcess.Count > 0)
ProcessAccount(accountsProcess);
}
public static List<Account> GenerateAccount()
{
var accounts = new List<Account>();
var first = "First";
var second = "Second";
for (int i = 0; i <= 1000; i++)
{
var account = new Account();
account.first = first + i;
account.second = second + i;
accounts.Add(account);
}
return accounts;
}
public static void ProcessAccount(List<Account> accounts)
{
Console.WriteLine(accounts.Count);
foreach (var account in accounts)
{
Console.WriteLine(account.first + account.second);
}
}
}
public class Account
{
public string first;
public string second;
}

foreach (var account in accounts)
{
if (count == batch)
{
taskList.Add(new Task(() => ProcessAccount(accountsProcess)));
count = 0;
accountsProcess.Clear();
}
count++;
accountsProcess.Add(account);
}
The issue is that all of the Tasks are sharing the same List<Account> object.
I would suggest changing the code to:
foreach (var account in accounts)
{
if (count == batch)
{
var bob = accountsProcess;
taskList.Add(new Task(() => ProcessAccount(bob)));
count = 0;
accountsProcess = new List<Account>();
}
count++;
accountsProcess.Add(account);
}
By using bob and assigning a new List to accountsProcess we ensure each Task gets its own List - rather than sharing a single List.
Also, consider using MoreLINQ's Batch rather than rolling your own.

Returning data from foreach data

I am trying to return that data from the foreach "hostedId" Can someone help?
public static string GetHostedRecordSet()
{
var request = new ListHostedZonesRequest()
{
MaxItems = "1"
};
var list = client.ListHostedZones(request);
foreach (var hostedId in list.HostedZones)
{
Console.WriteLine("\n Hosted ID is:");
Console.Write(hostedId.Id);
}
return hostedId;
}

It depends. If you want to return the first element:
return list.HostedZones.First().Id; // Not in a loop!
If you want to return several items, change the signature of the method:
public static IEnumerable<string> GetHostedRecordSet()
{
var request = new ListHostedZonesRequest()
{
MaxItems = "1"
};
var list = client.ListHostedZones(request);
return list.HostedZones
.Select(z => z.Id);
}

If you want to return all values as a single string you can concatenate them with a delimiter, such as ',':
public static string GetHostedRecordSet()
{
var request = new ListHostedZonesRequest()
{
MaxItems = "1"
};
var list = client.ListHostedZones(request);
StringBuilder result = new StringBuilder();
foreach (var hostedId in list.HostedZones)
{
result.Append(hostedId.Id).Append(",");
}
return result.ToString(0, Math.Max(0, result.Length - 1);
}

WaitForDrawerClose blocks printing

I'm writing an application that uses Pos for .Net, and I'm noticing that if you call WaitForDrawerClose, then you won't be able to print receipts until it has returned.
This is not desirable behavior. Is there another way to wait for the cash drawer to close without blocking the printer?
I've looked into the OnDrawerStateChanged Event, but that is a protected member of CashDrawerBase, and I'm not entirely sure how to access it.
Here is my SSCCE:
static void Main(string[] args)
{
var posExplorer = new PosExplorer();
var waitTask = WaitForCloseAsync(posExplorer);
System.Threading.Thread.Sleep(500);
PrintText(posExplorer);
waitTask.Wait();
}
public static Task WaitForCloseAsync(PosExplorer posExplorer)
{
var result = Task.Factory.StartNew(() =>
{
Console.WriteLine("waiting");
var cashDrawer = GetCashDrawer(posExplorer);
cashDrawer.Open();
cashDrawer.Claim(1000);
cashDrawer.DeviceEnabled = true;
cashDrawer.WaitForDrawerClose(10000, 4000, 500, 5000);
cashDrawer.Release();
cashDrawer.Close();
Console.WriteLine("waited");
});
return result;
}
public static void PrintText(PosExplorer posExplorer)
{
Console.WriteLine("printing");
var printer = GetPosPrinter(posExplorer);
printer.Open();
printer.Claim(1000);
printer.DeviceEnabled = true;
var text = "abc\x1B|1lF";
printer.PrintNormal(PrinterStation.Receipt, text);
printer.Release();
printer.Close();
Console.WriteLine("printed");
}
public static CashDrawer GetCashDrawer(PosExplorer posExplorer)
{
var deviceInfo = posExplorer.GetDevices(DeviceCompatibilities.Opos)
.Cast<DeviceInfo>()
.Where(d => d.Type == "CashDrawer")
.ToList();
var device = deviceInfo.FirstOrDefault(d => d.Compatibility == DeviceCompatibilities.Opos);
if (device == null)
{
return null;
}
else
return (CashDrawer)posExplorer.CreateInstance(device);
}
private static PosPrinter GetPosPrinter(PosExplorer posExplorer)
{
var deviceInfo = posExplorer.GetDevices(DeviceCompatibilities.Opos)
.Cast<DeviceInfo>()
.Where(d => d.Type == "PosPrinter")
.ToList();
var device = deviceInfo.FirstOrDefault(d => d.Compatibility == DeviceCompatibilities.Opos);
if (device == null)
{
return null;
}
else
{
return (PosPrinter)posExplorer.CreateInstance(device);
}
}

so, what I did was essentially this: Instead of using (WaitForClose), I just poll it like this:
for (var i = 0; i < 15; i++)
{
cashDrawer = GetCashDrawer(posExplorer);
cashDrawer.Open();
cashDrawer.Claim(1000);
cashDrawer.DeviceEnabled = true;
if (!cashDrawer.DrawerOpened)
{
Console.WriteLine("waited");
return;
}
cashDrawer.Release();
cashDrawer.Close();
System.Threading.Thread.Sleep(1500);
}
Console.WriteLine("timed out");
It's not ideal, but it doesn't lock the printer up either, so It'll have to to for now.

IObservable with Replay and FirstAsync + ForEachAsync weird side effects

I am having an issue with my code below. I have a cold source that begins when you subscribe. I want to run Observable once, so I am using a replay call on it. What I found is that when it hits the conditional branch to write the header, it starts the Observable on the call to FirstAsync, and then starts the Observable again in a new thread at the ForEachAsync call. I end up with the observable running concurently in two threads. I am not sure why this is occuring.
public async Task WriteToFileAsync(string filename, IObservable<IFormattableTestResults> source, bool overwrite)
{
ThrowIfInvalidFileName(filename);
var path = Path.Combine(_path, filename);
bool fileExists = File.Exists(path);
using (var writer = new StreamWriter(path, !overwrite))
{
var replay = source.Replay().RefCount();
if (overwrite || !fileExists)
{
var first = await replay.FirstAsync();
var header = GetCsvHeader(first.GetResults());
await writer.WriteLineAsync(header);
}
await replay.ForEachAsync(result => writer.WriteLineAsync(FormatCsv(result.GetResults())));
}
}
Edit 10/22/2015: Adding more code
private Task RunIVBoardCurrentAdjust(IVBoardAdjustment test)
{
logger.Info("Loading IV board current adjustment test.");
var testCases = _testCaseLoader.GetIVBoardCurrentAdjustTests().ToArray();
var source = test.RunCurrentAdjustment(testCases);
return _fileResultsService.WriteToFileAsync("IVBoardCurrentAdjust.csv", source, false);
}
public IObservable<IVBoardCurrentAdjustTestResults> RunCurrentAdjustment(IEnumerable<IVBoardCurrentAdjustTestCase> testCases)
{
return
testCases
.Select(RunCurrentAdjustment)
.Concat();
}
public IObservable<IVBoardCurrentAdjustTestResults> RunCurrentAdjustment(IVBoardCurrentAdjustTestCase testCase)
{
logger.Debug("Preparing IV board current adjustment procedure.");
return Observable.Create<IVBoardCurrentAdjustTestResults>(
(observer, cancelToken) =>
{
var results =
RunAdjustment(testCase)
.Do(result => logger.Trace(""))
.Select(
(output, i) =>
new IVBoardCurrentAdjustTestResults(i, testCase, output)
{
Component = "IV Board",
Description = "Characterization (Secant Method)"
})
.Replay();
results.Subscribe(observer, cancelToken);
var task = StoreResultInBTD(results, testCase, 1/testCase.Load);
results.Connect();
return task;
});
}
private IObservable<IRootFindingResult> RunAdjustment<T>(IVBoardAdjustTestCase<T> testCase) where T : DacCharacterizationSecantInput
{
logger.Debug("Initializing IV board test.");
SetupTest(testCase);
return
new DacCharacterization()
.RunSecantMethod(
code => _yellowCake.IVBoard.DacRegister.Value = code,
() => _dmm.Read(),
GetTestInputs(testCase));
}
private async Task StoreResultInBTD(IObservable<IVBoardAdjustTestResults> results, IVBoardAdjustTestCase testCase, double targetScalingFactor = 1)
{
var points =
results
.Select(
result =>
new IVBoardCharacteristicCurveTestPoint(
(result.Output.Target - result.Output.Error) * targetScalingFactor,
(int)result.Output.Root));
var curve = await points.ToArray();
_yellowCake.BoardTest.WriteIVBoardAjust(curve, testCase.Mode, testCase.Range);
_yellowCake.BoardTest.SaveToFile();
}
private IEnumerable<DacCharacterizationSecantInput> GetTestInputs<T>(IVBoardAdjustTestCase<T> testCase) where T : DacCharacterizationSecantInput
{
foreach (var input in testCase.Inputs)
{
logger.Debug("Getting next test input.");
_dmm.Config.PowerLineCycles.Value = input.IntegrationTime;
yield return input.Input;
}
}
public IObservable<IRootFindingResult> RunSecantMethod(
Action<int> setDacOutput,
Func<double> readMeanOutput,
IEnumerable<DacCharacterizationSecantInput> inputs)
{
var search = new SecantSearch();
var param = SecantMethodParameter.Create(setDacOutput, readMeanOutput);
return
Observable
.Create<IRootFindingResult>(
(observer, cancelToken) =>
Task.Run(() =>
{
foreach (var input in inputs)
{
cancelToken.ThrowIfCancellationRequested();
var result =
search.FindRoot(
param.SearchFunction,
input.FirstGuess,
input.SecondGuess,
input.Target,
input.SearchOptions,
cancelToken,
() => param.AdaptedDacCode);
if (!result.Converged)
{
observer.OnError(new FailedToConvergeException(result));
}
observer.OnNext(result);
}
}, cancelToken));
}