We Keep Coding

C# all common sequences in lists of strings

I am trying to find the longest common sequence of strings within the provided arrays.
I have 25,000 lists with sequences, with a total of 450,000 of words that I need to order by length, then by count.
List<string> listA = new List<string>() {"Step1", "Step3", "Process", "System", "Process"};
List<string> listB = new List<string>() {"Process", "System", "Process"};
List<string> listC = new List<string>() {"Terminal", "Step1", "Step3"};
...
The desired output that prints all possible sequences and their length and count is:
Sequence Length Count
Step1->Step3->Process->System->Process 5 1
Step1->Step3->Process->System 4 1
Step3->Process->System->Process 4 1
Process->System->Process 3 2
Step1->Step3->Process 3 1
Step3->Process->System 3 1
Terminal->Step1->Step3 3 1
Step1->Step3 2 2
Process->System 2 2
System->Process 2 2
Step3->Process 2 1
Terminal->Step1 2 1
Process 1 4
Step1 1 2
Step3 1 2
System 1 2
Terminal 1 1
I could only find an implementation of substrings, and not whole words that can take multiple lists as input.

Ok so you can actually overload GetHashCode and Equals to treat strings like chars in a string. Also it might be reasonable create list segment to prevent flooding runtime with multiple collections.
public class ListSegment<T>
{
private sealed class ListSegmentEqualityComparer : IEqualityComparer<ListSegment<T>>
{
public bool Equals(ListSegment<T> x, ListSegment<T> y)
{
if (x.Length != y.Length)
{
return false;
}
return x.Lst.Skip(x.Offset).Take(x.Length)
.SequenceEqual(y.Lst.Skip(y.Offset).Take(y.Length));
}
public int GetHashCode(ListSegment<T> obj)
{
unchecked
{
int hash = 17;
for (int i = obj.Offset; i < obj.Offset + obj.Length; i++)
{
hash = hash * 31 + obj.Lst[i].GetHashCode();
}
return hash;
}
}
}
public static IEqualityComparer<ListSegment<T>> Default { get; } = new ListSegmentEqualityComparer();
public List<T> Lst { get; set; }
public int Offset { get; set; }
public int Length { get; set; }
public IEnumerable<T> GetEnumerable()
{
return Lst.Skip(Offset).Take(Length);
}
public override string ToString()
{
return string.Join("->", GetEnumerable());
}
}
And then you run through list of lists counting number of occurrences
public List<KeyValuePair<ListSegment<string>, int>> GetOrderedPairs(List<List<string>> data)
{
var segmentsDictionary = new Dictionary<ListSegment<string>, int>(ListSegment<string>.Default);
foreach (var list in data)
{
for (int i = 0; i < list.Count; i++)
for (int j = i + 1; j <= list.Count; j++)
{
var segment = new ListSegment<string>
{
Lst = list,
Length = j-i,
Offset = i,
};
if (segmentsDictionary.TryGetValue(segment, out var val))
{
segmentsDictionary[segment] = val + 1;
}
else
{
segmentsDictionary[segment] = 1;
}
}
}
return segmentsDictionary.OrderByDescending(pair => pair.Key.Length).ToList();
}
To test it run following
List<string> listA = new List<string>() { "Step1", "Step3", "Process", "System", "Process" };
List<string> listB = new List<string>() { "Process", "System", "Process" };
List<string> listC = new List<string>() { "Terminal", "Step1", "Step3" };
var pairs = GetOrderedPairs(new List<List<string>>()
{
listA, listB, listC
});
foreach (var keyValuePair in pairs)
{
Console.WriteLine(keyValuePair.Key + " " + keyValuePair.Key.Length + " " + keyValuePair.Value);
}

Using some extension methods, you can create an IEQualityComparer that compares IEnumerable sequences. Using this, you can use LINQ Distinct to compare by sequences:
public static class IEnumerableExt {
public static IEnumerable<IEnumerable<T>> DistinctIE<T>(this IEnumerable<IEnumerable<T>> src) => src.Distinct(Make.IESequenceEqualityComparer<T>());
// IEnumerable<string>
public static string Join(this IEnumerable<string> src, string sep) => String.Join(sep, src);
}
public static class Make {
public static IEqualityComparer<IEnumerable<T>> IESequenceEqualityComparer<T>() => new IEnumerableSequenceEqualityComparer<T>();
public static IEqualityComparer<IEnumerable<T>> IESequenceEqualityComparer<T>(T _) => new IEnumerableSequenceEqualityComparer<T>();
public class IEnumerableSequenceEqualityComparer<T> : IEqualityComparer<IEnumerable<T>> {
public bool Equals(IEnumerable<T> x, IEnumerable<T> y) =>
Object.ReferenceEquals(x, y) || (x != null && y != null && (x.SequenceEqual(y)));
public int GetHashCode(IEnumerable<T> src) {
var hc = new HashCode();
foreach (var v in src)
hc.Add(v);
return hc.ToHashCode();
}
}
}
With these tools, you can create an extension method to generate all the subsequences of a List and all the distinct subsequences:
public static class ListExt {
public static IEnumerable<IEnumerable<T>> Subsequences<T>(this List<T> src) {
IEnumerable<T> Helper(int start, int end) {
for (int j3 = start; j3 <= end; ++j3)
yield return src[j3];
}
for (int j1 = 0; j1 < src.Count; ++j1) {
for (int j2 = j1; j2 < src.Count; ++j2)
yield return Helper(j1, j2);
}
}
public static IEnumerable<IEnumerable<T>> DistinctSubsequences<T>(this List<T> src) => src.Subsequences().DistinctIE();
}
Now you can compute the answer.
First, compute all the subsequences and combine them:
var ssA = listA.DistinctSubsequences();
var ssB = listB.DistinctSubsequences();
var ssC = listC.DistinctSubsequences();
var ssAll = ssA.Concat(ssB).Concat(ssC).DistinctIE();
Then, create some helpers for counting occurrences:
var hA = ssA.ToHashSet(Make.IESequenceEqualityComparer<string>());
var hB = ssB.ToHashSet(Make.IESequenceEqualityComparer<string>());
var hC = ssC.ToHashSet(Make.IESequenceEqualityComparer<string>());
Func<IEnumerable<string>, HashSet<IEnumerable<string>>, int> testIn = (s, h) => h.Contains(s) ? 1 : 0;
Func<IEnumerable<string>,int> countIn = s => testIn(s,hA)+testIn(s,hB)+testIn(s,hC);
Finally, compute the answer:
var ans = ssAll.Select(ss => new { Sequence = ss.Join("->"), Length = ss.Count(), Count = countIn(ss) }).OrderByDescending(sc => sc.Sequence.Length);

How To get Positional Occurrence of strings in List using Linq?

var a = new List<string>() {"aa","aa","bb","bb","bb","cc","aa","aa","cc","cc" };
I want to find occurrence count of all strings, for above list i want output like:
Index String Count
0 aa 2
2 bb 3
5 cc 1
6 aa 2
8 cc 2
but as i tried groupby it gives the total count not positional.
how i can get this?
edit: i expect more than 10 million entries in list.

You can do following.
Solution Using Linq
var a = new List<string>() {"aa","aa","bb","bb","bb","cc","aa","aa","cc","cc" };
var result = Enumerable.Range(0, a.Count())
.Where(x => x == 0 || a[x - 1] != a[x])
.Select((x,index) => new Stat<string>
{
Index =index,
StringValue = a[x],
Count = a.Skip(x).TakeWhile(c => c == a[x]).Count()
});
Where Stat is defined as
public class Stat<T>
{
public int Index{get;set;}
public T StringValue {get;set;}
public int Count {get;set;}
}
Update
Using Iterator
public IEnumerable<Stat<T>> CountOccurance<T>(IEnumerable<T> source)
{
var lastItem = source.First();
var count = 1;
var index= 0;
foreach(var item in source.Skip(1))
{
if(item.Equals(lastItem))
{
count++;
}
else
{
yield return new Stat<T>
{
Index = index,
StringValue = lastItem,
Count = count
};
count=1;
lastItem = item;
index++;
}
}
yield return new Stat<T>
{
Index = index,
StringValue = lastItem,
Count = count
};
}
You can then fetch the result as
var result = CountOccurance(a);

If you will use this logic multiple times you can consider writing your own extension method.
public static class ExtensionMethods
{
public static IEnumerable<SpecialGroup<TKey>> GroupAccordingToSuccessiveItems<TSource, TKey>(this IEnumerable<TSource> source, Func<TSource, TKey> keySelector)
{
int index = 0;
int count = 0;
TKey latestKey = default(TKey);
foreach (var item in source)
{
TKey key = keySelector(item);
if (index != 0 && !object.Equals(key, latestKey))
{
yield return new SpecialGroup<TKey>
{
Index = index - count,
Obj = latestKey,
Count = count
};
count = 0;
}
latestKey = key;
count++;
index++;
}
yield return new SpecialGroup<TKey>
{
Index = index - count,
Obj = latestKey,
Count = count
};
}
}
public class SpecialGroup<T>
{
public int Index { get; set; }
public T Obj { get; set; }
public int Count { get; set; }
}
Then you can call it,
var result =
a.
GroupAccordingToSuccessiveItems(i => i);
This will iterate all the list once.

How to use LINQ to group objects in two groups

Lets say that I have the class Invoice:
public class Invoice
{
public int PartNumber { get; set; }
public string PartDescription { get; set; }
public int Quantity { get; set; }
public decimal Price { get; set; }
}
and then I have an array of its objects in the variable arrayOfInvoices .
If I had to Group invoices in two groups- invoices with Unit price below 12 and invoices with unit price above or equal to 12 and display details about invoices in each group sorted in ascending order of the price, how could I do that ?

You can simply do something like this:
var results =
from inv in arrayOfInvoices
orderby inv.Price
group inv by inv.Price < 12;
Or if you prefer fluent syntax:
var results = arrayOfInvoices.OrderBy(inv => inv.Price)
.GroupBy(inv => inv.Price < 12);
To group the invoices into three or more 'buckets', you can use BinarySearch:
var priceBoundaries = new[] { 12m, 20m };
var results =
from inv in arrayOfInvoices
orderby inv.Price
let index = Array.BinarySearch(priceBoundaries, inv.Price)
group inv by (index < 0 ? ~index : index + 1);
Or use side effects, like this:
var priceBoundaries = new[] { 12m, 20m, Decimal.MaxValue }; // Note the addition of MaxValue
var i = 0;
var results =
from inv in arrayOfInvoices
orderby inv.Price
group inv by (inv.Price < priceBoundaries[i] ? i : ++i);
This is generally bad practice, but should perform better than the BinarySearch method above.

If using group function is a pain (sometimes it gets annoying), then you can also use "Where"
var invoices = new List<Invoice> ();
var group1= invoices.Where(i=> i.Price<12).Orderby(i=> i.Price).ToList();
var group2= invoices.Where(i=> i.Price>=12).Orderby(i=> i.Price).ToList();

You can encapsulate the concept of the range in a class:
private class PriceRange<T> : IEquatable<PriceRange<T>>
{
public T Min { get; set; }
public T Max { get; set; }
public override int GetHashCode()
{
unchecked
{
int hash = 17;
hash = hash * 31 + Min.GetHashCode();
hash = hash * 31 + Max.GetHashCode();
return hash;
}
}
public override bool Equals(object obj)
{
return Equals(obj as PriceRange<T>);
}
public bool Equals(PriceRange<T> other)
{
if (other == null) return false;
if (!Min.Equals(other.Min)) return false;
if (!Max.Equals(other.Max)) return false;
return true;
}
}
Then use a map function to map the prices of each of your invoices to the appropriate range:
private class PriceRangeFactory<T>
{
public PriceRangeFactory(T[] rangeCutoffs)
{
_RangeCutoffs = rangeCutoffs;
}
private T[] _RangeCutoffs;
public PriceRange<T> Map(T price)
{
var index = Array.BinarySearch(_RangeCutoffs, price);
// Assume that the _RangeCutoffs that we have fully cover all possible values for T.
if (index < 0) index = ~index;
return new PriceRange<T>() { Min = _RangeCutoffs[index - 1], Max = _RangeCutoffs[index] };
}
}
Then use GroupBy with that map function:
var rangeFactory = new PriceRangeFactory<decimal>(new decimal[] { decimal.MinValue, 12, 20, decimal.MaxValue });
var grouped = foos.GroupBy(a => rangeFactory.Map(a.Price));
You get a list of IGroupings each keyed by a range that you specified, with the appropriate objects that fit into that range attached.
Now, obviously the code above is not quite production level, but it should be enough to get you started. As for why its not production level:
There's no check to assert that the ranges supplied to the PriceRangeFactory actually do fully cover all possible values.
It always assumes that the range is described as > X and <= Y.
There's no checking around Range.Min < Range.Max.
There's no assert that the list of range cutoffs supplied to the PriceRangeFactory are ordered correctly (required for BinarySearch).
There are bound to be some edge cases I haven't covered.

Fastest way to check if two List<T> are equal

I have two Lists
ListA<Emp> and ListB<Emp>
both are having 1000 records.
Emp is an object of Employee Class. Below is my Employee class
public class Employee
{
int ID = 0;
string Name = String.Empty;
string Dept = String.Empty;
string Address = String.Empty;
int Age = 0;
string Email = String.Empty;
}
I want to verify if both the Lists are equal. The Emp objects may be placed in different order. Also, there might be several Emp objects which are having exactly same info in both the list. I have to verify those also.
I tried to sort the lists and compared using SequenceEqual
Enumerable.SequenceEqual(ListA.OrderBy(s => s), ListB.OrderBy(s => s)
I am getting below error
At least one object must implement IComparable.
Exception Stack trace is as below
at System.Collections.Comparer.Compare(Object a, Object b)
at System.Collections.Generic.ObjectComparer`1.Compare(T x, T y)
at System.Linq.EnumerableSorter`2.CompareKeys(Int32 index1, Int32 index2)
at System.Linq.EnumerableSorter`1.QuickSort(Int32[] map, Int32 left, Int32 right)
at System.Linq.EnumerableSorter`1.Sort(TElement[] elements, Int32 count)
at System.Linq.OrderedEnumerable`1.<GetEnumerator>d__0.MoveNext()
at System.Linq.Enumerable.SequenceEqual[TSource](IEnumerable`1 first, IEnumerable`1 second, IEqualityComparer`1 comparer)
at System.Linq.Enumerable.SequenceEqual[TSource](IEnumerable`1 first, IEnumerable`1 second)
How can I implement this ? Also it will be better if you guys can provide me the fastest way of doing this because the number of objects in List may grow to 10 million.
Thanks for your help !
EDIT: Every employee must be in both list, order does not matter. But, if ListA contains same employee object 5 times (that means some duplicate entries), and ListB contains the employee object 4 times, then ListA and ListB are not equal.

You can use SequenceEqual with a custom IEqualityComparer<Employee>:
class EmployeeComparer : IEqualityComparer<Employee>
{
public bool Equals(Employee x, Employee y)
{
if (x == null || y == null) return false;
bool equals = x.ID==y.ID && x.Name == y.Name && x.Dept == y.Dept
&& x.Address == y.Address && x.Age == y.Age && x.Email == y.Email;
return equals;
}
public int GetHashCode(Employee obj)
{
if (obj == null) return int.MinValue;
int hash = 19;
hash = hash + obj.ID.GetHashCode();
hash = hash + obj.Name.GetHashCode();
hash = hash + obj.Dept.GetHashCode();
hash = hash + obj.Address.GetHashCode();
hash = hash + obj.Age.GetHashCode();
hash = hash + obj.Email.GetHashCode();
return hash;
}
}
Now it's so simple:
listA.SequenceEqual(ListB, new EmployeeComparer());
If the order is not important and you only want to know if all employees are in both lists you can use HashSet<Employee>.SetEquals to determine if both lists contain the same people:
var empComparer = new EmployeeComparer();
bool bothEqual = new HashSet<Employee>(ListA, empComparer)
.SetEquals(new HashSet<Employee>(ListB, empComparer));

Best complexity is O(N)
Following realization with using HashSet:
Class with implementation of GetHashCode and Equals:
public class Employee
{
public int ID = 0;
public string Name = String.Empty;
public string Dept = String.Empty;
public string Address = String.Empty;
public int Age = 0;
public string Email = String.Empty;
public override int GetHashCode()
{
return
ID.GetHashCode() ^
(Name ?? String.Empty).GetHashCode() ^
(Dept ?? String.Empty).GetHashCode() ^
(Address ?? String.Empty).GetHashCode() ^
Age.GetHashCode() ^
(Email ?? String.Empty).GetHashCode()
;
}
public override bool Equals(object obj)
{
Employee other = obj as Employee;
if (obj == null)
return false;
return ID == other.ID &&
Name == other.Name &&
Dept == other.Dept &&
Address == other.Address &&
Age == other.Age &&
Email == other.Email;
}
}
Function to compare lists:
public static bool CompareLists(List<Employee> list1, List<Employee> list2)
{
if (list1 == null || list2 == null)
return list1 == list2;
if (list1.Count != list2.Count)
return false;
Dictionary<Employee, int> hash = new Dictionary<Employee, int>();
foreach (Employee employee in list1)
{
if (hash.ContainsKey(employee))
{
hash[employee]++;
}
else
{
hash.Add(employee, 1);
}
}
foreach (Employee employee in list2)
{
if (!hash.ContainsKey(employee) || hash[employee] == 0)
{
return false;
}
hash[employee]--;
}
return true;
}

If the numbers in the list are going to grow enormous (10M), you are probably going to have to consider parallelization of the look-up to get an acceptable query time.
Consider using PLINQ.
Some more clarity on what you mean by 'equal' would be good. How complex is the equivalence check? Are you checking that the objects are the same or that the objects values are the same?
Another consideration would be this; if the number of elements are going to become large, could you consider moving this check down from .NET into your database - perhaps as a stored procedure? You may find it executes more efficiently there.

reduce the list to a scalar type: int, string, ....
L1.Select(x => x.K).ToArray()
use the except method
L1.Select(x => x.K).ToArray().Except(L1.Select(x => x.K).ToArray())
If the count of the resulting set is 0 then the List are equals
L1.Select(x => x.K).ToArray().Except(L1.Select(x => x.K).ToArray()).Count()
All together
public class Program {
public static void Main(String[] args) {
List<O> L1 = new List<O>{
new O {K = 1, V = "abcd"},
new O {K = 2, V = "efgh"}
};
List<O> L2 = new List<O>{
new O {K = 1, V = "abcd"}
};
List<O> L3 = new List<O>{
new O {K = 1, V = "abcd"},
new O {K = 3, V = "ijkl"}
};
List<O> L4 = new List<O>{
new O {K = 2, V = "efgh"},
new O {K = 1, V = "abcd"}
};
Console.WriteLine(L1.Select(x => x.K).ToArray().Except(L1.Select(x => x.K).ToArray()).Count());
Console.WriteLine(L1.Select(x => x.K).ToArray().Except(L2.Select(x => x.K).ToArray()).Count());
Console.WriteLine(L1.Select(x => x.K).ToArray().Except(L3.Select(x => x.K).ToArray()).Count());
Console.WriteLine(L1.Select(x => x.K).ToArray().Except(L4.Select(x => x.K).ToArray()).Count());
}
}
public class O {
public int K { get; set; }
public String V { get; set; }
}

Exactly what it says.
Implement IComparable on the class Employee
Also need to override Equals
Due to potentially a large number of calls to GetHashCode save it and only calculate on changes.
Tested
IComparable Interface
public MainWindow()
{
InitializeComponent();
List<Person> PLa = new List<Person>();
List<Person> PLb = new List<Person>();
PLa.Add(new Person { Age = 3, Name = "Jim"});
PLa.Add(new Person { Age = 2, Name = "Jimmmy" });
PLa.Add(new Person { Age = 1, Name = "Jim" });
PLb.Add(new Person { Age = 1, Name = "Jim" });
PLb.Add(new Person { Age = 3, Name = "Jim" });
PLb.Add(new Person { Age = 2, Name = "Jimmmy" });
System.Diagnostics.Debug.WriteLine(ListSameIgnoreOrder(PLa, PLb));
}
public bool ListSameIgnoreOrder(List<Person> PLa, List<Person> PLb)
{
if (PLa.Count != PLb.Count) return false;
//PLa.Sort();
//PLb.Sort();
return Enumerable.SequenceEqual(PLa.OrderBy(s => s), PLb.OrderBy(s => s));
//for (int i = 0; i < PLa.Count; i++)
//{
// System.Diagnostics.Debug.WriteLine(
// PLa[i].Age.ToString() + " " + PLb[i].Age.ToString() + " " +
// PLa[i].Name + " " + PLb[i].Name);
// if (!PLa[i].Equals(PLb[i])) return false;
//}
//return true;
}
public class Person : object, IComparable
{
private int age = 0;
private string name = string.Empty;
private int hash;
public int Age
{
get { return age; }
set
{
if (age == value) return;
age = value;
CalcHash();
}
}
public string Name
{
get { return name; }
set
{
if (name == value) return;
name = value;
CalcHash();
}
}
public override bool Equals(Object obj)
{
//Check for null and compare run-time types.
if (obj == null || !(obj is Person)) return false;
Person f = (Person)obj;
if (f.Age != this.Age) return false;
return (string.Compare(f.name, this.name) == 0);
}
private void CalcHash()
{
hash = Age.GetHashCode() ^
(Name ?? String.Empty).GetHashCode();
}
public override int GetHashCode()
{
return hash;
//return age ^ name.GetHashCode();
}
public int CompareTo(object obj)
{
if (obj == null) return 1;
Person otherPerson = obj as Person;
if (otherPerson != null)
{
if (otherPerson.Age > this.Age) return -1;
if (otherPerson.Age < this.Age) return 1;
// compare all properties like above
return string.Compare(otherPerson.name, this.name);
}
else
throw new ArgumentException("Object is not a Person");
}
public Person() { CalcHash(); }
}

This works.
public bool EqualList(Dictionary<int, string> a, Dictionary<int, string> b)
{
if (a.Count == b.Count)
{
bool rs = false;
foreach (var i in a)
{
if (b.ContainsKey(i.Key))
{
rs = true;
}
else
{
rs = false;
break;
}
}
return rs;
}
else
{
return false;
}
Usage:
if(EqualList(List<A>.ToDictionary(k => k.Key, k => k.Value), List<B>.ToDictionary(k => k.Key, k => k.Value)){
}else{
}

Remove duplicates in a list (c#)

I want to remove duplicates in a list using following code, but it does not work. Anyone could enlighten me? Thanks.
public sealed class Pairing
{
public int Index { get; private set; }
public int Length { get; private set; }
public int Offset { get; private set; }
public Pairing(int index, int length, int offset)
{
Index = index;
Length = length;
Offset = offset;
}
}
class MyComparer : IEqualityComparer<Pairing>
{
public bool Equals(Pairing x, Pairing y)
{
return ((x.Index == y.Index) && (x.Length == y.Length) && (x.Offset == y.Offset));
}
public int GetHashCode(Pairing obj)
{
return obj.GetHashCode();
}
}
class Program
{
static void Main(string[] args)
{
List<Pairing> ps = new List<Pairing>();
ps.Add(new Pairing(2, 4, 14));
ps.Add(new Pairing(1, 2, 4));
ps.Add(new Pairing(2, 4, 14));
var unique = ps.Distinct(new MyComparer());
foreach (Pairing p in unique)
{
Console.WriteLine("{0}\t{1}\t{2}", p.Index, p.Length, p.Offset);
}
Console.ReadLine();
}
}

According to the example on the IEnumerable.Distinct page you will need to implement GetHashCode() so that the equal objects return the same hashcode. If you do not override GetHashCode() in your object it is not guaranteed to return the same hashcode.
// If Equals() returns true for a pair of objects
// then GetHashCode() must return the same value for these objects.
public int GetHashCode(Product product)
{
//Check whether the object is null
if (Object.ReferenceEquals(product, null)) return 0;
//Get hash code for the Name field if it is not null.
int hashProductName = product.Name == null ? 0 : product.Name.GetHashCode();
//Get hash code for the Code field.
int hashProductCode = product.Code.GetHashCode();
//Calculate the hash code for the product.
return hashProductName ^ hashProductCode;
}

Defining GetHashCode to return a unique answer causes the Distinct to work as expected;
public int GetHashCode(Pairing obj)
{
if (obj==null) return 0;
var hc1 = obj.Index.GetHashCode();
var hc2 = obj.Length.GetHashCode();
var hc3 = obj.Offset.GetHashCode();
return hc1 ^ hc2 ^ hc3;
}