C# – Read Text File, Remove Dupes, Write Text File

Text file saved as withDupes.txt:

John 39
John 39
Jack 22
Jack 22
John 39
John 39
Eddie 45
Sara 42
Dalia 30
Sam 31
Amy 19
Amy 19
Eddie 45
Amy 19
Terry 22
Jennifer 27
John 39
John 39

 

Code

* tested in Visual Studio 2019

// MrNetTek
// eddiejackson.net/blog
// 12/22/2019
// free for public use 
// free to claim as your own

using System;
using System.Collections.Generic;
using System.IO;

class RemoveDupes
{
    static void Main(string[] args)
    {
        if (args is null)
        {
            throw new ArgumentNullException(nameof(args));
        }

        using TextReader txtReader = File.OpenText(@"C:\csharp\RemoveDupes\withDupes.txt");
        using TextWriter txtWriter = File.CreateText(@"C:\csharp\RemoveDupes\withoutDupes.txt");
        
        string currLine;

        HashSet<string> prevLines = new HashSet<string>();

        while ((currLine = txtReader.ReadLine()) != null)
        {
            if (prevLines.Add(currLine))
            {
                txtWriter.WriteLine(currLine);
            }
        }

        Console.WriteLine("\nDuplicates have been removed!\n");

        Console.WriteLine("\nPress any key to continue...");
        Console.ReadKey();

    }
}

 

tags: C# Duplicates, C# Dupes, MrNetTek