Question

我有一个超过5百万行的文本文件。我需要逐行运行并删除某些行并替换某个字符串。我用C＃编写了一些“＃”工作原理＆＃39;但它可能需要将近一天的时间才能完成，因为在搜索中进行搜索和替换，++可以在几分钟内完成。但是我们需要自动化。

文件可以任意包含一行

"<-comment 1: (more text on the line here)"

和

"<-Another line (more text on the line here)"

我想删除以评论1或其他行开头的任何行...

还有一个字符串

<tag>&#x2014;</tag>

我想用下划线替换。这应仅出现在以＆＃34; LINK：＆＃34;

开头的行上

我到目前为止的代码是：

static void Main()
{
    const Int32 BufferSize = 128;
    int count = 0;
    int count2 = 0;
    string filename = @"C:\test\test.txt";
    string output = @"C:\text\output.txt";
    string Startcomment = @"<-comment 1:";
    string Startmoretext= @"<-Another line";
    string othercit = @"LINK:";
    string sub = @"<tag>&#x2014;</tag>";
    string subrepalce = @"_";

    string line;

    using (var filestream = File.OpenRead(filename))
    {
        Console.WriteLine("Start time: " + DateTime.Now.ToString());
        using (var streamreader = new StreamReader(filestream, Encoding.UTF8, true, BufferSize))
        {
            File.WriteAllText(output, "Clean text file" + Environment.NewLine);                    
            while ((line = streamreader.ReadLine()) != null)
            {
                count++;
                if(count % 10000 == 0)
                {
                    Console.WriteLine("Batch complete: " + DateTime.Now.ToString());
                    Console.WriteLine(count);
                }

                if(!line.StartsWith(Startcomment) && !line.StartsWith(Startmoretext))
                {
                    count2++;
                    if (line.StartsWith(othercit))
                    {
                        line = line.Replace(sub, subrepalce);
                    }
                    File.AppendAllText(output, line + Environment.NewLine);
                }
            }                    

        }                
        Console.WriteLine(DateTime.Now.ToString());
        Console.WriteLine(count + " Lines processed");
        Console.WriteLine(count2 + " Lines written back");
        Console.WriteLine("Finished!!!!!!");
        Console.Read();
    }
}

运行时间不可行。

我想让这个运行在一个正则表达式下运行，如果我们需要添加新的异常，我们可以在脚本之外维护一个配置文件，但它似乎也会永远运行。

static void Main()
{
    const Int32 BufferSize = 128;
    string filename = @"C:\test\test.txt";
    XmlDocument xdoc = new XmlDocument();
    xdoc.Load(@"C:\test\RegexConfig.xml");
    XmlElement xmlRoot = xdoc.DocumentElement;
    XmlNodeList xmlNodes = xmlRoot.SelectNodes("/root/line");
    int count = 0;
    string line;
    using (var filestream = File.OpenRead(filename))
    {
        Console.WriteLine(DateTime.Now.ToString());
        using (var streamreader = new StreamReader(filestream, Encoding.UTF8, true, BufferSize))
        {
            File.WriteAllText(@"C:\test\output.txt", "Clean file" + Environment.NewLine);
            while ((line = streamreader.ReadLine()) != null)
            {
                string output = line;
                foreach (XmlNode node in xmlNodes)
                {
                    string pattern = node["pattern"].InnerText;
                    string replacement = node["replacement"].InnerText;                           
                    Regex rgx = new Regex(pattern);
                    output = rgx.Replace(output, replacement);
                    rgx = null;
                }
                if (output.Length > 0)
                {
                    count++;
                    if (count % 10000 == 0)
                    {
                        Console.WriteLine(count);
                        Console.WriteLine(DateTime.Now.ToString());
                    }
                    File.AppendAllText(@"C:\test\test.txt", output + Environment.NewLine);
                }

            }

        }
        Console.WriteLine(DateTime.Now.ToString());
        Console.WriteLine("Finished!!!!!!");
        Console.Read();
    }
}

XML配置文件

<?xml version="1.0" encoding="UTF-8"?>
<root>
    <line>
        <pattern><![CDATA[<-comment 1:.*]]></pattern>
        <replacement><![CDATA[]]></replacement>
    </line> 
    <line>
        <pattern><![CDATA[<-Another line.*]]></pattern>
        <replacement><![CDATA[]]></replacement>
    </line> 
    <line>
        <pattern><![CDATA[<tag>&#x2014;</tag>]]></pattern>
        <replacement>_</replacement>
    </line> 
</root>

如何以最有效率的方式完成这样的工作？

Answer 1

我认为以下工作效率更高，因为@ C.Evenhuis部分建议......

using (FileStream fs = File.Open(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
using (BufferedStream bs = new BufferedStream(fs))
using (StreamReader sr = new StreamReader(bs))
using (StreamWriter writer = new StreamWriter("C:\test\test.txt"))
{
    string line;
    while ((line = sr.ReadLine()) != null)
    {
        string output = line;
        foreach (XmlNode node in xmlNodes)
        {
             string pattern = node["pattern"].InnerText;
             string replacement = node["replacement"].InnerText;                           
             Regex rgx = new Regex(pattern);
             output = rgx.Replace(output, replacement);
             rgx = null;
        }
        if (output.Length > 0)
        {
             count++;
             if (count % 10000 == 0)
             {
                  Console.WriteLine(count);
                  Console.WriteLine(DateTime.Now.ToString());
             }
             writer.WriteLine(output);
         }
    }
    writer.Close();
}

Answer 2

如果在内存中执行并应用并行怎么办？像这样：

    const Int32 BufferSize = 128;
    int count = 0;
    int count2 = 0;
    string filename = @"C:\test\test.txt";
    string output = @"C:\text\output.txt";
    string Startcomment = @"<-comment 1:";
    string Startmoretext= @"<-Another line";
    string othercit = @"LINK:";
    string sub = @"<tag>&#x2014;</tag>";
    string subrepalce = @"_";


    string line;
    string[] fileText = File.ReadAllLines(filename);

        Console.WriteLine("Start time: " + DateTime.Now.ToString());
    Parallel.For(0, fileText.Length, i=>{

      if(!fileText[i].StartsWith(Startcomment) && !fileText[i].StartsWith(Startmoretext))
                {
                    count2++;
                    if (fileText[i].StartsWith(othercit))
                    {
                        fileText[i]= fileText[i].Replace(sub, subrepalce);
                    }
                    File.WriteAllLines(yourPath, fileText);
                }
            }                    

        }                
        Console.WriteLine(DateTime.Now.ToString());
        Console.WriteLine(count + " Lines processed");
        Console.WriteLine(count2 + " Lines written back");
        Console.WriteLine("Finished!!!!!!");
        Console.Read();
    });

在内存中执行此操作可能会更快。但请确保您有足够的RAM存储空间。

删除行并替换文本文件中的字符串

2 个答案: