SQL Server DELETE TOP会随着时间的推移而减慢速度

Question

我正在运行以下代码，以便将数据从临时表（行存储）移动到生产表（列存储）中。我观察到这个过程随着时间的推移而变慢。我怀疑它可能是因为我执行INSERT SELECT FROM，然后DELETE直到表为空。有没有更好的方法来做到这一点或我该怎么做才能优化这个过程？我应该在中间重建指数吗？感谢您的意见。

数据库

• 按月分区的表格
• 简单恢复模式
• 8 CPU，16GB RAM

表定义

• 表中没有约束，没有触发器

• 每次转移时每月平均行数约为20亿行 开始

  CREATE TABLE [dbo].[AnalogT](
  [SourceId] [int] NOT NULL,
  [WindFarmId] [smallint] NOT NULL,
                  [StationId] [int] NOT NULL,
                  [TDIID] [int] NOT NULL,
                  [CTDIID] [int] NOT NULL,
                  [LogTime] [datetime2](0) NOT NULL,
                  [MeanValue] [real] NULL,
                  [MinValue] [real] NULL,
                  [MaxValue] [real] NULL,
                  [Stddev] [real] NULL,
                  [EndValue] [real] NULL,
                  [Published] [datetime2](3) NULL,
                  [TimeZoneOffset] [tinyint] NULL
              ) ON [Data]
              GO
  
      CREATE UNIQUE CLUSTERED INDEX [CIDX_AnalogT] ON [dbo].[AnalogT]
      (
          [LogTime] ASC,
          [CTDIID] ASC,
          [StationId] ASC,
          [WindFarmId] ASC
      ) ON [Data]
      GO
  

获取要传输的数据集

public class AnalogDeltaTransfer2ProductionService : DeltaTransfer2ProductionService
{
    public override TdiType GetTargetTdiType()
    {
        return TdiType.Analog;
    }

    public override string GetCommandText(int batchSize)
    {
        string commandText = $"SELECT TOP {batchSize} " +
                             "[SourceId]," +
                             "[WindFarmId]" +
                             ",[StationId]" +
                             ",[TDIID]" +
                             ",[CTDIID]" +
                             ",[LogTime]" +
                             ",[MeanValue]" +
                             ",[MinValue]" +
                             ",[MaxValue]" +
                             ",[Stddev]" +
                             ",[EndValue]" +
                             ",[Published]" +
                             ",[TimeZoneOffset]" +
                             ",GETUTCDATE() [DateInsertedUtc]" +
                             $"FROM[dbo].[{GetStagingTable()}] ORDER BY LogTime, CTDIID, StationId, WindFarmId ";
        return commandText;
    }

    public override string GetStagingTable()
    {
        var appSettings = new AppSettings();
        return appSettings.deltasync_staging_analog;
    }

    public override string GetProductionTable()
    {
        var appSettings = new AppSettings();
        return appSettings.deltasync_destination_analog;
    }

}

批量加载数据

//Bulkd insert data from staging into production table
private void BulkTransferData(int batchSize, int batchNo)
{
    var appSettings = new AppSettings();
    var tdiTypeName = Enum.GetName(typeof (TdiType), GetTargetTdiType());

    //source and destination databases is the same
    var sourceAndDestinationConnectionString =
        new StringBuilder(appSettings.deltasync_destination_connectionstring +
                          "Application Name=HPSDB DeltaSync DT2P {tdiType}")
            .Replace("{tdiType}", tdiTypeName.ToUpper().Substring(0, 3))
            .ToString();

    using (var stagingConnection = new SqlConnection(sourceAndDestinationConnectionString))
    {
        stagingConnection.Open();

        // get data from the source table as a stream datareader.
        var commandSourceData = new SqlCommand(GetCommandText(batchSize), stagingConnection);
        commandSourceData.CommandType = CommandType.Text;
        commandSourceData.CommandTimeout = appSettings.deltasync_deltatransfer_prod_commandtimeout_secs.AsInt();

        //prepare the fast reader
        var reader = commandSourceData.ExecuteReader();

        var transactionTimeOut =
            TimeSpan.FromSeconds(appSettings.deltasync_deltatransfer_prod_trnxntimeout_secs.AsInt());
        using (
            var transactionScope = new TransactionScope(TransactionScopeOption.RequiresNew, transactionTimeOut))
        {
            //establish connection to destination production table
            using (var destinationConnection = new SqlConnection(sourceAndDestinationConnectionString))
            {
                //grab connection from connection pool
                destinationConnection.Open();

                // set up the bulk copy object. 
                // note that the column positions in the source
                // data reader match the column positions in the destination table so there is no need to map columns.
                using (
                    var bulkCopy = new SqlBulkCopy(sourceAndDestinationConnectionString,
                        SqlBulkCopyOptions.TableLock))
                {
                    try
                    {
                        // write from the source to the destination.
                        bulkCopy.DestinationTableName = GetProductionTable();
                        bulkCopy.BatchSize = appSettings.deltasync_deltatransfer_staging_batchsize.AsInt();
                        bulkCopy.BulkCopyTimeout =
                            appSettings.deltasync_deltatransfer_prod_commandtimeout_secs.AsInt();
                        bulkCopy.EnableStreaming = true;
                        bulkCopy.WriteToServer(reader);
                    }
                    catch (Exception ex)
                    {
                        //interrupt transaction and allow rollback process to commence
                        Log.Error(
                            $"DeltaTransfer2ProductionService.BulkTransferData/{tdiTypeName}/BulkInsert/{batchNo}: Faulted with with ff error: {ex}");
                        throw;
                    }
                }
            }

            //establish connection to destination production table
            using (var destinationConnection = new SqlConnection(sourceAndDestinationConnectionString))
            {
                //grab connection from connection pool
                destinationConnection.Open();

                //delete the rows that has been moved to destination table
                try
                {
                    //delete top x number of rows we just moved into prod
                    var selectTopXSqlText = GetCommandText(batchSize);
                    var sqlText = $"WITH cte AS({selectTopXSqlText}) DELETE FROM cte;";

                    var commandDropData = new SqlCommand(sqlText, destinationConnection);
                    commandDropData.CommandType = CommandType.Text;
                    commandDropData.CommandTimeout =
                        appSettings.deltasync_deltatransfer_prod_commandtimeout_secs.AsInt();
                    commandDropData.ExecuteNonQuery();
                }
                catch (Exception ex)
                {
                    //interrupt transaction and allow rollback process to commence
                    Log.Error(
                        $"DeltaTransfer2ProductionService.BulkTransferData/{tdiTypeName}/DeleteTop/{batchNo}: Faulted with with ff error: {ex}");
                    throw;
                }
            }

            //commit all changes
            transactionScope.Complete();
            transactionScope.Dispose();
        }
    }
}