将记录从Oracle数据库移动到SQL Server的最快方法

Question

好吧，这是场景...我在Oracle中有一个表，它像一个队列... VB.net程序读取队列并调用存储在SQL Server中处理进程，然后将消息插入到另一个SQL Server表中，然后从oracle表中删除该记录。

我们使用DataReader从Oracle读取记录，然后调用每个记录的存储过程。该计划似乎有点慢。存储过程本身并不慢。 SP在循环中调用时可以在20秒内处理大约2000条记录。但是当从.Net程序中调用时，执行时间大约是每秒5条记录。 我已经看到，大部分时间消耗在调用存储过程并等待它返回。有没有更好的方法来做到这一点？

下面是实际的代码

Function StartDataXfer() As Boolean 
    Dim status As Boolean = False 
    Try 

    SqlConn.Open() 
    OraConn.Open() 
    c.ErrorLog(Now.ToString & "--Going to Get the messages from oracle", 1) 

    If GetMsgsFromOracle() Then 
    c.ErrorLog(Now.ToString & "--Got messages from oracle", 1) 

    If ProcessMessages() Then 
    c.ErrorLog(Now.ToString & "--Finished Processing all messages in the queue", 0) 
    status = True 
    Else 
    c.ErrorLog(Now.ToString & "--Failed to Process all messages in the queue", 0) 
    status = False 
    End If 
    Else 
    status = True 
    End If 
    StartDataXfer = status 
    Catch ex As Exception 
    Finally 
    SqlConn.Close() 
    OraConn.Close() 
    End Try 
End Function 
Private Function GetMsgsFromOracle() As Boolean 
    Try 
    OraDataAdapter = New OleDb.OleDbDataAdapter 
    OraDataTable = New System.Data.DataTable 
    OraSelCmd = New OleDb.OleDbCommand 
    GetMsgsFromOracle = False 
    With OraSelCmd 
    .CommandType = CommandType.Text 
    .Connection = OraConn 
    .CommandText = GetMsgSql 
    End With 
    OraDataAdapter.SelectCommand = OraSelCmd 
    OraDataAdapter.Fill(OraDataTable) 
    If OraDataTable.Rows.Count > 0 Then 
    GetMsgsFromOracle = True 
    End If 
    Catch ex As Exception 
    GetMsgsFromOracle = False 
    End Try 
End Function 

Private Function ProcessMessages() As Boolean 
    Try 
    ProcessMessages = False 
    PrepareSQLInsert() 
    PrepOraDel() 

    i = 0 
    Dim Method As Integer 
    Dim OraDataRow As DataRow 
    c.ErrorLog(Now.ToString & "--Going to call message sending procedure", 2) 
    For Each OraDataRow In OraDataTable.Rows 
    With OraDataRow 
    Method = GetMethod(.Item(0)) 
    SQLInsCmd.Parameters("RelLifeTime").Value = c.RelLifetime 
    SQLInsCmd.Parameters("Param1").Value = Nothing 
    SQLInsCmd.Parameters("ID").Value = GenerateTransactionID() ' Nothing 
    SQLInsCmd.Parameters("UID").Value = Nothing 
    SQLInsCmd.Parameters("Param").Value = Nothing 
    SQLInsCmd.Parameters("Credit").Value = 0 
    SQLInsCmd.ExecuteNonQuery() 

    'check the return value 
    If SQLInsCmd.Parameters("ReturnValue").Value = 1 And SQLInsCmd.Parameters("OutPutParam").Value = 0 Then 'success 
     'delete the input record from the source table once it is logged 
     c.ErrorLog(Now.ToString & "--Moved record successfully", 2) 
     OraDataAdapter.DeleteCommand.Parameters("P(0)").Value = OraDataRow.Item(6) 
     OraDataAdapter.DeleteCommand.ExecuteNonQuery() 
     c.ErrorLog(Now.ToString & "--Deleted record successfully", 2) 
     OraDataAdapter.Update(OraDataTable) 
     c.ErrorLog(Now.ToString & "--Committed record successfully", 2) 
     i = i + 1 
    Else 'failure 
     c.ErrorLog(Now.ToString & "--Failed to exec: " & c.DestIns & "Status: " & SQLInsCmd.Parameters("OutPutParam").Value & " and TrackId: " & SQLInsCmd.Parameters("TrackID").Value.ToString, 0) 
    End If 
    If File.Exists("stop.txt") Then 
     c.ErrorLog(Now.ToString & "--Stop File Found", 1) 
     'ProcessMessages = True 
     'Exit Function 
     Exit For 
    End If 
    End With 
    Next 
    OraDataAdapter.Update(OraDataTable) 
    c.ErrorLog(Now.ToString & "--Updated Oracle Table", 1) 
    c.ErrorLog(Now.ToString & "--Moved " & i & " records from Oracle to SQL Table", 1) 
    ProcessMessages = True 
    Catch ex As Exception 
    ProcessMessages = False 
    c.ErrorLog(Now.ToString & "--MoveMsgsToSQL: " & ex.Message, 0) 
    Finally 
    OraDataTable.Clear() 
    OraDataTable.Dispose() 
    OraDataAdapter.Dispose() 
    OraDelCmd.Dispose() 
    OraDelCmd = Nothing 
    OraSelCmd = Nothing 
    OraDataTable = Nothing 
    OraDataAdapter = Nothing 
    End Try 

End Function 


Public Function GenerateTransactionID() As Int64 
    Dim SeqNo As Int64 
    Dim qry As String 
    Dim SqlTransCmd As New OleDb.OleDbCommand 
    qry = " select seqno from StoreSeqNo" 
    SqlTransCmd.CommandType = CommandType.Text 
    SqlTransCmd.Connection = SqlConn 
    SqlTransCmd.CommandText = qry 
    SeqNo = SqlTransCmd.ExecuteScalar 

    If SeqNo > 2147483647 Then 
    qry = "update StoreSeqNo set seqno=1" 
    SqlTransCmd.CommandText = qry 
    SqlTransCmd.ExecuteNonQuery() 
    GenerateTransactionID = 1 
    Else 
    qry = "update StoreSeqNo set seqno=" & SeqNo + 1 
    SqlTransCmd.CommandText = qry 
    SqlTransCmd.ExecuteNonQuery() 
    GenerateTransactionID = SeqNo 
    End If 

End Function 
Private Function PrepareSQLInsert() As Boolean 
    'function to prepare the insert statement for the insert into the SQL stmt using 
    'the sql procedure SMSProcessAndDispatch 
    Try 
    Dim dr As DataRow 
    SQLInsCmd = New OleDb.OleDbCommand 
    With SQLInsCmd 
    .CommandType = CommandType.StoredProcedure 
    .Connection = SqlConn 
    .CommandText = SQLInsProc 

    .Parameters.Add("ReturnValue", OleDb.OleDbType.Integer) 
    .Parameters("ReturnValue").Direction = ParameterDirection.ReturnValue 
    .Parameters.Add("OutPutParam", OleDb.OleDbType.Integer) 
    .Parameters("OutPutParam").Direction = ParameterDirection.Output 
    .Parameters.Add("TrackID", OleDb.OleDbType.VarChar, 70) 
    .Parameters.Add("RelLifeTime", OleDb.OleDbType.TinyInt) 
    .Parameters("RelLifeTime").Direction = ParameterDirection.Input 
    .Parameters.Add("Param1", OleDb.OleDbType.VarChar, 160) 
    .Parameters("Param1").Direction = ParameterDirection.Input 
    .Parameters.Add("TransID", OleDb.OleDbType.VarChar, 70) 
    .Parameters("TransID").Direction = ParameterDirection.Input 
    .Parameters.Add("UID", OleDb.OleDbType.VarChar, 20) 
    .Parameters("UID").Direction = ParameterDirection.Input 
    .Parameters.Add("Param", OleDb.OleDbType.VarChar, 160) 
    .Parameters("Param").Direction = ParameterDirection.Input 
    .Parameters.Add("CheckCredit", OleDb.OleDbType.Integer) 
    .Parameters("CheckCredit").Direction = ParameterDirection.Input 
    .Prepare() 
    End With 
    Catch ex As Exception 
    c.ErrorLog(Now.ToString & "--PrepareSQLInsert: " & ex.Message) 
    End Try 
End Function 

Private Function PrepOraDel() As Boolean 
    OraDelCmd = New OleDb.OleDbCommand 
    Try 
    PrepOraDel = False 
    With OraDelCmd 
    .CommandType = CommandType.Text 
    .Connection = OraConn 
    .CommandText = DelSrcSQL 
    .Parameters.Add("P(0)", OleDb.OleDbType.VarChar, 160) 'RowID 
    .Parameters("P(0)").Direction = ParameterDirection.Input 
    .Prepare() 
    End With 
    OraDataAdapter.DeleteCommand = OraDelCmd 
    PrepOraDel = True 

    Catch ex As Exception 
    PrepOraDel = False 
    End Try 
End Function 

的片段是我想知道的是，是否有无论如何，以加快这一方案？ 任何意见/建议，将高度赞赏...

Regardss， 阿赫亚

Answer 1

由于Oracle和SQL Server可以参与分布式事务，为什么不写一个SSIS任务来查询Oracle和运行SQL Server存储过程，删除“排队”记录等？

创建链接的服务器到Oracle数据库一个SQL Server ...

[为了增加@Bob关于第一简介贾维斯意见，以确保瓶颈是你认为它是：EqaTec .NET代码分析器是免费的个人使用...]

Answer 2

如果存储过程不是问题，那么它必须是与调用存储过程相关的东西。由于我们无法修改/修复.Net本身，因此我们来看看我们可以改变的事情。

当调用存储过程时，您正在对StoreSeqNo表进行选择和更新（请参阅对GenerateTransactionID的调用）。这可能是减速的一部分，尤其是因为您正在查询没有WHERE子句的StoreSeqNo。此外，我建议您了解序列，这些对象旨在生成非重复的连续数字，并且可安全地跨事务，用户等使用。我不知道SQL Server是否支持它们（尚未使用SQL Server for现在有好几年了），但我确信它们在Oracle中得到了支持。

此外，还有“Method = GetMethod（.Item（0））”调用。 GetMethod是你的一个程序吗？它没有在代码片段中显示，但也许这是增加了一些开销？您可能想尝试添加一些代码来确定哪些线路花费最多时间。我怀疑有一个分析器用于.Net。

只是有些想法。祝你好运。