3
我必须计算包含在5GB csv文件中的向量的相关矩阵。每行包含一个观察每个随机变量。要做到这一点,我写了下面的:F#中的惰性关联矩阵计算
let getCorrMatrix data =
let getMatrixInfo nCol (count,crossProd:float array array,sumVector:float array,sqVector:float array) (newLine:float array) =
for i in 0..(nCol-1) do
sumVector.[i]<-sumVector.[i]+newLine.[i]
sqVector.[i]<-sqVector.[i]+(newLine.[i]*newLine.[i])
for j in (i+1)..(nCol-1) do
crossProd.[i].[j-(i+1)]<-crossProd.[i].[j-(i+1)]+newLine.[i]*newLine.[j]
let newCount = count+1
//(newCount,newMatrix,newSumVector,newSqVector)
(newCount,crossProd,sumVector,sqVector)
//Get number of columns
let nCol = data|>Seq.head|>Seq.length
//Initialize objects for the fold
let matrixStart = Array.init nCol (fun i -> Array.create (nCol-i-1) 0.0)
let sumVector = Array.init nCol (fun _ -> 0.0)
let sqVector = Array.init nCol (fun _ -> 0.0)
let init = (0,matrixStart,sumVector,sqVector)
//Run the fold and obtain all the elements to build te correlation matrix
let (count,crossProd,sum,sq) =
data
|>PSeq.fold(getMatrixInfo nCol) init
//Compute averages standard deviations, and finally correlations
let averages = sum|>Array.map(fun s ->s/(float count))
let std = Array.zip3 sum sq averages
|> Array.map(fun (elemSum,elemSq,av)-> let temp = elemSq-2.0*av*elemSum+float(count)*av*av
sqrt (temp/(float count-1.0)))
//Map allteh elements to correlation
let rec getCorr i j =
if i=j then
1.0
elif i<j then
(crossProd.[i].[j-(i+1)]-averages.[i]*sum.[j]-averages.[j]*sum.[i]+(float count*averages.[i]*averages.[j]))/((float count-1.0)*std.[i]*std.[j])
else
getCorr j i
let corrMatrix = Array2D.init nCol nCol (fun i j -> getCorr i j)
corrMatrix
我已经测试它反对研究计算和它匹配。由于我打算反复使用这个,如果你有一些反馈意见(或发现一个错误),将不胜感激。 (注意我发布这个是因为认为它可能对其他人有用)。
感谢
感谢代码运行得更快! (刚刚发现你的网站,非常酷!) – jlezard 2010-10-02 12:53:00
和'fun i j - > getCorr i j'只能是'getCorr'。 :-) – 2010-10-02 18:54:34