我正在使用嵌套的foreach循环中的data.table对象，但我无法按照自己喜欢的方式创建结果对象。
基本上我的想法是生成长度（intersect（set1，set2））。我还想生成长度（union（set1，set2））和其他一些指标。
下面的代码包含样本数据：
library(iterators)
library(data.table)
library(foreach)
#generate dummy data
set.seed(1212)
sample1 <- data.frame(parentid=round((runif(50000, min=1, max=50000))), childid=round(runif(100000, min=1, max=100000)))
length(unique(sample1$parentid))
#get unique parents
sample1uniq <- as.data.frame(unique(sample1$parentid))
names(sample1uniq) <- "parentid"
#convert original dataset to data.table
sample1 <- data.table(sample1)
setkey(sample1,parentid)
#convert unique ids to data.table
sample1uniq <- data.table(sample1uniq)
setkey(sample1uniq,parentid)
#a random sample of 5K to users to scan against
sample2uniq_idx <- sample(1:nrow(sample1uniq), size=5000)
sample2uniq <- sample1uniq[sample2uniq_idx]
sample2uniq <- data.table(sample2uniq)
setkey(sample2uniq,parentid)
#construct iterators
sample1uniq_iter <- iter(sample1uniq)
sample2uniq_iter <- iter(sample2uniq)
编辑12/5/2013以使我的问题更加清晰：
outerresults <- foreach (x = sample1uniq_iter, .combine=rbind, .packages=c('foreach','doParallel', 'data.table')) %dopar% {
b <- sample1[J(x)] #ith parent
b2 <- as.data.frame(b)[,2] #ith parent's children
foreach (y = sample2uniq_iter, .combine=rbind) %dopar% {
c <- sample1[J(y)] #jth parent
c2 <- as.data.frame(c)[,2] #jth parent's children
common <- length(intersect(b2, c2))
results <- list(u1=x, u2=y, inter=common)
}
}
我期待结果像这样（组成）：
u1 u2 inter
1 2 10
1 3 4
1 4 7
1 5 6
2 3 10
2 4 4
3 5 7
4 5 6
相反，它出现在一个列表中u1＆amp; u2作为前2个元素＆amp; inter作为SUM（length（intersect（set1，set2）））。
感谢任何想法...
答案 0 :(得分：2)
您的主要问题是迭代器。请记住，数据表被许多事物（例如迭代器）视为列表，因此您生成的两个迭代器将分别迭代一个项目，即每个数据表中的单个列。仔细观察结果：
> str(outerresults)
List of 3
$ u1 : num [1:31602] 2 3 5 6 7 8 10 11 12 14 ...
$ u2 : num [1:5000] 14 26 27 31 34 61 68 81 99 106 ...
$ inter: int 14778
u1基本上只是sample1unique，u2是sample2unique，而inter是：
> length(intersect(sample1[J(sample1uniq)][,childid], sample1[J(sample2uniq)][,childid]))
[1] 14778
换句话说，你实际上根本没有循环任何东西。
你遇到的另一个问题是这种方法（一旦针对上述问题得到修复）非常缓慢。您通过大约160MM次的列表来生长一个非常大的对象。这是坏消息。我修复了它（更改了迭代器）并以更小的尺寸运行它以给你一个想法（100 x 20，或原始大小的1/8000）：
#generate dummy data
set.seed(1212)
sample1 <- data.frame(parentid=round((runif(50, min=1, max=50))), childid=round(runif(100, min=1, max=100)))
length(unique(sample1$parentid))
#get unique parents
sample1uniq <- as.data.frame(unique(sample1$parentid))
names(sample1uniq) <- "parentid"
#convert original dataset to data.table
sample1 <- data.table(sample1)
setkey(sample1,parentid)
#convert unique ids to data.table
sample1uniq <- data.table(sample1uniq)
setkey(sample1uniq,parentid)
#a random sample of 5K to users to scan against
sample2uniq_idx <- sample(1:nrow(sample1uniq), size=20)
sample2uniq <- sample1uniq[sample2uniq_idx]
sample2uniq <- data.table(sample2uniq)
setkey(sample2uniq,parentid)
# Notice how we don't use iterator objects
outerresults <- foreach (x = sample1uniq$parentid, .combine=rbind, .packages=c('foreach','doParallel', 'data.table')) %dopar% {
b <- sample1[J(x)] #ith parent
b2 <- as.data.frame(b)[,2] #ith parent's children
results <- foreach (y = sample2uniq$parentid, .combine=rbind) %dopar% {
c <- sample1[J(y)] #jth parent
c2 <- as.data.frame(c)[,2] #jth parent's children
common <- length(intersect(b2, c2))
results <- list(u1=x, u2=y, inter=common)
results
}
}
# user system elapsed
# 1.57 0.00 1.60
head(outerresults)
# u1 u2 inter
# result.1 2 2 4
# result.2 2 4 0
# result.3 2 7 0
# result.4 2 7 0
# result.5 2 8 2
# result.6 2 8 2
假设一切正常，那么全尺寸将需要3个多小时。
我认为你最好完全排除循环并使用data.table四周：
# Prepare data in two data tables
vec.samp1 <- par.ids # exact copy of what we generated earlier
vec.samp1.child <- child.ids # exact copy of what we generated earlier
dt.s1 <- data.table(sample1=vec.samp1, sample1.child=vec.samp1.child, key="sample1")
vec.samp2 <- sample2.ids # exact copy of what we generated earlier
dt.s2 <- dt.s1[data.table(sample2=vec.samp2)]
setnames(dt.s2, c("sample2", "sample2.child"))
# Create the cartesian join of our data sets and then
# join to get the child values
combinations <- CJ(sample1=vec.samp1, sample2=vec.samp2)
setkey(combinations, "sample1")
combinations <- combinations[dt.s1, allow.cartesian=T]
setkey(combinations, "sample2")
combinations <- combinations[dt.s2, allow.cartesian=T]
# Compute intersect and union
combinations[order(sample1, sample2),
list(
intersect=length(intersect(sample1.child, sample2.child)),
union=length(union(sample1.child, sample2.child))
),
by=list(sample1, sample2)
]
# user system elapsed
# 0.06 0.00 0.06
# sample1 sample2 intersect union
# 1: 2 2 4 4
# 2: 2 4 0 6
# 3: 2 7 0 10
# 4: 2 8 2 10
# 5: 2 9 0 6
结果相同，但速度提高了25倍（但请注意，data.table版本仅报告sample1-sample2的唯一组合）。