Hi all, I'm trying to use the package read.table within a foreach loop. I'm grabbing 500M rows of data at a time from two different files and then doing an aggregate/tapply like function in read.table after that. I had planned on doing a foreach loop 39 times at once for the 39 files I have, but obviously that won't work until I figure out why the segfault is occurring. The sessionInfo, code, and error are pasted below. If you have any ideas, would love to hear them. (I have no control over the version of R - 2.13.0 - being used). Best Matt SESSION INFO: > sessionInfo() R version 2.13.0 (2011-04-13) Platform: x86_64-unknown-linux-gnu (64-bit) locale: [1] LC_CTYPE=en_US.UTF-8 LC_NUMERIC=C LC_TIME=en_US.UTF-8 LC_COLLATE=en_US.UTF-8 LC_MONETARY=C [6] LC_MESSAGES=en_US.UTF-8 LC_PAPER=en_US.UTF-8 LC_NAME=C LC_ADDRESS=C LC_TELEPHONE=C [11] LC_MEASUREMENT=en_US.UTF-8 LC_IDENTIFICATION=C attached base packages: [1] stats graphics grDevices utils datasets methods base other attached packages: [1] data.table_1.7.10 doMC_1.2.2 multicore_0.1-5 foreach_1.3.2 codetools_0.2-8 iterators_1.0.3 MY CODE: computeAllPairSums <- function(filename, nbindiv,nrows.to.read) { con <- file(filename, open="r") on.exit(close(con)) ans <- matrix(numeric(nbindiv * nbindiv), nrow=nbindiv) chunk <- 0L while (TRUE) { #read.table faster than scan df0 <- read.table(con,col.names=c("ID1", "ID2", "ignored", "sharing"), colClasses=c("integer", "integer", "NULL", "numeric"),nrows=nrows.to.read,comment.char="") DT <- data.table(df0) setkey(DT,ID1,ID2) ss <- DT[,sum(sharing),by="ID1,ID2"] if (nrow(df0) == 0L) break chunk <- chunk + 1L cat("Processing chunk", chunk, "... ") idd <- as.matrix(subset(ss,select=1:2)) newvec <- as.vector(as.matrix(subset(ss,select=3))) ans[idd] <- ans[idd] + newvec cat("OK\n") } ans } require(foreach) require(doMC) registerDoMC(cores=2) num <- 8891 nr <- 500000000L #500 million rows at a time MMM <- foreach(IT = 1:2) %dopar% { require(data.table) if (IT==1){ x <- system.time({computeAllPairSums( paste(GERMLINE,"bc.chr22.q.20.file",sep=''),num,nr)}) } #Run it on regular file PID 6489, 24 gb if (IT==2){ z <- system.time({computeAllPairSums.gz( paste(GERMLINE,"bc.chr22.q.20.gz",sep=''),num,nr)}) } #Run it on gz file PID 6490, 24 gb } MY R OUTPUT/ERROR: MMM <- foreach(IT = 1:2) %dopar% { + require(data.table) + if (IT==1){ x <- system.time({computeAllPairSums( paste(GERMLINE,"bc.chr22.q.20.file",sep=''),num,nr)}) } #Run it on regular file PID 6053, 5.9 gb + if (IT==2){ z <- system.time({computeAllPairSums.gz( paste(GERMLINE,"bc.chr22.q.20.gz",sep=''),num,nr)}) } #Run it on gz file PID 6054, 4 gb + } Loading required package: data.table Loading required package: data.table data.table 1.7.10 For help type: help("data.table") data.table 1.7.10 For help type: help("data.table") *** caught segfault *** address 0x2ae93df90000, cause 'memory not mapped' Traceback: 1: .Call("dogroups", x, xcols, o__, f__, len__, jsub, SDenv, testj, byretn, byval, i, as.integer(icols), i[1, ivars, with = FALSE], if (length(ivars)) paste("i.", ivars, sep = ""), is.na(nomatch), verbose, PACKAGE = "data.table") 2: `[.data.table`(DT, , sum(sharing), by = "ID1,ID2") 3: DT[, sum(sharing), by = "ID1,ID2"] 4: computeAllPairSums(paste(GERMLINE, "bc.chr22.q.20.file", sep = ""), num, nr) 5: system.time({ computeAllPairSums(paste(GERMLINE, "bc.chr22.q.20.file", sep = ""), num, nr)}) 6: eval(expr, envir, enclos) 7: eval(c.expr, envir = args, enclos = envir) 8: doTryCatch(return(expr), name, parentenv, handler) 9: tryCatchOne(expr, names, parentenv, handlers[[1L]]) 10: tryCatchList(expr, classes, parentenv, handlers) 11: tryCatch(eval(c.expr, envir = args, enclos = envir), error = function(e) e) 12: FUN(X[[1L]], ...) 13: lapply(S, FUN, ...) 14: doTryCatch(return(expr), name, parentenv, handler) 15: tryCatchOne(expr, names, parentenv, handlers[[1L]]) 16: tryCatchList(expr, classes, parentenv, handlers) 17: tryCatch(expr, error = function(e) { call <- conditionCall(e) if (!is.null(call)) { if (identical(call[[1L]], quote(doTryCatch))) call <- sys.call(-4L) dcall <- deparse(call)[1L] prefix <- paste("Error in", dcall, ": ") LONG <- 75L msg <- conditionMessage(e) sm <- strsplit(msg, "\n")[[1L]] w <- 14L + nchar(dcall, type = "w") + nchar(sm[1L], type = "w") if is.na(w)) w <- 14L + nchar(dcall, type = "b") + nchar(sm[1L], type = "b") if (w > LONG) prefix <- paste(prefix, "\n ", sep = "") } else prefix <- "Error : " msg <- paste(prefix, conditionMessage(e), "\n", sep = "") .Internal(seterrmessage(msg[1L])) if (!silent && identical(getOption("show.error.messages"), TRUE)) { cat(msg, file = stderr()) .Internal(printDeferredWarnings()) } invisible(structure(msg, class = "try-error"))}) 18: try(lapply(S, FUN, ...), silent = TRUE) 19: sendMaster(try(lapply(S, FUN, ...), silent = TRUE)) 20: FUN(1:2[[1L]], ...) 21: lapply(1:cores, inner.do) 22: mclapply(argsList, FUN, mc.preschedule = preschedule, mc.set.seed = set.seed, mc.silent = silent, mc.cores = cores) 23: e$fun(obj, substitute(ex), parent.frame(), e$data) 24: foreach(IT = 1:2) %dopar% { require(data.table) if (IT == 1) { x <- system.time({ computeAllPairSums(paste(GERMLINE, "bc.chr22.q.20.file", sep = ""), num, nr) }) } if (IT == 2) { z <- system.time({ computeAllPairSums.gz(paste(GERMLINE, "bc.chr22.q.20.gz", sep = ""), num, nr) }) }} Possible actions: 1: abort (with core dump, if enabled) 2: normal R exit 3: exit R without saving workspace 4: exit R saving workspace -- Matthew C Keller Asst. Professor of Psychology University of Colorado at Boulder www.matthewckeller.com