使用Groovy进行并发/并行数据库查询?

时间:2016-02-29 14:41:06

标签: groovy concurrency parallel-processing gpars

如何使用Groovy实现并发/并行数据库查询?

我想从数据库(navigationItem.rightBarButtonItem = instanceOfUIBarButtonItem )计算所有表中的行数,并将计数写入单独的文件。

一些表有数百万行,需要几分钟才能计算,而且许多表没有行。我希望教Groovy分开每个计数请求,等待结果,抓住列表中的下一个表,依此类推。我将此功能添加到现有的Groovy脚本中,该脚本是可靠且有效的。

这是我到目前为止所做的:

select count(*) from $TABLE

`

以上代码与必需品配对。原始代码包含用户名和密码(这是唯一的区别)返回此错误:

retrieve_table_count_list = { objParams ->

    aryTables = objParams.table_list

    objDB2DBRS = [:]
    objDB2DBRS["database_jdbc_url"] = get_config_setting(setting: "DB2DatabaseURL").toString()

    strReturnSQL = ""

    GParsPool.withPool(10) { pool ->
        currentPool = pool
        aryTables.eachWithIndexParallel { objTable, intTable ->

            intNumTables = aryTables.size()
            strTableName = objTable.table_name

            strSQL="select count(*) as \"row_count\" from ${strTableName}"

            aryRowCount = { db_commands(query: strSQL, params: objDB2DBRS) }
            objFastRowCount = aryRowCount.async()
            objResultRowCount = objFastRowCount()

            intRowCount = aryRowCount[0]["row_count"]

            strReturnSQL += "update tmp_TableCount set row_count=${intRowCount} where table_name='${strTableName}';\n"
        }
    }
    return strReturnSQL
}

1 个答案:

答案 0 :(得分:0)

有几个问题。 Jeremie B的回答有所帮助,但还有其他问题。我不确定是谁给出答案。欢迎任何意见!

// PROBLEM #1:
// db_commands was calling another closure to decrypt a password,
// but the decryption library was not thread safe
// solved by wrapping decryption in a "waiting loop:"
DECRYPT_AVAILABLE = true
// decrypt string
decrypt = { def objParams ->
    // if we're already decrypting a string, wait a short time, then test again
    while (DECRYPT_AVAILABLE == false) {
        Thread.sleep(100);
    }
    DECRYPT_AVAILABLE = false
    // ... decrypt text ...
    DECRYPT_AVAILABLE = true
    return decryptedText;
}

def count_table_rows = { def objParams ->
    def strTableName = objParams.table_name
    def objMSSQLDBRS = objParams.params
    def intTotal = objParams.total

    def String strSQL
    def boolean bolCountError
    def int intRowCount
    def String strRowCount
    def String strToLog
    def objRowCount
    def aryRowCount
    def strReturnSQL

    strSQL = "select count(*) as \"row_count\" from ${strTableName}"
    objRowCount = db_commands(query: strSQL, params: objMSSQLDBRS)

    if (objRowCount.containsKey("error") && objRowCount["error"] == true) {
        w(string: "ERROR on SQL: " + strSQL, style: "error");
        w(string: objRowCount["error_message"].toString(), style: "error");
        System.exit(0)
    }

    aryRowCount = objRowCount["data"]
    intRowCount = aryRowCount[0]["row_count"]

    strReturnSQL = "update tmp_TableCounts set row_count=${intRowCount} where table_name='${strTableName}';\n"

    return strReturnSQL
}

def count_all_table_rows = {
    def objPGDBRS = [:]
    def objMSSQLDBRS = [:]
    def strReturn
    objPGDBRS["database_jdbc_url"] = get_config_setting(setting: "PGDatabaseURL").toString()
    def strPGSchema = get_config_setting(setting: "PGDatabaseSchema").toString()

    def strSQL="select schemaname, tablename from pg_tables where schemaname='${strPGSchema}' order by tablename"

    // PROBLEM #2:
    // db_commands can't have any variables defined within the closure that are accessible outside the closure
    // (iow, all variables must be locally scoped)

    def objResults = db_commands(query: strSQL, params: objPGDBRS);

    if (objResults["error"] == true) {
        w(string: "ERROR on SQL: " + strSQL, style: "error");
        w(string: objResults["error_message"].toString(), style: "error");
        System.exit(0)
    }

    def aryTables = objResults["data"].table_name
    def strMSSQLSchema

    objMSSQLDBRS["database_jdbc_url"] = get_config_setting(setting: "MSSQLDatabaseURL").toString()

    GParsPool.withPool() {
        aryTables.eachParallel { def strTableName ->
            strReturn = count_table_rows(table_name: strTableName, params: objMSSQLDBRS, total: aryTables.size())
            strCommands += "$strReturn"
        }
    }

}