te')); return $arr; } /* 遍历用户所有主题 * @param $uid 用户ID * @param int $page 页数 * @param int $pagesize 每页记录条数 * @param bool $desc 排序方式 TRUE降序 FALSE升序 * @param string $key 返回的数组用那一列的值作为 key * @param array $col 查询哪些列 */ function thread_tid_find_by_uid($uid, $page = 1, $pagesize = 1000, $desc = TRUE, $key = 'tid', $col = array()) { if (empty($uid)) return array(); $orderby = TRUE == $desc ? -1 : 1; $arr = thread_tid__find($cond = array('uid' => $uid), array('tid' => $orderby), $page, $pagesize, $key, $col); return $arr; } // 遍历栏目下tid 支持数组 $fid = array(1,2,3) function thread_tid_find_by_fid($fid, $page = 1, $pagesize = 1000, $desc = TRUE) { if (empty($fid)) return array(); $orderby = TRUE == $desc ? -1 : 1; $arr = thread_tid__find($cond = array('fid' => $fid), array('tid' => $orderby), $page, $pagesize, 'tid', array('tid', 'verify_date')); return $arr; } function thread_tid_delete($tid) { if (empty($tid)) return FALSE; $r = thread_tid__delete(array('tid' => $tid)); return $r; } function thread_tid_count() { $n = thread_tid__count(); return $n; } // 统计用户主题数 大数量下严谨使用非主键统计 function thread_uid_count($uid) { $n = thread_tid__count(array('uid' => $uid)); return $n; } // 统计栏目主题数 大数量下严谨使用非主键统计 function thread_fid_count($fid) { $n = thread_tid__count(array('fid' => $fid)); return $n; } ?>workflow - How best to handle snakemake jobs where program produce the same file names - Stack Overflow

科技改变生活-雨落星辰 - 所有的伟大,都源于一个勇敢的开始

    最新消息：雨落星辰是一个专注网站SEO优化、网站SEO诊断、搜索引擎研究、网络营销推广、网站策划运营及站长类的自媒体原创博客
    你的位置： 首页>programmer>workflow - How best to handle snakemake jobs where program produce the same file names - Stack Overflow

    workflow - How best to handle snakemake jobs where program produce the same file names - Stack Overflow

    programmeradmin4浏览0评论

    I have an issue with a snakemake workflow. Due to limitations from a certain software, the output of one of my rule is always named SolFix.

    I handle this by moving the file to a different named file based on the output of my rule

    ... [*command to generate SolFix*]; mv SolFix {output.SolFix_ebv}

    Multiple rules within the same group {param.group} produces the SolFix file. Hence, there has been occassions where the SolFix file gets overwritten by parallel processes in the snakemake workflow before being moved.

    Until now, I have addressed some of these issues by running the program within subfolders.

    [mkdir -p subfolder_{params.breed}/rule_subfolder_{params.breed}]; [cd subfolder_{params.breed}/rule_subfolder_{params.breed}];[*command to generate SolFix*]; mv SolFix {output.SolFix_ebv}

    However, this is becoming too subdivided.

    I have also tried to limit these issues by creating dependencies based on what rules/jobs tend to finish around the same time. However, this only tends to serialize my workflow.

    Are there alternative solutions (that scale) I can explore to resolve this issue.

    I have an issue with a snakemake workflow. Due to limitations from a certain software, the output of one of my rule is always named SolFix.

    I handle this by moving the file to a different named file based on the output of my rule

    ... [*command to generate SolFix*]; mv SolFix {output.SolFix_ebv}

    Multiple rules within the same group {param.group} produces the SolFix file. Hence, there has been occassions where the SolFix file gets overwritten by parallel processes in the snakemake workflow before being moved.

    Until now, I have addressed some of these issues by running the program within subfolders.

    [mkdir -p subfolder_{params.breed}/rule_subfolder_{params.breed}]; [cd subfolder_{params.breed}/rule_subfolder_{params.breed}];[*command to generate SolFix*]; mv SolFix {output.SolFix_ebv}

    However, this is becoming too subdivided.

    I have also tried to limit these issues by creating dependencies based on what rules/jobs tend to finish around the same time. However, this only tends to serialize my workflow.

    Are there alternative solutions (that scale) I can explore to resolve this issue.

    • workflow
    • snakemake
    Share Improve this question edited 2 days ago oguz ismail 50.8k16 gold badges57 silver badges78 bronze badges asked 2 days ago Damilola DecarlsDamilola Decarls 295 bronze badges 1
    • I'll also go with sub-folders, can't think of a better way. Or mix-and-match, run a serialized workflow in each of several sub-folders, to more or less balance performance and awkwardness. – X Zhang Commented 2 days ago
    Add a comment  | 

    1 Answer 1

    Reset to default 0

    What person came along and down-voted this question?! I voted it up again. It's a common problem in Snakemake and it has a good answer - shadow rules:

    https://snakemake.readthedocs.io/en/stable/snakefiles/rules.html#shadow-rules

    Using shadow mode is not as mysterious as it may seem. It just does exactly what you are trying to do with subfolders but in a consistent and robust way. The "shadow" is just a temporary directory where the rule runs and makes whatever output, then Snakemake moves the output file back to the real working directory and deletes anything else. It's great for cleaning up temp files, and for resolving conflicts like you have.

    If you ever tried Nextflow, that system basically runs every step as a shadow rule.

    The short answer is, just add shadow: 'minimal' to your original rule (the simple version that did [*command to generate SolFix*]; mv SolFix {output.SolFix_ebv}) and then you should be golden. Let me know if you still have problems.

    与本文相关的文章

    发布评论

    评论列表(0)

    1. 暂无评论